#!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright (c) 2021 Huawei Device Co., Ltd. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys import argparse import errno import json import datetime import http.client as client from http.server import BaseHTTPRequestHandler from http.server import HTTPServer PYCACHE_PORT = 7970 # Ascii code for 'yp' LOCALHOST = '127.0.0.1' DEBUG = int(os.environ.get('PRINT_BUILD_EXPLANATIONS', 0)) class PycacheDaemonRequestHandler(BaseHTTPRequestHandler): # Suppress logs def log_message(self, format, *args): # pylint: disable=redefined-builtin if DEBUG: super().log_message(format, *args) else: pass def do_cache_hit(self): self.server.hit_times += 1 self.send_response(200) def do_cache_miss(self): self.server.miss_times += 1 self.send_response(200) def do_cache_manage(self): self.send_response(200) self.server.cache_manage() def do_show_statistics(self): self.send_response(200) self.server.show_statistics() def do_stop_service(self): self.send_response(200) self.server.stop_service = True class PycacheDaemon(HTTPServer): def __init__(self, *args, **kargs): self.hit_times = 0 self.miss_times = 0 self.stop_service = False self.pycache_dir = None self.pycache_config_file = None super().__init__(*args, **kargs) def serve_forever(self, poll_interval=0.5): while not self.stop_service: self.handle_request() os.unlink(self.pycache_config_file) def record_pycache_config(self, pycache_dir): root = os.path.realpath(pycache_dir) self.pycache_dir = root self.pycache_config_file = os.path.join(root, '.config') os.makedirs(root, exist_ok=True) host, port = self.server_address[:2] config = { 'root': root, 'config_file': self.pycache_config_file, 'debug': bool(DEBUG), 'host': host, 'port': port, } with open(self.pycache_config_file, 'w') as jsonfile: json.dump(config, jsonfile, indent=2, sort_keys=True) def cache_manage(self): now = datetime.datetime.now() days = 15 earlier_time = (now - datetime.timedelta(days)).timestamp() # Pycache pool holds as much as 40GB or as long as 15 days # cache objects while days > 0: disk_usage = 0 for root, _, files in os.walk(self.pycache_dir): for file in files: path = os.path.join(root, file) stat = os.stat(path) if stat.st_atime < int(earlier_time): os.unlink(path) else: disk_usage += stat.st_size if disk_usage >= 40 * 1024 * 1024 * 1024: days -= 1 earlier_time = (now - datetime.timedelta(days)).timestamp() else: break def show_statistics(self): actions = self.hit_times + self.miss_times if actions != 0: print('-' * 80) print('pycache statistics:') print('pycache hit targets: {}'.format(self.hit_times)) print('pycache miss targets: {}'.format(self.miss_times)) hit_rate = float(self.hit_times) / actions * 100 miss_rate = float(self.miss_times) / actions * 100 print('pycache hit rate: {:.2f}%'.format(hit_rate)) print('pycache miss rate: {:.2f}%'.format(miss_rate)) print('-' * 80) else: print('-' * 80) print('No pycache actions in pycache, skip statistics') print('-' * 80) def start_server(host, port, root): if root is None: print('Warning: missing pycache root directory') return server_address = (host, port) try: pyd = PycacheDaemon(server_address, PycacheDaemonRequestHandler) print('Starting pycache daemon at {}:{}'.format(host, port)) pyd.record_pycache_config(root) pyd.serve_forever() except OSError as err: if err.errno == errno.EADDRINUSE: start_server(host, port + 2, root) else: print('Warning: Failed to start pycache daemon process') def get_pyd(): cache_dir = os.environ.get('PYCACHE_DIR') daemon_config_file = '{}/.config'.format(cache_dir) if not os.path.exists(daemon_config_file): raise Exception('Warning: {} not exists'.format(daemon_config_file)) with open(daemon_config_file, 'r') as jsonfile: data = json.load(jsonfile) return data.get('host'), data.get('port') def stop_server(): try: host, port = get_pyd() conn = client.HTTPConnection(host, port) conn.request('stop_service', '/') conn.close() except: # noqa: E722 pylint: disable=bare-except pass def show_statistics(): try: host, port = get_pyd() conn = client.HTTPConnection(host, port) conn.request('show_statistics', '/') conn.close() except: # noqa: E722 pylint: disable=bare-except pass def manage_cache_contents(): try: host, port = get_pyd() conn = client.HTTPConnection(host, port) conn.request('cache_manage', '/') conn.close() except: # noqa: E722 pylint: disable=bare-except pass def main(args): parser = argparse.ArgumentParser() parser.add_argument('--root', help='path to pycache root directory') parser.add_argument('--port', default=PYCACHE_PORT, help='which port to listen') parser.add_argument('--start', action='store_true', help='start daemon process for pycache') parser.add_argument('--stop', action='store_true', help='stop pycache daemon process') parser.add_argument('--stat', action='store_true', help='report cache statistics') parser.add_argument('--manage', action='store_true', help='manage pycache contents') options = parser.parse_args(args) if options.start: start_server(LOCALHOST, int(options.port), options.root) if options.stop: stop_server() if options.stat: show_statistics() if options.manage: manage_cache_contents() if __name__ == '__main__': main(sys.argv[1:])