1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 # Copyright (c) 2021 Huawei Device Co., Ltd. 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 16 import os 17 import sys 18 import argparse 19 import errno 20 import json 21 import datetime 22 import http.client as client 23 24 from http.server import BaseHTTPRequestHandler 25 from http.server import HTTPServer 26 27 PYCACHE_PORT = 7970 # Ascii code for 'yp' 28 LOCALHOST = '127.0.0.1' 29 DEBUG = int(os.environ.get('PRINT_BUILD_EXPLANATIONS', 0)) 30 31 32 class PycacheDaemonRequestHandler(BaseHTTPRequestHandler): 33 # Suppress logs 34 def log_message(self, format, *args): # pylint: disable=redefined-builtin 35 if DEBUG: 36 super().log_message(format, *args) 37 else: 38 pass 39 40 def do_cache_hit(self): 41 self.server.hit_times += 1 42 self.send_response(200) 43 44 def do_cache_miss(self): 45 self.server.miss_times += 1 46 self.send_response(200) 47 48 def do_cache_manage(self): 49 self.send_response(200) 50 self.server.cache_manage() 51 52 def do_show_statistics(self): 53 self.send_response(200) 54 self.server.show_statistics() 55 56 def do_stop_service(self): 57 self.send_response(200) 58 self.server.stop_service = True 59 60 61 class PycacheDaemon(HTTPServer): 62 def __init__(self, *args, **kargs): 63 self.hit_times = 0 64 self.miss_times = 0 65 self.stop_service = False 66 self.pycache_dir = None 67 self.pycache_config_file = None 68 super().__init__(*args, **kargs) 69 70 def serve_forever(self, poll_interval=0.5): 71 while not self.stop_service: 72 self.handle_request() 73 os.unlink(self.pycache_config_file) 74 75 def record_pycache_config(self, pycache_dir): 76 root = os.path.realpath(pycache_dir) 77 self.pycache_dir = root 78 self.pycache_config_file = os.path.join(root, '.config') 79 os.makedirs(root, exist_ok=True) 80 host, port = self.server_address[:2] 81 config = { 82 'root': root, 83 'config_file': self.pycache_config_file, 84 'debug': bool(DEBUG), 85 'host': host, 86 'port': port, 87 } 88 with open(self.pycache_config_file, 'w') as jsonfile: 89 json.dump(config, jsonfile, indent=2, sort_keys=True) 90 91 def cache_manage(self): 92 now = datetime.datetime.now() 93 days = 15 94 earlier_time = (now - datetime.timedelta(days)).timestamp() 95 # Pycache pool holds as much as 40GB or as long as 15 days 96 # cache objects 97 while days > 0: 98 disk_usage = 0 99 for root, _, files in os.walk(self.pycache_dir): 100 for file in files: 101 path = os.path.join(root, file) 102 stat = os.stat(path) 103 if stat.st_atime < int(earlier_time): 104 os.unlink(path) 105 else: 106 disk_usage += stat.st_size 107 if disk_usage >= 40 * 1024 * 1024 * 1024: 108 days -= 1 109 earlier_time = (now - datetime.timedelta(days)).timestamp() 110 else: 111 break 112 113 def show_statistics(self): 114 actions = self.hit_times + self.miss_times 115 if actions != 0: 116 print('-' * 80) 117 print('pycache statistics:') 118 print('pycache hit targets: {}'.format(self.hit_times)) 119 print('pycache miss targets: {}'.format(self.miss_times)) 120 hit_rate = float(self.hit_times) / actions * 100 121 miss_rate = float(self.miss_times) / actions * 100 122 print('pycache hit rate: {:.2f}%'.format(hit_rate)) 123 print('pycache miss rate: {:.2f}%'.format(miss_rate)) 124 print('-' * 80) 125 else: 126 print('-' * 80) 127 print('No pycache actions in pycache, skip statistics') 128 print('-' * 80) 129 130 131 def start_server(host, port, root): 132 if root is None: 133 print('Warning: missing pycache root directory') 134 return 135 server_address = (host, port) 136 try: 137 pyd = PycacheDaemon(server_address, PycacheDaemonRequestHandler) 138 print('Starting pycache daemon at {}:{}'.format(host, port)) 139 pyd.record_pycache_config(root) 140 pyd.serve_forever() 141 except OSError as err: 142 if err.errno == errno.EADDRINUSE: 143 start_server(host, port + 2, root) 144 else: 145 print('Warning: Failed to start pycache daemon process') 146 147 148 def get_pyd(): 149 cache_dir = os.environ.get('PYCACHE_DIR') 150 daemon_config_file = '{}/.config'.format(cache_dir) 151 if not os.path.exists(daemon_config_file): 152 raise Exception('Warning: {} not exists'.format(daemon_config_file)) 153 with open(daemon_config_file, 'r') as jsonfile: 154 data = json.load(jsonfile) 155 return data.get('host'), data.get('port') 156 157 158 def stop_server(): 159 try: 160 host, port = get_pyd() 161 conn = client.HTTPConnection(host, port) 162 conn.request('stop_service', '/') 163 conn.close() 164 except: # noqa: E722 pylint: disable=bare-except 165 pass 166 167 168 def show_statistics(): 169 try: 170 host, port = get_pyd() 171 conn = client.HTTPConnection(host, port) 172 conn.request('show_statistics', '/') 173 conn.close() 174 except: # noqa: E722 pylint: disable=bare-except 175 pass 176 177 178 def manage_cache_contents(): 179 try: 180 host, port = get_pyd() 181 conn = client.HTTPConnection(host, port) 182 conn.request('cache_manage', '/') 183 conn.close() 184 except: # noqa: E722 pylint: disable=bare-except 185 pass 186 187 188 def main(args): 189 parser = argparse.ArgumentParser() 190 parser.add_argument('--root', help='path to pycache root directory') 191 parser.add_argument('--port', 192 default=PYCACHE_PORT, 193 help='which port to listen') 194 parser.add_argument('--start', 195 action='store_true', 196 help='start daemon process for pycache') 197 parser.add_argument('--stop', 198 action='store_true', 199 help='stop pycache daemon process') 200 parser.add_argument('--stat', 201 action='store_true', 202 help='report cache statistics') 203 parser.add_argument('--manage', 204 action='store_true', 205 help='manage pycache contents') 206 207 options = parser.parse_args(args) 208 if options.start: 209 start_server(LOCALHOST, int(options.port), options.root) 210 if options.stop: 211 stop_server() 212 if options.stat: 213 show_statistics() 214 if options.manage: 215 manage_cache_contents() 216 217 218 if __name__ == '__main__': 219 main(sys.argv[1:]) 220