• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3# Copyright (c) 2021 Huawei Device Co., Ltd.
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import os
17import sys
18import argparse
19import errno
20import json
21import datetime
22import http.client as client
23
24from http.server import BaseHTTPRequestHandler
25from http.server import HTTPServer
26
27PYCACHE_PORT = 7970  # Ascii code for 'yp'
28LOCALHOST = '127.0.0.1'
29DEBUG = int(os.environ.get('PRINT_BUILD_EXPLANATIONS', 0))
30
31
32class PycacheDaemonRequestHandler(BaseHTTPRequestHandler):
33    # Suppress logs
34    def log_message(self, format, *args):  # pylint: disable=redefined-builtin
35        if DEBUG:
36            super().log_message(format, *args)
37        else:
38            pass
39
40    def do_cache_hit(self):
41        self.server.hit_times += 1
42        self.send_response(200)
43
44    def do_cache_miss(self):
45        self.server.miss_times += 1
46        self.send_response(200)
47
48    def do_cache_manage(self):
49        self.send_response(200)
50        self.server.cache_manage()
51
52    def do_show_statistics(self):
53        self.send_response(200)
54        self.server.show_statistics()
55
56    def do_stop_service(self):
57        self.send_response(200)
58        self.server.stop_service = True
59
60
61class PycacheDaemon(HTTPServer):
62    def __init__(self, *args, **kargs):
63        self.hit_times = 0
64        self.miss_times = 0
65        self.stop_service = False
66        self.pycache_dir = None
67        self.pycache_config_file = None
68        super().__init__(*args, **kargs)
69
70    def serve_forever(self, poll_interval=0.5):
71        while not self.stop_service:
72            self.handle_request()
73        os.unlink(self.pycache_config_file)
74
75    def record_pycache_config(self, pycache_dir):
76        root = os.path.realpath(pycache_dir)
77        self.pycache_dir = root
78        self.pycache_config_file = os.path.join(root, '.config')
79        os.makedirs(root, exist_ok=True)
80        host, port = self.server_address[:2]
81        config = {
82            'root': root,
83            'config_file': self.pycache_config_file,
84            'debug': bool(DEBUG),
85            'host': host,
86            'port': port,
87        }
88        with open(self.pycache_config_file, 'w') as jsonfile:
89            json.dump(config, jsonfile, indent=2, sort_keys=True)
90
91    def cache_manage(self):
92        now = datetime.datetime.now()
93        days = 15
94        earlier_time = (now - datetime.timedelta(days)).timestamp()
95        # Pycache pool holds as much as 40GB  or as long as 15 days
96        # cache objects
97        while days > 0:
98            disk_usage = 0
99            for root, _, files in os.walk(self.pycache_dir):
100                for file in files:
101                    path = os.path.join(root, file)
102                    stat = os.stat(path)
103                    if stat.st_atime < int(earlier_time):
104                        os.unlink(path)
105                    else:
106                        disk_usage += stat.st_size
107            if disk_usage >= 40 * 1024 * 1024 * 1024:
108                days -= 1
109                earlier_time = (now - datetime.timedelta(days)).timestamp()
110            else:
111                break
112
113    def show_statistics(self):
114        actions = self.hit_times + self.miss_times
115        if actions != 0:
116            print('-' * 80)
117            print('pycache statistics:')
118            print('pycache hit targets: {}'.format(self.hit_times))
119            print('pycache miss targets: {}'.format(self.miss_times))
120            hit_rate = float(self.hit_times) / actions * 100
121            miss_rate = float(self.miss_times) / actions * 100
122            print('pycache hit rate: {:.2f}%'.format(hit_rate))
123            print('pycache miss rate: {:.2f}%'.format(miss_rate))
124            print('-' * 80)
125        else:
126            print('-' * 80)
127            print('No pycache actions in pycache, skip statistics')
128            print('-' * 80)
129
130
131def start_server(host, port, root):
132    if root is None:
133        print('Warning: missing pycache root directory')
134        return
135    server_address = (host, port)
136    try:
137        pyd = PycacheDaemon(server_address, PycacheDaemonRequestHandler)
138        print('Starting pycache daemon at {}:{}'.format(host, port))
139        pyd.record_pycache_config(root)
140        pyd.serve_forever()
141    except OSError as err:
142        if err.errno == errno.EADDRINUSE:
143            start_server(host, port + 2, root)
144        else:
145            print('Warning: Failed to start pycache daemon process')
146
147
148def get_pyd():
149    cache_dir = os.environ.get('PYCACHE_DIR')
150    daemon_config_file = '{}/.config'.format(cache_dir)
151    if not os.path.exists(daemon_config_file):
152        raise Exception('Warning: {} not exists'.format(daemon_config_file))
153    with open(daemon_config_file, 'r') as jsonfile:
154        data = json.load(jsonfile)
155        return data.get('host'), data.get('port')
156
157
158def stop_server():
159    try:
160        host, port = get_pyd()
161        conn = client.HTTPConnection(host, port)
162        conn.request('stop_service', '/')
163        conn.close()
164    except:  # noqa: E722 pylint: disable=bare-except
165        pass
166
167
168def show_statistics():
169    try:
170        host, port = get_pyd()
171        conn = client.HTTPConnection(host, port)
172        conn.request('show_statistics', '/')
173        conn.close()
174    except:  # noqa: E722 pylint: disable=bare-except
175        pass
176
177
178def manage_cache_contents():
179    try:
180        host, port = get_pyd()
181        conn = client.HTTPConnection(host, port)
182        conn.request('cache_manage', '/')
183        conn.close()
184    except:  # noqa: E722 pylint: disable=bare-except
185        pass
186
187
188def main(args):
189    parser = argparse.ArgumentParser()
190    parser.add_argument('--root', help='path to pycache root directory')
191    parser.add_argument('--port',
192                        default=PYCACHE_PORT,
193                        help='which port to listen')
194    parser.add_argument('--start',
195                        action='store_true',
196                        help='start daemon process for pycache')
197    parser.add_argument('--stop',
198                        action='store_true',
199                        help='stop pycache daemon process')
200    parser.add_argument('--stat',
201                        action='store_true',
202                        help='report cache statistics')
203    parser.add_argument('--manage',
204                        action='store_true',
205                        help='manage pycache contents')
206
207    options = parser.parse_args(args)
208    if options.start:
209        start_server(LOCALHOST, int(options.port), options.root)
210    if options.stop:
211        stop_server()
212    if options.stat:
213        show_statistics()
214    if options.manage:
215        manage_cache_contents()
216
217
218if __name__ == '__main__':
219    main(sys.argv[1:])
220