#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (c) 2021 Huawei Device Co., Ltd.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import argparse
import errno
import json
import datetime
import http.client as client
from http.server import BaseHTTPRequestHandler
from http.server import HTTPServer
PYCACHE_PORT = 7970 # Ascii code for 'yp'
LOCALHOST = '127.0.0.1'
DEBUG = int(os.environ.get('PRINT_BUILD_EXPLANATIONS', 0))
class PycacheDaemonRequestHandler(BaseHTTPRequestHandler):
# Suppress logs
def log_message(self, format, *args): # pylint: disable=redefined-builtin
if DEBUG:
super().log_message(format, *args)
else:
pass
def do_cache_hit(self):
self.server.hit_times += 1
self.send_response(200)
def do_cache_miss(self):
self.server.miss_times += 1
self.send_response(200)
def do_cache_manage(self):
self.send_response(200)
self.server.cache_manage()
def do_show_statistics(self):
self.send_response(200)
self.server.show_statistics()
def do_stop_service(self):
self.send_response(200)
self.server.stop_service = True
class PycacheDaemon(HTTPServer):
def __init__(self, *args, **kargs):
self.hit_times = 0
self.miss_times = 0
self.stop_service = False
self.pycache_dir = None
self.pycache_config_file = None
super().__init__(*args, **kargs)
def serve_forever(self, poll_interval=0.5):
while not self.stop_service:
self.handle_request()
os.unlink(self.pycache_config_file)
def record_pycache_config(self, pycache_dir):
root = os.path.realpath(pycache_dir)
self.pycache_dir = root
self.pycache_config_file = os.path.join(root, '.config')
os.makedirs(root, exist_ok=True)
host, port = self.server_address[:2]
config = {
'root': root,
'config_file': self.pycache_config_file,
'debug': bool(DEBUG),
'host': host,
'port': port,
}
with open(self.pycache_config_file, 'w') as jsonfile:
json.dump(config, jsonfile, indent=2, sort_keys=True)
def cache_manage(self):
now = datetime.datetime.now()
days = 15
earlier_time = (now - datetime.timedelta(days)).timestamp()
# Pycache pool holds as much as 40GB or as long as 15 days
# cache objects
while days > 0:
disk_usage = 0
for root, _, files in os.walk(self.pycache_dir):
for file in files:
path = os.path.join(root, file)
stat = os.stat(path)
if stat.st_atime < int(earlier_time):
os.unlink(path)
else:
disk_usage += stat.st_size
if disk_usage >= 40 * 1024 * 1024 * 1024:
days -= 1
earlier_time = (now - datetime.timedelta(days)).timestamp()
else:
break
def show_statistics(self):
actions = self.hit_times + self.miss_times
if actions != 0:
print('-' * 80)
print('pycache statistics:')
print('pycache hit targets: {}'.format(self.hit_times))
print('pycache miss targets: {}'.format(self.miss_times))
hit_rate = float(self.hit_times) / actions * 100
miss_rate = float(self.miss_times) / actions * 100
print('pycache hit rate: {:.2f}%'.format(hit_rate))
print('pycache miss rate: {:.2f}%'.format(miss_rate))
print('-' * 80)
else:
print('-' * 80)
print('No pycache actions in pycache, skip statistics')
print('-' * 80)
def start_server(host, port, root):
if root is None:
print('Warning: missing pycache root directory')
return
server_address = (host, port)
try:
pyd = PycacheDaemon(server_address, PycacheDaemonRequestHandler)
print('Starting pycache daemon at {}:{}'.format(host, port))
pyd.record_pycache_config(root)
pyd.serve_forever()
except OSError as err:
if err.errno == errno.EADDRINUSE:
start_server(host, port + 2, root)
else:
print('Warning: Failed to start pycache daemon process')
def get_pyd():
cache_dir = os.environ.get('PYCACHE_DIR')
daemon_config_file = '{}/.config'.format(cache_dir)
if not os.path.exists(daemon_config_file):
raise Exception('Warning: {} not exists'.format(daemon_config_file))
with open(daemon_config_file, 'r') as jsonfile:
data = json.load(jsonfile)
return data.get('host'), data.get('port')
def stop_server():
try:
host, port = get_pyd()
conn = client.HTTPConnection(host, port)
conn.request('stop_service', '/')
conn.close()
except: # noqa: E722 pylint: disable=bare-except
pass
def show_statistics():
try:
host, port = get_pyd()
conn = client.HTTPConnection(host, port)
conn.request('show_statistics', '/')
conn.close()
except: # noqa: E722 pylint: disable=bare-except
pass
def manage_cache_contents():
try:
host, port = get_pyd()
conn = client.HTTPConnection(host, port)
conn.request('cache_manage', '/')
conn.close()
except: # noqa: E722 pylint: disable=bare-except
pass
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('--root', help='path to pycache root directory')
parser.add_argument('--port',
default=PYCACHE_PORT,
help='which port to listen')
parser.add_argument('--start',
action='store_true',
help='start daemon process for pycache')
parser.add_argument('--stop',
action='store_true',
help='stop pycache daemon process')
parser.add_argument('--stat',
action='store_true',
help='report cache statistics')
parser.add_argument('--manage',
action='store_true',
help='manage pycache contents')
options = parser.parse_args(args)
if options.start:
start_server(LOCALHOST, int(options.port), options.root)
if options.stop:
stop_server()
if options.stat:
show_statistics()
if options.manage:
manage_cache_contents()
if __name__ == '__main__':
main(sys.argv[1:])