• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2
3import contextlib
4import logging
5import os
6import re
7import shutil
8import sys
9import subprocess
10
11from datetime import datetime, timedelta
12from io import BytesIO
13from threading import Lock, Timer
14
15from watchdog.events import FileSystemEventHandler
16from watchdog.observers import Observer
17
18from http import HTTPStatus
19from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler
20
21CONFIG_FILE = 'serve_header.yml'
22MAKEFILE = 'Makefile'
23INCLUDE = 'include/nlohmann/'
24SINGLE_INCLUDE = 'single_include/nlohmann/'
25HEADER = 'json.hpp'
26
27DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S'
28
29JSON_VERSION_RE = re.compile(r'\s*#\s*define\s+NLOHMANN_JSON_VERSION_MAJOR\s+')
30
31class ExitHandler(logging.StreamHandler):
32    def __init__(self, level):
33        """."""
34        super().__init__()
35        self.level = level
36
37    def emit(self, record):
38        if record.levelno >= self.level:
39            sys.exit(1)
40
41def is_project_root(test_dir='.'):
42    makefile = os.path.join(test_dir, MAKEFILE)
43    include = os.path.join(test_dir, INCLUDE)
44    single_include = os.path.join(test_dir, SINGLE_INCLUDE)
45
46    return (os.path.exists(makefile)
47            and os.path.isfile(makefile)
48            and os.path.exists(include)
49            and os.path.exists(single_include))
50
51class DirectoryEventBucket:
52    def __init__(self, callback, delay=1.2, threshold=0.8):
53        """."""
54        self.delay = delay
55        self.threshold = timedelta(seconds=threshold)
56        self.callback = callback
57        self.event_dirs = set([])
58        self.timer = None
59        self.lock = Lock()
60
61    def start_timer(self):
62        if self.timer is None:
63            self.timer = Timer(self.delay, self.process_dirs)
64            self.timer.start()
65
66    def process_dirs(self):
67        result_dirs = []
68        event_dirs = set([])
69        with self.lock:
70            self.timer = None
71            while self.event_dirs:
72                time, event_dir = self.event_dirs.pop()
73                delta = datetime.now() - time
74                if delta < self.threshold:
75                    event_dirs.add((time, event_dir))
76                else:
77                    result_dirs.append(event_dir)
78            self.event_dirs = event_dirs
79            if result_dirs:
80                self.callback(os.path.commonpath(result_dirs))
81            if self.event_dirs:
82                self.start_timer()
83
84    def add_dir(self, path):
85        with self.lock:
86            # add path to the set of event_dirs if it is not a sibling of
87            # a directory already in the set
88            if not any(os.path.commonpath([path, event_dir]) == event_dir
89               for (_, event_dir) in self.event_dirs):
90                self.event_dirs.add((datetime.now(), path))
91                if self.timer is None:
92                    self.start_timer()
93
94class WorkTree:
95    make_command = 'make'
96
97    def __init__(self, root_dir, tree_dir):
98        """."""
99        self.root_dir = root_dir
100        self.tree_dir = tree_dir
101        self.rel_dir = os.path.relpath(tree_dir, root_dir)
102        self.name = os.path.basename(tree_dir)
103        self.include_dir = os.path.abspath(os.path.join(tree_dir, INCLUDE))
104        self.header = os.path.abspath(os.path.join(tree_dir, SINGLE_INCLUDE, HEADER))
105        self.rel_header = os.path.relpath(self.header, root_dir)
106        self.dirty = True
107        self.build_count = 0
108        t = os.path.getmtime(self.header)
109        t = datetime.fromtimestamp(t)
110        self.build_time = t.strftime(DATETIME_FORMAT)
111
112    def __hash__(self):
113        """."""
114        return hash((self.tree_dir))
115
116    def __eq__(self, other):
117        """."""
118        if not isinstance(other, type(self)):
119            return NotImplemented
120        return self.tree_dir == other.tree_dir
121
122    def update_dirty(self, path):
123        if self.dirty:
124            return
125
126        path = os.path.abspath(path)
127        if os.path.commonpath([path, self.include_dir]) == self.include_dir:
128            logging.info(f'{self.name}: working tree marked dirty')
129            self.dirty = True
130
131    def amalgamate_header(self):
132        if not self.dirty:
133            return
134
135        mtime = os.path.getmtime(self.header)
136        subprocess.run([WorkTree.make_command, 'amalgamate'], cwd=self.tree_dir,
137                       stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
138        if mtime == os.path.getmtime(self.header):
139            logging.info(f'{self.name}: no changes')
140        else:
141            self.build_count += 1
142            self.build_time = datetime.now().strftime(DATETIME_FORMAT)
143            logging.info(f'{self.name}: header amalgamated (build count {self.build_count})')
144
145        self.dirty = False
146
147class WorkTrees(FileSystemEventHandler):
148    def __init__(self, root_dir):
149        """."""
150        super().__init__()
151        self.root_dir = root_dir
152        self.trees = set([])
153        self.tree_lock = Lock()
154        self.scan(root_dir)
155        self.created_bucket = DirectoryEventBucket(self.scan)
156        self.observer = Observer()
157        self.observer.schedule(self, root_dir, recursive=True)
158        self.observer.start()
159
160    def scan(self, base_dir):
161        scan_dirs = set([base_dir])
162        # recursively scan base_dir for working trees
163
164        while scan_dirs:
165            scan_dir = os.path.abspath(scan_dirs.pop())
166            self.scan_tree(scan_dir)
167            try:
168                with os.scandir(scan_dir) as dir_it:
169                    for entry in dir_it:
170                        if entry.is_dir():
171                            scan_dirs.add(entry.path)
172            except FileNotFoundError as e:
173                logging.debug('path disappeared: %s', e)
174
175    def scan_tree(self, scan_dir):
176        if not is_project_root(scan_dir):
177            return
178
179        # skip source trees in build directories
180        # this check could be enhanced
181        if scan_dir.endswith('/_deps/json-src'):
182            return
183
184        tree = WorkTree(self.root_dir, scan_dir)
185        with self.tree_lock:
186            if not tree in self.trees:
187                if tree.name == tree.rel_dir:
188                    logging.info(f'adding working tree {tree.name}')
189                else:
190                    logging.info(f'adding working tree {tree.name} at {tree.rel_dir}')
191                url = os.path.join('/', tree.rel_dir, HEADER)
192                logging.info(f'{tree.name}: serving header at {url}')
193                self.trees.add(tree)
194
195    def rescan(self, path=None):
196        if path is not None:
197            path = os.path.abspath(path)
198        trees = set([])
199        # check if any working trees have been removed
200        with self.tree_lock:
201            while self.trees:
202                tree = self.trees.pop()
203                if ((path is None
204                    or os.path.commonpath([path, tree.tree_dir]) == tree.tree_dir)
205                    and not is_project_root(tree.tree_dir)):
206                    if tree.name == tree.rel_dir:
207                        logging.info(f'removing working tree {tree.name}')
208                    else:
209                        logging.info(f'removing working tree {tree.name} at {tree.rel_dir}')
210                else:
211                    trees.add(tree)
212            self.trees = trees
213
214    def find(self, path):
215        # find working tree for a given header file path
216        path = os.path.abspath(path)
217        with self.tree_lock:
218            for tree in self.trees:
219                if path == tree.header:
220                    return tree
221        return None
222
223    def on_any_event(self, event):
224        logging.debug('%s (is_dir=%s): %s', event.event_type,
225                      event.is_directory, event.src_path)
226        path = os.path.abspath(event.src_path)
227        if event.is_directory:
228            if event.event_type == 'created':
229                # check for new working trees
230                self.created_bucket.add_dir(path)
231            elif event.event_type == 'deleted':
232                # check for deleted working trees
233                self.rescan(path)
234        elif event.event_type == 'closed':
235            with self.tree_lock:
236                for tree in self.trees:
237                    tree.update_dirty(path)
238
239    def stop(self):
240        self.observer.stop()
241        self.observer.join()
242
243class HeaderRequestHandler(SimpleHTTPRequestHandler): # lgtm[py/missing-call-to-init]
244    def __init__(self, request, client_address, server):
245        """."""
246        self.worktrees = server.worktrees
247        self.worktree = None
248        try:
249            super().__init__(request, client_address, server,
250                             directory=server.worktrees.root_dir)
251        except ConnectionResetError:
252            logging.debug('connection reset by peer')
253
254    def translate_path(self, path):
255        path = os.path.abspath(super().translate_path(path))
256
257        # add single_include/nlohmann into path, if needed
258        header = os.path.join('/', HEADER)
259        header_path = os.path.join('/', SINGLE_INCLUDE, HEADER)
260        if (path.endswith(header)
261            and not path.endswith(header_path)):
262            path = os.path.join(os.path.dirname(path), SINGLE_INCLUDE, HEADER)
263
264        return path
265
266    def send_head(self):
267        # check if the translated path matches a working tree
268        # and fullfill the request; otherwise, send 404
269        path = self.translate_path(self.path)
270        self.worktree = self.worktrees.find(path)
271        if self.worktree is not None:
272            self.worktree.amalgamate_header()
273            logging.info(f'{self.worktree.name}; serving header (build count {self.worktree.build_count})')
274            return super().send_head()
275        logging.info(f'invalid request path: {self.path}')
276        super().send_error(HTTPStatus.NOT_FOUND, 'Not Found')
277        return None
278
279    def send_header(self, keyword, value):
280        # intercept Content-Length header; sent in copyfile later
281        if keyword == 'Content-Length':
282            return
283        super().send_header(keyword, value)
284
285    def end_headers (self):
286        # intercept; called in copyfile() or indirectly
287        # by send_head via super().send_error()
288        pass
289
290    def copyfile(self, source, outputfile):
291        injected = False
292        content = BytesIO()
293        length = 0
294        # inject build count and time into served header
295        for line in source:
296            line = line.decode('utf-8')
297            if not injected and JSON_VERSION_RE.match(line):
298                length += content.write(bytes('#define JSON_BUILD_COUNT '\
299                                              f'{self.worktree.build_count}\n', 'utf-8'))
300                length += content.write(bytes('#define JSON_BUILD_TIME '\
301                                              f'"{self.worktree.build_time}"\n\n', 'utf-8'))
302                injected = True
303            length += content.write(bytes(line, 'utf-8'))
304
305        # set content length
306        super().send_header('Content-Length', length)
307        # CORS header
308        self.send_header('Access-Control-Allow-Origin', '*')
309        # prevent caching
310        self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate')
311        self.send_header('Pragma', 'no-cache')
312        self.send_header('Expires', '0')
313        super().end_headers()
314
315        # send the header
316        content.seek(0)
317        shutil.copyfileobj(content, outputfile)
318
319    def log_message(self, format, *args):
320        pass
321
322class DualStackServer(ThreadingHTTPServer):
323    def __init__(self, addr, worktrees):
324        """."""
325        self.worktrees = worktrees
326        super().__init__(addr, HeaderRequestHandler)
327
328    def server_bind(self):
329        # suppress exception when protocol is IPv4
330        with contextlib.suppress(Exception):
331            self.socket.setsockopt(
332                socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
333        return super().server_bind()
334
335if __name__ == '__main__':
336    import argparse
337    import ssl
338    import socket
339    import yaml
340
341    # exit code
342    ec = 0
343
344    # setup logging
345    logging.basicConfig(format='[%(asctime)s] %(levelname)s: %(message)s',
346                        datefmt=DATETIME_FORMAT, level=logging.INFO)
347    log = logging.getLogger()
348    log.addHandler(ExitHandler(logging.ERROR))
349
350    # parse command line arguments
351    parser = argparse.ArgumentParser()
352    parser.add_argument('--make', default='make',
353                        help='the make command (default: make)')
354    args = parser.parse_args()
355
356    # propagate the make command to use for amalgamating headers
357    WorkTree.make_command = args.make
358
359    worktrees = None
360    try:
361        # change working directory to project root
362        os.chdir(os.path.realpath(os.path.join(sys.path[0], '../../')))
363
364        if not is_project_root():
365            log.error('working directory does not look like project root')
366
367        # load config
368        config = {}
369        config_file = os.path.abspath(CONFIG_FILE)
370        try:
371            with open(config_file, 'r') as f:
372                config = yaml.safe_load(f)
373        except FileNotFoundError:
374            log.info(f'cannot find configuration file: {config_file}')
375            log.info('using default configuration')
376
377        # find and monitor working trees
378        worktrees = WorkTrees(config.get('root', '.'))
379
380        # start web server
381        infos = socket.getaddrinfo(config.get('bind', None), config.get('port', 8443),
382                                   type=socket.SOCK_STREAM, flags=socket.AI_PASSIVE)
383        DualStackServer.address_family = infos[0][0]
384        HeaderRequestHandler.protocol_version = 'HTTP/1.0'
385        with DualStackServer(infos[0][4], worktrees) as httpd:
386            scheme = 'HTTP'
387            https = config.get('https', {})
388            if https.get('enabled', True):
389                cert_file = https.get('cert_file', 'localhost.pem')
390                key_file = https.get('key_file', 'localhost-key.pem')
391                ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
392                ssl_ctx.minimum_version = ssl.TLSVersion.TLSv1_2
393                ssl_ctx.maximum_version = ssl.TLSVersion.MAXIMUM_SUPPORTED
394                ssl_ctx.load_cert_chain(cert_file, key_file)
395                httpd.socket = ssl_ctx.wrap_socket(httpd.socket, server_side=True)
396                scheme = 'HTTPS'
397            host, port = httpd.socket.getsockname()[:2]
398            log.info(f'serving {scheme} on {host} port {port}')
399            log.info('press Ctrl+C to exit')
400            httpd.serve_forever()
401
402    except KeyboardInterrupt:
403        log.info('exiting')
404    except Exception:
405        ec = 1
406        log.exception('an error occurred:')
407    finally:
408        if worktrees is not None:
409            worktrees.stop()
410            sys.exit(ec)
411