1#!/usr/bin/env python 2 3# Copyright JS Foundation and other contributors, http://js.foundation 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16from __future__ import print_function 17 18import argparse 19import fnmatch 20import logging 21import os 22import re 23import sys 24 25 26class SourceMerger(object): 27 # pylint: disable=too-many-instance-attributes 28 29 _RE_INCLUDE = re.compile(r'\s*#include ("|<)(.*?)("|>)\n$') 30 31 def __init__(self, h_files, extra_includes=None, remove_includes=None, add_lineinfo=False): 32 self._log = logging.getLogger('sourcemerger') 33 self._last_builtin = None 34 self._processed = [] 35 self._output = [] 36 self._h_files = h_files 37 self._extra_includes = extra_includes or [] 38 self._remove_includes = remove_includes 39 self._add_lineinfo = add_lineinfo 40 # The copyright will be loaded from the first input file 41 self._copyright = {'lines': [], 'loaded': False} 42 43 def _process_non_include(self, line, file_level): 44 # Special case #2: Builtin include header name usage 45 if line.strip() == "#include BUILTIN_INC_HEADER_NAME": 46 assert self._last_builtin is not None, 'No previous BUILTIN_INC_HEADER_NAME definition' 47 self._log.debug('[%d] Detected usage of BUILTIN_INC_HEADER_NAME, including: %s', 48 file_level, self._last_builtin) 49 self.add_file(self._h_files[self._last_builtin]) 50 # return from the function as we have processed the included file 51 return 52 53 # Special case #1: Builtin include header name definition 54 if line.startswith('#define BUILTIN_INC_HEADER_NAME '): 55 # the line is in this format: #define BUILTIN_INC_HEADER_NAME "<filename>" 56 self._last_builtin = line.split('"', 2)[1] 57 self._log.debug('[%d] Detected definition of BUILTIN_INC_HEADER_NAME: %s', 58 file_level, self._last_builtin) 59 60 # the line is not anything special, just push it into the output 61 self._output.append(line) 62 63 def _emit_lineinfo(self, line_number, filename): 64 if not self._add_lineinfo: 65 return 66 67 normalized_path = repr(os.path.normpath(filename))[1:-1] 68 line_info = '#line %d "%s"\n' % (line_number, normalized_path) 69 70 if self._output and self._output[-1].startswith('#line'): 71 # Avoid emitting multiple line infos in sequence, just overwrite the last one 72 self._output[-1] = line_info 73 else: 74 self._output.append(line_info) 75 76 def add_file(self, filename, file_level=0): 77 if os.path.basename(filename) in self._processed: 78 self._log.warning('Tried to to process an already processed file: "%s"', filename) 79 return 80 81 if not file_level: 82 self._log.debug('Adding file: "%s"', filename) 83 84 file_level += 1 85 86 # mark the start of the new file in the output 87 self._emit_lineinfo(1, filename) 88 89 line_idx = 0 90 with open(filename, 'r') as input_file: 91 in_copyright = False 92 for line in input_file: 93 line_idx += 1 94 95 if not in_copyright and line.startswith('/* Copyright '): 96 in_copyright = True 97 if not self._copyright['loaded']: 98 self._copyright['lines'].append(line) 99 continue 100 101 if in_copyright: 102 if not self._copyright['loaded']: 103 self._copyright['lines'].append(line) 104 105 if line.strip().endswith('*/'): 106 in_copyright = False 107 self._copyright['loaded'] = True 108 # emit a line info so the line numbering can be tracked correctly 109 self._emit_lineinfo(line_idx + 1, filename) 110 111 continue 112 113 # check if the line is an '#include' line 114 match = SourceMerger._RE_INCLUDE.match(line) 115 if not match: 116 # the line is not a header 117 self._process_non_include(line, file_level) 118 continue 119 120 if match.group(1) == '<': 121 # found a "global" include 122 self._output.append(line) 123 continue 124 125 name = match.group(2) 126 127 if name in self._remove_includes: 128 self._log.debug('[%d] Removing include line (%s:%d): %s', 129 file_level, filename, line_idx, line.strip()) 130 # emit a line info so the line numbering can be tracked correctly 131 self._emit_lineinfo(line_idx + 1, filename) 132 continue 133 134 if name not in self._h_files: 135 self._log.warning('[%d] Include not found: "%s" in "%s:%d"', 136 file_level, name, filename, line_idx) 137 self._output.append(line) 138 continue 139 140 if name in self._processed: 141 self._log.debug('[%d] Already included: "%s"', 142 file_level, name) 143 # emit a line info so the line numbering can be tracked correctly 144 self._emit_lineinfo(line_idx + 1, filename) 145 continue 146 147 self._log.debug('[%d] Including: "%s"', 148 file_level, self._h_files[name]) 149 self.add_file(self._h_files[name], file_level) 150 151 # mark the continuation of the current file in the output 152 self._emit_lineinfo(line_idx + 1, filename) 153 154 if not name.endswith('.inc.h'): 155 # if the included file is not a "*.inc.h" file mark it as processed 156 self._processed.append(name) 157 158 file_level -= 1 159 if not filename.endswith('.inc.h'): 160 self._processed.append(os.path.basename(filename)) 161 162 def write_output(self, out_fp): 163 for line in self._copyright['lines']: 164 out_fp.write(line) 165 166 for include in self._extra_includes: 167 out_fp.write('#include "%s"\n' % include) 168 169 for line in self._output: 170 out_fp.write(line) 171 172 173def match_files(base_dir, pattern): 174 """ 175 Return the files matching the given pattern. 176 177 :param base_dir: directory to search in 178 :param pattern: file pattern to use 179 :returns generator: the generator which iterates the matching file names 180 """ 181 for path, _, files in os.walk(base_dir): 182 for name in files: 183 if fnmatch.fnmatch(name, pattern): 184 yield os.path.join(path, name) 185 186 187def collect_files(base_dir, pattern): 188 """ 189 Collect files in the provided base directory given a file pattern. 190 Will collect all files in the base dir recursively. 191 192 :param base_dir: directory to search in 193 :param pattern: file pattern to use 194 :returns dictionary: a dictionary file base name -> file path mapping 195 """ 196 name_mapping = {} 197 for fname in match_files(base_dir, pattern): 198 name = os.path.basename(fname) 199 200 if name in name_mapping: 201 print('Duplicate name detected: "%s" and "%s"' % (name, name_mapping[name])) 202 continue 203 204 name_mapping[name] = fname 205 206 return name_mapping 207 208 209def run_merger(args): 210 h_files = collect_files(args.base_dir, '*.h') 211 c_files = collect_files(args.base_dir, '*.c') 212 213 for name in args.remove_include: 214 c_files.pop(name, '') 215 h_files.pop(name, '') 216 217 merger = SourceMerger(h_files, args.push_include, args.remove_include, args.add_lineinfo) 218 for input_file in args.input_files: 219 merger.add_file(input_file) 220 221 if args.append_c_files: 222 # if the input file is in the C files list it should be removed to avoid 223 # double inclusion of the file 224 for input_file in args.input_files: 225 input_name = os.path.basename(input_file) 226 c_files.pop(input_name, '') 227 228 # Add the C files in reverse order to make sure that builtins are 229 # not at the beginning. 230 for fname in sorted(c_files.values(), reverse=True): 231 merger.add_file(fname) 232 233 with open(args.output_file, 'w') as output: 234 merger.write_output(output) 235 236 237def main(): 238 parser = argparse.ArgumentParser(description='Merge source/header files.') 239 parser.add_argument('--base-dir', metavar='DIR', type=str, dest='base_dir', 240 help='', default=os.path.curdir) 241 parser.add_argument('--input', metavar='FILES', type=str, action='append', dest='input_files', 242 help='Main input source/header files', default=[]) 243 parser.add_argument('--output', metavar='FILE', type=str, dest='output_file', 244 help='Output source/header file') 245 parser.add_argument('--append-c-files', dest='append_c_files', default=False, 246 action='store_true', help='Enable auto inclusion of c files under the base-dir') 247 parser.add_argument('--remove-include', action='append', default=[]) 248 parser.add_argument('--push-include', action='append', default=[]) 249 parser.add_argument('--add-lineinfo', action='store_true', default=False, 250 help='Enable #line macro insertion into the generated sources') 251 parser.add_argument('--verbose', '-v', action='store_true', default=False) 252 253 args = parser.parse_args() 254 255 log_level = logging.WARNING 256 if args.verbose: 257 log_level = logging.DEBUG 258 259 logging.basicConfig(level=log_level) 260 logging.debug('Starting merge with args: %s', str(sys.argv)) 261 262 run_merger(args) 263 264 265if __name__ == "__main__": 266 main() 267