• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0+
3#
4# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
5#
6
7"""
8Converter from Kconfig and MAINTAINERS to a board database.
9
10Run 'tools/genboardscfg.py' to create a board database.
11
12Run 'tools/genboardscfg.py -h' for available options.
13
14Python 2.6 or later, but not Python 3.x is necessary to run this script.
15"""
16
17import errno
18import fnmatch
19import glob
20import multiprocessing
21import optparse
22import os
23import sys
24import tempfile
25import time
26
27sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'buildman'))
28import kconfiglib
29
30### constant variables ###
31OUTPUT_FILE = 'boards.cfg'
32CONFIG_DIR = 'configs'
33SLEEP_TIME = 0.03
34COMMENT_BLOCK = '''#
35# List of boards
36#   Automatically generated by %s: don't edit
37#
38# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
39
40''' % __file__
41
42### helper functions ###
43def try_remove(f):
44    """Remove a file ignoring 'No such file or directory' error."""
45    try:
46        os.remove(f)
47    except OSError as exception:
48        # Ignore 'No such file or directory' error
49        if exception.errno != errno.ENOENT:
50            raise
51
52def check_top_directory():
53    """Exit if we are not at the top of source directory."""
54    for f in ('README', 'Licenses'):
55        if not os.path.exists(f):
56            sys.exit('Please run at the top of source directory.')
57
58def output_is_new(output):
59    """Check if the output file is up to date.
60
61    Returns:
62      True if the given output file exists and is newer than any of
63      *_defconfig, MAINTAINERS and Kconfig*.  False otherwise.
64    """
65    try:
66        ctime = os.path.getctime(output)
67    except OSError as exception:
68        if exception.errno == errno.ENOENT:
69            # return False on 'No such file or directory' error
70            return False
71        else:
72            raise
73
74    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
75        for filename in fnmatch.filter(filenames, '*_defconfig'):
76            if fnmatch.fnmatch(filename, '.*'):
77                continue
78            filepath = os.path.join(dirpath, filename)
79            if ctime < os.path.getctime(filepath):
80                return False
81
82    for (dirpath, dirnames, filenames) in os.walk('.'):
83        for filename in filenames:
84            if (fnmatch.fnmatch(filename, '*~') or
85                not fnmatch.fnmatch(filename, 'Kconfig*') and
86                not filename == 'MAINTAINERS'):
87                continue
88            filepath = os.path.join(dirpath, filename)
89            if ctime < os.path.getctime(filepath):
90                return False
91
92    # Detect a board that has been removed since the current board database
93    # was generated
94    with open(output, encoding="utf-8") as f:
95        for line in f:
96            if line[0] == '#' or line == '\n':
97                continue
98            defconfig = line.split()[6] + '_defconfig'
99            if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
100                return False
101
102    return True
103
104### classes ###
105class KconfigScanner:
106
107    """Kconfig scanner."""
108
109    ### constant variable only used in this class ###
110    _SYMBOL_TABLE = {
111        'arch' : 'SYS_ARCH',
112        'cpu' : 'SYS_CPU',
113        'soc' : 'SYS_SOC',
114        'vendor' : 'SYS_VENDOR',
115        'board' : 'SYS_BOARD',
116        'config' : 'SYS_CONFIG_NAME',
117        'options' : 'SYS_EXTRA_OPTIONS'
118    }
119
120    def __init__(self):
121        """Scan all the Kconfig files and create a Kconfig object."""
122        # Define environment variables referenced from Kconfig
123        os.environ['srctree'] = os.getcwd()
124        os.environ['UBOOTVERSION'] = 'dummy'
125        os.environ['KCONFIG_OBJDIR'] = ''
126        self._conf = kconfiglib.Kconfig(warn=False)
127
128    def __del__(self):
129        """Delete a leftover temporary file before exit.
130
131        The scan() method of this class creates a temporay file and deletes
132        it on success.  If scan() method throws an exception on the way,
133        the temporary file might be left over.  In that case, it should be
134        deleted in this destructor.
135        """
136        if hasattr(self, '_tmpfile') and self._tmpfile:
137            try_remove(self._tmpfile)
138
139    def scan(self, defconfig):
140        """Load a defconfig file to obtain board parameters.
141
142        Arguments:
143          defconfig: path to the defconfig file to be processed
144
145        Returns:
146          A dictionary of board parameters.  It has a form of:
147          {
148              'arch': <arch_name>,
149              'cpu': <cpu_name>,
150              'soc': <soc_name>,
151              'vendor': <vendor_name>,
152              'board': <board_name>,
153              'target': <target_name>,
154              'config': <config_header_name>,
155              'options': <extra_options>
156          }
157        """
158        # strip special prefixes and save it in a temporary file
159        fd, self._tmpfile = tempfile.mkstemp()
160        with os.fdopen(fd, 'w') as f:
161            for line in open(defconfig):
162                colon = line.find(':CONFIG_')
163                if colon == -1:
164                    f.write(line)
165                else:
166                    f.write(line[colon + 1:])
167
168        self._conf.load_config(self._tmpfile)
169        try_remove(self._tmpfile)
170        self._tmpfile = None
171
172        params = {}
173
174        # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
175        # Set '-' if the value is empty.
176        for key, symbol in list(self._SYMBOL_TABLE.items()):
177            value = self._conf.syms.get(symbol).str_value
178            if value:
179                params[key] = value
180            else:
181                params[key] = '-'
182
183        defconfig = os.path.basename(defconfig)
184        params['target'], match, rear = defconfig.partition('_defconfig')
185        assert match and not rear, '%s : invalid defconfig' % defconfig
186
187        # fix-up for aarch64
188        if params['arch'] == 'arm' and params['cpu'] == 'armv8':
189            params['arch'] = 'aarch64'
190
191        # fix-up options field. It should have the form:
192        # <config name>[:comma separated config options]
193        if params['options'] != '-':
194            params['options'] = params['config'] + ':' + \
195                                params['options'].replace(r'\"', '"')
196        elif params['config'] != params['target']:
197            params['options'] = params['config']
198
199        return params
200
201def scan_defconfigs_for_multiprocess(queue, defconfigs):
202    """Scan defconfig files and queue their board parameters
203
204    This function is intended to be passed to
205    multiprocessing.Process() constructor.
206
207    Arguments:
208      queue: An instance of multiprocessing.Queue().
209             The resulting board parameters are written into it.
210      defconfigs: A sequence of defconfig files to be scanned.
211    """
212    kconf_scanner = KconfigScanner()
213    for defconfig in defconfigs:
214        queue.put(kconf_scanner.scan(defconfig))
215
216def read_queues(queues, params_list):
217    """Read the queues and append the data to the paramers list"""
218    for q in queues:
219        while not q.empty():
220            params_list.append(q.get())
221
222def scan_defconfigs(jobs=1):
223    """Collect board parameters for all defconfig files.
224
225    This function invokes multiple processes for faster processing.
226
227    Arguments:
228      jobs: The number of jobs to run simultaneously
229    """
230    all_defconfigs = []
231    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
232        for filename in fnmatch.filter(filenames, '*_defconfig'):
233            if fnmatch.fnmatch(filename, '.*'):
234                continue
235            all_defconfigs.append(os.path.join(dirpath, filename))
236
237    total_boards = len(all_defconfigs)
238    processes = []
239    queues = []
240    for i in range(jobs):
241        defconfigs = all_defconfigs[total_boards * i // jobs :
242                                    total_boards * (i + 1) // jobs]
243        q = multiprocessing.Queue(maxsize=-1)
244        p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
245                                    args=(q, defconfigs))
246        p.start()
247        processes.append(p)
248        queues.append(q)
249
250    # The resulting data should be accumulated to this list
251    params_list = []
252
253    # Data in the queues should be retrieved preriodically.
254    # Otherwise, the queues would become full and subprocesses would get stuck.
255    while any([p.is_alive() for p in processes]):
256        read_queues(queues, params_list)
257        # sleep for a while until the queues are filled
258        time.sleep(SLEEP_TIME)
259
260    # Joining subprocesses just in case
261    # (All subprocesses should already have been finished)
262    for p in processes:
263        p.join()
264
265    # retrieve leftover data
266    read_queues(queues, params_list)
267
268    return params_list
269
270class MaintainersDatabase:
271
272    """The database of board status and maintainers."""
273
274    def __init__(self):
275        """Create an empty database."""
276        self.database = {}
277
278    def get_status(self, target):
279        """Return the status of the given board.
280
281        The board status is generally either 'Active' or 'Orphan'.
282        Display a warning message and return '-' if status information
283        is not found.
284
285        Returns:
286          'Active', 'Orphan' or '-'.
287        """
288        if not target in self.database:
289            print("WARNING: no status info for '%s'" % target, file=sys.stderr)
290            return '-'
291
292        tmp = self.database[target][0]
293        if tmp.startswith('Maintained'):
294            return 'Active'
295        elif tmp.startswith('Supported'):
296            return 'Active'
297        elif tmp.startswith('Orphan'):
298            return 'Orphan'
299        else:
300            print(("WARNING: %s: unknown status for '%s'" %
301                                  (tmp, target)), file=sys.stderr)
302            return '-'
303
304    def get_maintainers(self, target):
305        """Return the maintainers of the given board.
306
307        Returns:
308          Maintainers of the board.  If the board has two or more maintainers,
309          they are separated with colons.
310        """
311        if not target in self.database:
312            print("WARNING: no maintainers for '%s'" % target, file=sys.stderr)
313            return ''
314
315        return ':'.join(self.database[target][1])
316
317    def parse_file(self, file):
318        """Parse a MAINTAINERS file.
319
320        Parse a MAINTAINERS file and accumulates board status and
321        maintainers information.
322
323        Arguments:
324          file: MAINTAINERS file to be parsed
325        """
326        targets = []
327        maintainers = []
328        status = '-'
329        for line in open(file, encoding="utf-8"):
330            # Check also commented maintainers
331            if line[:3] == '#M:':
332                line = line[1:]
333            tag, rest = line[:2], line[2:].strip()
334            if tag == 'M:':
335                maintainers.append(rest)
336            elif tag == 'F:':
337                # expand wildcard and filter by 'configs/*_defconfig'
338                for f in glob.glob(rest):
339                    front, match, rear = f.partition('configs/')
340                    if not front and match:
341                        front, match, rear = rear.rpartition('_defconfig')
342                        if match and not rear:
343                            targets.append(front)
344            elif tag == 'S:':
345                status = rest
346            elif line == '\n':
347                for target in targets:
348                    self.database[target] = (status, maintainers)
349                targets = []
350                maintainers = []
351                status = '-'
352        if targets:
353            for target in targets:
354                self.database[target] = (status, maintainers)
355
356def insert_maintainers_info(params_list):
357    """Add Status and Maintainers information to the board parameters list.
358
359    Arguments:
360      params_list: A list of the board parameters
361    """
362    database = MaintainersDatabase()
363    for (dirpath, dirnames, filenames) in os.walk('.'):
364        if 'MAINTAINERS' in filenames:
365            database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
366
367    for i, params in enumerate(params_list):
368        target = params['target']
369        params['status'] = database.get_status(target)
370        params['maintainers'] = database.get_maintainers(target)
371        params_list[i] = params
372
373def format_and_output(params_list, output):
374    """Write board parameters into a file.
375
376    Columnate the board parameters, sort lines alphabetically,
377    and then write them to a file.
378
379    Arguments:
380      params_list: The list of board parameters
381      output: The path to the output file
382    """
383    FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
384              'options', 'maintainers')
385
386    # First, decide the width of each column
387    max_length = dict([ (f, 0) for f in FIELDS])
388    for params in params_list:
389        for f in FIELDS:
390            max_length[f] = max(max_length[f], len(params[f]))
391
392    output_lines = []
393    for params in params_list:
394        line = ''
395        for f in FIELDS:
396            # insert two spaces between fields like column -t would
397            line += '  ' + params[f].ljust(max_length[f])
398        output_lines.append(line.strip())
399
400    # ignore case when sorting
401    output_lines.sort(key=str.lower)
402
403    with open(output, 'w', encoding="utf-8") as f:
404        f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
405
406def gen_boards_cfg(output, jobs=1, force=False):
407    """Generate a board database file.
408
409    Arguments:
410      output: The name of the output file
411      jobs: The number of jobs to run simultaneously
412      force: Force to generate the output even if it is new
413    """
414    check_top_directory()
415
416    if not force and output_is_new(output):
417        print("%s is up to date. Nothing to do." % output)
418        sys.exit(0)
419
420    params_list = scan_defconfigs(jobs)
421    insert_maintainers_info(params_list)
422    format_and_output(params_list, output)
423
424def main():
425    try:
426        cpu_count = multiprocessing.cpu_count()
427    except NotImplementedError:
428        cpu_count = 1
429
430    parser = optparse.OptionParser()
431    # Add options here
432    parser.add_option('-f', '--force', action="store_true", default=False,
433                      help='regenerate the output even if it is new')
434    parser.add_option('-j', '--jobs', type='int', default=cpu_count,
435                      help='the number of jobs to run simultaneously')
436    parser.add_option('-o', '--output', default=OUTPUT_FILE,
437                      help='output file [default=%s]' % OUTPUT_FILE)
438    (options, args) = parser.parse_args()
439
440    gen_boards_cfg(options.output, jobs=options.jobs, force=options.force)
441
442if __name__ == '__main__':
443    main()
444