• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2#
3# Copyright (C) 2021 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18"""
19Dump new HALs that are introduced in each FCM version in a human-readable format.
20
21Example:
22hals_for_release.py
23  Show changes for each release, including new and deprecated HALs.
24hals_for_release.py -dua
25  Show changes as well as unchanged HALs for each release.
26hals_for_release.py -i
27  Show details about instance names and regex patterns as well.
28hals_for_release.py -p wifi
29  Show changes of Wi-Fi HALs for each release.
30"""
31
32import argparse
33import collections
34import enum
35import json
36import logging
37import os
38import subprocess
39from collections.abc import Sequence
40from typing import Any
41from typing import Optional
42
43import sys
44
45logging.basicConfig(format="%(levelname)s: %(message)s")
46logger = logging.getLogger(__name__)
47
48
49def ParseArgs() -> argparse.Namespace:
50  """
51  Parse arguments.
52  :return: arguments.
53  """
54  parser = argparse.ArgumentParser(description=__doc__,
55                                   formatter_class=argparse.RawTextHelpFormatter)
56  parser.add_argument("--analyze-matrix", help="Location of analyze_matrix")
57  parser.add_argument("input", metavar="INPUT", nargs="?",
58                      help="Directory of compatibility matrices.")
59  parser.add_argument("--deprecated", "-d",
60                      help="Show deprecated HALs. If none of deprecated, unchanged or introduced "
61                           "is specified, default is --deprecated and --introduced",
62                      action="store_true")
63  parser.add_argument("--unchanged", "-u",
64                      help="Show unchanged HALs. If none of deprecated, unchanged or introduced "
65                           "is specified, default is --deprecated and --introduced",
66                      action="store_true")
67  parser.add_argument("--introduced", "-a",
68                      help="Show deprecated HALs. If none of deprecated, unchanged or introduced "
69                           "is specified, default is --deprecated and --introduced",
70                      action="store_true")
71  parser.add_argument("--instances", "-i", action="store_true",
72                      help="Show instance names and regex patterns as well")
73  parser.add_argument("--packages", "-p", nargs="*", metavar="PACKAGE",
74                      help="Only print HALs where package contains the given substring. "
75                           "E.g. wifi, usb, health. Recommend to use with --unchanged.")
76  parser.add_argument("--verbose", "-v", action="store_true", help="Verbose mode")
77  parser.add_argument("--json", "-j", action="store_true", help="Print JSON")
78  parser.add_argument("--package-only", action="store_true", help="Analyze on the package level.")
79  args = parser.parse_args()
80
81  if args.verbose:
82    logger.setLevel(logging.DEBUG)
83
84  if not args.deprecated and not args.unchanged and not args.introduced:
85    args.deprecated = args.introduced = True
86
87  host_out = os.environ.get("ANDROID_HOST_OUT")
88  if host_out and not args.analyze_matrix:
89    analyze_matrix = os.path.join(host_out, "bin", "analyze_matrix")
90    if os.path.isfile(analyze_matrix):
91      args.analyze_matrix = analyze_matrix
92  if not args.analyze_matrix:
93    args.analyze_matrix = "analyze_matrix"
94
95  top = os.environ.get("ANDROID_BUILD_TOP")
96  if top and not args.input:
97    args.input = os.path.join(top, "hardware", "interfaces", "compatibility_matrices")
98  if not args.input:
99    logger.fatal("Unable to determine compatibility matrix dir, lunch or provide one explicitly.")
100    return None
101
102  logger.debug("Using analyze_matrix at path: %s", args.analyze_matrix)
103  logger.debug("Dumping compatibility matrices at path: %s", args.input)
104  logger.debug("Show deprecated HALs? %s", args.deprecated)
105  logger.debug("Show unchanged HALs? %s", args.unchanged)
106  logger.debug("Show introduced HALs? %s", args.introduced)
107  logger.debug("Only showing packages %s", args.packages)
108
109  return args
110
111
112def Analyze(analyze_matrix: str, file: str, args: Sequence[str],
113    ignore_errors=False) -> str:
114  """
115  Run analyze_matrix with
116  :param analyze_matrix: path of analyze_matrix
117  :param file: input file
118  :param arg: argument to analyze_matrix, e.g. "level"
119  :param ignore_errors: Whether errors during execution should be rased
120  :return: output of analyze_matrix
121  """
122  command = [analyze_matrix, "--input", file] + args
123  proc = subprocess.run(command,
124                        stdout=subprocess.PIPE,
125                        stderr=subprocess.DEVNULL if ignore_errors else subprocess.PIPE)
126  if not ignore_errors and proc.returncode != 0:
127    logger.warning("`%s` exits with code %d with the following error: %s", " ".join(command),
128                   proc.returncode, proc.stderr)
129    proc.check_returncode()
130  return proc.stdout.decode().strip()
131
132
133def GetLevel(analyze_matrix: str, file: str) -> Optional[int]:
134  """
135  :param analyze_matrix: Path of analyze_matrix
136  :param file: a file, possibly a compatibility matrix
137  :return: If it is a compatibility matrix, return an integer indicating the level.
138    If it is not a compatibility matrix, returns None.
139    For matrices with empty level, return None.
140  """
141  output = Analyze(analyze_matrix, file, ["--level"], ignore_errors=True)
142  # Ignore empty level matrices and non-matrices
143  if not output:
144    return None
145  try:
146    return int(output)
147  except ValueError:
148    logger.warning("Unknown level '%s' in file: %s", output, file)
149    return None
150
151
152def GetLevelName(analyze_matrix: str, file: str) -> str:
153  """
154  :param analyze_matrix: Path of analyze_matrix
155  :param file: a file, possibly a compatibility matrix
156  :return: If it is a compatibility matrix, return the level name.
157    If it is not a compatibility matrix, returns None.
158    For matrices with empty level, return "Level unspecified".
159  """
160  return Analyze(analyze_matrix, file, ["--level-name"], ignore_errors=True)
161
162
163class MatrixData(object):
164  def __init__(self, level: str, level_name: str, instances: Sequence[str]):
165    self.level = level
166    self.level_name = level_name
167    self.instances = instances
168
169  def GetInstancesKeyedOnPackage(self) -> dict[str, list[str]]:
170    return KeyOnPackage(self.instances)
171
172
173def ReadMatrices(args: argparse.Namespace) -> dict[int, MatrixData]:
174  """
175  :param args: parsed arguments from ParseArgs
176  :return: A dictionary. Key is an integer indicating the matrix level.
177  Value is (level name, a set of instances in that matrix).
178  """
179  matrices = dict()
180  for child in os.listdir(args.input):
181    file = os.path.join(args.input, child)
182    level, level_name = GetLevel(args.analyze_matrix, file), GetLevelName(args.analyze_matrix, file)
183    if level is None:
184      logger.debug("Ignoring file %s", file)
185      continue
186    action = "--instances" if args.instances else "--interfaces"
187    instances = Analyze(args.analyze_matrix, file, [action, "--requirement"]).split("\n")
188    instances = set(map(str.strip, instances)) - {""}
189    if level in matrices:
190      logger.warning("Found duplicated matrix for level %s, ignoring: %s", level, file)
191      continue
192    matrices[level] = MatrixData(level, level_name, instances)
193
194  return matrices
195
196
197class HalFormat(enum.Enum):
198  HIDL = 0
199  NATIVE = 1
200  AIDL = 2
201
202
203def GetHalFormat(instance: str) -> HalFormat:
204  """
205  Guess the HAL format of instance.
206  :param instance: two formats:
207    android.hardware.health.storage@1.0::IStorage/default optional
208    android.hardware.health.storage.IStorage/default (@1) optional
209    storage@5.0 optional
210  :return: HalFormat.HIDL for the first one, HalFormat.AIDL for the second,
211    HalFormat.NATIVE for the third.
212
213  >>> str(GetHalFormat("android.hardware.health.storage@1.0::IStorage/default optional"))
214  'HalFormat.HIDL'
215  >>> str(GetHalFormat("android.hardware.health.storage.IStorage/default (@1) optional"))
216  'HalFormat.AIDL'
217  >>> str(GetHalFormat("storage@5.0 optional"))
218  'HalFormat.NATIVE'
219  """
220  if "::" in instance:
221    return HalFormat.HIDL
222  elif "(@" in instance:
223    return HalFormat.AIDL
224  else:
225    return HalFormat.NATIVE
226
227
228def SplitInstance(instance: str) -> tuple[str, str, str]:
229  """
230  Split instance into parts.
231  :param instance:
232  :param instance: two formats:
233    android.hardware.health.storage@1.0::IStorage/default optional
234    android.hardware.health.storage.IStorage/default (@1) optional
235    storage@5.0 optional
236  :return: (package, version+interface+instance, requirement)
237
238  >>> SplitInstance("android.hardware.health.storage@1.0::IStorage/default optional")
239  ('android.hardware.health.storage', '@1.0::IStorage/default', 'optional')
240  >>> SplitInstance("android.hardware.health.storage.IStorage/default (@1) optional")
241  ('android.hardware.health.storage', 'IStorage/default (@1)', 'optional')
242  >>> SplitInstance("storage@5.0 optional")
243  ('storage', 'storage@5.0', 'optional')
244  """
245  format = GetHalFormat(instance)
246  if format == HalFormat.HIDL:
247    atPos = instance.find("@")
248    spacePos = instance.rfind(" ")
249    return instance[:atPos], instance[atPos:spacePos], instance[spacePos + 1:]
250  elif format == HalFormat.AIDL:
251    dotPos = instance.rfind(".")
252    spacePos = instance.rfind(" ")
253    return instance[:dotPos], instance[dotPos + 1:spacePos], instance[spacePos + 1:]
254  elif format == HalFormat.NATIVE:
255    atPos = instance.find("@")
256    spacePos = instance.rfind(" ")
257    return instance[:atPos], instance[:spacePos], instance[spacePos + 1:]
258
259
260def GetPackage(instance: str) -> str:
261  """
262  Guess the package of instance.
263  :param instance: two formats:
264    android.hardware.health.storage@1.0::IStorage/default
265    android.hardware.health.storage.IStorage/default (@1)
266  :return: The package. In the above example, return android.hardware.health.storage
267
268  >>> GetPackage("android.hardware.health.storage@1.0::IStorage/default")
269  'android.hardware.health.storage'
270  >>> GetPackage("android.hardware.health.storage.IStorage/default (@1)")
271  'android.hardware.health.storage'
272  """
273  return SplitInstance(instance)[0]
274
275
276def GetPackageAndHidlVersion(instance: str) -> str:
277  """
278  Guess the package and version of instance.
279  :param instance: two formats:
280    android.hardware.health.storage@1.0::IStorage/default
281    android.hardware.health.storage.IStorage/default (@1)
282    storage@5.0
283  :return: The package and HIDL version. In the above example, return
284    android.hardware.health.storage@1.0 for HIDL, storage@5.0 for NATIVE,
285    and android.hardware.health.storage for AIDL.
286
287  >>> GetPackageAndHidlVersion("android.hardware.health.storage@1.0::IStorage/default")
288  'android.hardware.health.storage@1.0'
289  >>> GetPackageAndHidlVersion("android.hardware.health.storage.IStorage/default (@1)")
290  'android.hardware.health.storage'
291  >>> GetPackageAndHidlVersion("storage@5.0")
292  'storage@5.0'
293  """
294  format = GetHalFormat(instance)
295  if format == HalFormat.HIDL:
296    colonPos = instance.find("::")
297    return instance[:colonPos]
298  elif format == HalFormat.AIDL:
299    dotPos = instance.rfind(".")
300    return instance[:dotPos]
301  elif format == HalFormat.NATIVE:
302    return instance
303
304
305
306def KeyOnPackage(instances: Sequence[str]) -> dict[str, list[str]]:
307  """
308  :param instances: A list of instances.
309  :return: A dictionary, where key is the package (see GetPackage), and
310    value is a list of instances in the provided list, where
311    GetPackage(instance) is the corresponding key.
312  """
313  d = collections.defaultdict(list)
314  for instance in instances:
315    package = GetPackage(instance)
316    d[package].append(instance)
317  return d
318
319
320class Report(object):
321  """
322  Base class for generating a report.
323  """
324  def __init__(self, matrixData1: MatrixData, matrixData2: MatrixData, args: argparse.Namespace):
325    """
326    Initialize the report with two matrices.
327    :param matrixData1: Data of the old matrix
328    :param matrixData2: Data of the new matrix
329    :param args: command-line arguments
330    """
331    self.args = args
332    self.matrixData1 = matrixData1
333    self.matrixData2 = matrixData2
334    self.instances_by_package1 = matrixData1.GetInstancesKeyedOnPackage()
335    self.instances_by_package2 = matrixData2.GetInstancesKeyedOnPackage()
336    self.all_packages = set(self.instances_by_package1.keys()) | set(
337      self.instances_by_package2.keys())
338
339  def GetReport(self) -> Any:
340    """
341    Generate the report
342    :return: An object representing the report. Type is implementation defined.
343    """
344    packages_report: dict[str, Any] = dict()
345    for package in self.all_packages:
346      package_instances1 = set(self.instances_by_package1.get(package, []))
347      package_instances2 = set(self.instances_by_package2.get(package, []))
348
349      if self.args.package_only:
350        package_instances1 = set(GetPackageAndHidlVersion(inst) for inst in package_instances1)
351        package_instances2 = set(GetPackageAndHidlVersion(inst) for inst in package_instances2)
352
353      deprecated = sorted(package_instances1 - package_instances2)
354      unchanged = sorted(package_instances1 & package_instances2)
355      introduced = sorted(package_instances2 - package_instances1)
356      package_report = self.DescribePackage(deprecated=deprecated,
357                                            unchanged=unchanged,
358                                            introduced=introduced)
359      if package_report:
360        packages_report[package] = package_report
361    return self.CombineReport(packages_report)
362
363  def DescribePackage(self, deprecated: Sequence[str], unchanged: Sequence[str],
364      introduced: Sequence[str]) -> Any:
365    """
366    Describe a package in a implementation-defined format, with the given
367    set of changes.
368    :param deprecated: set of deprecated HALs
369    :param unchanged:  set of unchanged HALs
370    :param introduced: set of new HALs
371    :return: An object that will later be passed into the values of the
372      packages_report argument of CombineReport
373    """
374    raise NotImplementedError
375
376  def CombineReport(self, packages_report: dict[str, Any]) -> Any:
377    """
378    Combine a set of reports for a package in an implementation-defined way.
379    :param packages_report: A dictionary, where key is the package
380      name, and value is the object generated by DescribePackage
381    :return: the report object
382    """
383    raise NotImplementedError
384
385
386class HumanReadableReport(Report):
387  def DescribePackage(self, deprecated: Sequence[str], unchanged: Sequence[str],
388      introduced: Sequence[str]) -> Any:
389    package_report = []
390    desc = lambda fmt, instance: fmt.format(GetHalFormat(instance).name,
391                                            *SplitInstance(instance))
392    if self.args.deprecated:
393      package_report += [desc("- {0} {2} can no longer be used", instance)
394                         for instance in deprecated]
395    if self.args.unchanged:
396      package_report += [desc("  {0} {2}", instance) for instance in
397                         unchanged]
398    if self.args.introduced:
399      package_report += [desc("+ {0} {2}", instance) for instance in
400                         introduced]
401
402    return package_report
403
404  def CombineReport(self, packages_report: dict[str, Any]) -> str:
405    report = ["============",
406              "Level %s (%s) (against Level %s (%s))" % (
407              self.matrixData2.level, self.matrixData2.level_name,
408              self.matrixData1.level, self.matrixData1.level_name),
409              "============"]
410    for package, lines in sorted(packages_report.items()):
411      report.append(package)
412      report += [("    " + e) for e in lines]
413
414    return "\n".join(report)
415
416
417class JsonReport(Report):
418  def DescribePackage(self, deprecated: Sequence[str], unchanged: Sequence[str],
419      introduced: Sequence[str]) -> Any:
420    package_report = collections.defaultdict(list)
421    if self.args.deprecated and deprecated:
422      package_report["deprecated"] += deprecated
423    if self.args.unchanged and unchanged:
424      package_report["unchanged"] += unchanged
425    if self.args.introduced and introduced:
426      package_report["introduced"] += introduced
427
428    return package_report
429
430  def CombineReport(self, packages_report: dict[str, Any]) -> dict[str, Any]:
431    final = collections.defaultdict(list)
432    for package_report in packages_report.values():
433      for key, lst in package_report.items():
434        final[key] += lst
435    for key in final:
436      final[key] = sorted(final[key])
437    final["__meta__"] = {
438        "old": {"level": self.matrixData1.level,
439                "level_name": self.matrixData1.level_name},
440        "new": {"level": self.matrixData2.level,
441                "level_name": self.matrixData2.level_name},
442    }
443    return final
444
445
446def PrintReport(matrices: dict[int, MatrixData], args: argparse.Namespace):
447  """
448  :param matrixData1: data of first matrix
449  :param matrixData2: data of second matrix
450  :return: A report of their difference.
451  """
452  sorted_matrices = sorted(matrices.items())
453  if not sorted_matrices:
454    logger.warning("Nothing to show, because no matrices found in '%s'.", args.input)
455
456  if args.json:
457    reports = []
458    for (level1, matrixData1), (level2, matrixData2) in zip(sorted_matrices, sorted_matrices[1:]):
459      reports.append(JsonReport(matrixData1, matrixData2, args).GetReport())
460    print(json.dumps(reports))
461    return
462
463  for (level1, matrixData1), (level2, matrixData2) in zip(sorted_matrices, sorted_matrices[1:]):
464    report = HumanReadableReport(matrixData1, matrixData2, args)
465    print(report.GetReport())
466
467
468def main():
469  sys.stderr.write("Generated with %s\n" % " ".join(sys.argv))
470  args = ParseArgs()
471  if args is None:
472    return 1
473  matrices = ReadMatrices(args)
474  PrintReport(matrices, args)
475  return 0
476
477
478if __name__ == "__main__":
479  sys.exit(main())
480