1# Copyright 2022, The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import json
16import os
17import subprocess
18import shutil
19import re
20import requests
21import zipfile
22import sys
23
24FETCH_ARTIFACT = "/google/data/ro/projects/android/fetch_artifact"
25BASS = "/google/data/ro/projects/android/bass"
26DEFAULT_CLONE_DEPTH = 100 # chosen arbitrarily, may need to be adjusted
27DEFAULT_BUILD_ARTIFACT_SEARCH_TIME_SPAN_IN_DAYS = 7 # chosen arbitrarily, may need to be adjusted
28GIT_LOG_URL = "https://android.googlesource.com/platform/frameworks/support/+log"
29IGNORE_PATHS = [
30 # includes timestamps
31  "*.xml",
32   # are different because the xml files include timestamps
33  "*.xml.*",
34  "*.sha*",
35  "*.md5",
36  # is different because it references the .sha* files.
37  "*.module"
38]
39
40
41def main(build_id):
42  """
43  This is a script to take a given build_id, and search for a build of the previous commit on ab/
44  If a commit is found, we download the top-of-tree-m2repository-all-{build_id}.zip file from both
45  builds and diff the contents. If an .aar / .jar is different, we will unzip those and diff the
46  contents as well.
47  """
48  staging_dir = prep_staging_dir()
49  build_info_file_path = fetch_build_info(build_id, staging_dir)
50
51  # presubmit BUILD_INFO files include the commit being built as well as the and parent commit
52  if is_presubmit_build(build_id):
53    previous_revision = get_previous_revision_from_build_info(build_info_file_path)
54  # other builds only include the commit being built (as far as I can tell),  we need to
55  # get the previous commit from git log
56  else:
57    current_revision = get_current_revision(build_info_file_path)
58    previous_revision = get_previous_revision_from_git_history(current_revision, staging_dir)
59  previous_build_id = get_previous_build_id(previous_revision)
60  (before, after) = download_and_unzip_repos(staging_dir, build_id, previous_build_id)
61  diff_repos(before, after, staging_dir)
62
63def prep_staging_dir():
64  """
65  remove and recreate the ./download_staging which is located as a sibling of this script.
66  """
67  current_dir = os.path.dirname(os.path.realpath(__file__))
68  staging_dir = current_dir + "/download_staging"
69  if os.path.isdir(staging_dir):
70    shutil.rmtree(staging_dir)
71  os.makedirs(staging_dir, exist_ok=True)
72  return staging_dir
73
74
75def fetch_m2repo(build_id, staging_dir):
76  file_path = f"top-of-tree-m2repository-all-{build_id}.zip"
77  if is_presubmit_build(build_id):
78    file_path = f"incremental/{file_path}"
79  return fetch_artifact(build_id, staging_dir, file_path)
80
81
82def fetch_build_info(build_id, staging_dir):
83  return fetch_artifact(build_id, staging_dir, "BUILD_INFO")
84
85
86def fetch_artifact(build_id, output_dir, file_path):
87  file_name = file_path.split("/")[-1]
88  print(f"fetching {file_name}")
89
90  if is_presubmit_build(build_id):
91    target = "androidx_incremental"
92  else:
93    target = "androidx"
94  return FetchArtifactService().fetch_artifact(build_id, "aosp-androidx-main", target, output_dir, file_path)
95
96
97def get_current_revision(build_info_file_path):
98  print("Getting current revision from BUILD_INFO")
99  with open(build_info_file_path) as f:
100    build_info = json.load(f)
101    support_project = next(
102        project for project in build_info["parsed_manifest"]["projects"] if
103        project["name"] == "platform/frameworks/support")
104    current_revision = support_project["revision"]
105    print(f"Found revision: {current_revision}")
106    return current_revision
107
108
109def get_previous_revision_from_build_info(build_info_file_path):
110  print("Getting previous revision from BUILD_INFO")
111  with open(build_info_file_path) as f:
112    build_info = json.load(f)
113    revision = build_info["git-pull"][0]["revisions"][0]["commit"]["parents"][0]["commitId"]
114    print(f"Found previous revision: {revision}")
115    return revision
116
117def get_previous_revision_from_git_history(current_revision, staging_dir):
118  """
119  Gets previous revision from git log endpoint for androidx-main.
120  """
121  response = requests.get(git_log_url(current_revision))
122  # endpoint returns  some junk in the first line making it invalid json
123  text_with_first_line_removed = "\n".join(response.text.split("\n")[1:])
124  response_json = json.loads(text_with_first_line_removed)
125  previous_revision = response_json["log"][0]["parents"][0]
126  print(f"Found previous revision: {previous_revision}")
127  return previous_revision
128
129
130def get_previous_build_id(previous_revision):
131  print("Searching Android Build server for build matching previous revision")
132  output = BassService().search_builds(
133    DEFAULT_BUILD_ARTIFACT_SEARCH_TIME_SPAN_IN_DAYS,
134    "aosp-androidx-main",
135    "androidx",
136    "BUILD_INFO",
137    previous_revision
138  )
139  match = re.search("BuildID\: (\d+)", output.stdout)
140  if match is None:
141    raise Exception(f"Couldn't find previous build ID for revision {previous_revision}")
142
143  previous_build_id = match.group(1)
144  print(f"Found build matching previous revision: {previous_build_id}")
145  return previous_build_id
146
147def download_and_unzip_repos(staging_dir, build_id, previous_build_id):
148  before_dir = staging_dir + "/before"
149  after_dir = staging_dir + "/after"
150  os.makedirs(before_dir)
151  os.makedirs(after_dir)
152  after_zip = fetch_m2repo(build_id, staging_dir)
153  before_zip = fetch_m2repo(previous_build_id, staging_dir)
154  return (unzip(before_zip, before_dir), unzip(after_zip, after_dir))
155
156def diff_repos(before, after, staging_dir):
157  output = DiffService().diff(before, after, IGNORE_PATHS)
158  for line in output.stdout.splitlines():
159    if line.startswith("Binary files "):
160      for (before_file, after_file) in re.findall("Binary files (.+) and (.+) differ", line):
161        diff_binary(before_file, after_file, staging_dir)
162    else:
163      print(line)
164
165
166def diff_binary(before, after, staging_dir):
167  file_name = before.split("/")[-1]
168  if is_unzippable(before) and is_unzippable(after):
169    before_contents = unzip(before, staging_dir + "/" + file_name + "-before")
170    after_contents = unzip(after, staging_dir + "/" + file_name + "-after")
171    output = DiffService().diff(before_contents, after_contents)
172    # sometimes the binary is "different" but the contents are identical.
173    # It might be interesting to add diff the metadata, but for now just ignore it.
174    if output.stdout.strip() != "":
175      print(output.stdout)
176  else:
177    print(f"Binary files {before} and {after} differ")
178
179def is_unzippable(filename):
180  return filename.endswith(".zip") or filename.endswith(".aar") or filename.endswith(".jar")
181
182def unzip(file, destination):
183  with zipfile.ZipFile(file, 'r') as zip:
184    zip.extractall(destination)
185    return destination
186
187def is_presubmit_build(build_id):
188    return build_id.startswith("P")
189
190def git_log_url(revision):
191    return f"{GIT_LOG_URL}/{revision}?format=JSON"
192
193class DiffService():
194    @staticmethod
195    def diff(before_dir, after_dir, exclude=[]):
196      args = ["diff", "-r"]
197      for pattern in exclude:
198        args.extend(["-x", pattern])
199      args.extend([before_dir, after_dir])
200      return subprocess.run(args, text=True, capture_output=True)
201
202
203class FetchArtifactService():
204    @staticmethod
205    def fetch_artifact(build_id, branch, target, output_dir, file_path):
206      file_name = file_path.split("/")[-1]
207      subprocess.run(
208        [
209          FETCH_ARTIFACT,
210          "--bid",
211          build_id,
212          "--branch",
213          branch,
214          "--target",
215          target,
216          file_path,
217        ],
218        cwd=output_dir,
219        capture_output=True,
220        check=True
221      )
222      return f"{output_dir}/{file_name}"
223
224class BassService():
225    @staticmethod
226    def search_builds(days, branch, target, file_name, query):
227     return subprocess.run([
228          BASS,
229          "--days",
230          str(days),
231          "--successful",
232          "true",
233          "--branch",
234          branch,
235          "--target",
236          target,
237          "--filename",
238          file_name,
239          "--query",
240          query
241      ],
242          capture_output=True,
243          text=True,
244          check=True
245      )
246
247if __name__ == "__main__":
248    main(sys.argv[1])