• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2018 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Functions for reading build information from GoogleStorage.
6
7This module contains functions providing access to basic data about
8Chrome OS builds:
9  * Functions for finding information about the Chrome OS versions
10    currently being served by Omaha for various boards/hardware models.
11  * Functions for finding information about the firmware delivered by
12    any given build of Chrome OS.
13
14The necessary data is stored in JSON files in well-known locations in
15GoogleStorage.
16"""
17
18import json
19import subprocess
20
21import common
22from autotest_lib.client.common_lib import utils
23from autotest_lib.server import frontend
24
25
26# _OMAHA_STATUS - URI of a file in GoogleStorage with a JSON object
27# summarizing all versions currently being served by Omaha.
28#
29# The principal data is in an array named 'omaha_data'.  Each entry
30# in the array contains information relevant to one image being
31# served by Omaha, including the following information:
32#   * The board name of the product, as known to Omaha.
33#   * The channel associated with the image.
34#   * The Chrome and Chrome OS version strings for the image
35#     being served.
36#
37_OMAHA_STATUS = 'gs://chromeos-build-release-console/omaha_status.json'
38
39
40# _BUILD_METADATA_PATTERN - Format string for the URI of a file in
41# GoogleStorage with a JSON object that contains metadata about
42# a given build.  The metadata includes the version of firmware
43# bundled with the build.
44#
45_BUILD_METADATA_PATTERN = 'gs://chromeos-image-archive/%s/metadata.json'
46
47
48# _FIRMWARE_UPGRADE_DENYLIST - a set of boards that are exempt from
49# automatic stable firmware version assignment.  This denylist is
50# here out of an abundance of caution, on the general principle of "if
51# it ain't broke, don't fix it."  Specifically, these are old, legacy
52# boards and:
53#   * They're working fine with whatever firmware they have in the lab
54#     right now.
55#   * Because of their age, we can expect that they will never get any
56#     new firmware updates in future.
57#   * Servo support is spotty or missing, so there's no certainty that
58#     DUTs bricked by a firmware update can be repaired.
59#   * Because of their age, they are somewhere between hard and
60#     impossible to replace.  In some cases, they are also already in
61#     short supply.
62#
63# N.B.  HARDCODED BOARD NAMES ARE EVIL!!!  This denylist uses hardcoded
64# names because it's meant to define a list of legacies that will shrivel
65# and die over time.
66#
67# DO NOT ADD TO THIS LIST.  If there's a new use case that requires
68# extending the denylist concept, you should find a maintainable
69# solution that deletes this code.
70#
71# TODO(jrbarnette):  When any board is past EOL, and removed from the
72# lab, it can be removed from the denylist.  When all the boards are
73# past EOL, the denylist should be removed.
74
75_FIRMWARE_UPGRADE_DENYLIST = set([
76        'butterfly',
77        'daisy',
78        'daisy_skate',
79        'daisy_spring',
80        'lumpy',
81        'parrot',
82        'parrot_ivb',
83        'peach_pi',
84        'peach_pit',
85        'stout',
86        'stumpy',
87        'x86-alex',
88        'x86-mario',
89        'x86-zgb',
90])
91
92
93def _read_gs_json_data(gs_uri):
94    """Read and parse a JSON file from GoogleStorage.
95
96    This is a wrapper around `gsutil cat` for the specified URI.
97    The standard output of the command is parsed as JSON, and the
98    resulting object returned.
99
100    @param gs_uri   URI of the JSON file in GoogleStorage.
101    @return A JSON object parsed from `gs_uri`.
102    """
103    with open('/dev/null', 'w') as ignore_errors:
104        sp = subprocess.Popen(['gsutil', 'cat', gs_uri],
105                              stdout=subprocess.PIPE,
106                              stderr=ignore_errors)
107        try:
108            json_object = json.load(sp.stdout)
109        finally:
110            sp.stdout.close()
111            sp.wait()
112    return json_object
113
114
115def _read_build_metadata(board, cros_version):
116    """Read and parse the `metadata.json` file for a build.
117
118    Given the board and version string for a potential CrOS image,
119    find the URI of the build in GoogleStorage, and return a Python
120    object for the associated `metadata.json`.
121
122    @param board         Board for the build to be read.
123    @param cros_version  Build version string.
124    """
125    image_path = frontend.format_cros_image_name(board, cros_version)
126    return _read_gs_json_data(_BUILD_METADATA_PATTERN % image_path)
127
128
129def _get_by_key_path(dictdict, key_path):
130    """Traverse a sequence of keys in a dict of dicts.
131
132    The `dictdict` parameter is a dict of nested dict values, and
133    `key_path` a list of keys.
134
135    A single-element key path returns `dictdict[key_path[0]]`, a
136    two-element path returns `dictdict[key_path[0]][key_path[1]]`, and
137    so forth.  If any key in the path is not found, return `None`.
138
139    @param dictdict   A dictionary of nested dictionaries.
140    @param key_path   The sequence of keys to look up in `dictdict`.
141    @return The value found by successive dictionary lookups, or `None`.
142    """
143    value = dictdict
144    for key in key_path:
145        value = value.get(key)
146        if value is None:
147            break
148    return value
149
150
151def _get_model_firmware_versions(metadata_json, board):
152    """Get the firmware version for all models in a unibuild board.
153
154    @param metadata_json    The metadata_json dict parsed from the
155                            metadata.json file generated by the build.
156    @param board            The board name of the unibuild.
157    @return If the board has no models, return {board: None}.
158            Otherwise, return a dict mapping each model name to its
159            firmware version.
160    """
161    model_firmware_versions = {}
162    key_path = ['board-metadata', board, 'models']
163    model_versions = _get_by_key_path(metadata_json, key_path)
164
165    if model_versions is not None:
166        for model, fw_versions in model_versions.iteritems():
167            fw_version = (fw_versions.get('main-readwrite-firmware-version') or
168                          fw_versions.get('main-readonly-firmware-version'))
169            model_firmware_versions[model] = fw_version
170    else:
171        model_firmware_versions[board] = None
172
173    return model_firmware_versions
174
175
176def get_omaha_version_map():
177    """Convert omaha versions data to a versions mapping.
178
179    Returns a dictionary mapping board names to the currently preferred
180    version for the Beta channel as served by Omaha.  The mappings are
181    provided by settings in the JSON object read from `_OMAHA_STATUS`.
182
183    The board names are the names as known to Omaha:  If the board name
184    in the AFE contains '_', the corresponding Omaha name uses '-'
185    instead.  The boards mapped may include boards not in the list of
186    managed boards in the lab.
187
188    @return A dictionary mapping Omaha boards to Beta versions.
189    """
190    def _entry_valid(json_entry):
191        return json_entry['channel'] == 'beta'
192
193    def _get_omaha_data(json_entry):
194        board = json_entry['board']['public_codename']
195        milestone = json_entry['milestone']
196        build = json_entry['chrome_os_version']
197        version = 'R%d-%s' % (milestone, build)
198        return (board, version)
199
200    omaha_status = _read_gs_json_data(_OMAHA_STATUS)
201    return dict(_get_omaha_data(e) for e in omaha_status['omaha_data']
202                    if _entry_valid(e))
203
204
205def get_omaha_upgrade(omaha_map, board, version):
206    """Get the later of a build in `omaha_map` or `version`.
207
208    Read the Omaha version for `board` from `omaha_map`, and compare it
209    to `version`.  Return whichever version is more recent.
210
211    N.B. `board` is the name of a board as known to the AFE.  Board
212    names as known to Omaha are different; see
213    `get_omaha_version_map()`, above.  This function is responsible
214    for translating names as necessary.
215
216    @param omaha_map  Mapping of Omaha board names to preferred builds.
217    @param board      Name of the board to look up, as known to the AFE.
218    @param version    Minimum version to be accepted.
219
220    @return Returns a Chrome OS version string in standard form
221            R##-####.#.#.  Will return `None` if `version` is `None` and
222            no Omaha entry is found.
223    """
224    omaha_version = omaha_map.get(board.replace('_', '-'))
225    if version is None:
226        return omaha_version
227    if omaha_version is not None:
228        if utils.compare_versions(version, omaha_version) < 0:
229            return omaha_version
230    return version
231
232
233def get_firmware_versions(board, cros_version):
234    """Get the firmware versions for a given board and CrOS version.
235
236    During the CrOS auto-update process, the system will check firmware
237    on the target device, and update that firmware if needed.  This
238    function finds the version string of the firmware that would be
239    installed from a given CrOS build.
240
241    A build may have firmware for more than one hardware model, so the
242    returned value is a dictionary mapping models to firmware version
243    strings.
244
245    The returned firmware version value will be `None` if the build
246    isn't found in storage, if there is no firmware found for the build,
247    or if the board is denylisted from firmware updates in the test
248    lab.
249
250    @param board          The board for the firmware version to be
251                          determined.
252    @param cros_version   The CrOS version bundling the firmware.
253    @return A dict mapping from board to firmware version string for
254            non-unibuild board, or a dict mapping from models to firmware
255            versions for a unibuild board (see return type of
256            _get_model_firmware_versions)
257    """
258    if board in _FIRMWARE_UPGRADE_DENYLIST:
259        return {board: None}
260    try:
261        metadata_json = _read_build_metadata(board, cros_version)
262        unibuild = bool(_get_by_key_path(metadata_json, ['unibuild']))
263        if unibuild:
264            return _get_model_firmware_versions(metadata_json, board)
265        else:
266            key_path = ['board-metadata', board, 'main-firmware-version']
267            return {board: _get_by_key_path(metadata_json, key_path)}
268    except Exception as e:
269        # TODO(jrbarnette): If we get here, it likely means that the
270        # build for this board doesn't exist.  That can happen if a
271        # board doesn't release on the Beta channel for at least 6 months.
272        #
273        # We can't allow this error to propagate up the call chain
274        # because that will kill assigning versions to all the other
275        # boards that are still OK, so for now we ignore it.  Probably,
276        # we should do better.
277        return {board: None}
278