xref: /build/scripts/util/md5_check.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import difflib
import hashlib
import itertools
import json
import os
import zipfile
from .pycache import pycache_enabled
from .pycache import pycache

# When set and a difference is detected, a diff of what changed is printed.
PRINT_EXPLANATIONS = int(os.environ.get('PRINT_BUILD_EXPLANATIONS', 0))

# An escape hatch that causes all targets to be rebuilt.
_FORCE_REBUILD = int(os.environ.get('FORCE_REBUILD', 0))


def get_new_metadata(input_strings, input_paths):
    new_metadata = _Metadata()
    new_metadata.add_strings(input_strings)

    for path in input_paths:
        if _is_zip_file(path):
            entries = _extract_zip_entries(path)
            new_metadata.add_zip_file(path, entries)
        else:
            new_metadata.add_file(path, _md5_for_path(path))
    return new_metadata


def get_old_metadata(record_path):
    old_metadata = None
    if os.path.exists(record_path):
        with open(record_path, 'r') as jsonfile:
            try:
                old_metadata = _Metadata.from_file(jsonfile)
            except:  # noqa: E722 pylint: disable=bare-except
                pass
    return old_metadata


def print_explanations(record_path, changes):
    if PRINT_EXPLANATIONS:
        print('=' * 80)
        print('Target is stale: %s' % record_path)
        print(changes.describe_difference())
        print('=' * 80)


def call_and_record_if_stale(
        function,  # pylint: disable=invalid-name
        record_path=None,
        input_paths=None,
        input_strings=None,
        output_paths=None,
        force=False,
        pass_changes=False):
    """Calls function if outputs are stale.

    Outputs are considered stale if:
    - any output_paths are missing, or
    - the contents of any file within input_paths has changed, or
    - the contents of input_strings has changed.

    To debug which files are out-of-date, set the environment variable:
        PRINT_MD5_DIFFS=1

    Args:
      function: The function to call.
      record_path: Path to record metadata.
        Defaults to output_paths[0] + '.md5.stamp'
      input_paths: List of paths to calculate a md5 sum on.
      input_strings: List of strings to record verbatim.
      output_paths: List of output paths.
      force: Whether to treat outputs as missing regardless of whether they
        actually are.
      pass_changes: Whether to pass a Changes instance to |function|.
    """
    assert record_path or output_paths
    input_paths = input_paths or []
    input_strings = input_strings or []
    output_paths = output_paths or []

    new_metadata = get_new_metadata(input_strings, input_paths)
    force = force or _FORCE_REBUILD
    missing_outputs = [
        x for x in output_paths if force or not os.path.exists(x)
    ]

    if pycache_enabled:
        # Input strings, input files and outputs names together compose
        # cache manifest, which is the only identifier of a python action.
        manifest = '-'.join(
            [new_metadata.strings_md5(),
             new_metadata.files_md5()] + sorted(output_paths))
        record_path = pycache.get_manifest_path('{}.manifest'.format(manifest))
        old_metadata = get_old_metadata(record_path)
    else:
        record_path = record_path or output_paths[0] + '.md5.stamp'
        # When outputs are missing, don't bother gathering change information.
        if not missing_outputs:
            old_metadata = get_old_metadata(record_path)
        else:
            old_metadata = None

    changes = Changes(old_metadata, new_metadata, force, missing_outputs)
    if not changes.has_changes():
        if not pycache_enabled:
            return
        if pycache_enabled and pycache.retrieve(output_paths, prefix=manifest):
            return

    print_explanations(record_path, changes)

    args = (changes, ) if pass_changes else ()
    function(*args)
    if pycache_enabled:
        try:
            pycache.report_cache_stat('cache_miss')
        except:  # noqa: E722 pylint: disable=bare-except
            pass
        pycache.save(output_paths, prefix=manifest)

    with open(record_path, 'w') as record:
        new_metadata.to_file(record)


class Changes(object):
    """Provides and API for querying what changed between runs."""
    def __init__(self, old_metadata, new_metadata, force, missing_outputs):
        self.old_metadata = old_metadata
        self.new_metadata = new_metadata
        self.force = force
        self.missing_outputs = missing_outputs

    def _get_old_tag(self, path, subpath=None):
        return self.old_metadata and self.old_metadata.get_tag(path, subpath)

    def has_changes(self):
        """Returns whether any changes exist."""
        return (
            self.force or not self.old_metadata or
            self.old_metadata.strings_md5() != self.new_metadata.strings_md5()
            or self.old_metadata.files_md5() != self.new_metadata.files_md5())

    def added_or_modified_only(self):
        """Returns whether the only changes were from added or modified (sub)files.

        No missing outputs, no removed paths/subpaths.
        """
        if (self.force or not self.old_metadata
                or self.old_metadata.strings_md5() !=
                self.new_metadata.strings_md5()):
            return False
        if any(self.iter_removed_paths()):
            return False
        for path in self.iter_modified_paths():
            if any(self.iter_removed_subpaths(path)):
                return False
        return True

    def iter_all_paths(self):
        """Generator for paths."""
        return self.new_metadata.iter_paths()

    def iter_all_subpaths(self, path):
        """Generator for subpaths."""
        return self.new_metadata.iter_subpaths(path)

    def iter_added_paths(self):
        """Generator for paths that were added."""
        for path in self.new_metadata.iter_paths():
            if self._get_old_tag(path) is None:
                yield path

    def iter_added_subpaths(self, path):
        """Generator for paths that were added within the given zip file."""
        for subpath in self.new_metadata.iter_subpaths(path):
            if self._get_old_tag(path, subpath) is None:
                yield subpath

    def iter_removed_paths(self):
        """Generator for paths that were removed."""
        if self.old_metadata:
            for path in self.old_metadata.iter_paths():
                if self.new_metadata.get_tag(path) is None:
                    yield path

    def iter_removed_subpaths(self, path):
        """Generator for paths that were removed within the given zip file."""
        if self.old_metadata:
            for subpath in self.old_metadata.iter_subpaths(path):
                if self.new_metadata.get_tag(path, subpath) is None:
                    yield subpath

    def iter_modified_paths(self):
        """Generator for paths whose contents have changed."""
        for path in self.new_metadata.iter_paths():
            old_tag = self._get_old_tag(path)
            new_tag = self.new_metadata.get_tag(path)
            if old_tag is not None and old_tag != new_tag:
                yield path

    def iter_modified_subpaths(self, path):
        """Generator for paths within a zip file whose contents have changed."""
        for subpath in self.new_metadata.iter_subpaths(path):
            old_tag = self._get_old_tag(path, subpath)
            new_tag = self.new_metadata.get_tag(path, subpath)
            if old_tag is not None and old_tag != new_tag:
                yield subpath

    def iter_changed_paths(self):
        """Generator for all changed paths (added/removed/modified)."""
        return itertools.chain(self.iter_removed_paths(),
                               self.iter_modified_paths(),
                               self.iter_added_paths())

    def iter_changed_subpaths(self, path):
        """Generator for paths within a zip that were added/removed/modified."""
        return itertools.chain(self.iter_removed_subpaths(path),
                               self.iter_modified_subpaths(path),
                               self.iter_added_subpaths(path))

    def describe_difference(self):
        """Returns a human-readable description of what changed."""
        if self.force:
            return 'force=True'
        elif self.old_metadata is None:
            return 'Previous stamp file not found.'

        if self.old_metadata.strings_md5() != self.new_metadata.strings_md5():
            ndiff = difflib.ndiff(self.old_metadata.get_strings(),
                                  self.new_metadata.get_strings())
            changed = [s for s in ndiff if not s.startswith(' ')]
            return 'Input strings changed:\n  ' + '\n  '.join(changed)

        if self.old_metadata.files_md5() == self.new_metadata.files_md5():
            return "There's no difference."

        lines = []
        lines.extend('Added: {}'.format(p for p in self.iter_added_paths()))
        lines.extend('Removed: {}'.format(p
                                          for p in self.iter_removed_paths()))
        for path in self.iter_modified_paths():
            lines.append('Modified: {}'.format(path))
            lines.extend('  -> Subpath added: {}'.format(
                p for p in self.iter_added_subpaths(path)))
            lines.extend('  -> Subpath removed: {}'.format(
                p for p in self.iter_removed_subpaths(path)))
            lines.extend('  -> Subpath modified: {}'.format(
                p for p in self.iter_modified_subpaths(path)))
        if lines:
            return 'Input files changed:\n  {}'.format('\n  '.join(lines))

        if self.missing_outputs:
            return 'Outputs do not exist:\n  {}'.format('\n  '.join(
                self.missing_outputs))

        return 'I have no idea what changed (there is a bug).'


class _Metadata(object):
    """Data model for tracking change metadata."""
    def __init__(self):
        self._files_md5 = None
        self._strings_md5 = None
        self._files = []
        self._strings = []
        # Map of (path, subpath) -> entry. Created upon first call to _get_entry().
        self._file_map = None

    @classmethod
    def from_file(cls, fileobj):
        """Returns a _Metadata initialized from a file object."""
        ret = cls()
        obj = json.load(fileobj)
        ret._files_md5 = obj['files-md5']
        ret._strings_md5 = obj['strings-md5']
        ret._files = obj['input-files']
        ret._strings = obj['input-strings']
        return ret

    def to_file(self, fileobj):
        """Serializes metadata to the given file object."""
        obj = {
            "files-md5": self.files_md5(),
            "strings-md5": self.strings_md5(),
            "input-files": self._files,
            "input-strings": self._strings,
        }
        json.dump(obj, fileobj, indent=2, sort_keys=True)

    def _assert_not_queried(self):
        assert self._files_md5 is None
        assert self._strings_md5 is None
        assert self._file_map is None

    def add_strings(self, values):
        self._assert_not_queried()
        self._strings.extend(str(v) for v in values)

    def add_file(self, path, tag):
        """Adds metadata for a non-zip file.

        Args:
          path: Path to the file.
          tag: A short string representative of the file contents.
        """
        self._assert_not_queried()
        self._files.append({
            'path': path,
            'tag': tag,
        })

    def add_zip_file(self, path, entries):
        """Adds metadata for a zip file.

        Args:
          path: Path to the file.
          entries: List of (subpath, tag) tuples for entries within the zip.
        """
        self._assert_not_queried()
        tag = _compute_inline_md5(
            itertools.chain((e[0] for e in entries), (e[1] for e in entries)))
        self._files.append({
            'path':
            path,
            'tag':
            tag,
            'entries': [{
                "path": e[0],
                "tag": e[1]
            } for e in entries],
        })

    def get_strings(self):
        """Returns the list of input strings."""
        return self._strings

    def files_md5(self):
        """Lazily computes and returns the aggregate md5 of input files."""
        if self._files_md5 is None:
            # Omit paths from md5 since temporary files have random names.
            self._files_md5 = _compute_inline_md5(
                self.get_tag(p) for p in sorted(self.iter_paths()))
        return self._files_md5

    def strings_md5(self):
        """Lazily computes and returns the aggregate md5 of input strings."""
        if self._strings_md5 is None:
            self._strings_md5 = _compute_inline_md5(self._strings)
        return self._strings_md5

    def _get_entry(self, path, subpath=None):
        """Returns the JSON entry for the given path / subpath."""
        if self._file_map is None:
            self._file_map = {}
            for entry in self._files:
                self._file_map[(entry['path'], None)] = entry
                for subentry in entry.get('entries', ()):
                    self._file_map[(entry['path'],
                                    subentry['path'])] = subentry
        return self._file_map.get((path, subpath))

    def get_tag(self, path, subpath=None):
        """Returns the tag for the given path / subpath."""
        ret = self._get_entry(path, subpath)
        return ret and ret['tag']

    def iter_paths(self):
        """Returns a generator for all top-level paths."""
        return (e['path'] for e in self._files)

    def iter_subpaths(self, path):
        """Returns a generator for all subpaths in the given zip.

        If the given path is not a zip file or doesn't exist, returns an empty
        iterable.
        """
        outer_entry = self._get_entry(path)
        if not outer_entry:
            return ()
        subentries = outer_entry.get('entries', [])
        return (entry['path'] for entry in subentries)


def _update_md5_for_file(md5, path, block_size=2**16):
    # record md5 of linkto for dead link.
    if os.path.islink(path):
        linkto = os.readlink(path)
        if not os.path.exists(linkto):
            md5.update(linkto.encode())
            return

    with open(path, 'rb') as infile:
        while True:
            data = infile.read(block_size)
            if not data:
                break
            md5.update(data)


def _update_md5_for_directory(md5, dir_path):
    for root, _, files in os.walk(dir_path):
        for f in files:
            _update_md5_for_file(md5, os.path.join(root, f))


def _md5_for_path(path):
    md5 = hashlib.md5()
    if os.path.isdir(path):
        _update_md5_for_directory(md5, path)
    else:
        _update_md5_for_file(md5, path)
    return md5.hexdigest()


def _compute_inline_md5(iterable):
    """Computes the md5 of the concatenated parameters."""
    md5 = hashlib.md5()
    for item in iterable:
        md5.update(str(item).encode())
    return md5.hexdigest()


def _is_zip_file(path):
    """Returns whether to treat the given file as a zip file."""
    return path[-4:] in ('.zip')


def _extract_zip_entries(path):
    """Returns a list of (path, CRC32) of all files within |path|."""
    entries = []
    with zipfile.ZipFile(path) as zip_file:
        for zip_info in zip_file.infolist():
            # Skip directories and empty files.
            if zip_info.CRC:
                entries.append(
                    (zip_info.filename, zip_info.CRC + zip_info.compress_type))
    return entries