• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import base64
6import io
7import gzip
8import os
9import re
10
11import six
12
13from devil import devil_env
14from devil.android import device_errors
15from devil.utils import cmd_helper
16
17MD5SUM_DEVICE_LIB_PATH = '/data/local/tmp/md5sum'
18MD5SUM_DEVICE_BIN_PATH = MD5SUM_DEVICE_LIB_PATH + '/md5sum_bin'
19
20_STARTS_WITH_CHECKSUM_RE = re.compile(r'^[0-9a-fA-F]{16}$')
21
22# We need to cap how many paths we send to the md5_sum binaries at once because
23# the ARG_MAX on Android devices is relatively small, typically 131072 bytes.
24# However, the more paths we use per invocation, the lower the overhead of
25# starting processes, so we want to maximize this number, but we can't compute
26# it exactly as we don't know how well our paths will compress.
27# 5000 is experimentally determined to be reasonable. 10000 fails, and 7500
28# works with existing usage, so 5000 seems like a pretty safe compromise.
29_MAX_PATHS_PER_INVOCATION = 5000
30
31
32def CalculateHostMd5Sums(paths):
33  """Calculates the MD5 sum value for all items in |paths|.
34
35  Directories are traversed recursively and the MD5 sum of each file found is
36  reported in the result.
37
38  Args:
39    paths: A list of host paths to md5sum.
40  Returns:
41    A dict mapping file paths to their respective md5sum checksums.
42  """
43  if isinstance(paths, six.string_types):
44    paths = [paths]
45  paths = list(paths)
46
47  md5sum_bin_host_path = devil_env.config.FetchPath('md5sum_host')
48  if not os.path.exists(md5sum_bin_host_path):
49    raise IOError('File not built: %s' % md5sum_bin_host_path)
50  out = ""
51  for i in range(0, len(paths), _MAX_PATHS_PER_INVOCATION):
52    mem_file = io.BytesIO()
53    compressed = gzip.GzipFile(fileobj=mem_file, mode="wb")
54    data = ";".join(
55          [os.path.realpath(p) for p in paths[i:i+_MAX_PATHS_PER_INVOCATION]])
56    if six.PY3:
57      data = data.encode('utf-8')
58    compressed.write(data)
59    compressed.close()
60    compressed_paths = base64.b64encode(mem_file.getvalue())
61    out += cmd_helper.GetCmdOutput(
62        [md5sum_bin_host_path, "-gz", compressed_paths])
63
64  return dict(zip(paths, out.splitlines()))
65
66
67def CalculateDeviceMd5Sums(paths, device):
68  """Calculates the MD5 sum value for all items in |paths|.
69
70  Directories are traversed recursively and the MD5 sum of each file found is
71  reported in the result.
72
73  Args:
74    paths: A list of device paths to md5sum.
75  Returns:
76    A dict mapping file paths to their respective md5sum checksums.
77  """
78  if not paths:
79    return {}
80
81  if isinstance(paths, six.string_types):
82    paths = [paths]
83  paths = list(paths)
84
85  md5sum_dist_path = devil_env.config.FetchPath('md5sum_device', device=device)
86
87  if os.path.isdir(md5sum_dist_path):
88    md5sum_dist_bin_path = os.path.join(md5sum_dist_path, 'md5sum_bin')
89  else:
90    md5sum_dist_bin_path = md5sum_dist_path
91
92  if not os.path.exists(md5sum_dist_path):
93    raise IOError('File not built: %s' % md5sum_dist_path)
94  md5sum_file_size = os.path.getsize(md5sum_dist_bin_path)
95
96  # For better performance, make the script as small as possible to try and
97  # avoid needing to write to an intermediary file (which RunShellCommand will
98  # do if necessary).
99  md5sum_script = 'a=%s;' % MD5SUM_DEVICE_BIN_PATH
100  # Check if the binary is missing or has changed (using its file size as an
101  # indicator), and trigger a (re-)push via the exit code.
102  md5sum_script += '! [[ $(ls -l $a) = *%d* ]]&&exit 2;' % md5sum_file_size
103  # Make sure it can find libbase.so
104  md5sum_script += 'export LD_LIBRARY_PATH=%s;' % MD5SUM_DEVICE_LIB_PATH
105  for i in range(0, len(paths), _MAX_PATHS_PER_INVOCATION):
106    mem_file = io.BytesIO()
107    compressed = gzip.GzipFile(fileobj=mem_file, mode="wb")
108    data = ";".join(paths[i:i+_MAX_PATHS_PER_INVOCATION])
109    if six.PY3:
110      data = data.encode('utf-8')
111    compressed.write(data)
112    compressed.close()
113    compressed_paths = base64.b64encode(mem_file.getvalue())
114    md5sum_script += '$a -gz %s;' % compressed_paths
115  try:
116    out = device.RunShellCommand(
117        md5sum_script, shell=True, check_return=True, large_output=True)
118  except device_errors.AdbShellCommandFailedError as e:
119    # Push the binary only if it is found to not exist
120    # (faster than checking up-front).
121    if e.status == 2:
122      # If files were previously pushed as root (adbd running as root), trying
123      # to re-push as non-root causes the push command to report success, but
124      # actually fail. So, wipe the directory first.
125      device.RunShellCommand(['rm', '-rf', MD5SUM_DEVICE_LIB_PATH],
126                             as_root=True,
127                             check_return=True)
128      if os.path.isdir(md5sum_dist_path):
129        device.adb.Push(md5sum_dist_path, MD5SUM_DEVICE_LIB_PATH)
130      else:
131        mkdir_cmd = 'a=%s;[[ -e $a ]] || mkdir $a' % MD5SUM_DEVICE_LIB_PATH
132        device.RunShellCommand(mkdir_cmd, shell=True, check_return=True)
133        device.adb.Push(md5sum_dist_bin_path, MD5SUM_DEVICE_BIN_PATH)
134
135      out = device.RunShellCommand(
136          md5sum_script, shell=True, check_return=True, large_output=True)
137    else:
138      raise
139
140  return dict(zip(paths, [l for l in out if _STARTS_WITH_CHECKSUM_RE.match(l)]))
141