• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import copy
16import errno
17import getopt
18import getpass
19import imp
20import os
21import platform
22import re
23import shlex
24import shutil
25import subprocess
26import sys
27import tempfile
28import threading
29import time
30import zipfile
31
32try:
33  from hashlib import sha1 as sha1
34except ImportError:
35  from sha import sha as sha1
36
37# missing in Python 2.4 and before
38if not hasattr(os, "SEEK_SET"):
39  os.SEEK_SET = 0
40
41class Options(object): pass
42OPTIONS = Options()
43OPTIONS.search_path = "out/host/linux-x86"
44OPTIONS.signapk_path = "framework/signapk.jar"  # Relative to search_path
45OPTIONS.extra_signapk_args = []
46OPTIONS.java_path = "java"  # Use the one on the path by default.
47OPTIONS.public_key_suffix = ".x509.pem"
48OPTIONS.private_key_suffix = ".pk8"
49OPTIONS.verbose = False
50OPTIONS.tempfiles = []
51OPTIONS.device_specific = None
52OPTIONS.extras = {}
53OPTIONS.info_dict = None
54
55
56# Values for "certificate" in apkcerts that mean special things.
57SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL")
58
59
60class ExternalError(RuntimeError): pass
61
62
63def Run(args, **kwargs):
64  """Create and return a subprocess.Popen object, printing the command
65  line on the terminal if -v was specified."""
66  if OPTIONS.verbose:
67    print "  running: ", " ".join(args)
68  return subprocess.Popen(args, **kwargs)
69
70
71def CloseInheritedPipes():
72  """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds
73  before doing other work."""
74  if platform.system() != "Darwin":
75    return
76  for d in range(3, 1025):
77    try:
78      stat = os.fstat(d)
79      if stat is not None:
80        pipebit = stat[0] & 0x1000
81        if pipebit != 0:
82          os.close(d)
83    except OSError:
84      pass
85
86
87def LoadInfoDict(zip):
88  """Read and parse the META/misc_info.txt key/value pairs from the
89  input target files and return a dict."""
90
91  d = {}
92  try:
93    for line in zip.read("META/misc_info.txt").split("\n"):
94      line = line.strip()
95      if not line or line.startswith("#"): continue
96      k, v = line.split("=", 1)
97      d[k] = v
98  except KeyError:
99    # ok if misc_info.txt doesn't exist
100    pass
101
102  # backwards compatibility: These values used to be in their own
103  # files.  Look for them, in case we're processing an old
104  # target_files zip.
105
106  if "mkyaffs2_extra_flags" not in d:
107    try:
108      d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip()
109    except KeyError:
110      # ok if flags don't exist
111      pass
112
113  if "recovery_api_version" not in d:
114    try:
115      d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip()
116    except KeyError:
117      raise ValueError("can't find recovery API version in input target-files")
118
119  if "tool_extensions" not in d:
120    try:
121      d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip()
122    except KeyError:
123      # ok if extensions don't exist
124      pass
125
126  if "fstab_version" not in d:
127    d["fstab_version"] = "1"
128
129  try:
130    data = zip.read("META/imagesizes.txt")
131    for line in data.split("\n"):
132      if not line: continue
133      name, value = line.split(" ", 1)
134      if not value: continue
135      if name == "blocksize":
136        d[name] = value
137      else:
138        d[name + "_size"] = value
139  except KeyError:
140    pass
141
142  def makeint(key):
143    if key in d:
144      d[key] = int(d[key], 0)
145
146  makeint("recovery_api_version")
147  makeint("blocksize")
148  makeint("system_size")
149  makeint("userdata_size")
150  makeint("cache_size")
151  makeint("recovery_size")
152  makeint("boot_size")
153  makeint("fstab_version")
154
155  d["fstab"] = LoadRecoveryFSTab(zip, d["fstab_version"])
156  d["build.prop"] = LoadBuildProp(zip)
157  return d
158
159def LoadBuildProp(zip):
160  try:
161    data = zip.read("SYSTEM/build.prop")
162  except KeyError:
163    print "Warning: could not find SYSTEM/build.prop in %s" % zip
164    data = ""
165
166  d = {}
167  for line in data.split("\n"):
168    line = line.strip()
169    if not line or line.startswith("#"): continue
170    name, value = line.split("=", 1)
171    d[name] = value
172  return d
173
174def LoadRecoveryFSTab(zip, fstab_version):
175  class Partition(object):
176    pass
177
178  try:
179    data = zip.read("RECOVERY/RAMDISK/etc/recovery.fstab")
180  except KeyError:
181    print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab in %s." % zip
182    data = ""
183
184  if fstab_version == 1:
185    d = {}
186    for line in data.split("\n"):
187      line = line.strip()
188      if not line or line.startswith("#"): continue
189      pieces = line.split()
190      if not (3 <= len(pieces) <= 4):
191        raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,))
192
193      p = Partition()
194      p.mount_point = pieces[0]
195      p.fs_type = pieces[1]
196      p.device = pieces[2]
197      p.length = 0
198      options = None
199      if len(pieces) >= 4:
200        if pieces[3].startswith("/"):
201          p.device2 = pieces[3]
202          if len(pieces) >= 5:
203            options = pieces[4]
204        else:
205          p.device2 = None
206          options = pieces[3]
207      else:
208        p.device2 = None
209
210      if options:
211        options = options.split(",")
212        for i in options:
213          if i.startswith("length="):
214            p.length = int(i[7:])
215          else:
216              print "%s: unknown option \"%s\"" % (p.mount_point, i)
217
218      d[p.mount_point] = p
219
220  elif fstab_version == 2:
221    d = {}
222    for line in data.split("\n"):
223      line = line.strip()
224      if not line or line.startswith("#"): continue
225      pieces = line.split()
226      if len(pieces) != 5:
227        raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,))
228
229      # Ignore entries that are managed by vold
230      options = pieces[4]
231      if "voldmanaged=" in options: continue
232
233      # It's a good line, parse it
234      p = Partition()
235      p.device = pieces[0]
236      p.mount_point = pieces[1]
237      p.fs_type = pieces[2]
238      p.device2 = None
239      p.length = 0
240
241      options = options.split(",")
242      for i in options:
243        if i.startswith("length="):
244          p.length = int(i[7:])
245        else:
246          # Ignore all unknown options in the unified fstab
247          continue
248
249      d[p.mount_point] = p
250
251  else:
252    raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,))
253
254  return d
255
256
257def DumpInfoDict(d):
258  for k, v in sorted(d.items()):
259    print "%-25s = (%s) %s" % (k, type(v).__name__, v)
260
261def BuildBootableImage(sourcedir, fs_config_file, info_dict=None):
262  """Take a kernel, cmdline, and ramdisk directory from the input (in
263  'sourcedir'), and turn them into a boot image.  Return the image
264  data, or None if sourcedir does not appear to contains files for
265  building the requested image."""
266
267  if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or
268      not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)):
269    return None
270
271  if info_dict is None:
272    info_dict = OPTIONS.info_dict
273
274  ramdisk_img = tempfile.NamedTemporaryFile()
275  img = tempfile.NamedTemporaryFile()
276
277  if os.access(fs_config_file, os.F_OK):
278    cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")]
279  else:
280    cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")]
281  p1 = Run(cmd, stdout=subprocess.PIPE)
282  p2 = Run(["minigzip"],
283           stdin=p1.stdout, stdout=ramdisk_img.file.fileno())
284
285  p2.wait()
286  p1.wait()
287  assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,)
288  assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,)
289
290  cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")]
291
292  fn = os.path.join(sourcedir, "cmdline")
293  if os.access(fn, os.F_OK):
294    cmd.append("--cmdline")
295    cmd.append(open(fn).read().rstrip("\n"))
296
297  fn = os.path.join(sourcedir, "base")
298  if os.access(fn, os.F_OK):
299    cmd.append("--base")
300    cmd.append(open(fn).read().rstrip("\n"))
301
302  fn = os.path.join(sourcedir, "pagesize")
303  if os.access(fn, os.F_OK):
304    cmd.append("--pagesize")
305    cmd.append(open(fn).read().rstrip("\n"))
306
307  args = info_dict.get("mkbootimg_args", None)
308  if args and args.strip():
309    cmd.extend(args.split())
310
311  cmd.extend(["--ramdisk", ramdisk_img.name,
312              "--output", img.name])
313
314  p = Run(cmd, stdout=subprocess.PIPE)
315  p.communicate()
316  assert p.returncode == 0, "mkbootimg of %s image failed" % (
317      os.path.basename(sourcedir),)
318
319  img.seek(os.SEEK_SET, 0)
320  data = img.read()
321
322  ramdisk_img.close()
323  img.close()
324
325  return data
326
327
328def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir,
329                     info_dict=None):
330  """Return a File object (with name 'name') with the desired bootable
331  image.  Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name
332  'prebuilt_name', otherwise construct it from the source files in
333  'unpack_dir'/'tree_subdir'."""
334
335  prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name)
336  if os.path.exists(prebuilt_path):
337    print "using prebuilt %s..." % (prebuilt_name,)
338    return File.FromLocalFile(name, prebuilt_path)
339  else:
340    print "building image from target_files %s..." % (tree_subdir,)
341    fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt"
342    return File(name, BuildBootableImage(os.path.join(unpack_dir, tree_subdir),
343                                         os.path.join(unpack_dir, fs_config),
344                                         info_dict))
345
346
347def UnzipTemp(filename, pattern=None):
348  """Unzip the given archive into a temporary directory and return the name.
349
350  If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a
351  temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES.
352
353  Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the
354  main file), open for reading.
355  """
356
357  tmp = tempfile.mkdtemp(prefix="targetfiles-")
358  OPTIONS.tempfiles.append(tmp)
359
360  def unzip_to_dir(filename, dirname):
361    cmd = ["unzip", "-o", "-q", filename, "-d", dirname]
362    if pattern is not None:
363      cmd.append(pattern)
364    p = Run(cmd, stdout=subprocess.PIPE)
365    p.communicate()
366    if p.returncode != 0:
367      raise ExternalError("failed to unzip input target-files \"%s\"" %
368                          (filename,))
369
370  m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE)
371  if m:
372    unzip_to_dir(m.group(1), tmp)
373    unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES"))
374    filename = m.group(1)
375  else:
376    unzip_to_dir(filename, tmp)
377
378  return tmp, zipfile.ZipFile(filename, "r")
379
380
381def GetKeyPasswords(keylist):
382  """Given a list of keys, prompt the user to enter passwords for
383  those which require them.  Return a {key: password} dict.  password
384  will be None if the key has no password."""
385
386  no_passwords = []
387  need_passwords = []
388  key_passwords = {}
389  devnull = open("/dev/null", "w+b")
390  for k in sorted(keylist):
391    # We don't need a password for things that aren't really keys.
392    if k in SPECIAL_CERT_STRINGS:
393      no_passwords.append(k)
394      continue
395
396    p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix,
397             "-inform", "DER", "-nocrypt"],
398            stdin=devnull.fileno(),
399            stdout=devnull.fileno(),
400            stderr=subprocess.STDOUT)
401    p.communicate()
402    if p.returncode == 0:
403      # Definitely an unencrypted key.
404      no_passwords.append(k)
405    else:
406      p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix,
407               "-inform", "DER", "-passin", "pass:"],
408              stdin=devnull.fileno(),
409              stdout=devnull.fileno(),
410              stderr=subprocess.PIPE)
411      stdout, stderr = p.communicate()
412      if p.returncode == 0:
413        # Encrypted key with empty string as password.
414        key_passwords[k] = ''
415      elif stderr.startswith('Error decrypting key'):
416        # Definitely encrypted key.
417        # It would have said "Error reading key" if it didn't parse correctly.
418        need_passwords.append(k)
419      else:
420        # Potentially, a type of key that openssl doesn't understand.
421        # We'll let the routines in signapk.jar handle it.
422        no_passwords.append(k)
423  devnull.close()
424
425  key_passwords.update(PasswordManager().GetPasswords(need_passwords))
426  key_passwords.update(dict.fromkeys(no_passwords, None))
427  return key_passwords
428
429
430def SignFile(input_name, output_name, key, password, align=None,
431             whole_file=False):
432  """Sign the input_name zip/jar/apk, producing output_name.  Use the
433  given key and password (the latter may be None if the key does not
434  have a password.
435
436  If align is an integer > 1, zipalign is run to align stored files in
437  the output zip on 'align'-byte boundaries.
438
439  If whole_file is true, use the "-w" option to SignApk to embed a
440  signature that covers the whole file in the archive comment of the
441  zip file.
442  """
443
444  if align == 0 or align == 1:
445    align = None
446
447  if align:
448    temp = tempfile.NamedTemporaryFile()
449    sign_name = temp.name
450  else:
451    sign_name = output_name
452
453  cmd = [OPTIONS.java_path, "-Xmx2048m", "-jar",
454         os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)]
455  cmd.extend(OPTIONS.extra_signapk_args)
456  if whole_file:
457    cmd.append("-w")
458  cmd.extend([key + OPTIONS.public_key_suffix,
459              key + OPTIONS.private_key_suffix,
460              input_name, sign_name])
461
462  p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
463  if password is not None:
464    password += "\n"
465  p.communicate(password)
466  if p.returncode != 0:
467    raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,))
468
469  if align:
470    p = Run(["zipalign", "-f", str(align), sign_name, output_name])
471    p.communicate()
472    if p.returncode != 0:
473      raise ExternalError("zipalign failed: return code %s" % (p.returncode,))
474    temp.close()
475
476
477def CheckSize(data, target, info_dict):
478  """Check the data string passed against the max size limit, if
479  any, for the given target.  Raise exception if the data is too big.
480  Print a warning if the data is nearing the maximum size."""
481
482  if target.endswith(".img"): target = target[:-4]
483  mount_point = "/" + target
484
485  if info_dict["fstab"]:
486    if mount_point == "/userdata": mount_point = "/data"
487    p = info_dict["fstab"][mount_point]
488    fs_type = p.fs_type
489    device = p.device
490    if "/" in device:
491      device = device[device.rfind("/")+1:]
492    limit = info_dict.get(device + "_size", None)
493  if not fs_type or not limit: return
494
495  if fs_type == "yaffs2":
496    # image size should be increased by 1/64th to account for the
497    # spare area (64 bytes per 2k page)
498    limit = limit / 2048 * (2048+64)
499  size = len(data)
500  pct = float(size) * 100.0 / limit
501  msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit)
502  if pct >= 99.0:
503    raise ExternalError(msg)
504  elif pct >= 95.0:
505    print
506    print "  WARNING: ", msg
507    print
508  elif OPTIONS.verbose:
509    print "  ", msg
510
511
512def ReadApkCerts(tf_zip):
513  """Given a target_files ZipFile, parse the META/apkcerts.txt file
514  and return a {package: cert} dict."""
515  certmap = {}
516  for line in tf_zip.read("META/apkcerts.txt").split("\n"):
517    line = line.strip()
518    if not line: continue
519    m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+'
520                 r'private_key="(.*)"$', line)
521    if m:
522      name, cert, privkey = m.groups()
523      public_key_suffix_len = len(OPTIONS.public_key_suffix)
524      private_key_suffix_len = len(OPTIONS.private_key_suffix)
525      if cert in SPECIAL_CERT_STRINGS and not privkey:
526        certmap[name] = cert
527      elif (cert.endswith(OPTIONS.public_key_suffix) and
528            privkey.endswith(OPTIONS.private_key_suffix) and
529            cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]):
530        certmap[name] = cert[:-public_key_suffix_len]
531      else:
532        raise ValueError("failed to parse line from apkcerts.txt:\n" + line)
533  return certmap
534
535
536COMMON_DOCSTRING = """
537  -p  (--path)  <dir>
538      Prepend <dir>/bin to the list of places to search for binaries
539      run by this script, and expect to find jars in <dir>/framework.
540
541  -s  (--device_specific) <file>
542      Path to the python module containing device-specific
543      releasetools code.
544
545  -x  (--extra)  <key=value>
546      Add a key/value pair to the 'extras' dict, which device-specific
547      extension code may look at.
548
549  -v  (--verbose)
550      Show command lines being executed.
551
552  -h  (--help)
553      Display this usage message and exit.
554"""
555
556def Usage(docstring):
557  print docstring.rstrip("\n")
558  print COMMON_DOCSTRING
559
560
561def ParseOptions(argv,
562                 docstring,
563                 extra_opts="", extra_long_opts=(),
564                 extra_option_handler=None):
565  """Parse the options in argv and return any arguments that aren't
566  flags.  docstring is the calling module's docstring, to be displayed
567  for errors and -h.  extra_opts and extra_long_opts are for flags
568  defined by the caller, which are processed by passing them to
569  extra_option_handler."""
570
571  try:
572    opts, args = getopt.getopt(
573        argv, "hvp:s:x:" + extra_opts,
574        ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=",
575         "java_path=", "public_key_suffix=", "private_key_suffix=",
576         "device_specific=", "extra="] +
577        list(extra_long_opts))
578  except getopt.GetoptError, err:
579    Usage(docstring)
580    print "**", str(err), "**"
581    sys.exit(2)
582
583  path_specified = False
584
585  for o, a in opts:
586    if o in ("-h", "--help"):
587      Usage(docstring)
588      sys.exit()
589    elif o in ("-v", "--verbose"):
590      OPTIONS.verbose = True
591    elif o in ("-p", "--path"):
592      OPTIONS.search_path = a
593    elif o in ("--signapk_path",):
594      OPTIONS.signapk_path = a
595    elif o in ("--extra_signapk_args",):
596      OPTIONS.extra_signapk_args = shlex.split(a)
597    elif o in ("--java_path",):
598      OPTIONS.java_path = a
599    elif o in ("--public_key_suffix",):
600      OPTIONS.public_key_suffix = a
601    elif o in ("--private_key_suffix",):
602      OPTIONS.private_key_suffix = a
603    elif o in ("-s", "--device_specific"):
604      OPTIONS.device_specific = a
605    elif o in ("-x", "--extra"):
606      key, value = a.split("=", 1)
607      OPTIONS.extras[key] = value
608    else:
609      if extra_option_handler is None or not extra_option_handler(o, a):
610        assert False, "unknown option \"%s\"" % (o,)
611
612  os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") +
613                        os.pathsep + os.environ["PATH"])
614
615  return args
616
617
618def Cleanup():
619  for i in OPTIONS.tempfiles:
620    if os.path.isdir(i):
621      shutil.rmtree(i)
622    else:
623      os.remove(i)
624
625
626class PasswordManager(object):
627  def __init__(self):
628    self.editor = os.getenv("EDITOR", None)
629    self.pwfile = os.getenv("ANDROID_PW_FILE", None)
630
631  def GetPasswords(self, items):
632    """Get passwords corresponding to each string in 'items',
633    returning a dict.  (The dict may have keys in addition to the
634    values in 'items'.)
635
636    Uses the passwords in $ANDROID_PW_FILE if available, letting the
637    user edit that file to add more needed passwords.  If no editor is
638    available, or $ANDROID_PW_FILE isn't define, prompts the user
639    interactively in the ordinary way.
640    """
641
642    current = self.ReadFile()
643
644    first = True
645    while True:
646      missing = []
647      for i in items:
648        if i not in current or not current[i]:
649          missing.append(i)
650      # Are all the passwords already in the file?
651      if not missing: return current
652
653      for i in missing:
654        current[i] = ""
655
656      if not first:
657        print "key file %s still missing some passwords." % (self.pwfile,)
658        answer = raw_input("try to edit again? [y]> ").strip()
659        if answer and answer[0] not in 'yY':
660          raise RuntimeError("key passwords unavailable")
661      first = False
662
663      current = self.UpdateAndReadFile(current)
664
665  def PromptResult(self, current):
666    """Prompt the user to enter a value (password) for each key in
667    'current' whose value is fales.  Returns a new dict with all the
668    values.
669    """
670    result = {}
671    for k, v in sorted(current.iteritems()):
672      if v:
673        result[k] = v
674      else:
675        while True:
676          result[k] = getpass.getpass("Enter password for %s key> "
677                                      % (k,)).strip()
678          if result[k]: break
679    return result
680
681  def UpdateAndReadFile(self, current):
682    if not self.editor or not self.pwfile:
683      return self.PromptResult(current)
684
685    f = open(self.pwfile, "w")
686    os.chmod(self.pwfile, 0600)
687    f.write("# Enter key passwords between the [[[ ]]] brackets.\n")
688    f.write("# (Additional spaces are harmless.)\n\n")
689
690    first_line = None
691    sorted = [(not v, k, v) for (k, v) in current.iteritems()]
692    sorted.sort()
693    for i, (_, k, v) in enumerate(sorted):
694      f.write("[[[  %s  ]]] %s\n" % (v, k))
695      if not v and first_line is None:
696        # position cursor on first line with no password.
697        first_line = i + 4
698    f.close()
699
700    p = Run([self.editor, "+%d" % (first_line,), self.pwfile])
701    _, _ = p.communicate()
702
703    return self.ReadFile()
704
705  def ReadFile(self):
706    result = {}
707    if self.pwfile is None: return result
708    try:
709      f = open(self.pwfile, "r")
710      for line in f:
711        line = line.strip()
712        if not line or line[0] == '#': continue
713        m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line)
714        if not m:
715          print "failed to parse password file: ", line
716        else:
717          result[m.group(2)] = m.group(1)
718      f.close()
719    except IOError, e:
720      if e.errno != errno.ENOENT:
721        print "error reading password file: ", str(e)
722    return result
723
724
725def ZipWriteStr(zip, filename, data, perms=0644):
726  # use a fixed timestamp so the output is repeatable.
727  zinfo = zipfile.ZipInfo(filename=filename,
728                          date_time=(2009, 1, 1, 0, 0, 0))
729  zinfo.compress_type = zip.compression
730  zinfo.external_attr = perms << 16
731  zip.writestr(zinfo, data)
732
733
734class DeviceSpecificParams(object):
735  module = None
736  def __init__(self, **kwargs):
737    """Keyword arguments to the constructor become attributes of this
738    object, which is passed to all functions in the device-specific
739    module."""
740    for k, v in kwargs.iteritems():
741      setattr(self, k, v)
742    self.extras = OPTIONS.extras
743
744    if self.module is None:
745      path = OPTIONS.device_specific
746      if not path: return
747      try:
748        if os.path.isdir(path):
749          info = imp.find_module("releasetools", [path])
750        else:
751          d, f = os.path.split(path)
752          b, x = os.path.splitext(f)
753          if x == ".py":
754            f = b
755          info = imp.find_module(f, [d])
756        self.module = imp.load_module("device_specific", *info)
757      except ImportError:
758        print "unable to load device-specific module; assuming none"
759
760  def _DoCall(self, function_name, *args, **kwargs):
761    """Call the named function in the device-specific module, passing
762    the given args and kwargs.  The first argument to the call will be
763    the DeviceSpecific object itself.  If there is no module, or the
764    module does not define the function, return the value of the
765    'default' kwarg (which itself defaults to None)."""
766    if self.module is None or not hasattr(self.module, function_name):
767      return kwargs.get("default", None)
768    return getattr(self.module, function_name)(*((self,) + args), **kwargs)
769
770  def FullOTA_Assertions(self):
771    """Called after emitting the block of assertions at the top of a
772    full OTA package.  Implementations can add whatever additional
773    assertions they like."""
774    return self._DoCall("FullOTA_Assertions")
775
776  def FullOTA_InstallBegin(self):
777    """Called at the start of full OTA installation."""
778    return self._DoCall("FullOTA_InstallBegin")
779
780  def FullOTA_InstallEnd(self):
781    """Called at the end of full OTA installation; typically this is
782    used to install the image for the device's baseband processor."""
783    return self._DoCall("FullOTA_InstallEnd")
784
785  def IncrementalOTA_Assertions(self):
786    """Called after emitting the block of assertions at the top of an
787    incremental OTA package.  Implementations can add whatever
788    additional assertions they like."""
789    return self._DoCall("IncrementalOTA_Assertions")
790
791  def IncrementalOTA_VerifyBegin(self):
792    """Called at the start of the verification phase of incremental
793    OTA installation; additional checks can be placed here to abort
794    the script before any changes are made."""
795    return self._DoCall("IncrementalOTA_VerifyBegin")
796
797  def IncrementalOTA_VerifyEnd(self):
798    """Called at the end of the verification phase of incremental OTA
799    installation; additional checks can be placed here to abort the
800    script before any changes are made."""
801    return self._DoCall("IncrementalOTA_VerifyEnd")
802
803  def IncrementalOTA_InstallBegin(self):
804    """Called at the start of incremental OTA installation (after
805    verification is complete)."""
806    return self._DoCall("IncrementalOTA_InstallBegin")
807
808  def IncrementalOTA_InstallEnd(self):
809    """Called at the end of incremental OTA installation; typically
810    this is used to install the image for the device's baseband
811    processor."""
812    return self._DoCall("IncrementalOTA_InstallEnd")
813
814class File(object):
815  def __init__(self, name, data):
816    self.name = name
817    self.data = data
818    self.size = len(data)
819    self.sha1 = sha1(data).hexdigest()
820
821  @classmethod
822  def FromLocalFile(cls, name, diskname):
823    f = open(diskname, "rb")
824    data = f.read()
825    f.close()
826    return File(name, data)
827
828  def WriteToTemp(self):
829    t = tempfile.NamedTemporaryFile()
830    t.write(self.data)
831    t.flush()
832    return t
833
834  def AddToZip(self, z):
835    ZipWriteStr(z, self.name, self.data)
836
837DIFF_PROGRAM_BY_EXT = {
838    ".gz" : "imgdiff",
839    ".zip" : ["imgdiff", "-z"],
840    ".jar" : ["imgdiff", "-z"],
841    ".apk" : ["imgdiff", "-z"],
842    ".img" : "imgdiff",
843    }
844
845class Difference(object):
846  def __init__(self, tf, sf, diff_program=None):
847    self.tf = tf
848    self.sf = sf
849    self.patch = None
850    self.diff_program = diff_program
851
852  def ComputePatch(self):
853    """Compute the patch (as a string of data) needed to turn sf into
854    tf.  Returns the same tuple as GetPatch()."""
855
856    tf = self.tf
857    sf = self.sf
858
859    if self.diff_program:
860      diff_program = self.diff_program
861    else:
862      ext = os.path.splitext(tf.name)[1]
863      diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
864
865    ttemp = tf.WriteToTemp()
866    stemp = sf.WriteToTemp()
867
868    ext = os.path.splitext(tf.name)[1]
869
870    try:
871      ptemp = tempfile.NamedTemporaryFile()
872      if isinstance(diff_program, list):
873        cmd = copy.copy(diff_program)
874      else:
875        cmd = [diff_program]
876      cmd.append(stemp.name)
877      cmd.append(ttemp.name)
878      cmd.append(ptemp.name)
879      p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
880      _, err = p.communicate()
881      if err or p.returncode != 0:
882        print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
883        return None
884      diff = ptemp.read()
885    finally:
886      ptemp.close()
887      stemp.close()
888      ttemp.close()
889
890    self.patch = diff
891    return self.tf, self.sf, self.patch
892
893
894  def GetPatch(self):
895    """Return a tuple (target_file, source_file, patch_data).
896    patch_data may be None if ComputePatch hasn't been called, or if
897    computing the patch failed."""
898    return self.tf, self.sf, self.patch
899
900
901def ComputeDifferences(diffs):
902  """Call ComputePatch on all the Difference objects in 'diffs'."""
903  print len(diffs), "diffs to compute"
904
905  # Do the largest files first, to try and reduce the long-pole effect.
906  by_size = [(i.tf.size, i) for i in diffs]
907  by_size.sort(reverse=True)
908  by_size = [i[1] for i in by_size]
909
910  lock = threading.Lock()
911  diff_iter = iter(by_size)   # accessed under lock
912
913  def worker():
914    try:
915      lock.acquire()
916      for d in diff_iter:
917        lock.release()
918        start = time.time()
919        d.ComputePatch()
920        dur = time.time() - start
921        lock.acquire()
922
923        tf, sf, patch = d.GetPatch()
924        if sf.name == tf.name:
925          name = tf.name
926        else:
927          name = "%s (%s)" % (tf.name, sf.name)
928        if patch is None:
929          print "patching failed!                                  %s" % (name,)
930        else:
931          print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
932              dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
933      lock.release()
934    except Exception, e:
935      print e
936      raise
937
938  # start worker threads; wait for them all to finish.
939  threads = [threading.Thread(target=worker)
940             for i in range(OPTIONS.worker_threads)]
941  for th in threads:
942    th.start()
943  while threads:
944    threads.pop().join()
945
946
947# map recovery.fstab's fs_types to mount/format "partition types"
948PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD",
949                    "ext4": "EMMC", "emmc": "EMMC" }
950
951def GetTypeAndDevice(mount_point, info):
952  fstab = info["fstab"]
953  if fstab:
954    return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device
955  else:
956    return None
957
958
959def ParseCertificate(data):
960  """Parse a PEM-format certificate."""
961  cert = []
962  save = False
963  for line in data.split("\n"):
964    if "--END CERTIFICATE--" in line:
965      break
966    if save:
967      cert.append(line)
968    if "--BEGIN CERTIFICATE--" in line:
969      save = True
970  cert = "".join(cert).decode('base64')
971  return cert
972