• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2017 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Helper functions to upload data to Google Storage.
6
7Text data should be streamed to logdog using |logdog_helper| module.
8Due to logdog not having image or HTML viewer, those instead should be uploaded
9to Google Storage directly using this module.
10"""
11
12import logging
13import os
14import sys
15import time
16try:
17  from urllib.parse import urlparse
18except ImportError:
19  from urlparse import urlparse
20
21from pylib.constants import host_paths
22from pylib.utils import decorators
23
24if host_paths.DEVIL_PATH not in sys.path:
25  sys.path.append(host_paths.DEVIL_PATH)
26from devil.utils import cmd_helper
27
28_GSUTIL_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party',
29                            'catapult', 'third_party', 'gsutil', 'gsutil')
30_PUBLIC_URL = 'https://storage.googleapis.com/%s/'
31_AUTHENTICATED_URL = 'https://storage.cloud.google.com/%s/'
32
33
34@decorators.NoRaiseException(default_return_value='')
35def upload(name, filepath, bucket, gs_args=None, command_args=None,
36           content_type=None, authenticated_link=True):
37  """Uploads data to Google Storage.
38
39  Args:
40    name: Name of the file on Google Storage.
41    filepath: Path to file you want to upload.
42    bucket: Bucket to upload file to.
43    content_type: Content type to upload as. If not specified, Google storage
44        will attempt to infer content type from file extension.
45    authenticated_link: Whether to return a link that requires user to
46        authenticate with a Google account. Setting this to false will return
47        a link that does not require user to be signed into Google account but
48        will only work for completely public storage buckets.
49  Returns:
50    Web link to item uploaded to Google Storage bucket.
51  """
52  bucket = _format_bucket_name(bucket)
53
54  gs_path = 'gs://%s/%s' % (bucket, name)
55  logging.info('Uploading %s to %s', filepath, gs_path)
56
57  cmd = [_GSUTIL_PATH, '-q']
58  cmd.extend(gs_args or [])
59  if content_type:
60    cmd.extend(['-h', 'Content-Type:%s' % content_type])
61  cmd.extend(['cp'] + (command_args or []) + [filepath, gs_path])
62
63  cmd_helper.RunCmd(cmd)
64
65  return get_url_link(name, bucket, authenticated_link)
66
67
68@decorators.NoRaiseException(default_return_value='')
69def read_from_link(link):
70  # Note that urlparse returns the path with an initial '/', so we only need to
71  # add one more after the 'gs;'
72  gs_path = 'gs:/%s' % urlparse(link).path
73  cmd = [_GSUTIL_PATH, '-q', 'cat', gs_path]
74  return cmd_helper.GetCmdOutput(cmd)
75
76
77@decorators.NoRaiseException(default_return_value=False)
78def exists(name, bucket):
79  bucket = _format_bucket_name(bucket)
80  gs_path = 'gs://%s/%s' % (bucket, name)
81
82  cmd = [_GSUTIL_PATH, '-q', 'stat', gs_path]
83  return_code = cmd_helper.RunCmd(cmd)
84  return return_code == 0
85
86
87# TODO(jbudorick): Delete this function. Only one user of it.
88def unique_name(basename, suffix='', timestamp=True, device=None):
89  """Helper function for creating a unique name for a file to store in GS.
90
91  Args:
92    basename: Base of the unique filename.
93    suffix: Suffix of filename.
94    timestamp: Whether or not to add a timestamp to name.
95    device: Device to add device serial of to name.
96  """
97  return '%s%s%s%s' % (
98      basename,
99      '_%s' % time.strftime('%Y_%m_%d_T%H_%M_%S-UTC', time.gmtime())
100          if timestamp else '',
101      '_%s' % device.serial if device else '',
102      suffix)
103
104
105def get_url_link(name, bucket, authenticated_link=True):
106  """Get url link before/without uploading.
107
108  Args:
109    name: Name of the file on Google Storage.
110    bucket: Bucket to upload file to.
111    authenticated_link: Whether to return a link that requires user to
112        authenticate with a Google account. Setting this to false will return
113        a link that does not require user to be signed into Google account but
114        will only work for completely public storage buckets.
115  Returns:
116    Web link to item to be uploaded to Google Storage bucket
117  """
118  bucket = _format_bucket_name(bucket)
119  url_template = _AUTHENTICATED_URL if authenticated_link else _PUBLIC_URL
120  return os.path.join(url_template % bucket, name)
121
122
123def _format_bucket_name(bucket):
124  if bucket.startswith('gs://'):
125    bucket = bucket[len('gs://'):]
126  if bucket.endswith('/'):
127    bucket = bucket[:-1]
128  return bucket
129