• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2016 - The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16"""A client that talks to Google Cloud Storage APIs."""
17
18import io
19import logging
20
21import apiclient
22
23from acloud import errors
24from acloud.internal.lib import base_cloud_client
25from acloud.internal.lib import utils
26
27logger = logging.getLogger(__name__)
28
29
30class StorageClient(base_cloud_client.BaseCloudApiClient):
31    """Client that talks to Google Cloud Storages."""
32
33    # API settings, used by BaseCloudApiClient.
34    API_NAME = "storage"
35    API_VERSION = "v1"
36    SCOPE = "https://www.googleapis.com/auth/devstorage.read_write"
37    GET_OBJ_MAX_RETRY = 3
38    GET_OBJ_RETRY_SLEEP = 5
39
40    # Other class variables.
41    OBJECT_URL_FMT = "https://storage.googleapis.com/%s/%s"
42
43    def Get(self, bucket_name, object_name):
44        """Get object in a bucket.
45
46        Args:
47            bucket_name: String, google cloud storage bucket name.
48            object_name: String, full path to the object within the bucket.
49
50        Returns:
51            A dictronary representing an object resource.
52        """
53        request = self.service.objects().get(
54            bucket=bucket_name, object=object_name)
55        return self.Execute(request)
56
57    def List(self, bucket_name, prefix=None):
58        """Lists objects in a bucket.
59
60        Args:
61            bucket_name: String, google cloud storage bucket name.
62            prefix: String, Filter results to objects whose names begin with
63                    this prefix.
64
65        Returns:
66            A list of google storage objects whose names match the prefix.
67            Each element is dictionary that wraps all the information about an object.
68        """
69        logger.debug("Listing storage bucket: %s, prefix: %s", bucket_name,
70                     prefix)
71        items = self.ListWithMultiPages(
72            api_resource=self.service.objects().list,
73            bucket=bucket_name,
74            prefix=prefix)
75        return items
76
77    def Upload(self, local_src, bucket_name, object_name, mime_type):
78        """Uploads a file.
79
80        Args:
81            local_src: string, a local path to a file to be uploaded.
82            bucket_name: string, google cloud storage bucket name.
83            object_name: string, the name of the remote file in storage.
84            mime_type: string, mime-type of the file.
85
86        Returns:
87            URL to the inserted artifact in storage.
88        """
89        logger.info("Uploading file: src: %s, bucket: %s, object: %s",
90                    local_src, bucket_name, object_name)
91        try:
92            with io.FileIO(local_src, mode="rb") as upload_file:
93                media = apiclient.http.MediaIoBaseUpload(upload_file, mime_type)
94                request = self.service.objects().insert(
95                    bucket=bucket_name, name=object_name, media_body=media)
96                response = self.Execute(request)
97            logger.info("Uploaded artifact: %s", response["selfLink"])
98            return response
99        except OSError as e:
100            logger.error("Uploading artifact fails: %s", str(e))
101            raise errors.DriverError(str(e))
102
103    def Delete(self, bucket_name, object_name):
104        """Deletes a file.
105
106        Args:
107            bucket_name: string, google cloud storage bucket name.
108            object_name: string, the name of the remote file in storage.
109        """
110        logger.info("Deleting file: bucket: %s, object: %s", bucket_name,
111                    object_name)
112        request = self.service.objects().delete(
113            bucket=bucket_name, object=object_name)
114        self.Execute(request)
115        logger.info("Deleted file: bucket: %s, object: %s", bucket_name,
116                    object_name)
117
118    def DeleteFiles(self, bucket_name, object_names):
119        """Deletes multiple files.
120
121        Args:
122            bucket_name: string, google cloud storage bucket name.
123            object_names: A list of strings, each of which is a name of a remote file.
124
125        Returns:
126            A tuple, (deleted, failed, error_msgs)
127            deleted: A list of names of objects that have been deleted.
128            faild: A list of names of objects that we fail to delete.
129            error_msgs: A list of failure messages.
130        """
131        deleted = []
132        failed = []
133        error_msgs = []
134        for object_name in object_names:
135            try:
136                self.Delete(bucket_name, object_name)
137                deleted.append(object_name)
138            except errors.DriverError as e:
139                failed.append(object_name)
140                error_msgs.append(str(e))
141        return deleted, failed, error_msgs
142
143    def GetUrl(self, bucket_name, object_name):
144        """Get information about a file object.
145
146        Args:
147            bucket_name: string, google cloud storage bucket name.
148            object_name: string, name of the file to look for.
149
150        Returns:
151            Value of "selfLink" field from the response, which represents
152            a url to the file.
153
154        Raises:
155            errors.ResourceNotFoundError: when file is not found.
156        """
157        item = utils.RetryExceptionType(
158            errors.ResourceNotFoundError,
159            self.GET_OBJ_MAX_RETRY,
160            self.Get,
161            self.GET_OBJ_RETRY_SLEEP,
162            utils.DEFAULT_RETRY_BACKOFF_FACTOR,
163            bucket_name=bucket_name,
164            object_name=object_name)
165        return item["selfLink"]
166