• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2016 - The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16"""A client that talks to Google Cloud Storage APIs."""
17
18import io
19import logging
20
21import apiclient
22
23from acloud import errors
24from acloud.internal.lib import base_cloud_client
25from acloud.internal.lib import utils
26
27
28logger = logging.getLogger(__name__)
29
30
31class StorageClient(base_cloud_client.BaseCloudApiClient):
32    """Client that talks to Google Cloud Storages."""
33
34    # API settings, used by BaseCloudApiClient.
35    API_NAME = "storage"
36    API_VERSION = "v1"
37    SCOPE = "https://www.googleapis.com/auth/devstorage.read_write"
38    GET_OBJ_MAX_RETRY = 3
39    GET_OBJ_RETRY_SLEEP = 5
40
41    # Other class variables.
42    OBJECT_URL_FMT = "https://storage.googleapis.com/%s/%s"
43
44    def Get(self, bucket_name, object_name):
45        """Get object in a bucket.
46
47        Args:
48            bucket_name: String, google cloud storage bucket name.
49            object_name: String, full path to the object within the bucket.
50
51        Returns:
52            A dictronary representing an object resource.
53        """
54        request = self.service.objects().get(
55            bucket=bucket_name, object=object_name)
56        return self.Execute(request)
57
58    def List(self, bucket_name, prefix=None):
59        """Lists objects in a bucket.
60
61        Args:
62            bucket_name: String, google cloud storage bucket name.
63            prefix: String, Filter results to objects whose names begin with
64                    this prefix.
65
66        Returns:
67            A list of google storage objects whose names match the prefix.
68            Each element is dictionary that wraps all the information about an object.
69        """
70        logger.debug("Listing storage bucket: %s, prefix: %s", bucket_name,
71                     prefix)
72        items = self.ListWithMultiPages(
73            api_resource=self.service.objects().list,
74            bucket=bucket_name,
75            prefix=prefix)
76        return items
77
78    def Upload(self, local_src, bucket_name, object_name, mime_type):
79        """Uploads a file.
80
81        Args:
82            local_src: string, a local path to a file to be uploaded.
83            bucket_name: string, google cloud storage bucket name.
84            object_name: string, the name of the remote file in storage.
85            mime_type: string, mime-type of the file.
86
87        Returns:
88            URL to the inserted artifact in storage.
89        """
90        logger.info("Uploading file: src: %s, bucket: %s, object: %s",
91                    local_src, bucket_name, object_name)
92        try:
93            with io.FileIO(local_src, mode="rb") as upload_file:
94                media = apiclient.http.MediaIoBaseUpload(upload_file, mime_type)
95                request = self.service.objects().insert(
96                    bucket=bucket_name, name=object_name, media_body=media)
97                response = self.Execute(request)
98            logger.info("Uploaded artifact: %s", response["selfLink"])
99            return response
100        except OSError as e:
101            logger.error("Uploading artifact fails: %s", str(e))
102            raise errors.DriverError(str(e))
103
104    def Delete(self, bucket_name, object_name):
105        """Deletes a file.
106
107        Args:
108            bucket_name: string, google cloud storage bucket name.
109            object_name: string, the name of the remote file in storage.
110        """
111        logger.info("Deleting file: bucket: %s, object: %s", bucket_name,
112                    object_name)
113        request = self.service.objects().delete(
114            bucket=bucket_name, object=object_name)
115        self.Execute(request)
116        logger.info("Deleted file: bucket: %s, object: %s", bucket_name,
117                    object_name)
118
119    def DeleteFiles(self, bucket_name, object_names):
120        """Deletes multiple files.
121
122        Args:
123            bucket_name: string, google cloud storage bucket name.
124            object_names: A list of strings, each of which is a name of a remote file.
125
126        Returns:
127            A tuple, (deleted, failed, error_msgs)
128            deleted: A list of names of objects that have been deleted.
129            faild: A list of names of objects that we fail to delete.
130            error_msgs: A list of failure messages.
131        """
132        deleted = []
133        failed = []
134        error_msgs = []
135        for object_name in object_names:
136            try:
137                self.Delete(bucket_name, object_name)
138                deleted.append(object_name)
139            except errors.DriverError as e:
140                failed.append(object_name)
141                error_msgs.append(str(e))
142        return deleted, failed, error_msgs
143
144    def GetUrl(self, bucket_name, object_name):
145        """Get information about a file object.
146
147        Args:
148            bucket_name: string, google cloud storage bucket name.
149            object_name: string, name of the file to look for.
150
151        Returns:
152            Value of "selfLink" field from the response, which represents
153            a url to the file.
154
155        Raises:
156            errors.ResourceNotFoundError: when file is not found.
157        """
158        item = utils.RetryExceptionType(
159            errors.ResourceNotFoundError,
160            self.GET_OBJ_MAX_RETRY,
161            self.Get,
162            self.GET_OBJ_RETRY_SLEEP,
163            utils.DEFAULT_RETRY_BACKOFF_FACTOR,
164            bucket_name=bucket_name,
165            object_name=object_name)
166        return item["selfLink"]
167