1# -*- coding: utf-8 -*- 2# Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/ 3# Copyright (c) 2010, Eucalyptus Systems, Inc. 4# Copyright (c) 2011, Nexenta Systems, Inc. 5# Copyright (c) 2012, Google, Inc. 6# All rights reserved. 7# 8# Permission is hereby granted, free of charge, to any person obtaining a 9# copy of this software and associated documentation files (the 10# "Software"), to deal in the Software without restriction, including 11# without limitation the rights to use, copy, modify, merge, publish, dis- 12# tribute, sublicense, and/or sell copies of the Software, and to permit 13# persons to whom the Software is furnished to do so, subject to the fol- 14# lowing conditions: 15# 16# The above copyright notice and this permission notice shall be included 17# in all copies or substantial portions of the Software. 18# 19# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- 21# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 22# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 23# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 24# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 25# IN THE SOFTWARE. 26 27""" 28Some integration tests for the GSConnection 29""" 30 31import os 32import re 33import StringIO 34import urllib 35import xml.sax 36 37from boto import handler 38from boto import storage_uri 39from boto.gs.acl import ACL 40from boto.gs.cors import Cors 41from boto.gs.lifecycle import LifecycleConfig 42from tests.integration.gs.testcase import GSTestCase 43 44 45CORS_EMPTY = '<CorsConfig></CorsConfig>' 46CORS_DOC = ('<CorsConfig><Cors><Origins><Origin>origin1.example.com' 47 '</Origin><Origin>origin2.example.com</Origin></Origins>' 48 '<Methods><Method>GET</Method><Method>PUT</Method>' 49 '<Method>POST</Method></Methods><ResponseHeaders>' 50 '<ResponseHeader>foo</ResponseHeader>' 51 '<ResponseHeader>bar</ResponseHeader></ResponseHeaders>' 52 '</Cors></CorsConfig>') 53 54LIFECYCLE_EMPTY = ('<?xml version="1.0" encoding="UTF-8"?>' 55 '<LifecycleConfiguration></LifecycleConfiguration>') 56LIFECYCLE_DOC = ('<?xml version="1.0" encoding="UTF-8"?>' 57 '<LifecycleConfiguration><Rule>' 58 '<Action><Delete/></Action>' 59 '<Condition><Age>365</Age>' 60 '<CreatedBefore>2013-01-15</CreatedBefore>' 61 '<NumberOfNewerVersions>3</NumberOfNewerVersions>' 62 '<IsLive>true</IsLive></Condition>' 63 '</Rule></LifecycleConfiguration>') 64LIFECYCLE_CONDITIONS = {'Age': '365', 65 'CreatedBefore': '2013-01-15', 66 'NumberOfNewerVersions': '3', 67 'IsLive': 'true'} 68 69# Regexp for matching project-private default object ACL. 70PROJECT_PRIVATE_RE = ('\s*<AccessControlList>\s*<Entries>\s*<Entry>' 71 '\s*<Scope type="GroupById"><ID>[0-9a-fA-F]+</ID></Scope>' 72 '\s*<Permission>FULL_CONTROL</Permission>\s*</Entry>\s*<Entry>' 73 '\s*<Scope type="GroupById"><ID>[0-9a-fA-F]+</ID></Scope>' 74 '\s*<Permission>FULL_CONTROL</Permission>\s*</Entry>\s*<Entry>' 75 '\s*<Scope type="GroupById"><ID>[0-9a-fA-F]+</ID></Scope>' 76 '\s*<Permission>READ</Permission></Entry>\s*</Entries>' 77 '\s*</AccessControlList>\s*') 78 79 80class GSBasicTest(GSTestCase): 81 """Tests some basic GCS functionality.""" 82 83 def test_read_write(self): 84 """Tests basic read/write to keys.""" 85 bucket = self._MakeBucket() 86 bucket_name = bucket.name 87 # now try a get_bucket call and see if it's really there 88 bucket = self._GetConnection().get_bucket(bucket_name) 89 key_name = 'foobar' 90 k = bucket.new_key(key_name) 91 s1 = 'This is a test of file upload and download' 92 k.set_contents_from_string(s1) 93 tmpdir = self._MakeTempDir() 94 fpath = os.path.join(tmpdir, key_name) 95 fp = open(fpath, 'wb') 96 # now get the contents from gcs to a local file 97 k.get_contents_to_file(fp) 98 fp.close() 99 fp = open(fpath) 100 # check to make sure content read from gcs is identical to original 101 self.assertEqual(s1, fp.read()) 102 fp.close() 103 # Use generate_url to get the contents 104 url = self._conn.generate_url(900, 'GET', bucket=bucket.name, key=key_name) 105 f = urllib.urlopen(url) 106 self.assertEqual(s1, f.read()) 107 f.close() 108 # check to make sure set_contents_from_file is working 109 sfp = StringIO.StringIO('foo') 110 k.set_contents_from_file(sfp) 111 self.assertEqual(k.get_contents_as_string(), 'foo') 112 sfp2 = StringIO.StringIO('foo2') 113 k.set_contents_from_file(sfp2) 114 self.assertEqual(k.get_contents_as_string(), 'foo2') 115 116 def test_get_all_keys(self): 117 """Tests get_all_keys.""" 118 phony_mimetype = 'application/x-boto-test' 119 headers = {'Content-Type': phony_mimetype} 120 tmpdir = self._MakeTempDir() 121 fpath = os.path.join(tmpdir, 'foobar1') 122 fpath2 = os.path.join(tmpdir, 'foobar') 123 with open(fpath2, 'w') as f: 124 f.write('test-data') 125 bucket = self._MakeBucket() 126 127 # First load some data for the first one, overriding content type. 128 k = bucket.new_key('foobar') 129 s1 = 'test-contents' 130 s2 = 'test-contents2' 131 k.name = 'foo/bar' 132 k.set_contents_from_string(s1, headers) 133 k.name = 'foo/bas' 134 k.set_contents_from_filename(fpath2) 135 k.name = 'foo/bat' 136 k.set_contents_from_string(s1) 137 k.name = 'fie/bar' 138 k.set_contents_from_string(s1) 139 k.name = 'fie/bas' 140 k.set_contents_from_string(s1) 141 k.name = 'fie/bat' 142 k.set_contents_from_string(s1) 143 # try resetting the contents to another value 144 md5 = k.md5 145 k.set_contents_from_string(s2) 146 self.assertNotEqual(k.md5, md5) 147 148 fp2 = open(fpath2, 'rb') 149 k.md5 = None 150 k.base64md5 = None 151 k.set_contents_from_stream(fp2) 152 fp = open(fpath, 'wb') 153 k.get_contents_to_file(fp) 154 fp.close() 155 fp2.seek(0, 0) 156 fp = open(fpath, 'rb') 157 self.assertEqual(fp2.read(), fp.read()) 158 fp.close() 159 fp2.close() 160 all = bucket.get_all_keys() 161 self.assertEqual(len(all), 6) 162 rs = bucket.get_all_keys(prefix='foo') 163 self.assertEqual(len(rs), 3) 164 rs = bucket.get_all_keys(prefix='', delimiter='/') 165 self.assertEqual(len(rs), 2) 166 rs = bucket.get_all_keys(maxkeys=5) 167 self.assertEqual(len(rs), 5) 168 169 def test_bucket_lookup(self): 170 """Test the bucket lookup method.""" 171 bucket = self._MakeBucket() 172 k = bucket.new_key('foo/bar') 173 phony_mimetype = 'application/x-boto-test' 174 headers = {'Content-Type': phony_mimetype} 175 k.set_contents_from_string('testdata', headers) 176 177 k = bucket.lookup('foo/bar') 178 self.assertIsInstance(k, bucket.key_class) 179 self.assertEqual(k.content_type, phony_mimetype) 180 k = bucket.lookup('notthere') 181 self.assertIsNone(k) 182 183 def test_metadata(self): 184 """Test key metadata operations.""" 185 bucket = self._MakeBucket() 186 k = self._MakeKey(bucket=bucket) 187 key_name = k.name 188 s1 = 'This is a test of file upload and download' 189 190 mdkey1 = 'meta1' 191 mdval1 = 'This is the first metadata value' 192 k.set_metadata(mdkey1, mdval1) 193 mdkey2 = 'meta2' 194 mdval2 = 'This is the second metadata value' 195 k.set_metadata(mdkey2, mdval2) 196 197 # Test unicode character. 198 mdval3 = u'föö' 199 mdkey3 = 'meta3' 200 k.set_metadata(mdkey3, mdval3) 201 k.set_contents_from_string(s1) 202 203 k = bucket.lookup(key_name) 204 self.assertEqual(k.get_metadata(mdkey1), mdval1) 205 self.assertEqual(k.get_metadata(mdkey2), mdval2) 206 self.assertEqual(k.get_metadata(mdkey3), mdval3) 207 k = bucket.new_key(key_name) 208 k.get_contents_as_string() 209 self.assertEqual(k.get_metadata(mdkey1), mdval1) 210 self.assertEqual(k.get_metadata(mdkey2), mdval2) 211 self.assertEqual(k.get_metadata(mdkey3), mdval3) 212 213 def test_list_iterator(self): 214 """Test list and iterator.""" 215 bucket = self._MakeBucket() 216 num_iter = len([k for k in bucket.list()]) 217 rs = bucket.get_all_keys() 218 num_keys = len(rs) 219 self.assertEqual(num_iter, num_keys) 220 221 def test_acl(self): 222 """Test bucket and key ACLs.""" 223 bucket = self._MakeBucket() 224 225 # try some acl stuff 226 bucket.set_acl('public-read') 227 acl = bucket.get_acl() 228 self.assertEqual(len(acl.entries.entry_list), 2) 229 bucket.set_acl('private') 230 acl = bucket.get_acl() 231 self.assertEqual(len(acl.entries.entry_list), 1) 232 k = self._MakeKey(bucket=bucket) 233 k.set_acl('public-read') 234 acl = k.get_acl() 235 self.assertEqual(len(acl.entries.entry_list), 2) 236 k.set_acl('private') 237 acl = k.get_acl() 238 self.assertEqual(len(acl.entries.entry_list), 1) 239 240 # Test case-insensitivity of XML ACL parsing. 241 acl_xml = ( 242 '<ACCESSControlList><EntrIes><Entry>' + 243 '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' + 244 '</Entry></EntrIes></ACCESSControlList>') 245 acl = ACL() 246 h = handler.XmlHandler(acl, bucket) 247 xml.sax.parseString(acl_xml, h) 248 bucket.set_acl(acl) 249 self.assertEqual(len(acl.entries.entry_list), 1) 250 aclstr = k.get_xml_acl() 251 self.assertGreater(aclstr.count('/Entry', 1), 0) 252 253 def test_logging(self): 254 """Test set/get raw logging subresource.""" 255 bucket = self._MakeBucket() 256 empty_logging_str="<?xml version='1.0' encoding='UTF-8'?><Logging/>" 257 logging_str = ( 258 "<?xml version='1.0' encoding='UTF-8'?><Logging>" 259 "<LogBucket>log-bucket</LogBucket>" + 260 "<LogObjectPrefix>example</LogObjectPrefix>" + 261 "</Logging>") 262 bucket.set_subresource('logging', logging_str) 263 self.assertEqual(bucket.get_subresource('logging'), logging_str) 264 # try disable/enable logging 265 bucket.disable_logging() 266 self.assertEqual(bucket.get_subresource('logging'), empty_logging_str) 267 bucket.enable_logging('log-bucket', 'example') 268 self.assertEqual(bucket.get_subresource('logging'), logging_str) 269 270 def test_copy_key(self): 271 """Test copying a key from one bucket to another.""" 272 # create two new, empty buckets 273 bucket1 = self._MakeBucket() 274 bucket2 = self._MakeBucket() 275 bucket_name_1 = bucket1.name 276 bucket_name_2 = bucket2.name 277 # verify buckets got created 278 bucket1 = self._GetConnection().get_bucket(bucket_name_1) 279 bucket2 = self._GetConnection().get_bucket(bucket_name_2) 280 # create a key in bucket1 and give it some content 281 key_name = 'foobar' 282 k1 = bucket1.new_key(key_name) 283 self.assertIsInstance(k1, bucket1.key_class) 284 k1.name = key_name 285 s = 'This is a test.' 286 k1.set_contents_from_string(s) 287 # copy the new key from bucket1 to bucket2 288 k1.copy(bucket_name_2, key_name) 289 # now copy the contents from bucket2 to a local file 290 k2 = bucket2.lookup(key_name) 291 self.assertIsInstance(k2, bucket2.key_class) 292 tmpdir = self._MakeTempDir() 293 fpath = os.path.join(tmpdir, 'foobar') 294 fp = open(fpath, 'wb') 295 k2.get_contents_to_file(fp) 296 fp.close() 297 fp = open(fpath) 298 # check to make sure content read is identical to original 299 self.assertEqual(s, fp.read()) 300 fp.close() 301 # delete keys 302 bucket1.delete_key(k1) 303 bucket2.delete_key(k2) 304 305 def test_default_object_acls(self): 306 """Test default object acls.""" 307 # create a new bucket 308 bucket = self._MakeBucket() 309 # get default acl and make sure it's project-private 310 acl = bucket.get_def_acl() 311 self.assertIsNotNone(re.search(PROJECT_PRIVATE_RE, acl.to_xml())) 312 # set default acl to a canned acl and verify it gets set 313 bucket.set_def_acl('public-read') 314 acl = bucket.get_def_acl() 315 # save public-read acl for later test 316 public_read_acl = acl 317 self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>' 318 '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' 319 '</Entry></Entries></AccessControlList>')) 320 # back to private acl 321 bucket.set_def_acl('private') 322 acl = bucket.get_def_acl() 323 self.assertEqual(acl.to_xml(), 324 '<AccessControlList></AccessControlList>') 325 # set default acl to an xml acl and verify it gets set 326 bucket.set_def_acl(public_read_acl) 327 acl = bucket.get_def_acl() 328 self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>' 329 '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' 330 '</Entry></Entries></AccessControlList>')) 331 # back to private acl 332 bucket.set_def_acl('private') 333 acl = bucket.get_def_acl() 334 self.assertEqual(acl.to_xml(), 335 '<AccessControlList></AccessControlList>') 336 337 def test_default_object_acls_storage_uri(self): 338 """Test default object acls using storage_uri.""" 339 # create a new bucket 340 bucket = self._MakeBucket() 341 bucket_name = bucket.name 342 uri = storage_uri('gs://' + bucket_name) 343 # get default acl and make sure it's project-private 344 acl = uri.get_def_acl() 345 self.assertIsNotNone(re.search(PROJECT_PRIVATE_RE, acl.to_xml())) 346 # set default acl to a canned acl and verify it gets set 347 uri.set_def_acl('public-read') 348 acl = uri.get_def_acl() 349 # save public-read acl for later test 350 public_read_acl = acl 351 self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>' 352 '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' 353 '</Entry></Entries></AccessControlList>')) 354 # back to private acl 355 uri.set_def_acl('private') 356 acl = uri.get_def_acl() 357 self.assertEqual(acl.to_xml(), 358 '<AccessControlList></AccessControlList>') 359 # set default acl to an xml acl and verify it gets set 360 uri.set_def_acl(public_read_acl) 361 acl = uri.get_def_acl() 362 self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>' 363 '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' 364 '</Entry></Entries></AccessControlList>')) 365 # back to private acl 366 uri.set_def_acl('private') 367 acl = uri.get_def_acl() 368 self.assertEqual(acl.to_xml(), 369 '<AccessControlList></AccessControlList>') 370 371 def test_cors_xml_bucket(self): 372 """Test setting and getting of CORS XML documents on Bucket.""" 373 # create a new bucket 374 bucket = self._MakeBucket() 375 bucket_name = bucket.name 376 # now call get_bucket to see if it's really there 377 bucket = self._GetConnection().get_bucket(bucket_name) 378 # get new bucket cors and make sure it's empty 379 cors = re.sub(r'\s', '', bucket.get_cors().to_xml()) 380 self.assertEqual(cors, CORS_EMPTY) 381 # set cors document on new bucket 382 bucket.set_cors(CORS_DOC) 383 cors = re.sub(r'\s', '', bucket.get_cors().to_xml()) 384 self.assertEqual(cors, CORS_DOC) 385 386 def test_cors_xml_storage_uri(self): 387 """Test setting and getting of CORS XML documents with storage_uri.""" 388 # create a new bucket 389 bucket = self._MakeBucket() 390 bucket_name = bucket.name 391 uri = storage_uri('gs://' + bucket_name) 392 # get new bucket cors and make sure it's empty 393 cors = re.sub(r'\s', '', uri.get_cors().to_xml()) 394 self.assertEqual(cors, CORS_EMPTY) 395 # set cors document on new bucket 396 cors_obj = Cors() 397 h = handler.XmlHandler(cors_obj, None) 398 xml.sax.parseString(CORS_DOC, h) 399 uri.set_cors(cors_obj) 400 cors = re.sub(r'\s', '', uri.get_cors().to_xml()) 401 self.assertEqual(cors, CORS_DOC) 402 403 def test_lifecycle_config_bucket(self): 404 """Test setting and getting of lifecycle config on Bucket.""" 405 # create a new bucket 406 bucket = self._MakeBucket() 407 bucket_name = bucket.name 408 # now call get_bucket to see if it's really there 409 bucket = self._GetConnection().get_bucket(bucket_name) 410 # get lifecycle config and make sure it's empty 411 xml = bucket.get_lifecycle_config().to_xml() 412 self.assertEqual(xml, LIFECYCLE_EMPTY) 413 # set lifecycle config 414 lifecycle_config = LifecycleConfig() 415 lifecycle_config.add_rule('Delete', None, LIFECYCLE_CONDITIONS) 416 bucket.configure_lifecycle(lifecycle_config) 417 xml = bucket.get_lifecycle_config().to_xml() 418 self.assertEqual(xml, LIFECYCLE_DOC) 419 420 def test_lifecycle_config_storage_uri(self): 421 """Test setting and getting of lifecycle config with storage_uri.""" 422 # create a new bucket 423 bucket = self._MakeBucket() 424 bucket_name = bucket.name 425 uri = storage_uri('gs://' + bucket_name) 426 # get lifecycle config and make sure it's empty 427 xml = uri.get_lifecycle_config().to_xml() 428 self.assertEqual(xml, LIFECYCLE_EMPTY) 429 # set lifecycle config 430 lifecycle_config = LifecycleConfig() 431 lifecycle_config.add_rule('Delete', None, LIFECYCLE_CONDITIONS) 432 uri.configure_lifecycle(lifecycle_config) 433 xml = uri.get_lifecycle_config().to_xml() 434 self.assertEqual(xml, LIFECYCLE_DOC) 435