1# -*- coding: utf-8 -*- 2# Copyright 2010 Google Inc. All Rights Reserved. 3# 4# Permission is hereby granted, free of charge, to any person obtaining a 5# copy of this software and associated documentation files (the 6# "Software"), to deal in the Software without restriction, including 7# without limitation the rights to use, copy, modify, merge, publish, dis- 8# tribute, sublicense, and/or sell copies of the Software, and to permit 9# persons to whom the Software is furnished to do so, subject to the fol- 10# lowing conditions: 11# 12# The above copyright notice and this permission notice shall be included 13# in all copies or substantial portions of the Software. 14# 15# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 16# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- 17# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 18# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 19# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21# IN THE SOFTWARE. 22"""Tests for parallel uploads ported from gsutil naming tests. 23 24Currently, the mock storage service is not thread-safe and therefore not 25suitable for multiprocess/multithreaded testing. Since parallel composite 26uploads necessarily create at least one worker thread outside of main, 27these tests are present in this file as temporary (slower) integration tests 28to provide validation for parallel composite uploads until a thread-safe 29mock storage service rewrite. 30 31Tests for relative paths are not included as integration_testcase does not 32support modifying the current working directory. 33""" 34 35import os 36 37import gslib.tests.testcase as testcase 38from gslib.tests.util import ObjectToURI as suri 39from gslib.tests.util import SequentialAndParallelTransfer 40from gslib.util import Retry 41 42 43class TestParallelCp(testcase.GsUtilIntegrationTestCase): 44 """Unit tests for gsutil naming logic.""" 45 46 @SequentialAndParallelTransfer 47 def testCopyingTopLevelFileToBucket(self): 48 """Tests copying one top-level file to a bucket.""" 49 src_file = self.CreateTempFile(file_name='f0') 50 dst_bucket_uri = self.CreateBucket() 51 self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri)]) 52 53 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1) 54 self.assertEqual(suri(dst_bucket_uri, 'f0'), lines[0]) 55 56 @SequentialAndParallelTransfer 57 def testCopyingMultipleFilesToBucket(self): 58 """Tests copying multiple files to a bucket.""" 59 src_file0 = self.CreateTempFile(file_name='f0') 60 src_file1 = self.CreateTempFile(file_name='f1') 61 dst_bucket_uri = self.CreateBucket() 62 self.RunGsUtil(['cp', src_file0, src_file1, suri(dst_bucket_uri)]) 63 64 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 2) 65 self.assertEqual(suri(dst_bucket_uri, 'f0'), lines[0]) 66 self.assertEqual(suri(dst_bucket_uri, 'f1'), lines[1]) 67 68 @SequentialAndParallelTransfer 69 def testCopyingNestedFileToBucketSubdir(self): 70 """Tests copying a nested file to a bucket subdir. 71 72 Tests that we correctly translate local FS-specific delimiters ('\' on 73 Windows) to bucket delimiter (/). 74 """ 75 tmpdir = self.CreateTempDir() 76 subdir = os.path.join(tmpdir, 'subdir') 77 os.mkdir(subdir) 78 src_file = self.CreateTempFile(tmpdir=tmpdir, file_name='obj', contents='') 79 dst_bucket_uri = self.CreateBucket() 80 # Make an object under subdir so next copy will treat subdir as a subdir. 81 self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri, 'subdir/a')]) 82 self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri, 'subdir')]) 83 84 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 2) 85 self.assertEqual(suri(dst_bucket_uri, 'subdir/a'), lines[0]) 86 self.assertEqual(suri(dst_bucket_uri, 'subdir/obj'), lines[1]) 87 88 @SequentialAndParallelTransfer 89 def testCopyingAbsolutePathDirToBucket(self): 90 """Tests recursively copying absolute path directory to a bucket.""" 91 dst_bucket_uri = self.CreateBucket() 92 src_dir_root = self.CreateTempDir(test_files=[ 93 'f0', 'f1', 'f2.txt', ('dir0', 'dir1', 'nested')]) 94 self.RunGsUtil(['cp', '-R', src_dir_root, suri(dst_bucket_uri)]) 95 src_tmpdir = os.path.split(src_dir_root)[1] 96 97 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 4) 98 self.assertEqual(suri(dst_bucket_uri, src_tmpdir, 99 'dir0', 'dir1', 'nested'), lines[0]) 100 self.assertEqual(suri(dst_bucket_uri, src_tmpdir, 'f0'), lines[1]) 101 self.assertEqual(suri(dst_bucket_uri, src_tmpdir, 'f1'), lines[2]) 102 self.assertEqual(suri(dst_bucket_uri, src_tmpdir, 'f2.txt'), lines[3]) 103 104 @SequentialAndParallelTransfer 105 def testCopyingDirContainingOneFileToBucket(self): 106 """Tests copying a directory containing 1 file to a bucket. 107 108 We test this case to ensure that correct bucket handling isn't dependent 109 on the copy being treated as a multi-source copy. 110 """ 111 dst_bucket_uri = self.CreateBucket() 112 src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'foo')]) 113 self.RunGsUtil(['cp', '-R', os.path.join(src_dir, 'dir0', 'dir1'), 114 suri(dst_bucket_uri)]) 115 116 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1) 117 self.assertEqual(suri(dst_bucket_uri, 'dir1', 'foo'), lines[0]) 118 119 @SequentialAndParallelTransfer 120 def testCopyingFileToObjectWithConsecutiveSlashes(self): 121 """Tests copying a file to an object containing consecutive slashes.""" 122 src_file = self.CreateTempFile(file_name='f0') 123 dst_bucket_uri = self.CreateBucket() 124 self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri) + '//obj']) 125 126 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1) 127 self.assertEqual(suri(dst_bucket_uri) + '//obj', lines[0]) 128 129 @SequentialAndParallelTransfer 130 def testCopyingObjsAndFilesToBucket(self): 131 """Tests copying objects and files to a bucket.""" 132 src_bucket_uri = self.CreateBucket() 133 self.CreateObject(src_bucket_uri, object_name='f1', contents='foo') 134 src_dir = self.CreateTempDir(test_files=['f2']) 135 dst_bucket_uri = self.CreateBucket() 136 self.RunGsUtil(['cp', '-R', suri(src_bucket_uri, '**'), 137 '%s%s**' % (src_dir, os.sep), suri(dst_bucket_uri)]) 138 139 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 2) 140 self.assertEqual(suri(dst_bucket_uri, 'f1'), lines[0]) 141 self.assertEqual(suri(dst_bucket_uri, 'f2'), lines[1]) 142 143 @SequentialAndParallelTransfer 144 def testCopyingSubdirRecursiveToNonexistentSubdir(self): 145 """Tests copying a directory with a single file recursively to a bucket. 146 147 The file should end up in a new bucket subdirectory with the file's 148 directory structure starting below the recursive copy point, as in Unix cp. 149 150 Example: 151 filepath: dir1/dir2/foo 152 cp -r dir1 dir3 153 Results in dir3/dir2/foo being created. 154 """ 155 src_dir = self.CreateTempDir() 156 self.CreateTempFile(tmpdir=src_dir + '/dir1/dir2', file_name='foo') 157 dst_bucket_uri = self.CreateBucket() 158 self.RunGsUtil(['cp', '-R', src_dir + '/dir1', 159 suri(dst_bucket_uri, 'dir3')]) 160 161 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1) 162 self.assertEqual(suri(dst_bucket_uri, 'dir3/dir2/foo'), lines[0]) 163 164 @SequentialAndParallelTransfer 165 def testCopyingWildcardedFilesToBucketSubDir(self): 166 """Tests copying wildcarded files to a bucket subdir.""" 167 # Test with and without final slash on dest subdir. 168 for final_dst_char in ('', '/'): 169 dst_bucket_uri = self.CreateBucket() 170 self.CreateObject(dst_bucket_uri, object_name='subdir0/existing', 171 contents='foo') 172 self.CreateObject(dst_bucket_uri, object_name='subdir1/existing', 173 contents='foo') 174 src_dir = self.CreateTempDir(test_files=['f0', 'f1', 'f2']) 175 176 for i in range(2): 177 self.RunGsUtil( 178 ['cp', os.path.join(src_dir, 'f?'), 179 suri(dst_bucket_uri, 'subdir%d' % i) + final_dst_char]) 180 181 @Retry(AssertionError, tries=3, timeout_secs=1) 182 def _Check1(): 183 """Validate files were copied to the correct destinations.""" 184 stdout = self.RunGsUtil(['ls', suri(dst_bucket_uri, 'subdir%d' % i, 185 '**')], 186 return_stdout=True) 187 lines = stdout.split('\n') 188 self.assertEqual(5, len(lines)) 189 self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'existing'), 190 lines[0]) 191 self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'f0'), lines[1]) 192 self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'f1'), lines[2]) 193 self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'f2'), lines[3]) 194 _Check1() 195 196 @SequentialAndParallelTransfer 197 def testCopyingOneNestedFileToBucketSubDir(self): 198 """Tests copying one nested file to a bucket subdir.""" 199 # Test with and without final slash on dest subdir. 200 for final_dst_char in ('', '/'): 201 202 dst_bucket_uri = self.CreateBucket() 203 self.CreateObject(dst_bucket_uri, object_name='d0/placeholder', 204 contents='foo') 205 self.CreateObject(dst_bucket_uri, object_name='d1/placeholder', 206 contents='foo') 207 208 for i in range(2): 209 src_dir = self.CreateTempDir(test_files=[('d3', 'd4', 'nested', 'f1')]) 210 self.RunGsUtil(['cp', '-r', suri(src_dir, 'd3'), 211 suri(dst_bucket_uri, 'd%d' % i) + final_dst_char]) 212 213 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 4) 214 self.assertEqual(suri(dst_bucket_uri, 'd0', 'd3', 'd4', 'nested', 'f1'), 215 lines[0]) 216 self.assertEqual(suri(dst_bucket_uri, 'd0', 'placeholder'), lines[1]) 217 self.assertEqual(suri(dst_bucket_uri, 'd1', 'd3', 'd4', 'nested', 'f1'), 218 lines[2]) 219 self.assertEqual(suri(dst_bucket_uri, 'd1', 'placeholder'), lines[3]) 220 221