Lines Matching +full:test +full:- +full:path
3 # Use of this source code is governed by a BSD-style license that can be
40 # Test value to use for `days_old`, if nothing else is required.
54 def is_fifo(path): argument
55 """Determines whether a path is a fifo.
57 @param path: fifo path string.
59 return stat.S_ISFIFO(os.lstat(path).st_mode)
106 'Fa:ke:ma:c0:12:34', 'rand0m-uu1d')
134 """Test default offloader options."""
147 """Test offloader handling for the --all option."""
149 offloader = gs_offloader.Offloader(_get_options(['--all']))
159 """Test offloader handling for the --hosts option."""
162 _get_options(['--hosts']))
173 """Test offloader handling for the --parallelism option."""
176 _get_options(['--parallelism', '2']))
187 """Test offloader handling for the --delete_only option."""
189 _get_options(['--delete_only']))
200 """Test offloader handling for the --days_old option."""
203 _get_options(['--days_old', '7']))
214 """Test offloader construction for Moblab."""
227 """Test enabling of --delete_only via global_config."""
235 """Test multiprocessing is set."""
237 offloader = gs_offloader.Offloader(_get_options(['-m']))
243 """Test multiprocessing is set."""
252 """Test multiprocessing is set."""
262 """Test multiprocessing is set."""
277 GLOB_PATTERN = '[0-9]*-*'
284 self.queue_args = [resultsdir, os.path.dirname(resultsdir), self._timestamp]
328 if not os.path.isdir(self.dirname):
341 if os.path.isdir(self.dirname):
358 * '-m' option (argv[1]) is on when the argument, multi, is True.
360 * The next-to-last argument (the source directory) is the
368 @param multi True when using '-m' option for gsutil.
371 test_bucket_uri = 'gs://a-test-bucket'
375 gs_path = os.path.join(test_bucket_uri, job.queue_args[1])
382 self.assertEqual(command[1], '-m')
383 self.assertEqual(command[-2], job.queue_args[0])
387 self.assertEqual(command[-1],
388 os.path.join(test_bucket_uri, job.queue_args[0]))
391 self.assertEqual(command[-1],
392 os.path.join(test_bucket_uri, job.queue_args[1]))
399 os.path.join(gs_path, '.finished_offload'))
403 """Test `_get_cmd_list()` as for a regular job."""
404 job = _MockJobDirectory('118-debug')
409 """Test `_get_cmd_list()` as for a special job."""
410 job = _MockJobDirectory('hosts/host1/118-reset')
415 """Test `_get_cmd_list()` as for a regular job."""
416 job = _MockJobDirectory('118-debug')
421 """Test `_get_cmd_list()` as for a special job."""
422 job = _MockJobDirectory('hosts/host1/118-reset')
427 """Test `_get_cmd_list()` as for a regular job with True multi."""
428 job = _MockJobDirectory('118-debug')
433 """Test `_get_cmd_list()` as for a special job with True multi."""
434 job = _MockJobDirectory('hosts/host1/118-reset')
442 '111-fubar', '112-fubar', '113-fubar', '114-snafu']
445 'hosts/host1/333-reset', 'hosts/host1/334-reset',
446 'hosts/host2/444-reset', 'hosts/host3/555-reset']
486 os.mkdir(os.path.join(hostsdir, host))
492 """Base Mox test class for tests using a temporary results directory."""
496 """Test the formatting of failed offloads log file."""
498 # deliberately hard-coded and then parsed to create the test data; the idea
502 # --+----1----+---- ----+ ----+----1----+----2----+----3
505 2014-03-14 15:09:26 1 118-fubar
506 2014-03-14 15:19:23 2 117-fubar
507 2014-03-14 15:29:20 6 116-fubar
508 2014-03-14 15:39:17 24 115-fubar
509 2014-03-14 15:49:14 120 114-fubar
510 2014-03-14 15:59:11 720 113-fubar
511 2014-03-14 16:09:08 5040 112-fubar
512 2014-03-14 16:19:05 40320 111-fubar
519 for line in self._SAMPLE_DIRECTORIES_REPORT.split('\n')[1 : -1]:
527 # test that the output will be sorted.
534 @param report_file: Path to report file
543 'Failed to find end-of-header marker in the report')
551 """Trigger an e-mail report and check its contents."""
552 log_file = os.path.join(self._resultsroot, 'failed_log')
560 log_file = os.path.join(self._resultsroot, 'failed_log')
581 self.mox.StubOutWithMock(models.test, 'parse_job_keyval')
609 self.mox.StubOutWithMock(os.path, 'isfile')
610 os.path.isfile(mox.IgnoreArg()).AndReturn(marker_initially_exists)
637 os.path.isdir(self._job.queue_args[0]))
642 os.path.exists(swarming_req_dir))
646 """Test that `offload_dir()` can succeed correctly."""
647 self._mock_offload_dir_calls(['test', '-d'],
649 os.path.isfile(mox.IgnoreArg()).AndReturn(True)
655 """Test that `offload_dir()` can fail correctly."""
656 self._mock_offload_dir_calls(['test', '!', '-d'],
662 """Test that `offload_dir()` can prune the empty swarming task dir."""
663 should_remove = os.path.join('results', 'swarming-123abc0')
664 self._job = self.make_job(os.path.join(should_remove, '1'))
665 self._mock_offload_dir_calls(['test', '-d'],
668 os.path.isfile(mox.IgnoreArg()).AndReturn(True)
675 """Test that `offload_dir()` keeps the non-empty swarming task dir."""
676 should_not_remove = os.path.join('results', 'swarming-456edf0')
677 self._job = self.make_job(os.path.join(should_not_remove, '1'))
678 self.make_job(os.path.join(should_not_remove, '2'))
679 self._mock_offload_dir_calls(['test', '-d'],
682 os.path.isfile(mox.IgnoreArg()).AndReturn(True)
689 """Test that folder/file name with invalid character can be corrected.
695 invalid_folder = os.path.join(
698 invalid_files.append(os.path.join(
701 good_folder = os.path.join(results_folder, 'valid_name_folder')
702 good_file = os.path.join(good_folder, 'valid_name_file')
709 symlink = os.path.join(invalid_folder, 'broken-link')
710 os.symlink(os.path.join(results_folder, 'no-such-file'),
712 fifo1 = os.path.join(results_folder, 'test_fifo1')
713 fifo2 = os.path.join(good_folder, 'test_fifo2')
714 fifo3 = os.path.join(invalid_folder, 'test_fifo3')
716 fifo4 = os.path.join(invalid_folder, invalid_fifo4_name)
727 self.assertTrue(os.path.exists(good_file))
729 self.assertTrue(os.path.exists(fifo1))
731 self.assertTrue(os.path.exists(fifo2))
733 corrected_folder = os.path.join(
735 corrected_fifo3 = os.path.join(
738 self.assertFalse(os.path.exists(fifo3))
739 self.assertTrue(os.path.exists(corrected_fifo3))
741 corrected_fifo4 = os.path.join(
743 self.assertFalse(os.path.exists(fifo4))
744 self.assertTrue(os.path.exists(corrected_fifo4))
747 corrected_symlink = os.path.join(
749 'broken-link')
750 self.assertFalse(os.path.lexists(symlink))
751 self.assertTrue(os.path.exists(corrected_symlink))
752 self.assertFalse(os.path.islink(corrected_symlink))
757 """Test that folder with too many files can be compressed.
759 @param is_test_job: True to check the method with test job result
763 host_folder = os.path.join(
765 'lab1-host1' if is_test_job else 'hosts/lab1-host1/1-repair')
766 debug_folder = os.path.join(host_folder, 'debug')
767 sysinfo_folder = os.path.join(host_folder, 'sysinfo')
771 with open(os.path.join(folder, str(i)), 'w') as f:
772 f.write('test')
777 self.assertTrue(os.path.exists(sysinfo_folder))
782 self.assertFalse(os.path.exists(sysinfo_folder))
783 self.assertTrue(os.path.exists(sysinfo_folder + '.tgz'))
784 self.assertTrue(os.path.exists(debug_folder))
790 """Test that folder with too many files can be compressed.
797 """Test _is_valid_result."""
798 release_build = 'veyron_minnie-cheets-release/R52-8248.0.0'
799 pfq_build = 'cyan-cheets-android-pfq/R54-8623.0.0-rc1'
800 trybot_build = 'trybot-samus-release/R54-8640.0.0-b5092'
801 trybot_2_build = 'trybot-samus-pfq/R54-8640.0.0-b5092'
802 release_2_build = 'test-trybot-release/R54-8640.0.0-b5092'
804 release_build, gs_offloader.CTS_RESULT_PATTERN, 'arc-cts'))
810 release_build, gs_offloader.CTS_RESULT_PATTERN, 'bvt-arc'))
812 release_build, gs_offloader.CTS_RESULT_PATTERN, 'bvt-perbuild'))
814 release_build, gs_offloader.CTS_RESULT_PATTERN, 'bvt-cq'))
816 release_build, gs_offloader.CTS_V2_RESULT_PATTERN, 'arc-gts'))
818 None, gs_offloader.CTS_RESULT_PATTERN, 'arc-cts'))
822 pfq_build, gs_offloader.CTS_RESULT_PATTERN, 'arc-cts'))
824 trybot_build, gs_offloader.CTS_RESULT_PATTERN, 'arc-cts'))
826 trybot_2_build, gs_offloader.CTS_RESULT_PATTERN, 'arc-cts'))
828 release_2_build, gs_offloader.CTS_RESULT_PATTERN, 'arc-cts'))
834 host_folder = os.path.join(results_folder, 'chromeos4-row9-rack11-host22')
835 debug_folder = os.path.join(host_folder, 'debug')
836 sysinfo_folder = os.path.join(host_folder, 'sysinfo')
837 cts_result_folder = os.path.join(
838 host_folder, 'cheets_CTS.android.dpi', 'results', 'cts-results')
839 cts_v2_result_folder = os.path.join(host_folder,
840 'cheets_CTS_N.CtsGraphicsTestCases', 'results', 'android-cts')
841 gts_result_folder = os.path.join(
842 host_folder, 'cheets_GTS.google.admin', 'results', 'android-gts')
844 timestamp_cts_folder = os.path.join(cts_result_folder, timestamp_str)
845 timestamp_cts_v2_folder = os.path.join(cts_v2_result_folder, timestamp_str)
846 timestamp_gts_folder = os.path.join(gts_result_folder, timestamp_str)
849 host_info_path = os.path.join(host_folder, 'host_keyvals')
852 dir_to_create = os.path.join(dir_to_create, tdir)
853 if not os.path.exists(dir_to_create):
855 with open(os.path.join(host_info_path, 'chromeos4-row9-rack11-host22'), 'w') as store_file:
858 'accel%3Acros-ec,'+
861 # .autoserv_execute file is needed for the test results package to look
863 autoserve_path = os.path.join(host_folder, '.autoserv_execute')
867 # Test results in cts_result_folder with a different time-stamp.
869 timestamp_cts_folder_2 = os.path.join(cts_result_folder, timestamp_str_2)
884 cts_zip_file = os.path.join(cts_result_folder, timestamp_str + '.zip')
885 cts_zip_file_2 = os.path.join(cts_result_folder, timestamp_str_2 + '.zip')
886 cts_v2_zip_file = os.path.join(cts_v2_result_folder, timestamp_str + '.zip')
887 gts_zip_file = os.path.join(gts_result_folder, timestamp_str + '.zip')
890 cts_result_file = os.path.join(timestamp_cts_folder, 'testResult.xml')
891 cts_result_file_2 = os.path.join(timestamp_cts_folder_2,
893 cts_result_compressed_file_2 = os.path.join(timestamp_cts_folder_2,
895 gts_result_file = os.path.join(timestamp_gts_folder, 'test_result.xml')
896 cts_v2_result_file = os.path.join(timestamp_cts_v2_folder,
898 cts_v2_result_compressed_file = os.path.join(timestamp_cts_v2_folder,
907 os.path.basename(file_path)]
909 f.write('test')
915 f.write('test')
921 """Test _upload_cts_testresult."""
928 ['test', '-d', host_folder])
932 ['test', '-d', host_folder])
936 ['test', '-d', host_folder])
945 # A autotest path
946 path = ('/317739475-chromeos-test/chromeos4-row9-rack11-host22/'
947 'cheets_CTS.android.dpi/results/cts-results/'
950 gs_offloader._parse_cts_job_results_file_path(path)
951 self.assertEqual('317739475-chromeos-test', job_id)
956 # A skylab path
957 path = ('/swarming-458e3a3a7fc6f210/1/autoserv_test/'
958 'cheets_CTS.android.dpi/results/cts-results/'
961 gs_offloader._parse_cts_job_results_file_path(path)
962 self.assertEqual('swarming-458e3a3a7fc6f210-1', job_id)
968 """Test upload_files"""
971 for path, pattern in path_pattern_pair:
972 models.test.parse_job_keyval(mox.IgnoreArg()).AndReturn({
973 'build': 'veyron_minnie-cheets-release/R52-8248.0.0',
974 'hostname': 'chromeos4-row9-rack11-host22',
976 'suite': 'arc-cts'
981 ['test', '-d', path])
984 ['test', '-d', path])
987 gs_offloader._upload_files(host_folder, path, pattern, False,
988 'gs://a-test-bucket/',
989 'gs://a-test-apfe-bucket/')
997 """Test method _get_metrics_fields."""
999 models.test.parse_job_keyval(mox.IgnoreArg()).AndReturn({
1000 'build': 'veyron_minnie-cheets-release/R52-8248.0.0',
1002 'suite': 'arc-cts'
1006 self.assertEqual({'board': 'veyron_minnie-cheets',
1032 stdout = os.path.join(results_dir, 'std.log')
1033 stderr = os.path.join(results_dir, 'std.err')
1036 'proxy_socket': '/file-system/foo-socket',
1037 'mysql_user': 'foo-user',
1038 'mysql_password_file': '/file-system/foo-password-file'
1042 'credentials_file': '/foo-creds'
1049 path = os.path.join(results_dir, 'side_effects_config.json')
1050 with open(path, 'w') as f:
1058 expect_dest).AndReturn(['test', '-d', expect_dest])
1077 """Test upload results to dev gs bucket and skip cts uploading."""
1079 gs_bucket = 'dev-bucket'
1085 """Test upload results to the prod gs bucket and also upload to cts."""
1087 gs_bucket = 'prod-bucket'
1093 """Test skip the 'gs://' prefix if already presented."""
1095 gs_bucket = 'gs://prod-bucket'
1106 to test that when `days_old` is 0, the job is always treated
1127 The test cases below are designed to exercise all of the
1157 This method tests assertions D and E regarding side-effects
1170 """Make calls to `enqueue_offload()` for a just-expired job.
1173 side-effects on `first_offload_start`.
1188 """Test a series of `enqueue_offload()` calls with `days_old` of 0.
1200 """Test a series of `enqueue_offload()` calls with `days_old` of 0.
1211 """Test a series of `enqueue_offload()` calls with `days_old` non-zero.
1226 """Test a series of `enqueue_offload()` calls with `days_old` non-zero.
1239 """Test a series of `enqueue_offload()` calls with `days_old` non-zero.
1251 """Test a series of `enqueue_offload()` calls with `days_old` non-zero.
1267 os.mkdir('not-a-job')
1268 open('not-a-dir', 'w').close()
1272 """Test `get_job_directories()` for the given class.
1284 """Test `RegularJobDirectory.get_job_directories()`."""
1290 """Test `SpecialJobDirectory.get_job_directories()`."""
1298 MOREJOBS = ['115-fubar', '116-fubar', '117-fubar', '118-snafu']
1305 self._offloader = gs_offloader.Offloader(_get_options(['-a']))
1310 """Basic test assertions for `_add_new_jobs()`.
1319 count = len(expected_key_set) - len(self._offloader._open_jobs)
1332 """Test adding jobs to an empty dictionary.
1342 """Test adding jobs to a non-empty dictionary.
1400 count = len(self._offloader._open_jobs) - len(new_open_jobs)
1431 function for this test is evidence that we need to pull out the local
1440 """Test `_report_failed_jobs()` with no open jobs.
1452 """Test `_report_failed_jobs()` with only complete jobs.
1467 """Test `_report_failed_jobs()` with only unfinished jobs.
1498 """Test that `offload_dir()` times out correctly.
1500 This test triggers timeout at the earliest possible moment,
1513 self.assertTrue(os.path.isdir(self._job.queue_args[0]))
1520 # This test is also kind of redundant since we are using the timeout
1524 """Test that `offload_dir()` times out correctly.
1526 This test triggers timeout at the latest possible moment, at
1536 get_cmd_list.return_value = ['test', '-d', self._job.queue_args[0]]
1542 self.assertTrue(os.path.isdir(self._job.queue_args[0]))