• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2021 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Implementation of a filestore using Github actions artifacts."""
15import logging
16import os
17import shutil
18import sys
19import tarfile
20import tempfile
21
22# pylint: disable=wrong-import-position,import-error
23sys.path.append(
24    os.path.join(os.path.pardir, os.path.pardir, os.path.pardir,
25                 os.path.dirname(os.path.abspath(__file__))))
26
27import utils
28import http_utils
29import filestore
30from filestore.github_actions import github_api
31
32UPLOAD_JS = os.path.join(os.path.dirname(__file__), 'upload.js')
33
34
35def tar_directory(directory, archive_path):
36  """Tars a |directory| and stores archive at |archive_path|. |archive_path|
37  must end in .tar"""
38  assert archive_path.endswith('.tar')
39  # Do this because make_archive will append the extension to archive_path.
40  archive_path = os.path.splitext(archive_path)[0]
41
42  root_directory = os.path.abspath(directory)
43  shutil.make_archive(archive_path,
44                      'tar',
45                      root_dir=root_directory,
46                      base_dir='./')
47
48
49class GithubActionsFilestore(filestore.BaseFilestore):
50  """Implementation of BaseFilestore using Github actions artifacts. Relies on
51  github_actions_toolkit for using the GitHub actions API and the github_api
52  module for using GitHub's standard API. We need to use both because the GitHub
53  actions API is the only way to upload an artifact but it does not support
54  downloading artifacts from other runs. The standard GitHub API does support
55  this however."""
56
57  ARTIFACT_PREFIX = 'cifuzz-'
58  BUILD_PREFIX = 'build-'
59  CRASHES_PREFIX = 'crashes-'
60  CORPUS_PREFIX = 'corpus-'
61  COVERAGE_PREFIX = 'coverage-'
62
63  def __init__(self, config):
64    super().__init__(config)
65    self.github_api_http_headers = github_api.get_http_auth_headers(config)
66
67  def _get_artifact_name(self, name):
68    """Returns |name| prefixed with |self.ARITFACT_PREFIX| if it isn't already
69    prefixed. Otherwise returns |name|."""
70    if name.startswith(self.ARTIFACT_PREFIX):
71      return name
72    return f'{self.ARTIFACT_PREFIX}{name}'
73
74  def _upload_directory(self, name, directory):  # pylint: disable=no-self-use
75    """Uploads |directory| as artifact with |name|."""
76    name = self._get_artifact_name(name)
77    with tempfile.TemporaryDirectory() as temp_dir:
78      archive_path = os.path.join(temp_dir, name + '.tar')
79      tar_directory(directory, archive_path)
80      _raw_upload_directory(name, temp_dir)
81
82  def upload_crashes(self, name, directory):
83    """Uploads the crashes at |directory| to |name|."""
84    return _raw_upload_directory(self.CRASHES_PREFIX + name, directory)
85
86  def upload_corpus(self, name, directory, replace=False):
87    """Uploads the corpus at |directory| to |name|."""
88    # Not applicable as the the entire corpus is uploaded under a single
89    # artifact name.
90    del replace
91    return self._upload_directory(self.CORPUS_PREFIX + name, directory)
92
93  def upload_build(self, name, directory):
94    """Uploads the build at |directory| to |name|."""
95    return self._upload_directory(self.BUILD_PREFIX + name, directory)
96
97  def upload_coverage(self, name, directory):
98    """Uploads the coverage report at |directory| to |name|."""
99    return self._upload_directory(self.COVERAGE_PREFIX + name, directory)
100
101  def download_corpus(self, name, dst_directory):  # pylint: disable=unused-argument,no-self-use
102    """Downloads the corpus located at |name| to |dst_directory|."""
103    return self._download_artifact(self.CORPUS_PREFIX + name, dst_directory)
104
105  def _find_artifact(self, name):
106    """Finds an artifact using the GitHub API and returns it."""
107    logging.debug('Listing artifacts.')
108    artifacts = self._list_artifacts()
109    artifact = github_api.find_artifact(name, artifacts)
110    logging.debug('Artifact: %s.', artifact)
111    return artifact
112
113  def _download_artifact(self, name, dst_directory):
114    """Downloads artifact with |name| to |dst_directory|. Returns True on
115    success."""
116    name = self._get_artifact_name(name)
117
118    with tempfile.TemporaryDirectory() as temp_dir:
119      if not self._raw_download_artifact(name, temp_dir):
120        logging.warning('Could not download artifact: %s.', name)
121        return False
122
123      artifact_tarfile_path = os.path.join(temp_dir, name + '.tar')
124      if not os.path.exists(artifact_tarfile_path):
125        logging.error('Artifact zip did not contain a tarfile.')
126        return False
127
128      # TODO(jonathanmetzman): Replace this with archive.unpack from
129      # libClusterFuzz so we can avoid path traversal issues.
130      with tarfile.TarFile(artifact_tarfile_path) as artifact_tarfile:
131        artifact_tarfile.extractall(dst_directory)
132    return True
133
134  def _raw_download_artifact(self, name, dst_directory):
135    """Downloads the artifact with |name| to |dst_directory|. Returns True on
136    success. Does not do any untarring or adding prefix to |name|."""
137    artifact = self._find_artifact(name)
138    if not artifact:
139      logging.warning('Could not find artifact: %s.', name)
140      return False
141    download_url = artifact['archive_download_url']
142    return http_utils.download_and_unpack_zip(
143        download_url, dst_directory, headers=self.github_api_http_headers)
144
145  def _list_artifacts(self):
146    """Returns a list of artifacts."""
147    return github_api.list_artifacts(self.config.project_repo_owner,
148                                     self.config.project_repo_name,
149                                     self.github_api_http_headers)
150
151  def download_build(self, name, dst_directory):
152    """Downloads the build with name |name| to |dst_directory|."""
153    return self._download_artifact(self.BUILD_PREFIX + name, dst_directory)
154
155  def download_coverage(self, name, dst_directory):
156    """Downloads the latest project coverage report."""
157    return self._download_artifact(self.COVERAGE_PREFIX + name, dst_directory)
158
159
160def _upload_artifact_with_upload_js(name, artifact_paths, directory):
161  """Uploads the artifacts in |artifact_paths| that are located in |directory|
162  to |name|, using the upload.js script."""
163  command = [UPLOAD_JS, name, directory] + artifact_paths
164  _, _, retcode = utils.execute(command)
165  return retcode == 0
166
167
168def _raw_upload_directory(name, directory):
169  """Uploads the artifacts located in |directory| to |name|. Does not do any
170  tarring or adding prefixes to |name|."""
171  # Get file paths.
172  artifact_paths = []
173  for root, _, curr_file_paths in os.walk(directory):
174    for file_path in curr_file_paths:
175      artifact_paths.append(os.path.join(root, file_path))
176  logging.debug('Artifact paths: %s.', artifact_paths)
177  return _upload_artifact_with_upload_js(name, artifact_paths, directory)
178