• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2021 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Module for a git based filestore."""
15
16from distutils import dir_util
17import logging
18import os
19import shutil
20import subprocess
21import sys
22import tempfile
23
24import filestore
25
26# pylint: disable=wrong-import-position
27INFRA_DIR = os.path.dirname(
28    os.path.dirname(os.path.dirname(os.path.dirname(
29        os.path.abspath(__file__)))))
30sys.path.append(INFRA_DIR)
31
32import retry
33
34_PUSH_RETRIES = 3
35_PUSH_BACKOFF = 1
36_GIT_EMAIL = 'cifuzz@clusterfuzz.com'
37_GIT_NAME = 'CIFuzz'
38_CORPUS_DIR = 'corpus'
39_COVERAGE_DIR = 'coverage'
40
41
42def git_runner(repo_path):
43  """Returns a gits runner for the repo_path."""
44
45  def func(*args):
46    return subprocess.check_call(('git', '-C', repo_path) + args)
47
48  return func
49
50
51# pylint: disable=unused-argument,no-self-use
52class GitFilestore(filestore.BaseFilestore):
53  """Generic git filestore. This still relies on another filestore provided by
54  the CI for larger artifacts or artifacts which make sense to be included as
55  the result of a workflow run."""
56
57  def __init__(self, config, ci_filestore):
58    super().__init__(config)
59    self.repo_path = tempfile.mkdtemp()
60    self._git = git_runner(self.repo_path)
61    self._clone(self.config.git_store_repo)
62
63    self._ci_filestore = ci_filestore
64
65  def __del__(self):
66    shutil.rmtree(self.repo_path)
67
68  def _clone(self, repo_url):
69    """Clones repo URL."""
70    self._git('clone', repo_url, '.')
71    self._git('config', '--local', 'user.email', _GIT_EMAIL)
72    self._git('config', '--local', 'user.name', _GIT_NAME)
73
74  def _reset_git(self, branch):
75    """Resets the git repo."""
76    self._git('fetch', 'origin')
77    try:
78      self._git('checkout', '-B', branch, 'origin/' + branch)
79      self._git('reset', '--hard', 'HEAD')
80    except subprocess.CalledProcessError:
81      self._git('checkout', '--orphan', branch)
82
83    self._git('clean', '-fxd')
84
85  # pylint: disable=too-many-arguments
86  @retry.wrap(_PUSH_RETRIES, _PUSH_BACKOFF)
87  def _upload_to_git(self,
88                     message,
89                     branch,
90                     upload_path,
91                     local_path,
92                     replace=False):
93    """Uploads a directory to git. If `replace` is True, then existing contents
94    in the upload_path is deleted."""
95    self._reset_git(branch)
96
97    full_repo_path = os.path.join(self.repo_path, upload_path)
98    if replace and os.path.exists(full_repo_path):
99      shutil.rmtree(full_repo_path)
100
101    dir_util.copy_tree(local_path, full_repo_path)
102    self._git('add', '.')
103    try:
104      self._git('commit', '-m', message)
105    except subprocess.CalledProcessError:
106      logging.debug('No changes, skipping git push.')
107      return
108
109    self._git('push', 'origin', branch)
110
111  def upload_crashes(self, name, directory):
112    """Uploads the crashes at |directory| to |name|."""
113    return self._ci_filestore.upload_crashes(name, directory)
114
115  def upload_corpus(self, name, directory, replace=False):
116    """Uploads the corpus at |directory| to |name|."""
117    self._upload_to_git('Corpus upload',
118                        self.config.git_store_branch,
119                        os.path.join(_CORPUS_DIR, name),
120                        directory,
121                        replace=replace)
122
123  def upload_build(self, name, directory):
124    """Uploads the build at |directory| to |name|."""
125    return self._ci_filestore.upload_build(name, directory)
126
127  def upload_coverage(self, name, directory):
128    """Uploads the coverage report at |directory| to |name|."""
129    self._upload_to_git('Coverage upload',
130                        self.config.git_store_branch_coverage,
131                        os.path.join(_COVERAGE_DIR, name),
132                        directory,
133                        replace=True)
134
135  def download_corpus(self, name, dst_directory):
136    """Downloads the corpus located at |name| to |dst_directory|."""
137    self._reset_git(self.config.git_store_branch)
138    path = os.path.join(self.repo_path, _CORPUS_DIR, name)
139    if not os.path.exists(path):
140      logging.debug('Corpus does not exist at %s.', path)
141      return False
142
143    dir_util.copy_tree(path, dst_directory)
144    return True
145
146  def download_build(self, name, dst_directory):
147    """Downloads the build with |name| to |dst_directory|."""
148    return self._ci_filestore.download_build(name, dst_directory)
149
150  def download_coverage(self, name, dst_directory):
151    """Downloads the latest project coverage report."""
152    self._reset_git(self.config.git_store_branch_coverage)
153    path = os.path.join(self.repo_path, _COVERAGE_DIR, name)
154    if not os.path.exists(path):
155      logging.debug('Coverage does not exist at %s.', path)
156      return False
157
158    dir_util.copy_tree(path, dst_directory)
159    return True
160