1# Copyright 2020 Google Inc. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# 15################################################################################ 16"""Unit tests for Cloud Function sync, which syncs the list of github projects 17and uploads them to the Cloud Datastore.""" 18 19import os 20import sys 21import unittest 22 23from google.cloud import ndb 24 25sys.path.append(os.path.dirname(__file__)) 26# pylint: disable=wrong-import-position 27 28from datastore_entities import Project 29from project_sync import get_github_creds 30from project_sync import get_projects 31from project_sync import ProjectMetadata 32from project_sync import sync_projects 33import test_utils 34 35# pylint: disable=no-member 36 37 38# pylint: disable=too-few-public-methods 39class Repository: 40 """Mocking Github Repository.""" 41 42 def __init__(self, name, file_type, path, contents=None): 43 self.contents = contents or [] 44 self.name = name 45 self.type = file_type 46 self.path = path 47 self.decoded_content = b"name: test" 48 49 def get_contents(self, path): 50 """"Get contents of repository.""" 51 if self.path == path: 52 return self.contents 53 54 for content_file in self.contents: 55 if content_file.path == path: 56 return content_file.contents 57 58 return None 59 60 def set_yaml_contents(self, decoded_content): 61 """Set yaml_contents.""" 62 self.decoded_content = decoded_content 63 64 65class CloudSchedulerClient: 66 """Mocking cloud scheduler client.""" 67 68 def __init__(self): 69 self.schedulers = [] 70 71 # pylint: disable=no-self-use 72 def location_path(self, project_id, location_id): 73 """Return project path.""" 74 return f'projects/{project_id}/location/{location_id}' 75 76 def create_job(self, parent, job): 77 """Simulate create job.""" 78 del parent 79 self.schedulers.append(job) 80 81 # pylint: disable=no-self-use 82 def job_path(self, project_id, location_id, name): 83 """Return job path.""" 84 return f'projects/{project_id}/location/{location_id}/jobs/{name}' 85 86 def delete_job(self, name): 87 """Simulate delete jobs.""" 88 for job in self.schedulers: 89 if job['name'] == name: 90 self.schedulers.remove(job) 91 break 92 93 def update_job(self, job, update_mask): 94 """Simulate update jobs.""" 95 for existing_job in self.schedulers: 96 if existing_job == job: 97 job['schedule'] = update_mask['schedule'] 98 99 100class TestDataSync(unittest.TestCase): 101 """Unit tests for sync.""" 102 103 @classmethod 104 def setUpClass(cls): 105 cls.ds_emulator = test_utils.start_datastore_emulator() 106 test_utils.wait_for_emulator_ready(cls.ds_emulator, 'datastore', 107 test_utils.DATASTORE_READY_INDICATOR) 108 test_utils.set_gcp_environment() 109 110 def setUp(self): 111 test_utils.reset_ds_emulator() 112 113 def test_sync_projects_update(self): 114 """Testing sync_projects() updating a schedule.""" 115 cloud_scheduler_client = CloudSchedulerClient() 116 117 with ndb.Client().context(): 118 Project(name='test1', 119 schedule='0 8 * * *', 120 project_yaml_contents='', 121 dockerfile_contents='').put() 122 Project(name='test2', 123 schedule='0 9 * * *', 124 project_yaml_contents='', 125 dockerfile_contents='').put() 126 127 projects = { 128 'test1': ProjectMetadata('0 8 * * *', '', ''), 129 'test2': ProjectMetadata('0 7 * * *', '', '') 130 } 131 sync_projects(cloud_scheduler_client, projects) 132 133 projects_query = Project.query() 134 self.assertEqual({ 135 'test1': '0 8 * * *', 136 'test2': '0 7 * * *' 137 }, {project.name: project.schedule for project in projects_query}) 138 139 def test_sync_projects_create(self): 140 """"Testing sync_projects() creating new schedule.""" 141 cloud_scheduler_client = CloudSchedulerClient() 142 143 with ndb.Client().context(): 144 Project(name='test1', 145 schedule='0 8 * * *', 146 project_yaml_contents='', 147 dockerfile_contents='').put() 148 149 projects = { 150 'test1': ProjectMetadata('0 8 * * *', '', ''), 151 'test2': ProjectMetadata('0 7 * * *', '', '') 152 } 153 sync_projects(cloud_scheduler_client, projects) 154 155 projects_query = Project.query() 156 self.assertEqual({ 157 'test1': '0 8 * * *', 158 'test2': '0 7 * * *' 159 }, {project.name: project.schedule for project in projects_query}) 160 161 self.assertCountEqual([ 162 { 163 'name': 'projects/test-project/location/us-central1/jobs/' 164 'test2-scheduler-fuzzing', 165 'pubsub_target': { 166 'topic_name': 'projects/test-project/topics/request-build', 167 'data': b'test2' 168 }, 169 'schedule': '0 7 * * *' 170 }, 171 { 172 'name': 'projects/test-project/location/us-central1/jobs/' 173 'test2-scheduler-coverage', 174 'pubsub_target': { 175 'topic_name': 176 'projects/test-project/topics/request-coverage-build', 177 'data': 178 b'test2' 179 }, 180 'schedule': '0 6 * * *' 181 }, 182 ], cloud_scheduler_client.schedulers) 183 184 def test_sync_projects_delete(self): 185 """Testing sync_projects() deleting.""" 186 cloud_scheduler_client = CloudSchedulerClient() 187 188 with ndb.Client().context(): 189 Project(name='test1', 190 schedule='0 8 * * *', 191 project_yaml_contents='', 192 dockerfile_contents='').put() 193 Project(name='test2', 194 schedule='0 9 * * *', 195 project_yaml_contents='', 196 dockerfile_contents='').put() 197 198 projects = {'test1': ProjectMetadata('0 8 * * *', '', '')} 199 sync_projects(cloud_scheduler_client, projects) 200 201 projects_query = Project.query() 202 self.assertEqual( 203 {'test1': '0 8 * * *'}, 204 {project.name: project.schedule for project in projects_query}) 205 206 def test_get_projects_yaml(self): 207 """Testing get_projects() yaml get_schedule().""" 208 209 repo = Repository('oss-fuzz', 'dir', 'projects', [ 210 Repository('test0', 'dir', 'projects/test0', [ 211 Repository('Dockerfile', 'file', 'projects/test0/Dockerfile'), 212 Repository('project.yaml', 'file', 'projects/test0/project.yaml') 213 ]), 214 Repository('test1', 'dir', 'projects/test1', [ 215 Repository('Dockerfile', 'file', 'projects/test1/Dockerfile'), 216 Repository('project.yaml', 'file', 'projects/test1/project.yaml') 217 ]) 218 ]) 219 repo.contents[0].contents[1].set_yaml_contents(b'builds_per_day: 2') 220 repo.contents[1].contents[1].set_yaml_contents(b'builds_per_day: 3') 221 222 self.assertEqual( 223 get_projects(repo), { 224 'test0': 225 ProjectMetadata('0 6,18 * * *', 'builds_per_day: 2', 226 'name: test'), 227 'test1': 228 ProjectMetadata('0 6,14,22 * * *', 'builds_per_day: 3', 229 'name: test') 230 }) 231 232 def test_get_projects_no_docker_file(self): 233 """Testing get_projects() with missing dockerfile""" 234 235 repo = Repository('oss-fuzz', 'dir', 'projects', [ 236 Repository('test0', 'dir', 'projects/test0', [ 237 Repository('Dockerfile', 'file', 'projects/test0/Dockerfile'), 238 Repository('project.yaml', 'file', 'projects/test0/project.yaml') 239 ]), 240 Repository('test1', 'dir', 'projects/test1') 241 ]) 242 243 self.assertEqual( 244 get_projects(repo), 245 {'test0': ProjectMetadata('0 6 * * *', 'name: test', 'name: test')}) 246 247 def test_get_projects_invalid_project_name(self): 248 """Testing get_projects() with invalid project name""" 249 250 repo = Repository('oss-fuzz', 'dir', 'projects', [ 251 Repository('test0', 'dir', 'projects/test0', [ 252 Repository('Dockerfile', 'file', 'projects/test0/Dockerfile'), 253 Repository('project.yaml', 'file', 'projects/test0/project.yaml') 254 ]), 255 Repository('test1@', 'dir', 'projects/test1', [ 256 Repository('Dockerfile', 'file', 'projects/test1/Dockerfile'), 257 Repository('project.yaml', 'file', 'projects/test0/project.yaml') 258 ]) 259 ]) 260 261 self.assertEqual( 262 get_projects(repo), 263 {'test0': ProjectMetadata('0 6 * * *', 'name: test', 'name: test')}) 264 265 def test_get_projects_non_directory_type_project(self): 266 """Testing get_projects() when a file in projects/ is not of type 'dir'.""" 267 268 repo = Repository('oss-fuzz', 'dir', 'projects', [ 269 Repository('test0', 'dir', 'projects/test0', [ 270 Repository('Dockerfile', 'file', 'projects/test0/Dockerfile'), 271 Repository('project.yaml', 'file', 'projects/test0/project.yaml') 272 ]), 273 Repository('test1', 'file', 'projects/test1') 274 ]) 275 276 self.assertEqual( 277 get_projects(repo), 278 {'test0': ProjectMetadata('0 6 * * *', 'name: test', 'name: test')}) 279 280 def test_invalid_yaml_format(self): 281 """Testing invalid yaml schedule parameter argument.""" 282 283 repo = Repository('oss-fuzz', 'dir', 'projects', [ 284 Repository('test0', 'dir', 'projects/test0', [ 285 Repository('Dockerfile', 'file', 'projects/test0/Dockerfile'), 286 Repository('project.yaml', 'file', 'projects/test0/project.yaml') 287 ]) 288 ]) 289 repo.contents[0].contents[1].set_yaml_contents( 290 b'builds_per_day: some-string') 291 292 self.assertEqual(get_projects(repo), {}) 293 294 def test_yaml_out_of_range(self): 295 """Testing invalid yaml schedule parameter argument.""" 296 297 repo = Repository('oss-fuzz', 'dir', 'projects', [ 298 Repository('test0', 'dir', 'projects/test0', [ 299 Repository('Dockerfile', 'file', 'projects/test0/Dockerfile'), 300 Repository('project.yaml', 'file', 'projects/test0/project.yaml') 301 ]) 302 ]) 303 repo.contents[0].contents[1].set_yaml_contents(b'builds_per_day: 5') 304 305 self.assertEqual(get_projects(repo), {}) 306 307 def test_get_github_creds(self): 308 """Testing get_github_creds().""" 309 with ndb.Client().context(): 310 self.assertRaises(RuntimeError, get_github_creds) 311 312 @classmethod 313 def tearDownClass(cls): 314 test_utils.cleanup_emulator(cls.ds_emulator) 315 316 317if __name__ == '__main__': 318 unittest.main(exit=False) 319