1package controllers 2 3import ( 4 "fmt" 5 "os" 6 "os/exec" 7 "path/filepath" 8 "strings" 9 "time" 10 11 "github.com/pkg/errors" 12 13 ent "repodiff/entities" 14 "repodiff/interactors" 15 "repodiff/mappers" 16 "repodiff/persistence/filesystem" 17 "repodiff/repositories" 18) 19 20var expectedOutputFilenames = []string{ 21 "project.csv", 22 "commit.csv", 23} 24 25// Executes all of the differentials specified in the application config. 26// While each target is executed synchronously, the differential script is already multi-threaded 27// across all of the local machine's cores, so there is no benefit to parallelizing multiple differential 28// targets 29func ExecuteDifferentials(config ent.ApplicationConfig) error { 30 err := createWorkingPath(config.OutputDirectory) 31 if err != nil { 32 return errors.Wrap(err, "Could not create working path") 33 } 34 35 commonManifest, err := defineCommonManifest(config) 36 if err != nil { 37 return err 38 } 39 40 for _, target := range config.DiffTargets { 41 fmt.Printf("Processing differential from %s to %s\n", target.Upstream.Branch, target.Downstream.Branch) 42 err = clearOutputDirectory(config) 43 commitCSV, projectCSV, err := runPyScript(config, target) 44 if err != nil { 45 return errors.Wrap(err, "Error running python differential script") 46 } 47 err = TransferScriptOutputToDownstream(config, target, projectCSV, commitCSV, commonManifest) 48 if err != nil { 49 return errors.Wrap(err, "Error transferring script output to downstream") 50 } 51 } 52 return nil 53} 54 55func defineCommonManifest(config ent.ApplicationConfig) (*ent.ManifestFile, error) { 56 workingDirectory := filepath.Join(config.OutputDirectory, "common_upstream") 57 if err := createWorkingPath(workingDirectory); err != nil { 58 return nil, err 59 } 60 cmd := exec.Command( 61 "bash", 62 "-c", 63 fmt.Sprintf( 64 "repo init -u %s -b %s", 65 config.CommonUpstream.URL, 66 config.CommonUpstream.Branch, 67 ), 68 ) 69 cmd.Dir = workingDirectory 70 if _, err := cmd.Output(); err != nil { 71 return nil, err 72 } 73 74 var manifest ent.ManifestFile 75 err := filesystem.ReadXMLAsEntity( 76 // the output of repo init will generate a manifest file at this location 77 filepath.Join(workingDirectory, ".repo/manifest.xml"), 78 &manifest, 79 ) 80 return &manifest, err 81} 82 83func createWorkingPath(folderPath string) error { 84 return os.MkdirAll(folderPath, os.ModePerm) 85} 86 87func printFunctionDuration(fnLabel string, start time.Time) { 88 fmt.Printf("Finished '%s' in %s\n", fnLabel, time.Now().Sub(start)) 89} 90 91func clearOutputDirectory(config ent.ApplicationConfig) error { 92 return exec.Command( 93 "/bin/sh", 94 "-c", 95 fmt.Sprintf("rm -rf %s/*", config.OutputDirectory), 96 ).Run() 97} 98 99func setupCommand(pyScript string, config ent.ApplicationConfig, target ent.DiffTarget) *exec.Cmd { 100 cmd := exec.Command( 101 "python", 102 pyScript, 103 "--manifest-url", 104 target.Downstream.URL, 105 "--manifest-branch", 106 target.Downstream.Branch, 107 "--upstream-manifest-url", 108 target.Upstream.URL, 109 "--upstream-manifest-branch", 110 target.Upstream.Branch, 111 ) 112 cmd.Dir = config.OutputDirectory 113 return cmd 114} 115 116func runPyScript(config ent.ApplicationConfig, target ent.DiffTarget) (projectCSV string, commitCSV string, err error) { 117 pyScript := filepath.Join( 118 config.AndroidProjectDir, 119 config.DiffScript, 120 ) 121 outFilesBefore := filesystem.FindFnamesInDir(config.OutputDirectory, expectedOutputFilenames...) 122 err = diffTarget(pyScript, config, target) 123 if err != nil { 124 return "", "", err 125 } 126 outFilesAfter := filesystem.FindFnamesInDir(config.OutputDirectory, expectedOutputFilenames...) 127 newFiles := interactors.DistinctValues(outFilesBefore, outFilesAfter) 128 if len(newFiles) != 2 { 129 return "", "", errors.New("Expected 1 new output filent. A race condition exists") 130 } 131 return newFiles[0], newFiles[1], nil 132} 133 134func diffTarget(pyScript string, config ent.ApplicationConfig, target ent.DiffTarget) error { 135 defer printFunctionDuration("Run Differential", time.Now()) 136 cmd := setupCommand(pyScript, config, target) 137 138 displayStr := strings.Join(cmd.Args, " ") 139 fmt.Printf("Executing command:\n\n%s\n\n", displayStr) 140 141 return errors.Wrap( 142 cmd.Run(), 143 fmt.Sprintf( 144 "Failed to execute (%s). Ensure glogin has been run or update application config to provide correct parameters", 145 displayStr, 146 ), 147 ) 148} 149 150// SBL need to add test coverage here 151func TransferScriptOutputToDownstream( 152 config ent.ApplicationConfig, 153 target ent.DiffTarget, 154 projectCSVFile, commitCSVFile string, 155 common *ent.ManifestFile) error { 156 157 diffRows, commitRows, err := readCSVFiles(projectCSVFile, commitCSVFile) 158 if err != nil { 159 return err 160 } 161 162 manifestFileGroup, err := loadTargetManifests(config, common) 163 if err != nil { 164 return err 165 } 166 analyzedDiffRows, analyzedCommitRows := interactors.ApplyApplicationMutations( 167 interactors.AppProcessingParameters{ 168 DiffRows: diffRows, 169 CommitRows: commitRows, 170 Manifests: manifestFileGroup, 171 }, 172 ) 173 return persistEntities(target, analyzedDiffRows, analyzedCommitRows) 174} 175 176func loadTargetManifests(config ent.ApplicationConfig, common *ent.ManifestFile) (*ent.ManifestFileGroup, error) { 177 var upstream, downstream ent.ManifestFile 178 dirToLoadAddress := map[string]*ent.ManifestFile{ 179 "upstream": &upstream, 180 "downstream": &downstream, 181 } 182 183 for dir, addr := range dirToLoadAddress { 184 if err := filesystem.ReadXMLAsEntity( 185 filepath.Join(config.OutputDirectory, dir, ".repo/manifest.xml"), 186 addr, 187 ); err != nil { 188 return nil, err 189 } 190 } 191 192 return &ent.ManifestFileGroup{ 193 Common: *common, 194 Upstream: upstream, 195 Downstream: downstream, 196 }, nil 197} 198 199func readCSVFiles(projectCSVFile, commitCSVFile string) ([]ent.DiffRow, []ent.CommitRow, error) { 200 diffRows, err := csvFileToDiffRows(projectCSVFile) 201 if err != nil { 202 return nil, nil, errors.Wrap(err, "Error converting CSV file to entities") 203 } 204 commitRows, err := CSVFileToCommitRows(commitCSVFile) 205 if err != nil { 206 return nil, nil, errors.Wrap(err, "Error converting CSV file to entities") 207 } 208 return diffRows, commitRows, nil 209} 210 211func persistEntities(target ent.DiffTarget, diffRows []ent.AnalyzedDiffRow, commitRows []ent.AnalyzedCommitRow) error { 212 sourceRepo, err := repositories.NewSourceRepository() 213 if err != nil { 214 return errors.Wrap(err, "Error initializing Source Repository") 215 } 216 mappedTarget, err := sourceRepo.DiffTargetToMapped(target) 217 if err != nil { 218 return errors.Wrap(err, "Error mapping diff targets; a race condition is possible") 219 } 220 err = persistDiffRowsDownstream(mappedTarget, diffRows) 221 if err != nil { 222 return errors.Wrap(err, "Error persisting diff rows") 223 } 224 225 return MaybeNullObjectCommitRepository( 226 mappedTarget, 227 ).InsertCommitRows( 228 commitRows, 229 ) 230} 231 232func csvFileToDiffRows(csvFile string) ([]ent.DiffRow, error) { 233 entities, err := filesystem.CSVFileToEntities( 234 csvFile, 235 func(cols []string) (interface{}, error) { 236 return mappers.CSVLineToDiffRow(cols) 237 }, 238 ) 239 if err != nil { 240 return nil, err 241 } 242 return toDiffRows(entities) 243} 244 245func toDiffRows(entities []interface{}) ([]ent.DiffRow, error) { 246 diffRows := make([]ent.DiffRow, len(entities)) 247 for i, entity := range entities { 248 diffRow, ok := entity.(*ent.DiffRow) 249 if !ok { 250 return nil, errors.New("Error casting to DiffRow") 251 } 252 diffRows[i] = *diffRow 253 } 254 return diffRows, nil 255} 256 257func CSVFileToCommitRows(csvFile string) ([]ent.CommitRow, error) { 258 entities, err := filesystem.CSVFileToEntities( 259 csvFile, 260 func(cols []string) (interface{}, error) { 261 return mappers.CSVLineToCommitRow(cols) 262 }, 263 ) 264 if err != nil { 265 return nil, err 266 } 267 return toCommitRows(entities) 268} 269 270func toCommitRows(entities []interface{}) ([]ent.CommitRow, error) { 271 commitRows := make([]ent.CommitRow, len(entities)) 272 for i, entity := range entities { 273 commitRow, ok := entity.(*ent.CommitRow) 274 if !ok { 275 return nil, errors.New("Error casting to CommitRow") 276 } 277 commitRows[i] = *commitRow 278 } 279 return commitRows, nil 280} 281 282func persistDiffRowsDownstream(mappedTarget ent.MappedDiffTarget, diffRows []ent.AnalyzedDiffRow) error { 283 p, err := repositories.NewProjectRepository(mappedTarget) 284 if err != nil { 285 return errors.Wrap(err, "Error instantiating a new project repository") 286 } 287 err = p.InsertDiffRows(diffRows) 288 if err != nil { 289 return errors.Wrap(err, "Error inserting rows from controller") 290 } 291 return nil 292} 293