1 /*
<lambda>null2  * Copyright 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.build.importMaven
18 
19 import okio.FileSystem
20 import okio.Path
21 import org.apache.logging.log4j.kotlin.logger
22 import org.jetbrains.kotlin.com.google.common.annotations.VisibleForTesting
23 import java.security.MessageDigest
24 import java.util.Locale
25 import java.util.concurrent.ConcurrentHashMap
26 
27 /**
28  * A [DownloadObserver] that will save all files into the given repository folders.
29  */
30 class LocalMavenRepoDownloader(
31     val fileSystem: FileSystem,
32     /**
33      * Path to the internal repo (e.g. prebuilts/androidx/internal)
34      */
35     val internalFolder: Path,
36     /**
37      * Path to the external repo (e.g. prebuilts/androidx/external)
38      */
39     val externalFolder: Path,
40 ) : DownloadObserver {
41     private val logger = logger("LocalMavenRepoDownloader")
42     private val licenseDownloader = LicenseDownloader(enableGithubApi = false)
43     private val writtenFilesMap: MutableMap<Path, Boolean> = ConcurrentHashMap<Path, Boolean>()
44     fun writtenFiles(): Set<Path> = writtenFilesMap.keys
45 
46     /**
47      * Returns the list of files we've downloaded.
48      */
49     fun getDownloadedFiles() = writtenFiles()
50 
51     override fun onDownload(path: String, bytes: ByteArray) {
52         if (path.substringAfterLast('.') in checksumExtensions) {
53             // we sign files locally, don't download them
54             logger.trace {
55                 "Skipping $path because we'll sign it locally"
56             }
57             return
58         }
59         val internal = isInternalArtifact(path)
60         val folder = if (internal) internalFolder else externalFolder
61         logger.trace {
62             "Downloading $path. internal? $internal"
63         }
64         folder.resolve(path).let { file ->
65             val targetFolder = file.parent ?: error("invalid parent for $file")
66             if (file.name.endsWith(".pom")) {
67                 // Keep original MD5 and SHA1 hashes
68                 copyWithDigest(targetFolder, fileName = file.name, bytes = bytes)
69                 if (internal) {
70                     val transformed = transformInternalPomFile(bytes)
71                     copyWithoutDigest(targetFolder, fileName = file.name, bytes = transformed)
72                 } else {
73                     // download licenses only for external poms
74                     licenseDownloader.fetchLicenseFromPom(bytes)?.let { licenseBytes ->
75                         copyWithoutDigest(targetFolder, "LICENSE", licenseBytes)
76                     }
77                 }
78             } else {
79                 copyWithDigest(
80                     targetFolder = targetFolder,
81                     fileName = file.name,
82                     bytes = bytes
83                 )
84             }
85         }
86     }
87 
88     /**
89      * Creates the file in the given [targetFolder]/[fileName] pair.
90      */
91     private fun copyWithoutDigest(
92         targetFolder: Path,
93         fileName: String,
94         bytes: ByteArray
95     ) {
96         fileSystem.writeBytes(targetFolder / fileName, bytes)
97     }
98 
99     /**
100      *  Creates the file in the given [targetFolder]/[fileName] pair and also creates the md5 and
101      *  sha1 checksums.
102      */
103     private fun copyWithDigest(
104         targetFolder: Path,
105         fileName: String,
106         bytes: ByteArray
107     ) {
108         copyWithoutDigest(targetFolder, fileName, bytes)
109         if (fileName.substringAfterLast('.') !in checksumExtensions) {
110             digest(bytes, fileName, "MD5").let { (name, contents) ->
111                 fileSystem.writeBytes(targetFolder / name, contents)
112             }
113             digest(bytes, fileName, "SHA1").let { (name, contents) ->
114                 fileSystem.writeBytes(targetFolder / name, contents)
115             }
116         }
117     }
118 
119     private fun FileSystem.writeBytes(
120         file: Path,
121         contents: ByteArray
122     ) {
123         writtenFilesMap.put(file.normalized(), true)
124         file.parent?.let(fileSystem::createDirectories)
125         write(
126             file = file,
127             mustCreate = false
128         ) {
129             write(contents)
130         }
131         logger.info {
132             "Saved ${file.normalized()} (${contents.size} bytes)"
133         }
134     }
135 
136     /**
137      * Certain prebuilts might have improper downloads.
138      * This method traverses all folders into which we've fetched an artifact and deletes files
139      * that are not fetched by us.
140      *
141      * Note that, sometimes we might pull a pom file but not download its artifacts (if there is a
142      * newer version). So before cleaning any file, we make sure we fetched one of
143      * [EXTENSIONS_FOR_CLENAUP].
144      *
145      * This should be used only if the local repositories are disabled in resolution. Otherwise, it
146      * might delete files that were resolved from the local repository.
147      */
148     fun cleanupLocalRepositories() {
149         val folders = writtenFiles().filter {
150             val isDirectory = fileSystem.metadata(it).isDirectory
151             !isDirectory && it.name.substringAfterLast(".") in EXTENSIONS_FOR_CLENAUP
152         }.mapNotNull {
153             it.parent
154         }.distinct()
155         logger.info {
156             "Cleaning up local repository. Folders to clean: ${folders.size}"
157         }
158 
159         // traverse all folders and make sure they are in the written files list
160         folders.forEachIndexed { index, folder ->
161             logger.trace {
162                 "Cleaning up $folder ($index of ${folders.size})"
163             }
164             fileSystem.list(folder).forEach { candidateToDelete ->
165                 if (!writtenFiles().contains(candidateToDelete.normalized())) {
166                     logger.trace {
167                         "Deleting $candidateToDelete since it is not re-downloaded"
168                     }
169                     fileSystem.delete(candidateToDelete)
170                 } else {
171                     logger.trace {
172                         "Keeping $candidateToDelete"
173                     }
174                 }
175             }
176         }
177     }
178 
179     /**
180      * Transforms POM files so we automatically comment out nodes with <type>aar</type>.
181      *
182      * We are doing this for all internal libraries to account for -Pandroidx.useMaxDepVersions
183      * which swaps out the dependencies of all androidx libraries with their respective ToT
184      * versions. For more information look at b/127495641.
185      *
186      * Instead of parsing the dom and re-writing it, we use a simple string replace to keep the file
187      * contents intact.
188      */
189     private fun transformInternalPomFile(bytes: ByteArray): ByteArray {
190         // using a simple line match rather than xml parsing because we want to keep file same as
191         // much as possible
192         return bytes.toString(Charsets.UTF_8).lineSequence().map {
193             it.replace("<type>aar</type>", "<!--<type>aar</type>-->")
194         }.joinToString("\n").toByteArray(Charsets.UTF_8)
195     }
196 
197     companion object {
198         val checksumExtensions = listOf("md5", "sha1")
199 
200         /**
201          * If we downloaded an artifact with one of these extensions, we can cleanup that folder
202          * for files that are not re-downloaded.
203          */
204         private val EXTENSIONS_FOR_CLENAUP = listOf(
205             "jar",
206             "aar",
207             "klib"
208         )
209         private val INTERNAL_ARTIFACT_PREFIXES = listOf(
210             "android/arch",
211             "com/android/support",
212             "androidx"
213         )
214 
215         // Need to exclude androidx.databinding
216         private val FORCE_EXTERNAL_PREFIXES = setOf(
217             "androidx/databinding"
218         )
219 
220         /**
221          * Checks if an artifact is *internal*.
222          */
223         fun isInternalArtifact(path: String): Boolean {
224             if (FORCE_EXTERNAL_PREFIXES.any {
225                     path.startsWith(it)
226                 }) {
227                 return false
228             }
229             return INTERNAL_ARTIFACT_PREFIXES.any {
230                 path.startsWith(it)
231             }
232         }
233 
234         /**
235          * Creates digest for the given contents.
236          *
237          * @param contents file contents
238          * @param fileName original file name
239          * @param algorithm Algorithm to use
240          *
241          * @return a pair if <new file name> : <digest bytes>
242          */
243         @VisibleForTesting
244         internal fun digest(
245             contents: ByteArray,
246             fileName: String,
247             algorithm: String
248         ): Pair<String, ByteArray> {
249             val messageDigest = MessageDigest.getInstance(algorithm)
250             val digestBytes = messageDigest.digest(contents)
251             val builder = StringBuilder()
252             for (byte in digestBytes) {
253                 builder.append(String.format("%02x", byte))
254             }
255             val signatureFileName = "$fileName.${algorithm.lowercase(Locale.US)}"
256             val resultBytes = builder.toString().toByteArray(Charsets.UTF_8)
257             return signatureFileName to resultBytes
258         }
259     }
260 }
261