1 /*
<lambda>null2  * Copyright 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.build.importMaven
18 
19 import androidx.build.importMaven.KmpConfig.SUPPORTED_HOSTS
20 import androidx.build.importMaven.KmpConfig.SUPPORTED_KONAN_TARGETS
21 import java.util.Properties
22 import kotlinx.coroutines.Dispatchers
23 import kotlinx.coroutines.ExperimentalCoroutinesApi
24 import kotlinx.coroutines.async
25 import kotlinx.coroutines.awaitAll
26 import kotlinx.coroutines.runBlocking
27 import okhttp3.OkHttpClient
28 import okhttp3.Request
29 import okhttp3.internal.closeQuietly
30 import okio.FileSystem
31 import okio.Path
32 import org.apache.logging.log4j.kotlin.logger
33 import org.gradle.api.Project
34 import org.jetbrains.kotlin.gradle.plugin.KotlinPluginWrapper
35 import org.jetbrains.kotlin.gradle.utils.NativeCompilerDownloader
36 import org.jetbrains.kotlin.konan.properties.resolvablePropertyList
37 import org.jetbrains.kotlin.konan.properties.resolvablePropertyString
38 import org.jetbrains.kotlin.konan.target.Architecture
39 import org.jetbrains.kotlin.konan.target.Distribution
40 import org.jetbrains.kotlin.konan.target.Family
41 import org.jetbrains.kotlin.konan.target.HostManager
42 
43 typealias KonanProperties = org.jetbrains.kotlin.konan.properties.Properties
44 
45 /**
46  * Utility class to download all konan prebuilts that are required to compile native code. These
47  * files are often downloaded into `/prebuilts/androidx/konan`.
48  */
49 class KonanPrebuiltsDownloader(
50     private val fileSystem: FileSystem,
51     private val downloadPath: Path,
52     private val testMode: Boolean = false
53 ) {
54     private val logger = logger("KonanPrebuiltsDownloader")
55     private val client = OkHttpClient()
56 
57     fun download(compilerVersion: String) {
58         val project = ProjectService.createProject()
59         project.initializeKotlin()
60 
61         val compiler = NativeCompilerDownloader(project = project)
62         // make sure we have the local compiler downloaded so we can find the konan.properties
63         compiler.downloadIfNeeded()
64         val distribution = Distribution(compiler.compilerDirectory.canonicalPath)
65         // base konan properties file for reference:
66         // https://github.com/JetBrains/kotlin/blob/master/kotlin-native/konan/konan.properties
67         // Note that [Distribution] might add overrides.
68         val compilationDependencies =
69             findCompilationDependencies(konanProperties = distribution.properties)
70         downloadDistributions(compilationDependencies)
71         downloadNativeCompiler(compilerVersion)
72     }
73 
74     /**
75      * NativeCompilerDownloader expects Kotlin plugin to be applied before the download starts. As
76      * this class is simulating the same environment, apply the Kotlin plugin manually.
77      */
78     private fun Project.initializeKotlin() {
79         project.pluginManager.apply(KotlinPluginWrapper::class.java)
80     }
81 
82     private fun downloadNativeCompiler(compilerVersion: String) {
83         SUPPORTED_HOSTS.forEach { host ->
84             HostManager.simpleOsName()
85             val osName =
86                 when (host.family) {
87                     Family.OSX -> "macos"
88                     Family.LINUX -> "linux"
89                     else -> "unsupported host family: $host"
90                 }
91             val archName =
92                 when (host.architecture) {
93                     Architecture.X64 -> "x86_64"
94                     Architecture.ARM64 -> "aarch64"
95                     else -> "unsupported architecture: $host"
96                 }
97             val platformName = "$osName-$archName"
98             val subPath =
99                 listOf(
100                         "releases",
101                         compilerVersion,
102                         platformName,
103                         "kotlin-native-prebuilt-$platformName-$compilerVersion.tar.gz"
104                     )
105                     .joinToString("/")
106             val url = listOf(NATIVE_COMPILERS_BASE_URL, subPath).joinToString("/")
107             downloadIfNecessary(
108                 url = url,
109                 localFile = downloadPath / "nativeCompilerPrebuilts" / subPath
110             )
111         }
112     }
113 
114     /**
115      * Finds the compilation dependencies of each supported host machine.
116      *
117      * There are 3 groups of distributions we need to compile offline:
118      * * llvm -> needed for all targets (host based) llvm.<host>.user
119      * * ffi -> needed for all targets (host based) llvm.<host>
120      * * target specific dependencies (host + target combinations) dependencies.<host>(-<target>)?
121      *
122      * Technically, a host machine might be capable of building for multiple targets. For instance,
123      * macOS can build for windows. To avoid excessive downloads, we only download artifacts that
124      * are in the [SUPPORTED_TARGETS] lists.
125      */
126     private fun findCompilationDependencies(konanProperties: KonanProperties): List<String> {
127         val hostDeps =
128             SUPPORTED_HOST_NAMES.flatMap { host -> listOf("llvm.$host.user", "libffiDir.$host") }
129         val dependencies = buildHostAndTargetKeys("dependencies")
130         val gccDeps = SUPPORTED_HOST_NAMES.flatMap { host -> listOf("gccToolchain.$host") }
131         val emulatorDependencies = buildHostAndTargetKeys("emulatorDependency")
132         return (hostDeps + dependencies + gccDeps + emulatorDependencies)
133             .flatMap { konanProperties.resolvablePropertyList(it) }
134             .distinct()
135     }
136 
137     private fun buildHostAndTargetKeys(prefix: String): List<String> {
138         return SUPPORTED_HOST_NAMES.flatMap { host ->
139             listOf("$prefix.$host") + SUPPORTED_TARGETS.map { target -> "$prefix.$host-$target" }
140         }
141     }
142 
143     @OptIn(ExperimentalCoroutinesApi::class)
144     private fun downloadDistributions(distributions: List<String>) {
145         runBlocking(Dispatchers.IO.limitedParallelism(4)) {
146             val results =
147                 distributions
148                     .map { dist ->
149                         // since we always build on linux/mac, we use the tar.gz artifacts.
150                         // if we ever add support for windows builds, we would need to use zip.
151                         val fileName = "$dist.tar.gz"
152                         val localFile = downloadPath / fileName
153                         async {
154                             val url = "$REPO_BASE_URL/$fileName"
155                             url to
156                                 runCatching {
157                                     downloadIfNecessary(url = url, localFile = localFile)
158                                 }
159                         }
160                     }
161                     .awaitAll()
162             val failures = results.filter { it.second.isFailure }
163             if (failures.isNotEmpty()) {
164                 error(
165                     buildString {
166                         appendLine("Couldn't fetch ${failures.size} of ${results.size} artifacts")
167                         appendLine("Failed artifacts:")
168                         failures.forEach { failure ->
169                             appendLine("${failure.first}:")
170                             appendLine(failure.second.exceptionOrNull()!!.stackTraceToString())
171                             appendLine("----")
172                         }
173                     }
174                 )
175             }
176         }
177     }
178 
179     /** Downloads a url into [localFile] if it does not already exists. */
180     private fun downloadIfNecessary(url: String, localFile: Path) {
181         if (fileSystem.exists(localFile)) {
182             logger.trace { "${localFile.name} exists, won't re-download" }
183         } else {
184             logger.trace { "will download $url into $localFile" }
185             val tmpFile = localFile.parent!! / "${localFile.name}.tmp"
186             if (fileSystem.exists(tmpFile)) {
187                 fileSystem.delete(tmpFile)
188             }
189             val response = client.newCall(Request.Builder().url(url).build()).execute()
190             try {
191                 check(response.isSuccessful) { "Failed to fetch $url" }
192                 fileSystem.createDirectories(localFile.parent!!)
193                 checkNotNull(response.body?.source()) { "No body while fetching $url" }
194                     .use { bodySource ->
195                         fileSystem.write(file = tmpFile, mustCreate = true) {
196                             if (testMode) {
197                                 // don't download the whole file for tests
198                                 this.write(bodySource.readByteArray(10))
199                             } else {
200                                 this.writeAll(bodySource)
201                             }
202                         }
203                     }
204                 response.body?.close()
205                 fileSystem.atomicMove(source = tmpFile, target = localFile)
206                 logger.trace { "Finished downloading $url into $localFile" }
207             } finally {
208                 response.closeQuietly()
209             }
210         }
211     }
212 
213     companion object {
214         // https://github.com/JetBrains/kotlin/releases/download/v2.0.10-RC/kotlin-native-prebuilt-linux-x86_64-2.0.10-RC.tar.gz
215         private const val REPO_BASE_URL = "https://download.jetbrains.com/kotlin/native"
216         private const val NATIVE_COMPILERS_BASE_URL = "$REPO_BASE_URL/builds"
217 
218         private val SUPPORTED_HOST_NAMES = SUPPORTED_HOSTS.map { it.name }
219 
220         // target architectures for which we might build artifacts for.
221         private val SUPPORTED_TARGETS =
222             SUPPORTED_HOST_NAMES + SUPPORTED_KONAN_TARGETS.map { it.name }
223     }
224 }
225