1/* This pipeline is used for release testing, so it runs rarely. 2 * 3 * Test objectives for this pipeline are: 4 * 5 * - Run the entire pipeline in less than 60 minutes. 6 * - Test builds on all supported operating systems. 7 * - Test builds on optimized compiler choices (i.e. prefer Clang over GCC). 8 * - Build only release variants. 9 * - Run full functional tests. 10 * - Run full image quality tests. 11 * - Code sign the binaries on supported operating systems. 12 * - Build the release package. 13 * 14 * The test matrix is not fully covered; e.g. we can assume compilers behave 15 * similarly on different operating systems, so we test one compiler per OS. 16 */ 17 18@Library('hive-infra-library@changes/86/295486/1') _ 19 20pipeline { 21 agent none 22 23 options { 24 ansiColor('xterm') 25 timestamps() 26 } 27 28 stages { 29 stage('Build All') { 30 parallel { 31 /* Run static analysis on Linux */ 32 stage('Coverity') { 33 agent { 34 kubernetes { 35 yaml ''' 36apiVersion: v1 37kind: Pod 38spec: 39 securityContext: 40 runAsUser: 1000 41 runAsGroup: 1000 42 imagePullSecrets: 43 - name: artifactory-ms-docker 44 containers: 45 - name: astcenc 46 image: mobile-studio--docker.eu-west-1.artifactory.aws.arm.com/astcenc:3.2.0 47 command: 48 - sleep 49 args: 50 - infinity 51 resources: 52 requests: 53 cpu: 4 54 memory: 8Gi 55''' 56 defaultContainer 'astcenc' 57 } 58 } 59 stages { 60 stage('Clean') { 61 steps { 62 sh 'git clean -fdx' 63 } 64 } 65 stage('Coverity') { 66 steps { 67 withCredentials([usernamePassword(credentialsId: 'jenkins-password', 68 usernameVariable: 'USERNAME', 69 passwordVariable: 'PASSWORD')]) { 70 sh script: '''#!/bin/bash 71 mkdir -p ${WORKSPACE}/occonfig 72 73 mkdir build_cov 74 cd build_cov 75 76 cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON .. 77 78 cov-configure --config ${WORKSPACE}/coverity.conf --template --compiler cc --comptype gcc 79 cov-configure --config ${WORKSPACE}/coverity.conf --template --compiler c++ --comptype g++ 80 cov-build --config ${WORKSPACE}/coverity.conf --dir ${WORKSPACE}/intermediate make install 81 cov-analyze --dir ${WORKSPACE}/intermediate 82 cov-commit-defects --dir ${WORKSPACE}/intermediate \\ 83 --stream astcenc-master \\ 84 --url https://coverity.cambridge.arm.com \\ 85 --user jenkins@arm.com --password ${PASSWORD} \\ 86 --strip-path ${WORKSPACE} 87 ''' 88 } 89 } 90 } 91 } 92 } 93 /* Build for Linux on x86-64 using Clang */ 94 stage('Linux') { 95 agent { 96 kubernetes { 97 yaml '''\ 98 apiVersion: v1 99 kind: Pod 100 spec: 101 securityContext: 102 runAsUser: 1000 103 runAsGroup: 1000 104 imagePullSecrets: 105 - name: artifactory-ms-docker 106 containers: 107 - name: astcenc 108 image: mobile-studio--docker.eu-west-1.artifactory.aws.arm.com/astcenc:3.0.0 109 command: 110 - sleep 111 args: 112 - infinity 113 resources: 114 requests: 115 cpu: 4 116 memory: 8Gi 117 limits: 118 cpu: 8 119 memory: 16Gi 120 '''.stripIndent() 121 defaultContainer 'astcenc' 122 } 123 } 124 stages { 125 stage('Clean') { 126 steps { 127 sh 'git clean -ffdx' 128 } 129 } 130 stage('Build astcenc R x64') { 131 steps { 132 sh ''' 133 export CXX=clang++-9 134 mkdir build_rel 135 cd build_rel 136 cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_PACKAGE=x64 .. 137 make install package -j4 138 ''' 139 } 140 } 141 stage('Build astcdec R x64') { 142 steps { 143 sh ''' 144 export CXX=clang++-9 145 mkdir build_reldec 146 cd build_reldec 147 cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_DECOMPRESSOR=ON .. 148 make -j4 149 ''' 150 } 151 } 152 stage('Stash') { 153 steps { 154 dir('build_rel') { 155 stash name: 'astcenc-linux-x64', includes: '*.zip' 156 stash name: 'astcenc-linux-x64-hash', includes: '*.zip.sha256' 157 } 158 } 159 } 160 stage('Test') { 161 steps { 162 sh ''' 163 python3 ./Test/astc_test_functional.py 164 python3 ./Test/astc_test_image.py --encoder=all-x86 --test-set Small 165 ''' 166 } 167 } 168 } 169 } 170 /* Build for Windows on x86-64 using MSVC ClangCL */ 171 stage('Windows') { 172 agent { 173 label 'Windows' 174 } 175 stages { 176 stage('Clean') { 177 steps { 178 bat 'git clean -ffdx' 179 } 180 } 181 stage('Build R x64') { 182 steps { 183 bat ''' 184 call c:\\progra~2\\micros~1\\2022\\buildtools\\vc\\auxiliary\\build\\vcvars64.bat 185 mkdir build_rel 186 cd build_rel 187 cmake -G "Visual Studio 17 2022" -T ClangCL -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_PACKAGE=x64 .. 188 msbuild astcencoder.sln -property:Configuration=Release 189 msbuild PACKAGE.vcxproj -property:Configuration=Release 190 msbuild INSTALL.vcxproj -property:Configuration=Release 191 ''' 192 } 193 } 194 stage('Build R Arm64') { 195 steps { 196 bat ''' 197 call c:\\progra~2\\micros~1\\2022\\buildtools\\vc\\auxiliary\\build\\vcvarsall.bat x64_arm64 198 mkdir build_rel_arm64 199 cd build_rel_arm64 200 cmake -G "Visual Studio 17 2022" -A ARM64 -T ClangCL -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_NEON=ON -DASTCENC_PACKAGE=arm64 .. 201 msbuild astcencoder.sln -property:Configuration=Release 202 msbuild PACKAGE.vcxproj -property:Configuration=Release 203 msbuild INSTALL.vcxproj -property:Configuration=Release 204 ''' 205 } 206 } 207 stage('Sign') { 208 steps { 209 dir('sign_tools') { 210 checkout changelog: false, 211 poll: false, 212 scm: [$class: 'GitSCM', 213 branches: [[name: '*/main']], 214 doGenerateSubmoduleConfigurations: false, 215 extensions: [], 216 submoduleCfg: [], 217 userRemoteConfigs: [[credentialsId: 'gerrit-jenkins-ssh', 218 url: 'ssh://mirror.eu-west-1.gerrit-eu01.aws.arm.com:29418/Hive/shared/signing']]] 219 } 220 withCredentials([usernamePassword(credentialsId: 'cepe-artifactory-jenkins', 221 usernameVariable: 'AF_USER', 222 passwordVariable: 'APIKEY')]) { 223 powershell 'C:\\Python311\\python.exe .\\sign_tools\\windows-client-wrapper.py -b $Env:BUILD_NUMBER -t $Env:APIKEY (Get-ChildItem -Filter build_rel\\*.zip)[0].FullName' 224 powershell 'C:\\Python311\\python.exe .\\sign_tools\\windows-client-wrapper.py -b $Env:BUILD_NUMBER -t $Env:APIKEY (Get-ChildItem -Filter build_rel_arm64\\*.zip)[0].FullName' 225 } 226 } 227 } 228 stage('Stash') { 229 steps { 230 dir('build_rel') { 231 stash name: 'astcenc-windows-x64', includes: '*.zip' 232 stash name: 'astcenc-windows-x64-hash', includes: '*.zip.sha256' 233 } 234 dir('build_rel_arm64') { 235 stash name: 'astcenc-windows-arm64', includes: '*.zip' 236 stash name: 'astcenc-windows-arm64-hash', includes: '*.zip.sha256' 237 } 238 } 239 } 240 stage('Test') { 241 steps { 242 bat ''' 243 set Path=c:\\Python38;c:\\Python38\\Scripts;%Path% 244 call python ./Test/astc_test_image.py --test-set Small 245 ''' 246 } 247 } 248 } 249 } 250 /* Build for macOS on x86-64 using Clang */ 251 stage('macOS') { 252 agent { 253 label 'mac && x86_64 && notarizer' 254 } 255 stages { 256 stage('Clean') { 257 steps { 258 sh 'git clean -ffdx' 259 } 260 } 261 stage('Build R') { 262 steps { 263 sh ''' 264 mkdir build_rel 265 cd build_rel 266 cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_PACKAGE=universal .. 267 make install package -j4 268 ''' 269 } 270 } 271 stage('Sign and notarize') { 272 environment { 273 NOTARIZATION_CREDS = credentials('notarization-account') 274 } 275 steps { 276 dir('build_rel') { 277 sh 'git clone ssh://eu-gerrit-1.euhpc.arm.com:29418/Hive/shared/signing' 278 withCredentials([usernamePassword(credentialsId: 'win-signing', 279 usernameVariable: 'USERNAME', 280 passwordVariable: 'PASSWORD')]) { 281 sh 'python3 ./signing/macos-client-wrapper.py ${USERNAME} *.zip' 282 sh 'rm -rf ./signing' 283 } 284 } 285 } 286 } 287 stage('Stash') { 288 steps { 289 dir('build_rel') { 290 stash name: 'astcenc-macos-universal', includes: '*.zip' 291 stash name: 'astcenc-macos-universal-hash', includes: '*.zip.sha256' 292 } 293 } 294 } 295 stage('Test') { 296 steps { 297 sh ''' 298 export PATH=/usr/local/bin:$PATH 299 python3 ./Test/astc_test_image.py --test-set Small --encoder universal 300 ''' 301 } 302 } 303 } 304 } 305 } 306 } 307 stage('Artifactory') { 308 agent { 309 kubernetes { 310 yaml ''' 311apiVersion: v1 312kind: Pod 313spec: 314 securityContext: 315 runAsUser: 1000 316 runAsGroup: 1000 317 imagePullSecrets: 318 - name: artifactory-ms-docker 319 containers: 320 - name: astcenc 321 image: mobile-studio--docker.eu-west-1.artifactory.aws.arm.com/astcenc:3.0.0 322 command: 323 - sleep 324 args: 325 - infinity 326 resources: 327 requests: 328 cpu: 1 329 memory: 4Gi 330''' 331 defaultContainer 'astcenc' 332 } 333 } 334 options { 335 skipDefaultCheckout true 336 } 337 stages { 338 stage('Unstash') { 339 steps { 340 dir('upload') { 341 unstash 'astcenc-windows-x64-hash' 342 unstash 'astcenc-windows-arm64-hash' 343 unstash 'astcenc-linux-x64-hash' 344 unstash 'astcenc-macos-universal-hash' 345 346 unstash 'astcenc-windows-x64' 347 unstash 'astcenc-windows-arm64' 348 unstash 'astcenc-linux-x64' 349 unstash 'astcenc-macos-universal' 350 351 sh 'cat *.sha256 > release-sha256.txt' 352 sh 'rm *.sha256' 353 } 354 } 355 } 356 stage('Upload') { 357 steps { 358 zip zipFile: 'astcenc.zip', dir: 'upload', archive: false 359 cepeArtifactoryUpload(sourcePattern: 'astcenc.zip') 360 } 361 } 362 } 363 post { 364 always { 365 deleteDir() 366 } 367 } 368 } 369 } 370} 371