• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1/* This pipeline is used for release testing, so it runs rarely.
2 *
3 * Test objectives for this pipeline are:
4 *
5 *   - Run the entire pipeline in less than 60 minutes.
6 *   - Test builds on all supported operating systems.
7 *   - Test builds on optimized compiler choices (i.e. prefer Clang over GCC).
8 *   - Build only release variants.
9 *   - Run full functional tests.
10 *   - Run full image quality tests.
11 *   - Code sign the binaries on supported operating systems.
12 *   - Build the release package.
13 *
14 * The test matrix is not fully covered; e.g. we can assume compilers behave
15 * similarly on different operating systems, so we test one compiler per OS.
16 */
17
18@Library('hive-infra-library@changes/86/295486/1') _
19
20pipeline {
21  agent none
22
23  options {
24    ansiColor('xterm')
25    timestamps()
26  }
27
28  stages {
29    stage('Build All') {
30      parallel {
31        /* Run static analysis on Linux */
32        stage('Coverity') {
33          agent {
34            kubernetes {
35              yaml '''
36apiVersion: v1
37kind: Pod
38spec:
39  securityContext:
40    runAsUser: 1000
41    runAsGroup: 1000
42  imagePullSecrets:
43    - name: artifactory-ms-docker
44  containers:
45    - name: astcenc
46      image: mobile-studio--docker.eu-west-1.artifactory.aws.arm.com/astcenc:3.1.0
47      command:
48        - sleep
49      args:
50        - infinity
51      resources:
52        requests:
53          cpu: 4
54          memory: 8Gi
55'''
56            defaultContainer 'astcenc'
57            }
58          }
59          stages {
60            stage('Clean') {
61              steps {
62                sh 'git clean -fdx'
63              }
64            }
65            stage('Coverity') {
66              steps {
67                withCredentials([usernamePassword(credentialsId: 'jenkins-password',
68                                                  usernameVariable: 'USERNAME',
69                                                  passwordVariable: 'PASSWORD')]) {
70                  sh script: '''#!/bin/bash
71                    mkdir -p ${WORKSPACE}/occonfig
72
73                    mkdir build_cov
74                    cd build_cov
75
76                    cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DISA_AVX2=ON ..
77
78                    cov-configure --config ${WORKSPACE}/coverity.conf --template --compiler cc --comptype gcc
79                    cov-configure --config ${WORKSPACE}/coverity.conf --template --compiler c++ --comptype g++
80                    cov-build --config ${WORKSPACE}/coverity.conf --dir ${WORKSPACE}/intermediate make install
81                    cov-analyze --dir ${WORKSPACE}/intermediate
82                    cov-commit-defects --dir ${WORKSPACE}/intermediate \\
83                                       --stream astcenc-master \\
84                                       --url https://coverity.cambridge.arm.com \\
85                                       --user jenkins@arm.com --password ${PASSWORD} \\
86                                       --strip-path ${WORKSPACE}
87                  '''
88                }
89              }
90            }
91          }
92        }
93        /* Build for Linux on x86-64 using Clang */
94        stage('Linux') {
95          agent {
96            kubernetes {
97              yaml '''\
98                apiVersion: v1
99                kind: Pod
100                spec:
101                  securityContext:
102                    runAsUser: 1000
103                    runAsGroup: 1000
104                  imagePullSecrets:
105                    - name: artifactory-ms-docker
106                  containers:
107                    - name: astcenc
108                      image: mobile-studio--docker.eu-west-1.artifactory.aws.arm.com/astcenc:3.0.0
109                      command:
110                        - sleep
111                      args:
112                        - infinity
113                      resources:
114                        requests:
115                          cpu: 4
116                          memory: 8Gi
117                        limits:
118                          cpu: 8
119                          memory: 16Gi
120              '''.stripIndent()
121              defaultContainer 'astcenc'
122            }
123          }
124          stages {
125            stage('Clean') {
126              steps {
127                sh 'git clean -ffdx'
128              }
129            }
130            stage('Build astcenc R') {
131              steps {
132                sh '''
133                  export CXX=clang++-9
134                  mkdir build_rel
135                  cd build_rel
136                  cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DISA_AVX2=ON -DISA_SSE41=ON -DISA_SSE2=ON -DISA_NONE=ON -DPACKAGE=x64 ..
137                  make install package -j4
138                '''
139              }
140            }
141            stage('Build astcdec R') {
142              steps {
143                sh '''
144                  export CXX=clang++-9
145                  mkdir build_reldec
146                  cd build_reldec
147                  cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DISA_AVX2=ON -DISA_SSE41=ON -DISA_SSE2=ON -DISA_NONE=ON -DDECOMPRESSOR=ON ..
148                  make -j4
149                '''
150              }
151            }
152            stage('Stash') {
153              steps {
154                dir('build_rel') {
155                  stash name: 'astcenc-linux-x64', includes: '*.zip'
156                  stash name: 'astcenc-linux-x64-hash', includes: '*.zip.sha256'
157                }
158              }
159            }
160            stage('Test') {
161              steps {
162                sh '''
163                  python3 ./Test/astc_test_functional.py
164                  python3 ./Test/astc_test_image.py --encoder=all-x86 --test-set Small
165                '''
166              }
167            }
168          }
169        }
170        /* Build for Windows on x86-64 using MSVC ClangCL */
171        stage('Windows') {
172          agent {
173            label 'Windows'
174          }
175          stages {
176            stage('Clean') {
177              steps {
178                bat 'git clean -ffdx'
179              }
180            }
181            stage('Build R') {
182              steps {
183                bat '''
184                  call c:\\progra~2\\micros~1\\2019\\buildtools\\vc\\auxiliary\\build\\vcvars64.bat
185                  mkdir build_rel
186                  cd build_rel
187                  cmake -G "Visual Studio 16 2019" -T ClangCL -DCMAKE_INSTALL_PREFIX=../ -DISA_AVX2=ON -DISA_SSE41=ON -DISA_SSE2=ON -DPACKAGE=x64 ..
188                  msbuild astcencoder.sln -property:Configuration=Release
189                  msbuild PACKAGE.vcxproj -property:Configuration=Release
190                  msbuild INSTALL.vcxproj -property:Configuration=Release
191                '''
192              }
193            }
194            stage('Stash') {
195              steps {
196                dir('build_rel') {
197                  stash name: 'astcenc-windows-x64', includes: '*.zip'
198                  stash name: 'astcenc-windows-x64-hash', includes: '*.zip.sha256'
199                }
200              }
201            }
202            stage('Test') {
203              steps {
204                bat '''
205                  set Path=c:\\Python38;c:\\Python38\\Scripts;%Path%
206                  call python ./Test/astc_test_image.py --test-set Small
207                '''
208              }
209            }
210          }
211        }
212        /* Build for macOS on x86-64 using Clang */
213        stage('macOS') {
214          agent {
215            label 'mac && notarizer'
216          }
217          stages {
218            stage('Clean') {
219              steps {
220                sh 'git clean -ffdx'
221              }
222            }
223            stage('Build R') {
224              steps {
225                sh '''
226                  mkdir build_rel
227                  cd build_rel
228                  cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DISA_AVX2=ON -DISA_SSE41=ON -DISA_SSE2=ON -DPACKAGE=x64 ..
229                  make install package -j4
230                '''
231              }
232            }
233            stage('Sign and notarize') {
234              environment {
235                NOTARIZATION_CREDS = credentials('notarization-account')
236              }
237              steps {
238                dir('build_rel') {
239                  sh 'git clone ssh://eu-gerrit-1.euhpc.arm.com:29418/Hive/shared/signing'
240                  withCredentials([usernamePassword(credentialsId: 'win-signing',
241                                                    usernameVariable: 'USERNAME',
242                                                    passwordVariable: 'PASSWORD')]) {
243                    sh 'python3 ./signing/macos-client-wrapper.py ${USERNAME} *.zip'
244                    sh 'rm -rf ./signing'
245                  }
246                }
247              }
248            }
249            stage('Stash') {
250              steps {
251                dir('build_rel') {
252                  stash name: 'astcenc-macos-x64', includes: '*.zip'
253                  stash name: 'astcenc-macos-x64-hash', includes: '*.zip.sha256'
254                }
255              }
256            }
257            stage('Test') {
258              steps {
259                sh '''
260                  export PATH=/usr/local/bin:$PATH
261                  python3 ./Test/astc_test_image.py --test-set Small
262                '''
263              }
264            }
265          }
266        }
267        /* Build for macOS on x86-64 using Clang */
268        stage('macOS arm64') {
269          agent {
270            label 'mac && notarizer'
271          }
272          stages {
273            stage('Clean') {
274              steps {
275                sh 'git clean -ffdx'
276              }
277            }
278            stage('Build R') {
279              steps {
280                sh '''
281                  mkdir build_rel
282                  cd build_rel
283                  cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DISA_NEON=ON -DPACKAGE=aarch64 ..
284                  make install package -j4
285                '''
286              }
287            }
288            stage('Sign and notarize') {
289              environment {
290                NOTARIZATION_CREDS = credentials('notarization-account')
291              }
292              steps {
293                dir('build_rel') {
294                  sh 'git clone ssh://eu-gerrit-1.euhpc.arm.com:29418/Hive/shared/signing'
295                  withCredentials([usernamePassword(credentialsId: 'win-signing',
296                                                    usernameVariable: 'USERNAME',
297                                                    passwordVariable: 'PASSWORD')]) {
298                    sh 'python3 ./signing/macos-client-wrapper.py ${USERNAME} *.zip'
299                    sh 'rm -rf ./signing'
300                  }
301                }
302              }
303            }
304            stage('Stash') {
305              steps {
306                dir('build_rel') {
307                  stash name: 'astcenc-macos-aarch64', includes: '*.zip'
308                  stash name: 'astcenc-macos-aarch64-hash', includes: '*.zip.sha256'
309                }
310              }
311            }
312            // TODO: Currently can't test automatically
313          }
314        }
315      }
316    }
317    stage('Artifactory') {
318      agent {
319        kubernetes {
320          yaml '''
321apiVersion: v1
322kind: Pod
323spec:
324  securityContext:
325    runAsUser: 1000
326    runAsGroup: 1000
327  imagePullSecrets:
328    - name: artifactory-ms-docker
329  containers:
330    - name: astcenc
331      image: mobile-studio--docker.eu-west-1.artifactory.aws.arm.com/astcenc:3.0.0
332      command:
333        - sleep
334      args:
335        - infinity
336      resources:
337        requests:
338          cpu: 1
339          memory: 4Gi
340'''
341          defaultContainer 'astcenc'
342        }
343      }
344      options {
345        skipDefaultCheckout true
346      }
347      stages {
348        stage('Unstash') {
349          steps {
350            dir('upload') {
351              unstash 'astcenc-linux-x64-hash'
352              unstash 'astcenc-macos-x64-hash'
353              unstash 'astcenc-macos-aarch64-hash'
354
355              unstash 'astcenc-linux-x64'
356              unstash 'astcenc-macos-x64'
357              unstash 'astcenc-macos-aarch64'
358            }
359            dir('upload/windows-x64') {
360              unstash 'astcenc-windows-x64'
361              dir('signing') {
362                checkout changelog: false,
363                         poll: false,
364                         scm: [$class: 'GitSCM',
365                               branches: [[name: '*/master']],
366                               doGenerateSubmoduleConfigurations: false,
367                               extensions: [],
368                               submoduleCfg: [],
369                               userRemoteConfigs: [[credentialsId: 'gerrit-jenkins-ssh',
370                                                    url: 'ssh://mirror.eu-west-1.gerrit-eu01.aws.arm.com:29418/Hive/shared/signing']]]
371              }
372              withCredentials([usernamePassword(credentialsId: 'win-signing',
373                                                usernameVariable: 'USERNAME',
374                                                passwordVariable: 'PASSWORD')]) {
375                sh 'python3 ./signing/windows-client-wrapper.py ${USERNAME} *.zip'
376                sh 'mv *.zip.sha256 ../'
377                sh 'mv *.zip ../'
378              }
379            }
380            dir('upload') {
381              sh 'rm -rf ./windows-x64'
382              sh 'cat *.sha256 > release-sha256.txt'
383              sh 'rm *.sha256'
384            }
385          }
386        }
387        stage('Upload') {
388          steps {
389            zip zipFile: 'astcenc.zip', dir: 'upload', archive: false
390            cepeArtifactoryUpload(sourcePattern: 'astcenc.zip')
391          }
392        }
393      }
394      post {
395        always {
396          deleteDir()
397        }
398      }
399    }
400  }
401  post {
402    failure {
403      script {
404        slackSend channel: '#dsg-eng-astcenc', color: 'danger', message: "Build ${JOB_NAME} ${BUILD_NUMBER} failed. (<${BUILD_URL}|Open>)", teamDomain: 'arm-dsg', tokenCredentialId: 'jenkins-slack', username: 'jenkins'
405      }
406    }
407  }
408}
409