Jenkins: lib: sync library with "current" branch

2 files changed, 158 insertions, 141 deletions

diff --git a/vars/buildPackage.groovy b/vars/buildPackage.groovy
index 1384682a..09964cf3 100644
--- a/vars/buildPackage.groovy
+++ b/vars/buildPackage.groovy
@@ -1,6 +1,5 @@
 #!/usr/bin/env groovy
-
-// Copyright (C) 2020 VyOS maintainers and contributors
+// Copyright (C) 2020-2021 VyOS maintainers and contributors
 //
 // This program is free software; you can redistribute it and/or modify
 // in order to easy exprort images built to "external" world
@@ -22,172 +21,125 @@ def call(description=null, pkgList=null, buildCmd=null) {
     // - buildCmd: replace default build command "dpkg-buildpackage -uc -us -tc -b"
     //   with this custom version
 
-    // Only keep the 10 most recent builds
-    def projectProperties = [
-        [$class: 'BuildDiscarderProperty',strategy: [$class: 'LogRotator', numToKeepStr: '10']],
-    ]
-
-    properties(projectProperties)
     setDescription(description)
 
-    node('Docker') {
-        stage('Define Agent') {
-           script {
-               // create container name on demand
-               def branchName = getGitBranchName()
-               // Adjust PR target branch name so we can re-map it to the proper Docker image. 
-               if (isPullRequest()) {
-                   branchName = "${env.CHANGE_TARGET}".toLowerCase()
-               }
-               if (branchName.equals("master")) {
-                   branchName = "current"
-               }
-               env.DOCKER_IMAGE = "vyos/vyos-build:" + branchName
-           }
-        }
-    }
     pipeline {
-        agent {
-            docker {
-                reuseNode true
-                args "--sysctl net.ipv6.conf.lo.disable_ipv6=0 -e GOSU_UID=1006 -e GOSU_GID=1006"
-                image "${env.DOCKER_IMAGE}"
-                alwaysPull true
-            }
-        }
-        environment {
-            // get relative directory path to Jenkinsfile
-            BASE_DIR = getJenkinsfilePath()
-            CHANGESET_DIR = getChangeSetPath()
-            DEBIAN_ARCH = sh(returnStdout: true, script: 'dpkg --print-architecture').trim()
-        }
+        agent none
         options {
             disableConcurrentBuilds()
             skipDefaultCheckout()
-            timeout(time: 60, unit: 'MINUTES')
+            timeout(time: 120, unit: 'MINUTES')
             timestamps()
+            buildDiscarder(logRotator(numToKeepStr: '20'))
         }
         stages {
-            stage('Fetch Source') {
+            stage('Define Agent') {
+                agent {
+                    label "ec2_amd64"
+                }
                 steps {
                     script {
-                        // package build must be done in "any" subdir. Without it the Debian build system
-                        // is unable to generate the *.deb files in the sources parent directory, which
-                        // will cause a "Permission denied" error.
-                        dir ('build') {
-                            // checkout git repository which hold 'Jenkinsfile'
-                            checkout scm
+                        // create container name on demand
+                        def branchName = getGitBranchName()
+                        // Adjust PR target branch name so we can re-map it to the proper Docker image.
+                        if (isPullRequest())
+                            branchName = env.CHANGE_TARGET.toLowerCase()
+                        if (branchName.equals('master'))
+                            branchName = 'current'
 
-                            // Display Git commit Id used with the Jenkinsfile on the Job "Build History" pane
-                            def commitId = sh(returnStdout: true, script: 'git rev-parse --short=11 HEAD').trim()
-                            currentBuild.description = sprintf('Git SHA1: %s', commitId[-11..-1])
+                        env.DOCKER_IMAGE = 'vyos/vyos-build:' + branchName
 
-                            if (pkgList) {
-                                // Fetch individual package source code, but only if a URL is defined, this will
-                                // let us reuse this script for packages like vyos-1x which ship a Jenkinfile in
-                                // their repositories root folder.
-                                pkgList.each { pkg ->
-                                    dir(env.BASE_DIR + pkg.name) {
-                                        checkout([$class: 'GitSCM',
-                                            doGenerateSubmoduleConfigurations: false,
-                                            extensions: [[$class: 'CleanCheckout']],
-                                            branches: [[name: pkg.scmCommit]],
-                                            userRemoteConfigs: [[url: pkg.scmUrl]]])
-                                    }
-                                }
-                            }
-                        }
+                        // Get the current UID and GID from the jenkins agent to allow use of the same UID inside Docker
+                        env.USR_ID = sh(returnStdout: true, script: 'id -u').toString().trim()
+                        env.GRP_ID = sh(returnStdout: true, script: 'id -g').toString().trim()
+                        env.DOCKER_ARGS = '--sysctl net.ipv6.conf.lo.disable_ipv6=0 -e GOSU_UID=' + env.USR_ID + ' -e GOSU_GID=' + env.GRP_ID
                     }
                 }
             }
-            stage('Build Source') {
-                when {
-                    beforeOptions true
-                    beforeAgent true
-                    anyOf {
-                        changeset pattern: "${env.CHANGESET_DIR}"
-                        expression { isPullRequest() }
-                        triggeredBy 'TimerTrigger'
-                        triggeredBy cause: "UserIdCause"
-                    }
-                }
-                steps {
-                    script {
-                        // package build must be done in "any" subdir. Without it the Debian build system
-                        // is unable to generate the *.deb files in the sources parent directory, which
-                        // will cause a "Permission denied" error.
-                        dir ('build') {
-                            if (pkgList) {
-                                pkgList.each { pkg ->
-                                    dir(env.BASE_DIR + pkg.name) {
-                                        sh pkg.buildCmd
-                                    }
-                                }
-                            } else if (buildCmd) {
-                                sh buildCmd
-                            } else {
-                                sh "dpkg-buildpackage -uc -us -tc -b"
+            stage('Build Code') {
+                parallel {
+                    stage('amd64') {
+                        agent {
+                            docker {
+                                label "ec2_amd64"
+                                args "${env.DOCKER_ARGS}"
+                                image "${env.DOCKER_IMAGE}"
+                                alwaysPull true
+                                reuseNode true
+                            }
+                        }
+                        environment {
+                            // get relative directory path to Jenkinsfile
+                            BASE_DIR = getJenkinsfilePath()
+                            CHANGESET_DIR = getChangeSetPath()
+                        }
+                        steps {
+                            script {
+                                cloneAndBuild(description, 'amd64', pkgList, buildCmd)
+                                stash includes: '**/*.deb', name: 'binary-amd64'
+                            }
+                        }
+                        post {
+                            cleanup {
+                                deleteDir()
                             }
                         }
                     }
                 }
             }
-        }
-        post {
-            cleanup {
-                deleteDir()
-            }
-            success {
-                script {
-                    // package build must be done in "any" subdir. Without it the Debian build system
-                    // is unable to generate the *.deb files in the sources parent directory, which
-                    // will cause a "Permission denied" error.
-                    def BIN_DIR = ''
-                    if (env.BASE_DIR) {
-                        BIN_DIR = 'build/' + env.BASE_DIR
-                    }
-                    dir (BIN_DIR) {
+            stage("Finalize") {
+                agent {
+                    label "ec2_amd64"
+                }
+                steps {
+                    script {
+                        // Unpack files for amd64
+                        unstash 'binary-amd64'
+
                         if (isCustomBuild()) {
-                            // archive *.deb artifact on custom builds, deploy to repo otherwise
-                            archiveArtifacts artifacts: '**/*.deb', allowEmptyArchive: true
-                        } else {
-                            // publish build result, using SSH-dev.packages.vyos.net Jenkins Credentials
-                            sshagent(['SSH-dev.packages.vyos.net']) {
-                                // build up some fancy groovy variables so we do not need to write/copy
-                                // every option over and over again!
-                                def RELEASE = getGitBranchName()
-                                if (getGitBranchName() == "master") {
-                                    RELEASE = 'current'
-                                }
+                            echo "Build not started from official Git repository! Artifacts are not uploaded to external repository"
+                            return
+                        }
+                        echo "Uploading Artifacts to external repository"
+                        copyArtifacts fingerprintArtifacts: true, projectName: '${JOB_NAME}', selector: specific('${BUILD_NUMBER}')
+
+                        // build up some fancy groovy variables so we do not need to write/copy
+                        // every option over and over again!
+                        def RELEASE = getGitBranchName()
+                        if (getGitBranchName() == "master")
+                            RELEASE = 'current'
+
+                        def VYOS_REPO_PATH = '/home/sentrium/web/dev.packages.vyos.net/public_html/repositories/' + RELEASE
+                        if (getGitBranchName() == "crux")
+                            VYOS_REPO_PATH += '/vyos'
 
-                                def VYOS_REPO_PATH = '/home/sentrium/web/dev.packages.vyos.net/public_html/repositories/' + RELEASE + '/'
-                                if (getGitBranchName() == "crux")
-                                    VYOS_REPO_PATH += 'vyos/'
+                        def SSH_OPTS = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR'
+                        def SSH_REMOTE = env.DEV_PACKAGES_VYOS_NET_HOST // defined as global variable
+                        def SSH_DIR = '~/VyOS/' + RELEASE
 
-                                def SSH_OPTS = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR'
-                                def SSH_REMOTE = env.DEV_PACKAGES_VYOS_NET_HOST // defined as global variable
+                        // publish build result, using SSH-dev.packages.vyos.net Jenkins Credentials
+                        sshagent(['SSH-dev.packages.vyos.net']) {
+                            files = findFiles(glob: '**/*.deb')
+                            if (files) {
+                                sh(script: "ssh ${SSH_OPTS} ${SSH_REMOTE} -t \"bash --login -c 'mkdir -p ${SSH_DIR}'\"")
+                                echo "Uploading package(s) and updating package(s) in the repository ..."
+                                files.each { FILE ->
+                                    // NOTE: Groovy is a pain in the ass and " quotes differ from ', so all shell code must use " in the beginning
+                                    def PACKAGE = sh(returnStdout: true, script: "dpkg-deb -f ${FILE} Package").trim()
+                                    def PACKAGE_ARCH = sh(returnStdout: true, script: "dpkg-deb -f ${FILE} Architecture").trim()
+                                    def ARCH = ''
+                                    if (PACKAGE_ARCH != 'all')
+                                        ARCH = '-A ' + PACKAGE_ARCH
 
-                                def SSH_DIR = '~/VyOS/' + RELEASE + '/' + env.DEBIAN_ARCH
-                                def ARCH_OPT = ''
-                                if (env.DEBIAN_ARCH != 'all')
-                                    ARCH_OPT = '-A ' + env.DEBIAN_ARCH
+                                    sh(script: "scp ${SSH_OPTS} ${FILE} ${SSH_REMOTE}:${SSH_DIR}")
+                                    sh(script: "ssh ${SSH_OPTS} ${SSH_REMOTE} -t \"uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} ${ARCH} remove ${RELEASE} ${PACKAGE}'\"")
 
-                                files = findFiles(glob: '*.deb')
-                                if (files) {
-                                    echo "Uploading package(s) and updating package(s) in the repository ..."
-                                    files.each { FILE ->
-                                        def PKG = sh(returnStdout: true, script: "dpkg-deb -f ${FILE} Package").trim()
-                                        // No need to explicitly check the return code. The pipeline
-                                        // will fail if sh returns a noni-zero exit code
-                                        sh """
-                                            ssh ${SSH_OPTS} ${SSH_REMOTE} -t "bash --login -c 'mkdir -p ${SSH_DIR}'"
-                                            scp ${SSH_OPTS} ${FILE} ${SSH_REMOTE}:${SSH_DIR}/
-                                            ssh ${SSH_OPTS} ${SSH_REMOTE} -t "uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} ${ARCH_OPT} remove ${RELEASE} ${PKG}'"
-                                            ssh ${SSH_OPTS} ${SSH_REMOTE} -t "uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} deleteunreferenced'"
-                                            ssh ${SSH_OPTS} ${SSH_REMOTE} -t "uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} ${ARCH_OPT} includedeb ${RELEASE} ${SSH_DIR}/${FILE}'"
-                                        """
-                                    }
+                                    // Packages like FRR produce their binary in a nested path e.g. packages/frr/frr-rpki-rtrlib-dbgsym_7.5_arm64.deb,
+                                    // thus we will only extract the filename portion from FILE as the binary is scp'ed to SSH_DIR without any subpath.
+                                    def FILENAME = FILE.toString().tokenize('/')[-1]
+                                    sh(script: "ssh ${SSH_OPTS} ${SSH_REMOTE} -t \"uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} ${ARCH} includedeb ${RELEASE} ${SSH_DIR}/${FILENAME}'\"")
                                 }
+                                sh(script: "ssh ${SSH_OPTS} ${SSH_REMOTE} -t \"uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} deleteunreferenced'\"")
                             }
                         }
                     }
diff --git a/vars/cloneAndBuild.groovy b/vars/cloneAndBuild.groovy
new file mode 100644
index 00000000..f945ba75
--- /dev/null
+++ b/vars/cloneAndBuild.groovy
@@ -0,0 +1,65 @@
+#!/usr/bin/env groovy

+// Copyright (C) 2021 VyOS maintainers and contributors

+//

+// This program is free software; you can redistribute it and/or modify

+// in order to easy exprort images built to "external" world

+// it under the terms of the GNU General Public License version 2 or later as

+// published by the Free Software Foundation.

+//

+// This program is distributed in the hope that it will be useful,

+// but WITHOUT ANY WARRANTY; without even the implied warranty of

+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+// GNU General Public License for more details.

+//

+// You should have received a copy of the GNU General Public License

+// along with this program.  If not, see <http://www.gnu.org/licenses/>.

+

+def call(description, architecture, pkgList, buildCmd) {

+    // package build must be done in "any" subdir. Without it the Debian build system

+    // is unable to generate the *.deb files in the sources parent directory, which

+    // will cause a "Permission denied" error.

+    dir ("build-${architecture}") {

+        // cleanup

+        deleteDir()

+

+        // checkout git repository which hold 'Jenkinsfile'

+        checkout scm

+

+        // Display Git commit Id used with the Jenkinsfile on the Job "Build History" pane

+        def commitId = sh(returnStdout: true, script: 'git rev-parse --short=11 HEAD').trim()

+        currentBuild.description = sprintf('Git SHA1: %s', commitId[-11..-1])

+

+        if (pkgList) {

+            // Fetch individual package source code, but only if a URL is defined, this will

+            // let us reuse this script for packages like vyos-1x which ship a Jenkinfile in

+            // their repositories root folder.

+            pkgList.each { pkg ->

+                dir(env.BASE_DIR + pkg.name) {

+                    checkout([$class: 'GitSCM',

+                        doGenerateSubmoduleConfigurations: false,

+                        extensions: [[$class: 'CleanCheckout']],

+                        branches: [[name: pkg.scmCommit]],

+                        userRemoteConfigs: [[url: pkg.scmUrl]]])

+                }

+            }

+        }

+

+        // compile the source(s) ...

+        if (pkgList) {

+            pkgList.each { pkg ->

+                dir(env.BASE_DIR + pkg.name) {

+                    sh pkg.buildCmd

+                }

+            }

+        } else if (buildCmd) {

+            sh buildCmd

+        } else {

+            sh 'dpkg-buildpackage -uc -us -tc -b'

+        }

+    }

+    if (architecture == 'amd64') {

+        archiveArtifacts artifacts: "**/*.deb", fingerprint: true

+    } else {

+        archiveArtifacts artifacts: "**/*_${architecture}.deb", fingerprint: true

+    }

+}