Jenkins: sync build library and Pipeline to equuleus branch

6 files changed, 388 insertions, 229 deletions

diff --git a/Jenkinsfile b/Jenkinsfile
index 6291d7d7..a978747a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,5 +1,5 @@
 #!/usr/bin/env groovy
-// Copyright (C) 2019 VyOS maintainers and contributors
+// Copyright (C) 2019-2021 VyOS maintainers and contributors
 //
 // This program is free software; you can redistribute it and/or modify
 // in order to easy exprort images built to "external" world
@@ -13,112 +13,92 @@
 //
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 @NonCPS
 
-def getGitBranchName() {
-    def branch = scm.branches[0].name
-    return branch.split('/')[-1]
-}
-
-def getGitRepoURL() {
-    return scm.userRemoteConfigs[0].url
-}
-
-def getGitRepoName() {
-    return getGitRepoURL().split('/').last()
-}
-
-// Returns true if this is a custom build launched on any project fork.
-// Returns false if this is build from git@github.com:vyos/<reponame>.
-// <reponame> can be e.g. vyos-1x.git or vyatta-op.git
-def isCustomBuild() {
-    // GitHub organisation base URL
-    def gitURI = 'git@github.com:vyos/' + getGitRepoName()
-    def httpURI = 'https://github.com/vyos/' + getGitRepoName()
-
-    return ! ((getGitRepoURL() == gitURI) || (getGitRepoURL() == httpURI))
-}
-
-def setDescription() {
-    def item = Jenkins.instance.getItemByFullName(env.JOB_NAME)
-
-    // build up the main description text
-    def description = ""
-    description += "<h2>Build VyOS ISO image</h2>"
-
-    if (isCustomBuild()) {
-        description += "<p style='border: 3px dashed red; width: 50%;'>"
-        description += "<b>Build not started from official Git repository!</b><br>"
-        description += "<br>"
-        description += "Repository: <font face = 'courier'>" + getGitRepoURL() + "</font><br>"
-        description += "Branch: <font face = 'courier'>" + getGitBranchName() + "</font><br>"
-        description += "</p>"
-    } else {
-        description += "Sources taken from Git branch: <font face = 'courier'>" + getGitBranchName() + "</font><br>"
-    }
-
-    item.setDescription(description)
-    item.save()
-}
-
-// Only keep the 10 most recent builds
-def projectProperties = [
-    [$class: 'BuildDiscarderProperty',strategy: [$class: 'LogRotator', numToKeepStr: '10']],
-]
-
-properties(projectProperties)
+// Using a version specifier library, use 'current' branch. The underscore (_)
+// is not a typo! You need this underscore if the line immediately after the
+// @Library annotation is not an import statement!
+@Library('vyos-build@crux')_
 setDescription()
 
-// Due to long build times on DockerHub we rather build the container by ourself
-// and publish it later on.
 node('Docker') {
-    stage('Build Container') {
+    stage('Setup Container') {
         script {
-            git branch: getGitBranchName(),
-                url: getGitRepoURL()
-
             // create container name on demand
-            env.DOCKER_IMAGE = "vyos/vyos-build:" + getGitBranchName()
-            sh "docker build -t ${env.DOCKER_IMAGE} docker"
-            withDockerRegistry([credentialsId: "DockerHub"]) {
-                sh "docker push ${env.DOCKER_IMAGE}"
-            }
+            def branchName = getGitBranchName()
+            // Adjust PR target branch name so we can re-map it to the proper Docker image.
+            if (isPullRequest())
+                branchName = env.CHANGE_TARGET.toLowerCase()
+            if (branchName.equals('master'))
+                branchName = 'current'
+
+            env.DOCKER_IMAGE = 'vyos/vyos-build:' + branchName
+
+            // Get the current UID and GID from the jenkins agent to allow use of the same UID inside Docker
+            env.USR_ID = sh(returnStdout: true, script: 'id -u').toString().trim()
+            env.GRP_ID = sh(returnStdout: true, script: 'id -g').toString().trim()
+            env.DOCKER_ARGS = '--privileged --sysctl net.ipv6.conf.lo.disable_ipv6=0 -e GOSU_UID=' + env.USR_ID + ' -e GOSU_GID=' + env.GRP_ID
+            env.BASE_VERSION = '1.2-stable-'
         }
     }
 }
 
 pipeline {
-    options {
-        skipDefaultCheckout()
-        disableConcurrentBuilds()
-        timeout(time: 90, unit: 'MINUTES')
-        parallelsAlwaysFailFast()
-        timestamps()
+    agent {
+        docker {
+            label "Docker"
+            args "${env.DOCKER_ARGS}"
+            image "${env.DOCKER_IMAGE}"
+            alwaysPull true
+            reuseNode true
+        }
     }
     triggers {
         cron('H 6 * * *')
     }
-    agent {
-        dockerfile {
-            filename 'Dockerfile'
-            dir 'docker'
-            args '--privileged --sysctl net.ipv6.conf.lo.disable_ipv6=0 -e GOSU_UID=1006 -e GOSU_GID=1006'
-        }
+    parameters {
+        string(name: 'BUILD_BY', defaultValue: 'autobuild@vyos.net', description: 'Builder identifier (e.g. jrandomhacker@example.net)')
+        string(name: 'BUILD_VERSION', defaultValue: env.BASE_VERSION + 'ISO8601-TIMESTAMP', description: 'Version number (release builds only)')
+        booleanParam(name: 'BUILD_PUBLISH', defaultValue: false, description: 'Publish this build to downloads.vyos.io and AWS S3')
+        booleanParam(name: 'BUILD_SMOKETESTS', defaultValue: true, description: 'Include Smoketests in ISO image')
+        booleanParam(name: 'BUILD_SNAPSHOT', defaultValue: false, description: 'Upload image to AWS S3 snapshot bucket')
+    }
+    options {
+        disableConcurrentBuilds()
+        timeout(time: 180, unit: 'MINUTES')
+        timestamps()
+        buildDiscarder(logRotator(numToKeepStr: '20'))
     }
     stages {
         stage('Build ISO') {
+            when {
+                beforeOptions true
+                beforeAgent true
+                // Only run ISO image build process of explicit user request or
+                // once a night triggered by the timer.
+                anyOf {
+                    triggeredBy 'TimerTrigger'
+                    triggeredBy cause: "UserIdCause"
+                }
+            }
             steps {
                 script {
+                    // Display Git commit Id used with the Jenkinsfile on the Job "Build History" pane
                     def commitId = sh(returnStdout: true, script: 'git rev-parse --short=11 HEAD').trim()
                     currentBuild.description = sprintf('Git SHA1: %s', commitId[-11..-1])
 
+                    def CUSTOM_PACKAGES = ''
+
+                    def VYOS_VERSION = params.BUILD_BY
+                    if (params.BUILD_VERSION == env.BASE_VERSION + 'ISO8601-TIMESTAMP')
+                        VYOS_VERSION = env.BASE_VERSION + sh(returnStdout: true, script: 'date -u +%Y%m%d%H%M').toString().trim()
+
                     sh """
                         ./configure \
-                            --build-by autobuild@vyos.net \
-                            --debian-mirror http://ftp.us.debian.org/debian/ \
+                            --build-by "${params.BUILD_BY}" \
+                            --debian-mirror http://deb.debian.org/debian/ \
                             --build-type release \
-                            --version 1.2-stable-\$(date +%Y%m%d%H%M)
+                            --version "${VYOS_VERSION}" ${CUSTOM_PACKAGES}
                         sudo make iso
                     """
 
@@ -128,26 +108,80 @@ pipeline {
                 }
             }
         }
-        stage('QEMU') {
+        stage('Test') {
             when {
-                expression { fileExists 'build/live-image-amd64.hybrid.iso' }
+                expression { return params.BUILD_SMOKETESTS }
             }
-            steps {
-                sh "sudo make test"
+            parallel {
+                stage('Smoketests') {
+                    when {
+                        expression { fileExists 'build/live-image-amd64.hybrid.iso' }
+                    }
+                    steps {
+                        sh "sudo make test"
+                    }
+                }
             }
         }
     }
     post {
+        success {
+            script {
+                // only deploy ISO if build from official repository
+                if (isCustomBuild())
+                    return
+
+                // only deploy ISO if requested via parameter
+                if (! params.BUILD_PUBLISH)
+                    return
+
+                files = findFiles(glob: 'build/vyos*.iso')
+                // Publish ISO image to daily builds bucket
+                if (files) {
+                    // Publish ISO image to snapshot bucket
+                    if (files && params.BUILD_SNAPSHOT) {
+                        withAWS(region: 'us-east-1', credentials: 's3-vyos-downloads-rolling-rw') {
+                            s3Upload(bucket: 's3-us.vyos.io', path: 'snapshot/' + params.BUILD_VERSION + '/', workingDir: 'build', includePathPattern: 'vyos*.iso')
+                        }
+                    } else {
+                        // Publish build result to AWS S3 rolling bucket
+                        withAWS(region: 'us-east-1', credentials: 's3-vyos-downloads-rolling-rw') {
+                            s3Upload(bucket: 's3-us.vyos.io', path: 'rolling/' + getGitBranchName() + '/',
+                                     workingDir: 'build', includePathPattern: 'vyos*.iso')
+                            s3Copy(fromBucket: 's3-us.vyos.io', fromPath: 'rolling/' + getGitBranchName() + '/' + files[0].name,
+                                   toBucket: 's3-us.vyos.io', toPath: getGitBranchName() + '/vyos-rolling-latest.iso')
+                        }
+                    }
+
+                    // Trigger GitHub action which will re-build the static community website which
+                    // also holds the AWS download links to the generated ISO images
+                    withCredentials([string(credentialsId: 'GitHub-API-Token', variable: 'TOKEN')]) {
+                        sh '''
+                            curl -X POST --header "Accept: application/vnd.github.v3+json" \
+                            --header "authorization: Bearer $TOKEN" --data '{"ref": "production"}' \
+                            https://api.github.com/repos/vyos/community.vyos.net/actions/workflows/main.yml/dispatches
+                        '''
+                    }
+                }
+
+                // Publish ISO image to snapshot bucket
+                if (files && params.BUILD_SNAPSHOT) {
+                    withAWS(region: 'us-east-1', credentials: 's3-vyos-downloads-rolling-rw') {
+                        s3Upload(bucket: 's3-us.vyos.io', path: 'snapshot/',
+                                 workingDir: 'build', includePathPattern: 'vyos*.iso')
+                    }
+                }
+            }
+        }
+        failure {
+            archiveArtifacts artifacts: '**/build/vyos-*.iso, **/build/vyos-*.qcow2',
+                allowEmptyArchive: true
+        }
         cleanup {
             echo 'One way or another, I have finished'
             // the 'build' directory got elevated permissions during the build
             // cdjust permissions so it can be cleaned up by the regular user
-            sh '''
-                #!/bin/bash
-                if [ -d build ]; then
-                    sudo chmod -R 777 build/
-                fi
-            '''
+            sh 'sudo make purge'
             deleteDir() /* cleanup our workspace */
         }
     }
diff --git a/Jenkinsfile.docker b/Jenkinsfile.docker
new file mode 100644
index 00000000..8ea65b16
--- /dev/null
+++ b/Jenkinsfile.docker
@@ -0,0 +1,65 @@
+#!/usr/bin/env groovy
+// Copyright (C) 2019-2021 VyOS maintainers and contributors
+//
+// This program is free software; you can redistribute it and/or modify
+// in order to easy exprort images built to "external" world
+// it under the terms of the GNU General Public License version 2 or later as
+// published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+@NonCPS
+
+// Using a version specifier library, use 'current' branch. The underscore (_)
+// is not a typo! You need this underscore if the line immediately after the
+// @Library annotation is not an import statement!
+@Library('vyos-build@crux')_
+setDescription()
+
+pipeline {
+    agent none
+    options {
+        disableConcurrentBuilds()
+        timeout(time: 240, unit: 'MINUTES')
+        timestamps()
+        buildDiscarder(logRotator(numToKeepStr: '20'))
+    }
+    stages {
+        stage('Build containers') {
+            when {
+                beforeOptions true
+                beforeAgent true
+                // Only run ISO image build process of explicit user request or
+                // once a night triggered by the timer.
+                anyOf {
+                    changeset pattern: "**/docker/*"
+                    changeset pattern: "**/Jenkinsfile.docker"
+                    triggeredBy cause: "UserIdCause"
+                }
+            }
+            parallel {
+                stage('x86_64') {
+                    agent {
+                        label "ec2_amd64"
+                    }
+                    steps {
+                        script {
+                            DOCKER_IMAGE = "vyos/vyos-build:" + getGitBranchName()
+                            sh "docker build -t ${DOCKER_IMAGE} docker"
+                            if (! isCustomBuild()) {
+                                withDockerRegistry([credentialsId: "DockerHub"]) {
+                                    sh "docker push ${DOCKER_IMAGE}"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/vars/buildPackage.groovy b/vars/buildPackage.groovy
index d0e76c78..15ba7e42 100644
--- a/vars/buildPackage.groovy
+++ b/vars/buildPackage.groovy
@@ -1,6 +1,5 @@
 #!/usr/bin/env groovy
-
-// Copyright (C) 2020 VyOS maintainers and contributors
+// Copyright (C) 2020-2021 VyOS maintainers and contributors
 //
 // This program is free software; you can redistribute it and/or modify
 // in order to easy exprort images built to "external" world
@@ -15,185 +14,160 @@
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-def call(description=null, pkgList=null, buildCmd=null) {
+def call(description=null, pkgList=null, buildCmd=null, changesPattern="**") {
     // - description: Arbitrary text to print on Jenkins Job Description
     //   instead of package name
     // - pkgList: Multiple packages can be build at once in a single Pipeline run
     // - buildCmd: replace default build command "dpkg-buildpackage -uc -us -tc -b"
     //   with this custom version
+    // - changesPattern: package will only be build if a change file matching this
+    //   pattern is found
 
-    // Only keep the 10 most recent builds
-    def projectProperties = [
-        [$class: 'BuildDiscarderProperty',strategy: [$class: 'LogRotator', numToKeepStr: '10']],
-    ]
-
-    properties(projectProperties)
     setDescription(description)
 
-    node('Docker') {
-        stage('Define Agent') {
-           script {
-               // create container name on demand
-               def branchName = getGitBranchName()
-               // Adjust PR target branch name so we can re-map it to the proper
-               // Docker image. CHANGE_ID is set only for pull requests, so it is
-               // safe to access the pullRequest global variable
-               if (env.CHANGE_ID) {
-                   branchName = "${env.CHANGE_TARGET}".toLowerCase()
-               }
-               if (branchName.equals("master")) {
-                   branchName = "current"
-               }
-               env.DOCKER_IMAGE = "vyos/vyos-build:" + branchName
-           }
-        }
-    }
     pipeline {
-        agent {
-            docker {
-                args "--sysctl net.ipv6.conf.lo.disable_ipv6=0 -e GOSU_UID=1006 -e GOSU_GID=1006"
-                image "${env.DOCKER_IMAGE}"
-                alwaysPull true
-            }
-        }
-        environment {
-            // get relative directory path to Jenkinsfile
-            BASE_DIR = getJenkinsfilePath()
-            CHANGESET_DIR = getChangeSetPath()
-            DEBIAN_ARCH = sh(returnStdout: true, script: 'dpkg --print-architecture').trim()
-        }
+        agent none
         options {
-            skipDefaultCheckout()
-            timeout(time: 60, unit: 'MINUTES')
+            disableConcurrentBuilds()
+            timeout(time: 120, unit: 'MINUTES')
             timestamps()
+            buildDiscarder(logRotator(numToKeepStr: '20'))
         }
         stages {
-            stage('Fetch Source') {
+            stage('Define Agent') {
+                agent {
+                    label "ec2_amd64"
+                }
+                when {
+                    anyOf {
+                        changeset pattern: changesPattern, caseSensitive: true
+                        triggeredBy cause: "UserIdCause"
+                    }
+                }
                 steps {
                     script {
-                        // package build must be done in "any" subdir. Without it the Debian build system
-                        // is unable to generate the *.deb files in the sources parent directory, which
-                        // will cause a "Permission denied" error.
-                        dir ('build') {
-                            // checkout git repository which hold 'Jenkinsfile'
-                            checkout scm
+                        // create container name on demand
+                        def branchName = getGitBranchName()
+                        // Adjust PR target branch name so we can re-map it to the proper Docker image.
+                        if (isPullRequest())
+                            branchName = env.CHANGE_TARGET.toLowerCase()
+                        if (branchName.equals('master'))
+                            branchName = 'current'
 
-                            // Display Git commit Id used with the Jenkinsfile on the Job "Build History" pane
-                            def commitId = sh(returnStdout: true, script: 'git rev-parse --short=11 HEAD').trim()
-                            currentBuild.description = sprintf('Git SHA1: %s', commitId[-11..-1])
+                        env.DOCKER_IMAGE = 'vyos/vyos-build:' + branchName
 
-                            if (pkgList) {
-                                // Fetch individual package source code, but only if a URL is defined, this will
-                                // let us reuse this script for packages like vyos-1x which ship a Jenkinfile in
-                                // their repositories root folder.
-                                pkgList.each { pkg ->
-                                    dir(env.BASE_DIR + pkg.name) {
-                                        checkout([$class: 'GitSCM',
-                                            doGenerateSubmoduleConfigurations: false,
-                                            extensions: [[$class: 'CleanCheckout']],
-                                            branches: [[name: pkg.scmCommit]],
-                                            userRemoteConfigs: [[url: pkg.scmUrl]]])
-                                    }
-                                }
-                            }
-                        }
+                        // Get the current UID and GID from the jenkins agent to allow use of the same UID inside Docker
+                        env.USR_ID = sh(returnStdout: true, script: 'id -u').toString().trim()
+                        env.GRP_ID = sh(returnStdout: true, script: 'id -g').toString().trim()
+                        env.DOCKER_ARGS = '--sysctl net.ipv6.conf.lo.disable_ipv6=0 -e GOSU_UID=' + env.USR_ID + ' -e GOSU_GID=' + env.GRP_ID
                     }
                 }
             }
-            stage('Build Source') {
+            stage('Build Code') {
                 when {
-                    beforeOptions true
-                    beforeAgent true
                     anyOf {
-                        changeset pattern: "${env.CHANGESET_DIR}"
-                        triggeredBy 'TimerTrigger'
+                        changeset pattern: changesPattern, caseSensitive: true
                         triggeredBy cause: "UserIdCause"
                     }
                 }
-                steps {
-                    script {
-                        // package build must be done in "any" subdir. Without it the Debian build system
-                        // is unable to generate the *.deb files in the sources parent directory, which
-                        // will cause a "Permission denied" error.
-                        dir ('build') {
-                            if (pkgList) {
-                                pkgList.each { pkg ->
-                                    dir(env.BASE_DIR + pkg.name) {
-                                        sh pkg.buildCmd
-                                    }
-                                }
-                            } else if (buildCmd) {
-                                sh buildCmd
-                            } else {
-                                sh "dpkg-buildpackage -uc -us -tc -b"
+                parallel {
+                    stage('amd64') {
+                        agent {
+                            docker {
+                                label "ec2_amd64"
+                                args "${env.DOCKER_ARGS}"
+                                image "${env.DOCKER_IMAGE}"
+                                alwaysPull true
+                                reuseNode true
+                            }
+                        }
+                        environment {
+                            // get relative directory path to Jenkinsfile
+                            BASE_DIR = getJenkinsfilePath()
+                            CHANGESET_DIR = getChangeSetPath()
+                        }
+                        steps {
+                            script {
+                                cloneAndBuild(description, 'amd64', pkgList, buildCmd)
+                                stash includes: '**/*.deb', name: 'binary-amd64'
+                            }
+                        }
+                        post {
+                            cleanup {
+                                deleteDir()
                             }
                         }
                     }
                 }
             }
-        }
-        post {
-            cleanup {
-                deleteDir()
-            }
-            success {
-                script {
-                    // package build must be done in "any" subdir. Without it the Debian build system
-                    // is unable to generate the *.deb files in the sources parent directory, which
-                    // will cause a "Permission denied" error.
-                    def BIN_DIR = ''
-                    if (env.BASE_DIR) {
-                        BIN_DIR = 'build/' + env.BASE_DIR
+            stage("Finalize") {
+                when {
+                    anyOf {
+                        changeset pattern: changesPattern, caseSensitive: true
+                        triggeredBy cause: "UserIdCause"
                     }
-                    dir (BIN_DIR) {
+                }
+                agent {
+                    label "ec2_amd64"
+                }
+                steps {
+                    script {
+                        // Unpack files for amd64
+                        unstash 'binary-amd64'
+
                         if (isCustomBuild()) {
-                            // archive *.deb artifact on custom builds, deploy to repo otherwise
-                            archiveArtifacts artifacts: '**/*.deb', allowEmptyArchive: true
-                        } else {
-                            // publish build result, using SSH-dev.packages.vyos.net Jenkins Credentials
-                            sshagent(['SSH-dev.packages.vyos.net']) {
-                                // build up some fancy groovy variables so we do not need to write/copy
-                                // every option over and over again!
-                                def RELEASE = getGitBranchName()
-                                if (getGitBranchName() == "master") {
-                                    RELEASE = 'current'
-                                }
+                            echo "Build not started from official Git repository! Artifacts are not uploaded to external repository"
+                            return
+                        }
+                        echo "Uploading Artifacts to external repository"
+                        copyArtifacts fingerprintArtifacts: true, projectName: '${JOB_NAME}', selector: specific('${BUILD_NUMBER}')
 
-                                def VYOS_REPO_PATH = '/home/sentrium/web/dev.packages.vyos.net/public_html/repositories/' + RELEASE + '/'
-                                if (getGitBranchName() == "crux")
-                                    VYOS_REPO_PATH += 'vyos/'
+                        // build up some fancy groovy variables so we do not need to write/copy
+                        // every option over and over again!
+                        def RELEASE = getGitBranchName()
+                        if (getGitBranchName() == "master")
+                            RELEASE = 'current'
 
-                                def SSH_OPTS = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR'
-                                def SSH_REMOTE = env.DEV_PACKAGES_VYOS_NET_HOST // defined as global variable
+                        def VYOS_REPO_PATH = '/home/sentrium/web/dev.packages.vyos.net/public_html/repositories/' + RELEASE
+                        if (getGitBranchName() == "crux")
+                            VYOS_REPO_PATH += '/vyos'
 
-                                def SSH_DIR = '~/VyOS/' + RELEASE + '/' + env.DEBIAN_ARCH
+                        def SSH_OPTS = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR'
+                        def SSH_REMOTE = env.DEV_PACKAGES_VYOS_NET_HOST // defined as global variable
+                        def SSH_DIR = '~/VyOS/' + RELEASE
 
-                                files = findFiles(glob: '*.deb')
-                                if (files) {
-                                    echo "Uploading package(s) and updating package(s) in the repository ..."
-                                    files.each { FILE ->
-                                        def PKG = sh(returnStdout: true, script: "dpkg-deb -f ${FILE} Package").trim()
-                                        // No need to explicitly check the return code. The pipeline
-                                        // will fail if sh returns a noni-zero exit code
+                        // publish build result, using SSH-dev.packages.vyos.net Jenkins Credentials
+                        sshagent(['SSH-dev.packages.vyos.net']) {
+                            files = findFiles(glob: '**/*.deb')
+                            if (files) {
+                                sh(script: "ssh ${SSH_OPTS} ${SSH_REMOTE} -t \"bash --login -c 'mkdir -p ${SSH_DIR}'\"")
+                                echo "Uploading package(s) and updating package(s) in the repository ..."
+                                files.each { FILE ->
+                                    // NOTE: Groovy is a pain in the ass and " quotes differ from ', so all shell code must use " in the beginning
+                                    def PACKAGE = sh(returnStdout: true, script: "dpkg-deb -f ${FILE} Package").trim()
+                                    def PACKAGE_ARCH = sh(returnStdout: true, script: "dpkg-deb -f ${FILE} Architecture").trim()
+                                    def ARCH = ''
+                                    if (PACKAGE_ARCH != 'all')
+                                        ARCH = '-A ' + PACKAGE_ARCH
 
-                                        def PACKAGE_ARCH = sh(returnStdout: true, script: "dpkg-deb -f ${FILE} Architecture").trim()
-                                        def ARCH = ''
-                                        if (PACKAGE_ARCH != 'all')
-                                            ARCH = '-A ' + PACKAGE_ARCH
+                                    sh(script: "scp ${SSH_OPTS} ${FILE} ${SSH_REMOTE}:${SSH_DIR}")
+                                    sh(script: "ssh ${SSH_OPTS} ${SSH_REMOTE} -t \"uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} ${ARCH} remove ${RELEASE} ${PACKAGE}'\"")
 
-                                        sh """
-                                            ssh ${SSH_OPTS} ${SSH_REMOTE} -t "bash --login -c 'mkdir -p ${SSH_DIR}'"
-                                            scp ${SSH_OPTS} ${FILE} ${SSH_REMOTE}:${SSH_DIR}/
-                                            ssh ${SSH_OPTS} ${SSH_REMOTE} -t "uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} ${ARCH} remove ${RELEASE} ${PKG}'"
-                                            ssh ${SSH_OPTS} ${SSH_REMOTE} -t "uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} deleteunreferenced'"
-                                            ssh ${SSH_OPTS} ${SSH_REMOTE} -t "uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} ${ARCH} includedeb ${RELEASE} ${SSH_DIR}/${FILE}'"
-                                        """
-                                    }
+                                    // Packages like FRR produce their binary in a nested path e.g. packages/frr/frr-rpki-rtrlib-dbgsym_7.5_arm64.deb,
+                                    // thus we will only extract the filename portion from FILE as the binary is scp'ed to SSH_DIR without any subpath.
+                                    def FILENAME = FILE.toString().tokenize('/')[-1]
+                                    sh(script: "ssh ${SSH_OPTS} ${SSH_REMOTE} -t \"uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} ${ARCH} includedeb ${RELEASE} ${SSH_DIR}/${FILENAME}'\"")
                                 }
+                                sh(script: "ssh ${SSH_OPTS} ${SSH_REMOTE} -t \"uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} deleteunreferenced'\"")
                             }
                         }
                     }
                 }
+                post {
+                    cleanup {
+                        deleteDir()
+                    }
+                }
             }
         }
     }
diff --git a/vars/cloneAndBuild.groovy b/vars/cloneAndBuild.groovy
new file mode 100644
index 00000000..f945ba75
--- /dev/null
+++ b/vars/cloneAndBuild.groovy
@@ -0,0 +1,65 @@
+#!/usr/bin/env groovy

+// Copyright (C) 2021 VyOS maintainers and contributors

+//

+// This program is free software; you can redistribute it and/or modify

+// in order to easy exprort images built to "external" world

+// it under the terms of the GNU General Public License version 2 or later as

+// published by the Free Software Foundation.

+//

+// This program is distributed in the hope that it will be useful,

+// but WITHOUT ANY WARRANTY; without even the implied warranty of

+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+// GNU General Public License for more details.

+//

+// You should have received a copy of the GNU General Public License

+// along with this program.  If not, see <http://www.gnu.org/licenses/>.

+

+def call(description, architecture, pkgList, buildCmd) {

+    // package build must be done in "any" subdir. Without it the Debian build system

+    // is unable to generate the *.deb files in the sources parent directory, which

+    // will cause a "Permission denied" error.

+    dir ("build-${architecture}") {

+        // cleanup

+        deleteDir()

+

+        // checkout git repository which hold 'Jenkinsfile'

+        checkout scm

+

+        // Display Git commit Id used with the Jenkinsfile on the Job "Build History" pane

+        def commitId = sh(returnStdout: true, script: 'git rev-parse --short=11 HEAD').trim()

+        currentBuild.description = sprintf('Git SHA1: %s', commitId[-11..-1])

+

+        if (pkgList) {

+            // Fetch individual package source code, but only if a URL is defined, this will

+            // let us reuse this script for packages like vyos-1x which ship a Jenkinfile in

+            // their repositories root folder.

+            pkgList.each { pkg ->

+                dir(env.BASE_DIR + pkg.name) {

+                    checkout([$class: 'GitSCM',

+                        doGenerateSubmoduleConfigurations: false,

+                        extensions: [[$class: 'CleanCheckout']],

+                        branches: [[name: pkg.scmCommit]],

+                        userRemoteConfigs: [[url: pkg.scmUrl]]])

+                }

+            }

+        }

+

+        // compile the source(s) ...

+        if (pkgList) {

+            pkgList.each { pkg ->

+                dir(env.BASE_DIR + pkg.name) {

+                    sh pkg.buildCmd

+                }

+            }

+        } else if (buildCmd) {

+            sh buildCmd

+        } else {

+            sh 'dpkg-buildpackage -uc -us -tc -b'

+        }

+    }

+    if (architecture == 'amd64') {

+        archiveArtifacts artifacts: "**/*.deb", fingerprint: true

+    } else {

+        archiveArtifacts artifacts: "**/*_${architecture}.deb", fingerprint: true

+    }

+}

diff --git a/vars/isCustomBuild.groovy b/vars/isCustomBuild.groovy
index 301fdd42..c5e5fab7 100644
--- a/vars/isCustomBuild.groovy
+++ b/vars/isCustomBuild.groovy
@@ -22,5 +22,5 @@ def call() {
     def gitURI = 'git@github.com:vyos/' + getGitRepoName()
     def httpURI = 'https://github.com/vyos/' + getGitRepoName()
 
-    return !((getGitRepoURL() == gitURI) || (getGitRepoURL() == httpURI)) || env.CHANGE_ID
+    return !((getGitRepoURL() == gitURI) || (getGitRepoURL() == httpURI)) || isPullRequest()
 }
diff --git a/vars/isPullRequest.groovy b/vars/isPullRequest.groovy
new file mode 100644
index 00000000..813341bc
--- /dev/null
+++ b/vars/isPullRequest.groovy
@@ -0,0 +1,21 @@
+// Copyright (C) 2020 VyOS maintainers and contributors
+//
+// This program is free software; you can redistribute it and/or modify
+// in order to easy exprort images built to "external" world
+// it under the terms of the GNU General Public License version 2 or later as
+// published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+def call() {
+    if (env.CHANGE_ID) {
+        return true
+    }
+    return false
+}