From 17d8a6bf5bb53f427613bde49297d46c4038dda0 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Tue, 4 Jan 2022 07:25:29 +0100 Subject: Jenkins: sync build library and Pipeline to equuleus branch --- vars/buildPackage.groovy | 250 +++++++++++++++++++++------------------------- vars/cloneAndBuild.groovy | 65 ++++++++++++ vars/isCustomBuild.groovy | 2 +- vars/isPullRequest.groovy | 21 ++++ 4 files changed, 199 insertions(+), 139 deletions(-) create mode 100644 vars/cloneAndBuild.groovy create mode 100644 vars/isPullRequest.groovy (limited to 'vars') diff --git a/vars/buildPackage.groovy b/vars/buildPackage.groovy index d0e76c78..15ba7e42 100644 --- a/vars/buildPackage.groovy +++ b/vars/buildPackage.groovy @@ -1,6 +1,5 @@ #!/usr/bin/env groovy - -// Copyright (C) 2020 VyOS maintainers and contributors +// Copyright (C) 2020-2021 VyOS maintainers and contributors // // This program is free software; you can redistribute it and/or modify // in order to easy exprort images built to "external" world @@ -15,185 +14,160 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -def call(description=null, pkgList=null, buildCmd=null) { +def call(description=null, pkgList=null, buildCmd=null, changesPattern="**") { // - description: Arbitrary text to print on Jenkins Job Description // instead of package name // - pkgList: Multiple packages can be build at once in a single Pipeline run // - buildCmd: replace default build command "dpkg-buildpackage -uc -us -tc -b" // with this custom version + // - changesPattern: package will only be build if a change file matching this + // pattern is found - // Only keep the 10 most recent builds - def projectProperties = [ - [$class: 'BuildDiscarderProperty',strategy: [$class: 'LogRotator', numToKeepStr: '10']], - ] - - properties(projectProperties) setDescription(description) - node('Docker') { - stage('Define Agent') { - script { - // create container name on demand - def branchName = getGitBranchName() - // Adjust PR target branch name so we can re-map it to the proper - // Docker image. CHANGE_ID is set only for pull requests, so it is - // safe to access the pullRequest global variable - if (env.CHANGE_ID) { - branchName = "${env.CHANGE_TARGET}".toLowerCase() - } - if (branchName.equals("master")) { - branchName = "current" - } - env.DOCKER_IMAGE = "vyos/vyos-build:" + branchName - } - } - } pipeline { - agent { - docker { - args "--sysctl net.ipv6.conf.lo.disable_ipv6=0 -e GOSU_UID=1006 -e GOSU_GID=1006" - image "${env.DOCKER_IMAGE}" - alwaysPull true - } - } - environment { - // get relative directory path to Jenkinsfile - BASE_DIR = getJenkinsfilePath() - CHANGESET_DIR = getChangeSetPath() - DEBIAN_ARCH = sh(returnStdout: true, script: 'dpkg --print-architecture').trim() - } + agent none options { - skipDefaultCheckout() - timeout(time: 60, unit: 'MINUTES') + disableConcurrentBuilds() + timeout(time: 120, unit: 'MINUTES') timestamps() + buildDiscarder(logRotator(numToKeepStr: '20')) } stages { - stage('Fetch Source') { + stage('Define Agent') { + agent { + label "ec2_amd64" + } + when { + anyOf { + changeset pattern: changesPattern, caseSensitive: true + triggeredBy cause: "UserIdCause" + } + } steps { script { - // package build must be done in "any" subdir. Without it the Debian build system - // is unable to generate the *.deb files in the sources parent directory, which - // will cause a "Permission denied" error. - dir ('build') { - // checkout git repository which hold 'Jenkinsfile' - checkout scm + // create container name on demand + def branchName = getGitBranchName() + // Adjust PR target branch name so we can re-map it to the proper Docker image. + if (isPullRequest()) + branchName = env.CHANGE_TARGET.toLowerCase() + if (branchName.equals('master')) + branchName = 'current' - // Display Git commit Id used with the Jenkinsfile on the Job "Build History" pane - def commitId = sh(returnStdout: true, script: 'git rev-parse --short=11 HEAD').trim() - currentBuild.description = sprintf('Git SHA1: %s', commitId[-11..-1]) + env.DOCKER_IMAGE = 'vyos/vyos-build:' + branchName - if (pkgList) { - // Fetch individual package source code, but only if a URL is defined, this will - // let us reuse this script for packages like vyos-1x which ship a Jenkinfile in - // their repositories root folder. - pkgList.each { pkg -> - dir(env.BASE_DIR + pkg.name) { - checkout([$class: 'GitSCM', - doGenerateSubmoduleConfigurations: false, - extensions: [[$class: 'CleanCheckout']], - branches: [[name: pkg.scmCommit]], - userRemoteConfigs: [[url: pkg.scmUrl]]]) - } - } - } - } + // Get the current UID and GID from the jenkins agent to allow use of the same UID inside Docker + env.USR_ID = sh(returnStdout: true, script: 'id -u').toString().trim() + env.GRP_ID = sh(returnStdout: true, script: 'id -g').toString().trim() + env.DOCKER_ARGS = '--sysctl net.ipv6.conf.lo.disable_ipv6=0 -e GOSU_UID=' + env.USR_ID + ' -e GOSU_GID=' + env.GRP_ID } } } - stage('Build Source') { + stage('Build Code') { when { - beforeOptions true - beforeAgent true anyOf { - changeset pattern: "${env.CHANGESET_DIR}" - triggeredBy 'TimerTrigger' + changeset pattern: changesPattern, caseSensitive: true triggeredBy cause: "UserIdCause" } } - steps { - script { - // package build must be done in "any" subdir. Without it the Debian build system - // is unable to generate the *.deb files in the sources parent directory, which - // will cause a "Permission denied" error. - dir ('build') { - if (pkgList) { - pkgList.each { pkg -> - dir(env.BASE_DIR + pkg.name) { - sh pkg.buildCmd - } - } - } else if (buildCmd) { - sh buildCmd - } else { - sh "dpkg-buildpackage -uc -us -tc -b" + parallel { + stage('amd64') { + agent { + docker { + label "ec2_amd64" + args "${env.DOCKER_ARGS}" + image "${env.DOCKER_IMAGE}" + alwaysPull true + reuseNode true + } + } + environment { + // get relative directory path to Jenkinsfile + BASE_DIR = getJenkinsfilePath() + CHANGESET_DIR = getChangeSetPath() + } + steps { + script { + cloneAndBuild(description, 'amd64', pkgList, buildCmd) + stash includes: '**/*.deb', name: 'binary-amd64' + } + } + post { + cleanup { + deleteDir() } } } } } - } - post { - cleanup { - deleteDir() - } - success { - script { - // package build must be done in "any" subdir. Without it the Debian build system - // is unable to generate the *.deb files in the sources parent directory, which - // will cause a "Permission denied" error. - def BIN_DIR = '' - if (env.BASE_DIR) { - BIN_DIR = 'build/' + env.BASE_DIR + stage("Finalize") { + when { + anyOf { + changeset pattern: changesPattern, caseSensitive: true + triggeredBy cause: "UserIdCause" } - dir (BIN_DIR) { + } + agent { + label "ec2_amd64" + } + steps { + script { + // Unpack files for amd64 + unstash 'binary-amd64' + if (isCustomBuild()) { - // archive *.deb artifact on custom builds, deploy to repo otherwise - archiveArtifacts artifacts: '**/*.deb', allowEmptyArchive: true - } else { - // publish build result, using SSH-dev.packages.vyos.net Jenkins Credentials - sshagent(['SSH-dev.packages.vyos.net']) { - // build up some fancy groovy variables so we do not need to write/copy - // every option over and over again! - def RELEASE = getGitBranchName() - if (getGitBranchName() == "master") { - RELEASE = 'current' - } + echo "Build not started from official Git repository! Artifacts are not uploaded to external repository" + return + } + echo "Uploading Artifacts to external repository" + copyArtifacts fingerprintArtifacts: true, projectName: '${JOB_NAME}', selector: specific('${BUILD_NUMBER}') - def VYOS_REPO_PATH = '/home/sentrium/web/dev.packages.vyos.net/public_html/repositories/' + RELEASE + '/' - if (getGitBranchName() == "crux") - VYOS_REPO_PATH += 'vyos/' + // build up some fancy groovy variables so we do not need to write/copy + // every option over and over again! + def RELEASE = getGitBranchName() + if (getGitBranchName() == "master") + RELEASE = 'current' - def SSH_OPTS = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR' - def SSH_REMOTE = env.DEV_PACKAGES_VYOS_NET_HOST // defined as global variable + def VYOS_REPO_PATH = '/home/sentrium/web/dev.packages.vyos.net/public_html/repositories/' + RELEASE + if (getGitBranchName() == "crux") + VYOS_REPO_PATH += '/vyos' - def SSH_DIR = '~/VyOS/' + RELEASE + '/' + env.DEBIAN_ARCH + def SSH_OPTS = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR' + def SSH_REMOTE = env.DEV_PACKAGES_VYOS_NET_HOST // defined as global variable + def SSH_DIR = '~/VyOS/' + RELEASE - files = findFiles(glob: '*.deb') - if (files) { - echo "Uploading package(s) and updating package(s) in the repository ..." - files.each { FILE -> - def PKG = sh(returnStdout: true, script: "dpkg-deb -f ${FILE} Package").trim() - // No need to explicitly check the return code. The pipeline - // will fail if sh returns a noni-zero exit code + // publish build result, using SSH-dev.packages.vyos.net Jenkins Credentials + sshagent(['SSH-dev.packages.vyos.net']) { + files = findFiles(glob: '**/*.deb') + if (files) { + sh(script: "ssh ${SSH_OPTS} ${SSH_REMOTE} -t \"bash --login -c 'mkdir -p ${SSH_DIR}'\"") + echo "Uploading package(s) and updating package(s) in the repository ..." + files.each { FILE -> + // NOTE: Groovy is a pain in the ass and " quotes differ from ', so all shell code must use " in the beginning + def PACKAGE = sh(returnStdout: true, script: "dpkg-deb -f ${FILE} Package").trim() + def PACKAGE_ARCH = sh(returnStdout: true, script: "dpkg-deb -f ${FILE} Architecture").trim() + def ARCH = '' + if (PACKAGE_ARCH != 'all') + ARCH = '-A ' + PACKAGE_ARCH - def PACKAGE_ARCH = sh(returnStdout: true, script: "dpkg-deb -f ${FILE} Architecture").trim() - def ARCH = '' - if (PACKAGE_ARCH != 'all') - ARCH = '-A ' + PACKAGE_ARCH + sh(script: "scp ${SSH_OPTS} ${FILE} ${SSH_REMOTE}:${SSH_DIR}") + sh(script: "ssh ${SSH_OPTS} ${SSH_REMOTE} -t \"uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} ${ARCH} remove ${RELEASE} ${PACKAGE}'\"") - sh """ - ssh ${SSH_OPTS} ${SSH_REMOTE} -t "bash --login -c 'mkdir -p ${SSH_DIR}'" - scp ${SSH_OPTS} ${FILE} ${SSH_REMOTE}:${SSH_DIR}/ - ssh ${SSH_OPTS} ${SSH_REMOTE} -t "uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} ${ARCH} remove ${RELEASE} ${PKG}'" - ssh ${SSH_OPTS} ${SSH_REMOTE} -t "uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} deleteunreferenced'" - ssh ${SSH_OPTS} ${SSH_REMOTE} -t "uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} ${ARCH} includedeb ${RELEASE} ${SSH_DIR}/${FILE}'" - """ - } + // Packages like FRR produce their binary in a nested path e.g. packages/frr/frr-rpki-rtrlib-dbgsym_7.5_arm64.deb, + // thus we will only extract the filename portion from FILE as the binary is scp'ed to SSH_DIR without any subpath. + def FILENAME = FILE.toString().tokenize('/')[-1] + sh(script: "ssh ${SSH_OPTS} ${SSH_REMOTE} -t \"uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} ${ARCH} includedeb ${RELEASE} ${SSH_DIR}/${FILENAME}'\"") } + sh(script: "ssh ${SSH_OPTS} ${SSH_REMOTE} -t \"uncron-add 'reprepro -v -b ${VYOS_REPO_PATH} deleteunreferenced'\"") } } } } + post { + cleanup { + deleteDir() + } + } } } } diff --git a/vars/cloneAndBuild.groovy b/vars/cloneAndBuild.groovy new file mode 100644 index 00000000..f945ba75 --- /dev/null +++ b/vars/cloneAndBuild.groovy @@ -0,0 +1,65 @@ +#!/usr/bin/env groovy +// Copyright (C) 2021 VyOS maintainers and contributors +// +// This program is free software; you can redistribute it and/or modify +// in order to easy exprort images built to "external" world +// it under the terms of the GNU General Public License version 2 or later as +// published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +def call(description, architecture, pkgList, buildCmd) { + // package build must be done in "any" subdir. Without it the Debian build system + // is unable to generate the *.deb files in the sources parent directory, which + // will cause a "Permission denied" error. + dir ("build-${architecture}") { + // cleanup + deleteDir() + + // checkout git repository which hold 'Jenkinsfile' + checkout scm + + // Display Git commit Id used with the Jenkinsfile on the Job "Build History" pane + def commitId = sh(returnStdout: true, script: 'git rev-parse --short=11 HEAD').trim() + currentBuild.description = sprintf('Git SHA1: %s', commitId[-11..-1]) + + if (pkgList) { + // Fetch individual package source code, but only if a URL is defined, this will + // let us reuse this script for packages like vyos-1x which ship a Jenkinfile in + // their repositories root folder. + pkgList.each { pkg -> + dir(env.BASE_DIR + pkg.name) { + checkout([$class: 'GitSCM', + doGenerateSubmoduleConfigurations: false, + extensions: [[$class: 'CleanCheckout']], + branches: [[name: pkg.scmCommit]], + userRemoteConfigs: [[url: pkg.scmUrl]]]) + } + } + } + + // compile the source(s) ... + if (pkgList) { + pkgList.each { pkg -> + dir(env.BASE_DIR + pkg.name) { + sh pkg.buildCmd + } + } + } else if (buildCmd) { + sh buildCmd + } else { + sh 'dpkg-buildpackage -uc -us -tc -b' + } + } + if (architecture == 'amd64') { + archiveArtifacts artifacts: "**/*.deb", fingerprint: true + } else { + archiveArtifacts artifacts: "**/*_${architecture}.deb", fingerprint: true + } +} diff --git a/vars/isCustomBuild.groovy b/vars/isCustomBuild.groovy index 301fdd42..c5e5fab7 100644 --- a/vars/isCustomBuild.groovy +++ b/vars/isCustomBuild.groovy @@ -22,5 +22,5 @@ def call() { def gitURI = 'git@github.com:vyos/' + getGitRepoName() def httpURI = 'https://github.com/vyos/' + getGitRepoName() - return !((getGitRepoURL() == gitURI) || (getGitRepoURL() == httpURI)) || env.CHANGE_ID + return !((getGitRepoURL() == gitURI) || (getGitRepoURL() == httpURI)) || isPullRequest() } diff --git a/vars/isPullRequest.groovy b/vars/isPullRequest.groovy new file mode 100644 index 00000000..813341bc --- /dev/null +++ b/vars/isPullRequest.groovy @@ -0,0 +1,21 @@ +// Copyright (C) 2020 VyOS maintainers and contributors +// +// This program is free software; you can redistribute it and/or modify +// in order to easy exprort images built to "external" world +// it under the terms of the GNU General Public License version 2 or later as +// published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +def call() { + if (env.CHANGE_ID) { + return true + } + return false +} -- cgit v1.2.3