path: root/.github/workflows/vyos-rolling-nightly-build.yml

name: VyOS rolling nightly build

on:
  schedule:
    - cron: "0 0 * * *"

  workflow_dispatch:
    inputs:
      BUILD_BY:
        description: 'Builder identifier (if empty autobuild@vyos.net is used)'
        default: ''
      BUILD_VERSION:
        description: 'Version number (if empty 1.5-rolling-$(date -u +%Y%m%d%H%M) is used)'
        default: ''
      FAKE_BUILDING_PROCESS:
        description: 'Skip all Smoketests and fake building process'
        required: true
        type: boolean
        default: false
      SKIP_SMOKETEST_RAID1:
        description: 'Skip RAID1 Smoketest'
        required: true
        type: boolean
        default: false
      SKIP_SMOKETEST_SYSTEM:
        description: 'Skip system Smoketest'
        required: true
        type: boolean
        default: false
      SKIP_SMOKETEST_CONFIG:
        description: 'Skip config Smoketest'
        required: true
        type: boolean
        default: false
      SKIP_SNAPSHOT_S3_UPLOAD:
        description: 'Skip snapshot upload to S3'
        required: true
        type: boolean
        default: false
      SKIP_RELEASE_PUBLISHING:
        description: 'Skip release publishing'
        required: true
        type: boolean
        default: false
      SKIP_SLACK_NOTIFICATIONS:
        description: 'Skip Slack notifications'
        required: true
        type: boolean
        default: false
      SKIP_DOWNLOADS_PAGE_UPDATE:
        description: 'Skip downloads page update'
        required: true
        type: boolean
        default: false

jobs:
  build-iso:
    runs-on: ubuntu-latest
    permissions:
      contents: write
    concurrency:
      group: main
    env:
      BUILD_BY: autobuild@vyos.net
      DEBIAN_MIRROR: http://deb.debian.org/debian/
      VYOS_MIRROR: https://rolling-packages.vyos.net/current/
      DOCKER_CALL: docker run --rm --privileged -v ./vyos-build/:/vyos -w /vyos vyos/vyos-build:current
      DOCKER_CALL_ON_KVM_HOST: docker run --rm --privileged -v ~/vyos-build:/vyos -w /vyos vyos/vyos-build:current
      DOCKER_CALL_ON_KVM_HOST_WITH_IMAGE_PULL: docker run --rm --privileged --pull=always -v ~/vyos-build:/vyos -w /vyos vyos/vyos-build:current
    steps:

### Initialization ###

      - uses: actions/checkout@v4

      - name: "Initialization: set env variables"
        id: set_env_variables
        run: |
          if [ -n "${{ github.event.inputs.BUILD_BY }}" ]; then
            echo "BUILD_BY=${{ github.event.inputs.BUILD_BY }}" >> $GITHUB_ENV
          fi
          if [ -z "${{ github.event.inputs.BUILD_VERSION }}" ]; then
            echo "BUILD_VERSION=1.5-rolling-$(date -u +%Y%m%d%H%M)" >> $GITHUB_ENV
          else
            echo "BUILD_VERSION=${{ github.event.inputs.BUILD_VERSION }}" >> $GITHUB_ENV
          fi
          echo "TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_ENV
          echo "PREVIOUS_SUCCESS_BUILD_TIMESTAMP=$(cat version.json | jq -r '.[0].timestamp')" >> $GITHUB_ENV

      - name: "Initialization: git clone vyos-build"
        id: git_clone_vyos-build
        run: git clone -b current --single-branch https://github.com/vyos/vyos-build

      - name: "Initialization: git clone vyos-1x"
        id: git_clone_vyos-1x
        if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
        run: git clone -b current --single-branch https://github.com/vyos/vyos-1x

### Smoketest ###

      - name: "Smoketest: build generic ISO image"
        id: build_iso_for_smoketest
        if: ${{ !inputs.FAKE_BUILDING_PROCESS && (!inputs.SKIP_SMOKETEST_RAID1 || !inputs.SKIP_SMOKETEST_SYSTEM || !inputs.SKIP_SMOKETEST_CONFIG) }}
        run: |
          ${{ env.DOCKER_CALL }} \
          sudo --preserve-env ./build-vyos-image \
          --architecture amd64 \
          --build-by $BUILD_BY \
          --build-type release \
          --custom-package vyos-1x-smoketest \
          --debian-mirror $DEBIAN_MIRROR \
          --version $BUILD_VERSION \
          --vyos-mirror $VYOS_MIRROR \
          generic

      - name: "Smoketest: create generic ISO artifact"
        id: upload_iso_artifact_smoketest_image
        if: ${{ !inputs.FAKE_BUILDING_PROCESS && (!inputs.SKIP_SMOKETEST_RAID1 || !inputs.SKIP_SMOKETEST_SYSTEM || !inputs.SKIP_SMOKETEST_CONFIG) }}
        uses: actions/upload-artifact@v4
        with:
          name: vyos-${{ env.BUILD_VERSION }}-amd64-smoketest.iso
          path: ./vyos-build/build/live-image-amd64.hybrid.iso
          retention-days: 30
          if-no-files-found: error

      - name: "Smoketest: upload generic ISO to KVM host"
        if: ${{ !inputs.FAKE_BUILDING_PROCESS && (!inputs.SKIP_SMOKETEST_RAID1 || !inputs.SKIP_SMOKETEST_SYSTEM || !inputs.SKIP_SMOKETEST_CONFIG) }}
        id: copy_smoketest_iso_to_kvm_host
        uses: cross-the-world/ssh-scp-ssh-pipelines@latest
        env:
          BUILD_VERSION: ${{ env.BUILD_VERSION }}
        with:
          host: ${{ secrets.SSH_HOST }}
          user: ${{ secrets.SSH_USER }}
          port: ${{ secrets.SSH_PORT }}
          key: ${{ secrets.SSH_KEY }}
          connect_timeout: 600s
          first_ssh: |
            rm -rf vyos-build || true
            git clone -b current --single-branch https://github.com/vyos/vyos-build
            mkdir -p ~/vyos-build/build/
          scp: |
            './vyos-build/build/live-image-amd64.hybrid.iso' => '~/vyos-build/build/'
        timeout-minutes: 300

      - name: "Smoketest: RAID"
        if: ${{ !inputs.FAKE_BUILDING_PROCESS && !inputs.SKIP_SMOKETEST_RAID1 }}
        id: smoketest_raid
        uses: appleboy/ssh-action@v1.0.3
        with:
          host: ${{ secrets.SSH_HOST }}
          username: ${{ secrets.SSH_USER }}
          port: ${{ secrets.SSH_PORT }}
          key: ${{ secrets.SSH_KEY }}
          timeout: 15m
          command_timeout: 180m
          script: |
            ${{ env.DOCKER_CALL_ON_KVM_HOST_WITH_IMAGE_PULL }} /bin/bash -c "set -o pipefail && sudo make testraid | tee smoketest_raid.log"

      - name: "Smoketest: system"
        if: ${{ !inputs.FAKE_BUILDING_PROCESS && !inputs.SKIP_SMOKETEST_SYSTEM }}
        id: smoketest_system
        uses: appleboy/ssh-action@v1.0.3
        with:
          host: ${{ secrets.SSH_HOST }}
          username: ${{ secrets.SSH_USER }}
          port: ${{ secrets.SSH_PORT }}
          key: ${{ secrets.SSH_KEY }}
          timeout: 15m
          command_timeout: 180m
          script: |
            ${{ env.DOCKER_CALL_ON_KVM_HOST }} /bin/bash -c "set -o pipefail && sudo make test | tee smoketest_system.log"

      - name: "Smoketest: configuration"
        if: ${{ !inputs.FAKE_BUILDING_PROCESS && !inputs.SKIP_SMOKETEST_CONFIG }}
        id: smoketest_configuration
        uses: appleboy/ssh-action@v1.0.3
        with:
          host: ${{ secrets.SSH_HOST }}
          username: ${{ secrets.SSH_USER }}
          port: ${{ secrets.SSH_PORT }}
          key: ${{ secrets.SSH_KEY }}
          timeout: 15m
          command_timeout: 180m
          script: |
            ${{ env.DOCKER_CALL_ON_KVM_HOST }} /bin/bash -c "set -o pipefail && sudo make testc | tee smoketest_configuration.log"

      - name: "Smoketest: clear files"
        if: ${{ !inputs.FAKE_BUILDING_PROCESS && (!inputs.SKIP_SMOKETEST_RAID1 || !inputs.SKIP_SMOKETEST_SYSTEM || !inputs.SKIP_SMOKETEST_CONFIG) }}
        id: clear_smoketest_files
        uses: appleboy/ssh-action@v1.0.3
        with:
          host: ${{ secrets.SSH_HOST }}
          username: ${{ secrets.SSH_USER }}
          port: ${{ secrets.SSH_PORT }}
          key: ${{ secrets.SSH_KEY }}
          timeout: 15m
          command_timeout: 45m
          script: |
            ${{ env.DOCKER_CALL_ON_KVM_HOST }} sudo rm -rf ./*
            rm -rf ~/vyos-build

### Building ###

      - name: "Building: create ISO image"
        if: ${{ !inputs.FAKE_BUILDING_PROCESS }}
        id: build_iso
        run: |
          ${{ env.DOCKER_CALL }} \
          sudo --preserve-env ./build-vyos-image \
          --architecture amd64 \
          --build-by $BUILD_BY \
          --build-type release \
          --debian-mirror $DEBIAN_MIRROR \
          --version $BUILD_VERSION \
          --vyos-mirror $VYOS_MIRROR \
          generic

      - name: "Building: copy a generic ISO image"
        if: ${{ !inputs.FAKE_BUILDING_PROCESS }}
        id: copy_iso
        run: |
          cp ./vyos-build/build/live-image-amd64.hybrid.iso ./vyos-$BUILD_VERSION-amd64.iso

      - name: "Building: fake a generic ISO image"
        if: ${{ inputs.FAKE_BUILDING_PROCESS }}
        id: fake_iso
        run: |
          echo FAKE > ./vyos-$BUILD_VERSION-amd64.iso

      - name: "Building: sign an ISO image with Minisign"
        id: sign_iso_minisign
        run: |
          echo "${{ secrets.minisign_private_key }}" > /tmp/minisign.key
          shasum /tmp/minisign.key
          echo ${{ secrets.minisign_password }} | $GITHUB_WORKSPACE/bin/minisign -s /tmp/minisign.key -Sm ./vyos-$BUILD_VERSION-amd64.iso
          echo "${{ secrets.minisign_public_key }}" > /tmp/minisign.pub
          $GITHUB_WORKSPACE/bin/minisign -Vm ./vyos-$BUILD_VERSION-amd64.iso -x ./vyos-$BUILD_VERSION-amd64.iso.minisig -p /tmp/minisign.pub
          rm /tmp/minisign.key /tmp/minisign.pub

### Uploading artifacts ###

      - name: "Uploading artifacts: ISO image to S3 Glacier"
        id: upload_iso_to_s3_glacier
        if: ${{ !inputs.SKIP_SNAPSHOT_S3_UPLOAD }}
        uses: keithweaver/aws-s3-github-action@v1.0.0
        with:
          command: cp
          source: ./vyos-${{ env.BUILD_VERSION }}-amd64.iso
          destination: s3://${{ secrets.aws_s3_bucket }}/vyos-${{ env.BUILD_VERSION }}-amd64.iso
          aws_access_key_id: ${{ secrets.aws_access_key_id }}
          aws_secret_access_key: ${{ secrets.aws_secret_access_key }}
          aws_region: us-east-1
          flags: --storage-class GLACIER

      - name: "Uploading artifacts: ISO image's signature to S3 Glacier"
        id: upload_iso_signature_to_s3_glacier
        if: ${{ !inputs.SKIP_SNAPSHOT_S3_UPLOAD }}
        uses: keithweaver/aws-s3-github-action@v1.0.0
        with:
          command: cp
          source: ./vyos-${{ env.BUILD_VERSION }}-amd64.iso.minisig
          destination: s3://${{ secrets.aws_s3_bucket }}/vyos-${{ env.BUILD_VERSION }}-amd64.iso.minisig
          aws_access_key_id: ${{ secrets.aws_access_key_id }}
          aws_secret_access_key: ${{ secrets.aws_secret_access_key }}
          aws_region: us-east-1
          flags: --storage-class GLACIER

      - name: "Uploading artifacts: ISO image to GitHub"
        id: upload_iso_artifact
        uses: actions/upload-artifact@v4
        with:
          name: vyos-${{ env.BUILD_VERSION }}-amd64.iso
          path: ./vyos-${{ env.BUILD_VERSION }}-amd64.iso
          retention-days: 30
          if-no-files-found: error

      - name: "Uploading artifacts: ISO image's signature to GitHub"
        id: upload_iso_minisign_artifact
        uses: actions/upload-artifact@v4
        with:
          name: vyos-${{ env.BUILD_VERSION }}-amd64.iso.minisig
          path: ./vyos-${{ env.BUILD_VERSION }}-amd64.iso.minisig
          retention-days: 30
          if-no-files-found: error

### Release publishing ###

      - name: "Release publishing: retrieve the latest success build for vyos-build and vyos-1x"
        id: retrieve_the_latest_success_build_for_vyos-build_vyos-1x
        if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
        run: |
          cd ./vyos-build
          echo "CHANGELOG_COMMIT_build=$(git log --since "${{ env.PREVIOUS_SUCCESS_BUILD_TIMESTAMP }}" --format="%H" --reverse | head -n1)" >> $GITHUB_ENV
          cd ../vyos-1x
          echo "CHANGELOG_COMMIT_1x=$(git log --since "${{ env.PREVIOUS_SUCCESS_BUILD_TIMESTAMP }}" --format="%H" --reverse | head -n1)" >> $GITHUB_ENV
        env:
          GH_TOKEN: ${{ github.token }}

      - name: "Release publishing: generate changelog for vyos-1x"
        if: ${{ !inputs.SKIP_RELEASE_PUBLISHING && env.CHANGELOG_COMMIT_1x != '' }}
        id: generate_changelog_for_vyos-1x
        uses: mikepenz/release-changelog-builder-action@v4.1.0
        with:
          owner: "vyos"
          repo: "vyos-1x"
          fetchReviewers: false
          fromTag: ${{ env.CHANGELOG_COMMIT_1x }}
          toTag: HEAD
          configurationJson: |
            {
              "categories": [{"title": "", "labels": []}],
              "template": "#{{CHANGELOG}}",
              "pr_template": "- #{{TITLE}}\n   - PR: vyos/vyos-1x##{{NUMBER}}"
            }

      - name: "Release publishing: generate changelog for vyos-build"
        if: ${{ !inputs.SKIP_RELEASE_PUBLISHING && env.CHANGELOG_COMMIT_build }}
        id: generate_changelog_for_vyos-build
        uses: mikepenz/release-changelog-builder-action@v4.1.0
        with:
          owner: "vyos"
          repo: "vyos-build"
          fetchReviewers: false
          fromTag: ${{ env.CHANGELOG_COMMIT_build }}
          toTag: HEAD
          configurationJson: |
            {
              "categories": [{"title": "", "labels": []}],
              "template": "#{{CHANGELOG}}",
              "pr_template": "- #{{TITLE}}\n   - PR: vyos/vyos-build##{{NUMBER}}"
            }

      - name: "Release publishing: generate CHANGELOG.md"
        id: generate_changelog_md
        if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
        run: |
          cat <<EOF > CHANGELOG.md
          ## vyos-1x
          ${{ steps.generate_changelog_for_vyos-1x.outputs.changelog }}
          ## vyos-build
          ${{ steps.generate-build-changelog.outputs.changelog }}
          EOF
          cat CHANGELOG.md

      - name: "Release publishing: create version.json"
        if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
        id: create_version_json
        uses: jsdaniell/create-json@v1.2.3
        with:
          name: "version.json"
          json: |
            [
              {
                "url": "https://github.com/vyos/vyos-rolling-nightly-builds/releases/download/${{ env.BUILD_VERSION }}/vyos-${{ env.BUILD_VERSION }}-amd64.iso",
                "version": "${{ env.BUILD_VERSION }}",
                "timestamp": "${{ env.TIMESTAMP }}"
              }
            ]

      - name: "Release publishing: update Minisign public key"
        if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
        id: update_minisign_public_key
        run: |
          echo "${{ secrets.minisign_public_key }}" > minisign.pub

      - name: "Release publishing: check if the repository was modified during runtime to prevent autocommit failure"
        id: check_if_the_repository_was_modified_during_runtime
        if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
        run: |
          sudo chown -R $(whoami):$(whoami) ./*
          git pull --autostash --rebase

      - name: "Release publishing: create autocommit and tag"
        id: create_autocommit_and_tag
        if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
        uses: stefanzweifel/git-auto-commit-action@v5
        with:
          tagging_message: ${{ env.BUILD_VERSION }}
          commit_message: ${{ env.BUILD_VERSION }}

      - name: "Release publishing: publish release"
        id: publish_release
        if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
        uses: softprops/action-gh-release@v1
        with:
          body_path: CHANGELOG.md
          tag_name: ${{ env.BUILD_VERSION }}
          fail_on_unmatched_files: true
          files: |
            ./vyos-${{ env.BUILD_VERSION }}-amd64.iso
            ./vyos-${{ env.BUILD_VERSION }}-amd64.iso.minisig

      - name: "Release publishing: remove old releases"
        id: remove_old_releases
        if: ${{ !inputs.SKIP_RELEASE_PUBLISHING }}
        uses: dev-drprasad/delete-older-releases@v0.3.2
        with:
          keep_latest: 30
          delete_tags: true
        env:
          GITHUB_TOKEN: ${{ secrets.CUSTOM_GITHUB_TOKEN }}

### Slack notification ###

      - name: "Slack notification: identify a failed step"
        id: identify_a_failed_step
        if: ${{ failure() && !inputs.SKIP_SLACK_NOTIFICATIONS }}
        run: |
          echo FAILED_STEP=$((cat <<EOF
          ${{ toJson(steps) }}
          EOF
          ) | jq -r 'to_entries | map(select(.value.conclusion == "failure")) | .[].key' | head -n1) >> "$GITHUB_OUTPUT"

      - name: "Slack notification: download Smoketest failed logs"
        id: download_smoketest_failed_logs
        if: ${{ failure() && !inputs.SKIP_SLACK_NOTIFICATIONS && (steps.identify_a_failed_step.outputs.FAILED_STEP == 'smoketest_raid' || steps.identify_a_failed_step.outputs.FAILED_STEP == 'smoketest_system' || steps.identify_a_failed_step.outputs.FAILED_STEP == 'smoketest_configuration') }}
        uses: nicklasfrahm/scp-action@main
        with:
          direction: download
          host: ${{ secrets.SSH_HOST }}
          username: ${{ secrets.SSH_USER }}
          port: ${{ secrets.SSH_PORT }}
          key: ${{ secrets.SSH_KEY }}
          insecure_ignore_fingerprint: true
          source: "~/vyos-build/${{ steps.identify_a_failed_step.outputs.FAILED_STEP }}.log"
          target: "${{ github.workspace }}/${{ steps.identify_a_failed_step.outputs.FAILED_STEP }}.log"

      - name: "Slack notification: filter Smoketest failed logs"
        id: filter_smoketest_failed_logs
        if: ${{ failure() && !inputs.SKIP_SLACK_NOTIFICATIONS && (steps.identify_a_failed_step.outputs.FAILED_STEP == 'smoketest_raid' || steps.identify_a_failed_step.outputs.FAILED_STEP == 'smoketest_system' || steps.identify_a_failed_step.outputs.FAILED_STEP == 'smoketest_configuration') }}
        run: |
          cat ${{ github.workspace }}/${{ steps.identify_a_failed_step.outputs.FAILED_STEP }}.log | grep '... FAIL' > ${{ github.workspace }}/smoketest_filtered.log || true
          cat ${{ github.workspace }}/${{ steps.identify_a_failed_step.outputs.FAILED_STEP }}.log | grep 'ERROR -' >> ${{ github.workspace }}/smoketest_filtered.log || true
          cat ${{ github.workspace }}/${{ steps.identify_a_failed_step.outputs.FAILED_STEP }}.log | grep 'Error:' -A12 >> ${{ github.workspace }}/smoketest_filtered.log || true

      - name: "Slack notification: send to channel"
        id: send_a_slack_failure_notification
        if: ${{ failure() && !inputs.SKIP_SLACK_NOTIFICATIONS }}
        uses: slackapi/slack-github-action@v1.25.0
        with:
          payload: |
            {
              "text": "*VyOS nightly build automation didn't pass the step with ID \"${{ steps.identify_a_failed_step.outputs.FAILED_STEP }}\"*",
              "attachments": [
                {
                  "color": "FF6600",
                  "fields": [
                    {
                      "title": "Repository",
                      "short": true,
                      "value": "${{ github.repository }}"
                    },
                    {
                      "title": "Workflow",
                      "short": true,
                      "value": "${{ github.workflow }}"
                    },
                    {
                      "title": "Trigger",
                      "short": true,
                      "value": "${{ github.event_name }}"
                    },
                    {
                      "title": "Run number",
                      "short": true,
                      "value": "${{ github.run_number }}"
                    },
                    {
                      "title": "URL",
                      "short": false,
                      "value": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
                    }
                  ]
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

      - name: "Slack notification: send Smoketest log excerpt"
        id: send_smoketest_log_excerpt_to_slack
        if: ${{ failure() && !inputs.SKIP_SLACK_NOTIFICATIONS }}
        uses: MeilCli/slack-upload-file@v4
        with:
          slack_token: ${{ secrets.SLACK_BOT_TOKEN }}
          channel_id: ${{ secrets.SLACK_CHANNEL_ID }}
          file_type: 'text'
          file_path: ${{ github.workspace }}/smoketest_filtered.log
          initial_comment: Smoketest log excerpt
          if_no_files_found: ignore

### Update download page ###

  update-downloads-page:
    if: ${{ !inputs.SKIP_DOWNLOADS_PAGE_UPDATE }}
    needs: build-iso
    uses: vyos/community.vyos.net/.github/workflows/main.yml@production
    with:
      branch: production
    secrets:
        NETLIFY_TOKEN: ${{ secrets.NETLIFY_TOKEN }}
        GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}