Merge pull request #462 from c-po/current

T2640: update VyOS in docker image to current version

9 files changed, 166 insertions, 55 deletions

diff --git a/.gitignore b/.gitignore
index 36643800..23101b27 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,4 @@ packages/*
 !packages/*/
 testinstall*.img
 *.qcow2
+*.tar
diff --git a/Makefile b/Makefile
index f9b5903f..393ef3b7 100644
--- a/Makefile
+++ b/Makefile
@@ -59,6 +59,11 @@ testraid: checkiso
 qemu-live: checkiso
 	scripts/check-qemu-install --qemu-cmd build/live-image-amd64.hybrid.iso
 
+.PHONE: oci
+.ONESHELL:
+oci: checkiso
+	scripts/iso-to-oci build/live-image-amd64.hybrid.iso
+
 .PHONY: clean
 .ONESHELL:
 clean:
diff --git a/docker-vyos/Dockerfile b/docker-vyos/Dockerfile
index cfe505ef..ef7621c6 100644
--- a/docker-vyos/Dockerfile
+++ b/docker-vyos/Dockerfile
@@ -1,6 +1,6 @@
 # syntax = docker/dockerfile:1
 
-# Copyright (C) 2020 VyOS maintainers and contributors
+# Copyright (C) 2020-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -17,29 +17,48 @@
 # Define arguments for VyOS image
 ARG VYOS_VERSION
 ARG BUILD_DATE
-ARG DEBIAN_VERSION
 
 # Use Debian as base layer
-FROM debian:${DEBIAN_VERSION}-slim
+FROM debian:bookworm-slim
+
+LABEL authors="VyOS Maintainers <maintainers@vyos.io>"
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN /bin/echo -e 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommends
+
+# Base packaged needed to build packages and their package dependencies
+RUN apt-get update && apt-get install -y \
+      ca-certificates \
+      gnupg \
+      curl \
+      fuse-overlayfs \
+      jq \
+      yq
+
 # Copy installer script and default build settings
-COPY [ "data/defaults.json", "data/live-build-config/archives/*", "docker-vyos/vyos_install_common.sh", "docker-vyos/vyos_install_stage_01.sh", "/tmp/" ]
+COPY [ "data/defaults.toml", \
+       "data/architectures/amd64.toml", \
+       "data/live-build-config/archives/*", \
+       "docker-vyos/vyos_install_common.sh", \
+       "docker-vyos/vyos_install_stage_01.sh", \
+       "/tmp/"]
 COPY [ "data/live-build-config/hooks/live/*", "/tmp/hooks/" ]
 
-
 # Install VyOS dependencies
 WORKDIR /tmp
 RUN bash /tmp/vyos_install_stage_01.sh
 
 
 # Install VyOS specific software
-COPY [ "data/defaults.json", "docker-vyos/vyos_install_common.sh", "docker-vyos/vyos_install_stage_02.sh", "/tmp/" ]
+COPY [ "data/defaults.toml", \
+       "data/architectures/amd64.toml", \
+       "docker-vyos/vyos_install_common.sh", \
+       "docker-vyos/vyos_install_stage_02.sh", "/tmp/" ]
 RUN bash /tmp/vyos_install_stage_02.sh
 
 
 # Tune system for VyOS
 COPY [ "docker-vyos/vyos_install_common.sh", "docker-vyos/vyos_install_stage_03.sh", "/tmp/" ]
-# Copy skel for bash profile
-COPY data/live-build-config/includes.chroot/etc/skel/.bashrc /etc/skel/.bashrc
 # Copy default config
 COPY data/live-build-config/includes.chroot/opt/vyatta/etc/config.boot.default /opt/vyatta/etc/
 
diff --git a/docker-vyos/README.md b/docker-vyos/README.md
index 7c1fb6a1..ad0c3881 100644
--- a/docker-vyos/README.md
+++ b/docker-vyos/README.md
@@ -2,58 +2,81 @@
 
 VyOS can be run as a Docker container on a Linux host with a compatible kernel.
 
+## Build Container
 
-## Building Docker image
+To build a Docker image you need to have the whole `vyos-build` repository, not
+only a folder with Dockerfile, because some files from this repository are
+required for building.
 
-To build a Docker image you need to have the whole `vyos-build` repository, not only a folder with Dockerfile, because some files from this repository are required for building.
 Docker image with VyOS can be built on Linux host with the next command:
 
-```
-docker build --compress -f Dockerfile -t vyos:version-`date -u +%Y%m%d%H%M%S` --build-arg BUILD_DATE="`date -u --rfc-3339=seconds`" --build-arg VYOS_VERSION=version --build-arg DEBIAN_VERSION=debian --progress plain ..
+```console
+docker build --compress --file Dockerfile \
+  --tag vyos:version-`date -u +%Y%m%d%H%M%S` \
+  --build-arg BUILD_DATE="`date -u --rfc-3339=seconds`" \
+  --build-arg VYOS_VERSION=version \
+  --build-arg DEBIAN_VERSION=debian \
+  --progress plain ..
 ```
 
 Or, if you want to rebuild completely from the scratch (without cache):
 
-```
-docker build --no-cache --pull --compress -f Dockerfile -t vyos:version-`date -u +%Y%m%d%H%M%S` --build-arg BUILD_DATE="`date -u --rfc-3339=seconds`" --build-arg VYOS_VERSION=version --build-arg DEBIAN_VERSION=debian --progress plain ..
+```console
+docker build --no-cache --pull --compress --file Dockerfile \
+  --tag vyos:version-`date -u +%Y%m%d%H%M%S` \
+  --build-arg BUILD_DATE="`date -u --rfc-3339=seconds`" \
+  --build-arg VYOS_VERSION=version \
+  --build-arg DEBIAN_VERSION=debian \
+  --progress plain ..
 ```
 
-> **NOTE:** You must use proper version value for `DEBIAN_VERSION` variable. It can be only `jessie` (for VyOS 1.2) or `buster` (for VyOS 1.3).
+> **_NOTE:_** You must use proper version value for `DEBIAN_VERSION` variable.
+  It can be only `jessie` (for VyOS 1.2) or `buster` (for VyOS 1.3).
 
-## Running Docker image
+## Run Container
 
 Docker container with VyOS can be running with the next command:
 
-```
-docker run -v /lib/modules:/lib/modules --privileged --name vyos_inside_docker -d vyos:version
+```console
+docker run --privileged --detach \
+  --volume /lib/modules:/lib/modules \
+  --name vyos_inside_docker vyos:version
 ```
 
-You need to use the `--privileged` flag because the system actively interacts with a host kernel to perform routing operations and tune networking options.
-
+You need to use the `--privileged` flag because the system actively interacts
+with a host kernel to perform routing operations and tune networking options.
 
 **Experimantal:** You can limit access to some system resources with:
 
-```
-docker run --tmpfs /tmp --tmpfs /run --tmpfs /run/lock -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /lib/modules:/lib/modules --privileged --name vyos_inside_docker -d vyos:version
+```console
+docker run --privileged --detach \
+   --tmpfs /tmp \
+   --tmpfs /run \
+   --tmpfs /run/lock \
+   --volume /lib/modules:/lib/modules:ro \
+   --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \
+   --name vyos_inside_docker vyos:version
 ```
 
-## Logging into a VyOS container
+### Log into container
 
-To open VyOS CLI, you can use SSH connection to the Docker container or run on host:
+To open VyOS CLI, you can use SSH connection to the Docker container or run
+on host:
 
-```
+```console
 docker exec -it vyos_inside_docker su vyos
 ```
 
-
 ## Troubleshooting
 
-If in VyOS appears IPv6-related errors, for example, it cannot assign an IPv6 for an interface, it is necessary to enable IPv6 support in Docker. This can be done, by editing `/etc/docker/daemon.json`:
+If in VyOS appears IPv6-related errors, for example, it cannot assign an IPv6
+address for an interface, it is necessary to enable IPv6 support in Docker.
 
-```
+This can be done, by editing `/etc/docker/daemon.json`:
+
+```console
 {
     "ipv6": true,
     "fixed-cidr-v6": "fe80::/64"
 }
-
 ```
diff --git a/docker-vyos/vyos_install_common.sh b/docker-vyos/vyos_install_common.sh
index 7c5360c0..7d2ed7e3 100644
--- a/docker-vyos/vyos_install_common.sh
+++ b/docker-vyos/vyos_install_common.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (C) 2020 VyOS maintainers and contributors
+# Copyright (C) 2020-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -14,7 +14,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
 # Set environment variables
 export DEBIAN_FRONTEND="noninteractive"
 
@@ -23,21 +22,21 @@ function prepare_apt() {
     # Update packages list
     apt-get update
 
-    # Install jq (required to easily extract variables from defaults.json)
-    apt-get install -y --no-install-recommends jq gnupg
-
     # Add VyOS repository to the system
-    local APT_VYOS_MIRROR=`jq --raw-output .vyos_mirror /tmp/defaults.json`
-    local APT_VYOS_BRANCH=`jq --raw-output .vyos_branch /tmp/defaults.json`
-    local APT_ADDITIONAL_REPOS=`jq --raw-output .additional_repositories[] /tmp/defaults.json`
-    local RELEASE_TRAIN=`jq --raw-output .release_train /tmp/defaults.json`
+    local APT_VYOS_MIRROR=$(tomlq --raw-output .vyos_mirror /tmp/defaults.toml)
+    local APT_VYOS_BRANCH=$(tomlq --raw-output .vyos_branch /tmp/defaults.toml)
+    local APT_ADDITIONAL_REPOS=$(tomlq --raw-output .additional_repositories[] /tmp/amd64.toml)
+    local RELEASE_TRAIN=$(tomlq --raw-output .release_train /tmp/defaults.toml)
 
-    if [[ "${RELEASE_TRAIN}" == "crux" ]]; then
-        echo -e "deb ${APT_VYOS_MIRROR}/vyos ${APT_VYOS_BRANCH} main\ndeb ${APT_VYOS_MIRROR}/debian ${APT_VYOS_BRANCH} main\n${APT_ADDITIONAL_REPOS}" > /etc/apt/sources.list.d/vyos.list
-    fi
+    echo "APT_VYOS_MIRROR      : $APT_VYOS_MIRROR"
+    echo "APT_VYOS_BRANCH      : $APT_VYOS_BRANCH"
+    echo "APT_ADDITIONAL_REPOS : $APT_ADDITIONAL_REPOS"
+    echo "RELEASE_TRAIN        : $RELEASE_TRAIN"
+
+    echo -e "deb ${APT_VYOS_MIRROR} ${APT_VYOS_BRANCH} main\n${APT_ADDITIONAL_REPOS}" > /etc/apt/sources.list.d/vyos.list
+    cat /etc/apt/sources.list.d/vyos.list
 
-    if [[ "${RELEASE_TRAIN}" == "equuleus" || "${RELEASE_TRAIN}" == "sagitta" ]]; then
-        echo -e "deb ${APT_VYOS_MIRROR} ${APT_VYOS_BRANCH} main\n${APT_ADDITIONAL_REPOS}" > /etc/apt/sources.list.d/vyos.list
+    if [ ${RELEASE_TRAIN} == "equuleus" ]; then
         # Add backports repository
         echo -e "deb http://deb.debian.org/debian buster-backports main\ndeb http://deb.debian.org/debian buster-backports non-free" >> /etc/apt/sources.list.d/vyos.list
     fi
@@ -65,8 +64,6 @@ function prepare_apt() {
 
 # Cleanup APT after finish
 function cleanup_apt() {
-    # delete jq tool
-    dpkg -P jq
     # Clear APT cache
     apt-get clean
     rm -rf /var/lib/apt/lists/*
diff --git a/docker-vyos/vyos_install_stage_01.sh b/docker-vyos/vyos_install_stage_01.sh
index c793aa4e..ce9e9b53 100644
--- a/docker-vyos/vyos_install_stage_01.sh
+++ b/docker-vyos/vyos_install_stage_01.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (C) 2020 VyOS maintainers and contributors
+# Copyright (C) 2020-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -58,6 +58,7 @@ ignore_list=(
     "cluster-glue"
     "resource-agents"
     "heartbeat"
+    "podman"
     )
 
 # Get list of packages from VYOS repository
diff --git a/docker-vyos/vyos_install_stage_02.sh b/docker-vyos/vyos_install_stage_02.sh
index 1ec1676c..81a0975b 100644
--- a/docker-vyos/vyos_install_stage_02.sh
+++ b/docker-vyos/vyos_install_stage_02.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (C) 2020 VyOS maintainers and contributors
+# Copyright (C) 2020-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -35,6 +35,8 @@ vyos_packages_filtered=("$(filter_list vyos_packages[@] vyos_packages_filter[@])
 vyos_packages_filtered+=(
     "uuid"
     "jq"
+    "yq"
+    "systemd"
     )
 
 echo "Packages for installing: ${vyos_packages_filtered[@]}"
@@ -43,12 +45,12 @@ echo "Installing VyOS packages"
 apt-get install -y --no-install-recommends ${vyos_packages_filtered[@]}
 
 # Create VyOS version file
-RELEASAE_TRAIN=`jq --raw-output .release_train /tmp/defaults.json`
+RELEASAE_TRAIN=$(tomlq --raw-output .release_train /tmp/defaults.toml)
 apt-cache show vyos-1x | awk -v release_train=${RELEASAE_TRAIN} '{ if ($1 == "Version:") version = $2 } END { build_git = "unknown" ; built_by = "Sentrium S.L." ; built_on = strftime("%F %T UTC", systime(), utc) ; "uuid -v 4" | getline build_uuid ; printf("{\"version\": \"%s\", \"build_git\": \"%s\", \"built_on\": \"%s\", \"built_by\": \"%s\", \"build_uuid\": \"%s\", \"release_train\": \"%s\"}", version, build_git, built_on, built_by, build_uuid, release_train) }' | json_pp > /usr/share/vyos/version.json
 
 # Delete what we do not need inside Docker image (this step makes packages database inconsistent)
 echo "Deleting what is needless in containers"
-dpkg -P --force-depends dosfstools efibootmgr jq gdisk grub-common grub-efi-amd64-bin initscripts installation-report laptop-detect libossp-uuid16 libparted2 libwireshark-data libwireshark5 mdadm parted tshark uuid vyos-qat-kernel-modules wireguard-modules
+dpkg -P --force-depends dosfstools efibootmgr yq jq gdisk grub-common grub-efi-amd64-bin initscripts installation-report laptop-detect libossp-uuid16 libparted2 libwireshark-data libwireshark5 mdadm parted tshark uuid
 dpkg -l | awk '/linux-image-/ { system("dpkg -P --force-depends " $2) }'
 
 # Delete documentation
diff --git a/docker-vyos/vyos_install_stage_03.sh b/docker-vyos/vyos_install_stage_03.sh
index 90003a4f..e51ff089 100644
--- a/docker-vyos/vyos_install_stage_03.sh
+++ b/docker-vyos/vyos_install_stage_03.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (C) 2020 VyOS maintainers and contributors
+# Copyright (C) 2020-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -20,6 +20,7 @@
 . vyos_install_common.sh
 
 # Add config partition marker
+mkdir -p /opt/vyatta/etc/config
 touch /opt/vyatta/etc/config/.vyatta_config
 
 # create folder for configuration mounting
@@ -54,10 +55,9 @@ for hook in ${hooks_list[@]}; do
 done
 
 # Delete needless options from CLI
-# CLI_DELETION=(
-#     "/opt/vyatta/share/vyatta-cfg/templates/system/host-name/"
-#     )
-# rm -rf ${CLI_DELETION[@]}
-
+ CLI_DELETION=(
+     "/opt/vyatta/share/vyatta-cfg/templates/container/"
+     )
+ rm -rf ${CLI_DELETION[@]}
 
 exit 0
diff --git a/scripts/iso-to-oci b/scripts/iso-to-oci
new file mode 100755
index 00000000..c9396290
--- /dev/null
+++ b/scripts/iso-to-oci
@@ -0,0 +1,63 @@
+#!/bin/bash
+
+function cleanup() {
+    if [[ -d $ROOTFS ]]; then
+        rm -rf $ROOTFS
+    fi
+    if [[ -d $UNSQUASHFS ]]; then
+        rm -rf $UNSQUASHFS
+    fi
+}
+
+if [[ $(/usr/bin/id -u) -ne 0 ]]; then
+    echo "Not running as root"
+    exit
+fi
+
+if [ "$#" -ne 1 ]; then
+    echo "Illegal number of parameters"
+fi
+
+ISO=$1
+ROOTFS=rootfs
+UNSQUASHFS=unsquashfs
+
+# ensure clean working directory
+cleanup
+
+mkdir $ROOTFS $UNSQUASHFS
+echo "I: mount ISO $ISO"
+mount -t iso9660 -o loop $ISO $ROOTFS/ >/dev/null 2>&1
+
+# create directory, unpack squashfs filesystem, get ISO version
+# and unmount ISO
+echo "I: extracting squashfs content"
+unsquashfs -follow -dest $UNSQUASHFS/ $ROOTFS/live/filesystem.squashfs >/dev/null 2>&1
+VERSION=$(jq --raw-output .version $ROOTFS/version.json)
+umount $ROOTFS/
+
+# fix locales for correct system configuration loading
+sed -i 's/^LANG=.*$/LANG=C.UTF-8/' $UNSQUASHFS/etc/default/locale
+
+# optional step: Decrease docker image size by deleting not necessary files for container
+rm -rf $UNSQUASHFS/boot/*.img
+rm -rf $UNSQUASHFS/boot/*vyos*
+rm -rf $UNSQUASHFS/boot/vmlinuz
+rm -rf $UNSQUASHFS/lib/firmware/
+rm -rf $UNSQUASHFS/usr/lib/x86_64-linux-gnu/libwireshark.so*
+rm -rf $UNSQUASHFS/lib/modules/*amd64-vyos
+rm -rf $UNSQUASHFS/root/.gnupg
+
+# create a symbolic link to the configuration
+ln -s /opt/vyatta/etc/config $UNSQUASHFS/config
+
+# create docker image
+echo "I: generate OCI container image vyos-$VERSION.tar"
+tar -C unsquashfs -c . -f vyos-$VERSION.tar
+
+echo "I: to import the previously generated OCI image to your local images run:"
+echo ""
+echo "   docker import vyos-$VERSION.tar vyos:$VERSION --change 'CMD ["/sbin/init"]'"
+echo ""
+
+cleanup