summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChristian Breunig <christian@breunig.cc>2024-09-25 20:24:21 +0200
committerChristian Breunig <christian@breunig.cc>2024-09-25 20:24:21 +0200
commitd235b31a095f9b8fdb2d5c231935c8b4b4c3da6c (patch)
tree0a4256d787fcdda0bea8308f6a76c65ef1e7ad1b
parentb93672d9fb294e94804f16153428cb450696f4df (diff)
downloadvyos-build-d235b31a095f9b8fdb2d5c231935c8b4b4c3da6c.tar.gz
vyos-build-d235b31a095f9b8fdb2d5c231935c8b4b4c3da6c.zip
T861: sign all Kernel modules with an ephemeral key
The shim review board (which is the secure boot base loader) recommends using ephemeral keys when signing the Linux Kernel. This commit enables the Kernel build system to generate a one-time ephemeral key that is used to: * sign all build-in Kernel modules * sign all other out-of-tree Kernel modules The key lives in /tmp and is destroyed after the build container exits and is named: "VyOS build time autogenerated kernel key". In addition the Kernel now uses CONFIG_MODULE_SIG_FORCE. This now makes it unable to load any Kernel Module to the image that is NOT signed by the ephemeral key.
-rwxr-xr-xdata/live-build-config/hooks/live/19-kernel_symlinks.chroot5
-rwxr-xr-xdata/live-build-config/hooks/live/93-sb-sign-kernel.chroot22
-rwxr-xr-xdata/live-build-config/hooks/live/93-sign-kernel.chroot18
-rw-r--r--data/live-build-config/includes.chroot/var/lib/shim-signed/mok/README.md13
-rw-r--r--packages/linux-kernel/.gitignore2
-rw-r--r--packages/linux-kernel/arch/x86/configs/vyos_defconfig18
-rwxr-xr-xpackages/linux-kernel/build-accel-ppp.sh8
-rwxr-xr-xpackages/linux-kernel/build-intel-ixgbe.sh3
-rwxr-xr-xpackages/linux-kernel/build-intel-ixgbevf.sh3
-rwxr-xr-xpackages/linux-kernel/build-intel-qat.sh3
-rwxr-xr-xpackages/linux-kernel/build-jool.py4
-rwxr-xr-xpackages/linux-kernel/build-kernel.sh46
-rwxr-xr-xpackages/linux-kernel/build-mellanox-ofed.sh12
-rwxr-xr-xpackages/linux-kernel/build-nat-rtsp.sh8
-rwxr-xr-xpackages/linux-kernel/build-openvpn-dco.sh8
-rwxr-xr-xpackages/linux-kernel/sign-modules.sh15
-rwxr-xr-xscripts/check-qemu-install5
-rwxr-xr-xscripts/package-build/linux-kernel/build-accel-ppp.sh8
-rwxr-xr-xscripts/package-build/linux-kernel/build-intel-ixgbe.sh6
-rwxr-xr-xscripts/package-build/linux-kernel/build-intel-ixgbevf.sh7
-rwxr-xr-xscripts/package-build/linux-kernel/build-intel-qat.sh6
-rwxr-xr-xscripts/package-build/linux-kernel/build-jool.py7
-rwxr-xr-xscripts/package-build/linux-kernel/build-kernel.sh53
-rwxr-xr-xscripts/package-build/linux-kernel/build-mellanox-ofed.sh140
-rwxr-xr-xscripts/package-build/linux-kernel/build-nat-rtsp.sh12
-rwxr-xr-xscripts/package-build/linux-kernel/build-openvpn-dco.sh8
-rwxr-xr-xscripts/package-build/linux-kernel/build.py7
-rw-r--r--scripts/package-build/linux-kernel/package.toml5
-rwxr-xr-xscripts/package-build/linux-kernel/sign-modules.sh15
29 files changed, 392 insertions, 75 deletions
diff --git a/data/live-build-config/hooks/live/19-kernel_symlinks.chroot b/data/live-build-config/hooks/live/19-kernel_symlinks.chroot
index e63ca263..a7e95e0e 100755
--- a/data/live-build-config/hooks/live/19-kernel_symlinks.chroot
+++ b/data/live-build-config/hooks/live/19-kernel_symlinks.chroot
@@ -1,6 +1,9 @@
#!/bin/sh
-echo I: Creating kernel symlinks.
+echo I: Creating Linux Kernel symbolic links
cd /boot
ln -s initrd.img-* initrd.img
ln -s vmlinuz-* vmlinuz
+
+echo I: Remove Linux Kernel symbolic link to source folder
+rm -rf /lib/modules/*/build
diff --git a/data/live-build-config/hooks/live/93-sb-sign-kernel.chroot b/data/live-build-config/hooks/live/93-sb-sign-kernel.chroot
new file mode 100755
index 00000000..1dc03186
--- /dev/null
+++ b/data/live-build-config/hooks/live/93-sb-sign-kernel.chroot
@@ -0,0 +1,22 @@
+#!/bin/sh
+SIGN_FILE=$(find /usr/lib -name sign-file)
+MOK_KEY="/var/lib/shim-signed/mok/MOK.key"
+MOK_CERT="/var/lib/shim-signed/mok/MOK.pem"
+VMLINUZ=$(readlink /boot/vmlinuz)
+
+# All Linux Kernel modules need to be cryptographically signed
+find /lib/modules -type f -name \*.ko | while read MODULE; do
+ modinfo ${MODULE} | grep -q "signer:"
+ if [ $? != 0 ]; then
+ echo "E: Module ${MODULE} is not signed!"
+ read -n 1 -s -r -p "Press any key to continue"
+ fi
+done
+
+if [ ! -f ${MOK_KEY} ]; then
+ echo "I: Signing key for Linux Kernel not found - Secure Boot not possible"
+else
+ echo "I: Signing Linux Kernel for Secure Boot"
+ sbsign --key ${MOK_KEY} --cert ${MOK_CERT} /boot/${VMLINUZ} --output /boot/${VMLINUZ}
+ sbverify --list /boot/${VMLINUZ}
+fi
diff --git a/data/live-build-config/hooks/live/93-sign-kernel.chroot b/data/live-build-config/hooks/live/93-sign-kernel.chroot
deleted file mode 100755
index 031db10d..00000000
--- a/data/live-build-config/hooks/live/93-sign-kernel.chroot
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/sh
-SIGN_FILE=$(find /usr/lib -name sign-file)
-MOK_KEY="/var/lib/shim-signed/mok/kernel.key"
-MOK_CERT="/var/lib/shim-signed/mok/kernel.pem"
-kernel_elf=$(readlink /boot/vmlinuz)
-
-if [ ! -f ${MOK_KEY} ]; then
- echo "I: Signing key for Linux Kernel not found - Secure Boot not possible"
-else
- echo "I: Signing Linux Kernel for Secure Boot"
-
- sbsign --key $MOK_KEY --cert $MOK_CERT /boot/${kernel_elf} --output /boot/${kernel_elf}
- sbverify --list /boot/${kernel_elf}
-
- find /lib/modules -type f -name \*.ko -o -name \*.ko.xz | while read module; do
- $SIGN_FILE sha512 $MOK_KEY $MOK_CERT $module
- done
-fi
diff --git a/data/live-build-config/includes.chroot/var/lib/shim-signed/mok/README.md b/data/live-build-config/includes.chroot/var/lib/shim-signed/mok/README.md
index 5a6edbba..abaaa97a 100644
--- a/data/live-build-config/includes.chroot/var/lib/shim-signed/mok/README.md
+++ b/data/live-build-config/includes.chroot/var/lib/shim-signed/mok/README.md
@@ -6,17 +6,6 @@ Create Certificate Authority used for Kernel signing. CA is loaded into the
Machine Owner Key store on the target system.
```bash
-openssl req -new -x509 -newkey rsa:2048 -keyout MOK.key -outform DER -out MOK.der -days 36500 -subj "/CN=VyOS Secure Boot CA/" -nodes
+openssl req -new -x509 -newkey rsa:4096 -keyout MOK.key -outform DER -out MOK.der -days 36500 -subj "/CN=VyOS Secure Boot CA/" -nodes
openssl x509 -inform der -in MOK.der -out MOK.pem
```
-
-## Kernel Module Signing Key
-
-We do not make use of ephemeral keys for Kernel module signing. Instead a key
-is generated and signed by the VyOS Secure Boot CA which signs all the Kernel
-modules during ISO assembly if present.
-
-```bash
-openssl req -newkey rsa:2048 -keyout kernel.key -out kernel.csr -subj "/CN=VyOS Secure Boot Signer 2024 - linux/" -nodes
-openssl x509 -req -in kernel.csr -CA MOK.pem -CAkey MOK.key -CAcreateserial -out kernel.pem -days 730 -sha256
-```
diff --git a/packages/linux-kernel/.gitignore b/packages/linux-kernel/.gitignore
index a3e9257c..4b2ab254 100644
--- a/packages/linux-kernel/.gitignore
+++ b/packages/linux-kernel/.gitignore
@@ -13,6 +13,8 @@
/QAT*
*.tar.xz
/*.postinst
+/ephemeral.key
+/ephemeral.pem
# Intel Driver source
i40e-*/
diff --git a/packages/linux-kernel/arch/x86/configs/vyos_defconfig b/packages/linux-kernel/arch/x86/configs/vyos_defconfig
index e4daa23b..4c1a4adc 100644
--- a/packages/linux-kernel/arch/x86/configs/vyos_defconfig
+++ b/packages/linux-kernel/arch/x86/configs/vyos_defconfig
@@ -842,6 +842,7 @@ CONFIG_FUNCTION_ALIGNMENT=16
CONFIG_RT_MUTEXES=y
CONFIG_BASE_SMALL=0
+CONFIG_MODULE_SIG_FORMAT=y
CONFIG_MODULES=y
# CONFIG_MODULE_DEBUG is not set
CONFIG_MODULE_FORCE_LOAD=y
@@ -851,7 +852,15 @@ CONFIG_MODULE_FORCE_UNLOAD=y
CONFIG_MODVERSIONS=y
CONFIG_ASM_MODVERSIONS=y
# CONFIG_MODULE_SRCVERSION_ALL is not set
-# CONFIG_MODULE_SIG is not set
+CONFIG_MODULE_SIG=y
+CONFIG_MODULE_SIG_FORCE=y
+CONFIG_MODULE_SIG_ALL=y
+# CONFIG_MODULE_SIG_SHA1 is not set
+# CONFIG_MODULE_SIG_SHA224 is not set
+# CONFIG_MODULE_SIG_SHA256 is not set
+# CONFIG_MODULE_SIG_SHA384 is not set
+CONFIG_MODULE_SIG_SHA512=y
+CONFIG_MODULE_SIG_HASH="sha512"
CONFIG_MODULE_COMPRESS_NONE=y
# CONFIG_MODULE_COMPRESS_GZIP is not set
# CONFIG_MODULE_COMPRESS_XZ is not set
@@ -5888,8 +5897,11 @@ CONFIG_SIGNED_PE_FILE_VERIFICATION=y
#
# Certificates for signature checking
#
-CONFIG_SYSTEM_TRUSTED_KEYRING=y
-CONFIG_SYSTEM_TRUSTED_KEYS=""
+CONFIG_MODULE_SIG_KEY="certs/signing_key.pem"
+CONFIG_MODULE_SIG_KEY_TYPE_RSA=y
+# CONFIG_MODULE_SIG_KEY_TYPE_ECDSA is not set
+# CONFIG_SYSTEM_TRUSTED_KEYRING is not set
+# CONFIG_SYSTEM_TRUSTED_KEYS is not set
# CONFIG_SYSTEM_EXTRA_CERTIFICATE is not set
# CONFIG_SECONDARY_TRUSTED_KEYRING is not set
# CONFIG_SYSTEM_BLACKLIST_KEYRING is not set
diff --git a/packages/linux-kernel/build-accel-ppp.sh b/packages/linux-kernel/build-accel-ppp.sh
index 1685ff8d..a2f8df52 100755
--- a/packages/linux-kernel/build-accel-ppp.sh
+++ b/packages/linux-kernel/build-accel-ppp.sh
@@ -13,6 +13,10 @@ if [ ! -f ${KERNEL_VAR_FILE} ]; then
exit 1
fi
+cd ${ACCEL_SRC}
+git reset --hard HEAD
+git clean --force -d -x
+
PATCH_DIR=${CWD}/patches/accel-ppp
if [ -d $PATCH_DIR ]; then
cd ${ACCEL_SRC}
@@ -36,6 +40,10 @@ cmake -DBUILD_IPOE_DRIVER=TRUE \
-DMODULES_KDIR=${KERNEL_VERSION}${KERNEL_SUFFIX} \
-DCPACK_TYPE=Debian12 ..
make
+
+# Sign generated Kernel modules
+${CWD}/sign-modules.sh .
+
cpack -G DEB
# rename resulting Debian package according git description
diff --git a/packages/linux-kernel/build-intel-ixgbe.sh b/packages/linux-kernel/build-intel-ixgbe.sh
index ab44f551..797ed60f 100755
--- a/packages/linux-kernel/build-intel-ixgbe.sh
+++ b/packages/linux-kernel/build-intel-ixgbe.sh
@@ -80,6 +80,9 @@ fi
echo "I: Building Debian package vyos-intel-${DRIVER_NAME}"
cd ${CWD}
+# Sign generated Kernel modules
+${CWD}/sign-modules.sh ${DEBIAN_DIR}
+
# delete non required files which are also present in the kernel package
# und thus lead to duplicated files
find ${DEBIAN_DIR} -name "modules.*" | xargs rm -f
diff --git a/packages/linux-kernel/build-intel-ixgbevf.sh b/packages/linux-kernel/build-intel-ixgbevf.sh
index 39803852..7d389832 100755
--- a/packages/linux-kernel/build-intel-ixgbevf.sh
+++ b/packages/linux-kernel/build-intel-ixgbevf.sh
@@ -72,6 +72,9 @@ fi
echo "I: Building Debian package vyos-intel-${DRIVER_NAME}"
cd ${CWD}
+# Sign generated Kernel modules
+${CWD}/sign-modules.sh ${DEBIAN_DIR}
+
# delete non required files which are also present in the kernel package
# und thus lead to duplicated files
find ${DEBIAN_DIR} -name "modules.*" | xargs rm -f
diff --git a/packages/linux-kernel/build-intel-qat.sh b/packages/linux-kernel/build-intel-qat.sh
index 5b0e023f..708398d7 100755
--- a/packages/linux-kernel/build-intel-qat.sh
+++ b/packages/linux-kernel/build-intel-qat.sh
@@ -84,6 +84,9 @@ fi
echo "I: Building Debian package vyos-intel-${DRIVER_NAME}"
cd ${CWD}
+# Sign generated Kernel modules
+${CWD}/sign-modules.sh ${DEBIAN_DIR}
+
# delete non required files which are also present in the kernel package
# und thus lead to duplicated files
find ${DEBIAN_DIR} -name "modules.*" | xargs rm -f
diff --git a/packages/linux-kernel/build-jool.py b/packages/linux-kernel/build-jool.py
index 3f8fd3a5..21363c04 100755
--- a/packages/linux-kernel/build-jool.py
+++ b/packages/linux-kernel/build-jool.py
@@ -65,7 +65,7 @@ MODULES_DIR := extra
# main packaging script based on dh7 syntax
%:
- dh $@
+ dh $@
override_dh_clean:
dh_clean --exclude=debian/{PACKAGE_NAME}.substvars
@@ -87,7 +87,7 @@ override_dh_auto_install:
install -D -m 644 src/mod/common/jool_common.ko ${{PACKAGE_BUILD_DIR}}/lib/modules/${{KVER}}/${{MODULES_DIR}}/jool_common.ko
install -D -m 644 src/mod/nat64/jool.ko ${{PACKAGE_BUILD_DIR}}/lib/modules/${{KVER}}/${{MODULES_DIR}}/jool.ko
install -D -m 644 src/mod/siit/jool_siit.ko ${{PACKAGE_BUILD_DIR}}/lib/modules/${{KVER}}/${{MODULES_DIR}}/jool_siit.ko
-
+ ${{KERNEL_DIR}}/../sign-modules.sh ${{PACKAGE_BUILD_DIR}}/lib
'''
bild_rules = Path(f'{PACKAGE_DIR}/debian/rules')
bild_rules.write_text(build_rules_text)
diff --git a/packages/linux-kernel/build-kernel.sh b/packages/linux-kernel/build-kernel.sh
index f7b0c597..f9298c28 100755
--- a/packages/linux-kernel/build-kernel.sh
+++ b/packages/linux-kernel/build-kernel.sh
@@ -9,13 +9,16 @@ if [ ! -d ${KERNEL_SRC} ]; then
exit 1
fi
-echo "I: Copy Kernel config (x86_64_vyos_defconfig) to Kernel Source"
-cp -rv arch/ ${KERNEL_SRC}/
-
cd ${KERNEL_SRC}
-echo "I: clean modified files"
-git reset --hard HEAD
+if [ -d .git ]; then
+ echo "I: Clean modified files - reset Git repo"
+ git reset --hard HEAD
+ git clean --force -d -x
+fi
+
+echo "I: Copy Kernel config (x86_64_vyos_defconfig) to Kernel Source"
+cp -rv ${CWD}/arch/ .
KERNEL_VERSION=$(make kernelversion)
KERNEL_SUFFIX=-$(awk -F "= " '/kernel_flavor/ {print $2}' ../../../data/defaults.toml | tr -d \")
@@ -32,6 +35,9 @@ do
patch -p1 < ${PATCH_DIR}/${patch}
done
+# Change name of Signing Cert
+sed -i -e "s/CN =.*/CN=VyOS build time autogenerated kernel key/" certs/default_x509.genkey
+
TRUSTED_KEYS_FILE=trusted_keys.pem
# start with empty key file
echo -n "" > $TRUSTED_KEYS_FILE
@@ -41,16 +47,8 @@ if [ ! -z "${CERTS}" ]; then
for file in $CERTS; do
cat $file >> $TRUSTED_KEYS_FILE
done
-
# Force Kernel module signing and embed public keys
- echo "CONFIG_MODULE_SIG_FORMAT=y" >> $KERNEL_CONFIG
- echo "CONFIG_MODULE_SIG=y" >> $KERNEL_CONFIG
- echo "CONFIG_MODULE_SIG_FORCE=y" >> $KERNEL_CONFIG
- echo "# CONFIG_MODULE_SIG_ALL is not set" >> $KERNEL_CONFIG
- echo "CONFIG_MODULE_SIG_SHA512=y" >> $KERNEL_CONFIG
- echo "CONFIG_MODULE_SIG_HASH=\"sha512\"" >> $KERNEL_CONFIG
- echo "CONFIG_MODULE_SIG_KEY=\"\"" >> $KERNEL_CONFIG
- echo "CONFIG_MODULE_SIG_KEY_TYPE_RSA=y" >> $KERNEL_CONFIG
+ echo "CONFIG_SYSTEM_TRUSTED_KEYRING" >> $KERNEL_CONFIG
echo "CONFIG_SYSTEM_TRUSTED_KEYS=\"$TRUSTED_KEYS_FILE\"" >> $KERNEL_CONFIG
fi
@@ -59,21 +57,31 @@ echo "I: make vyos_defconfig"
make vyos_defconfig
echo "I: Generate environment file containing Kernel variable"
+EPHEMERAL_KEY="/tmp/ephemeral.key"
+EPHEMERAL_PEM="/tmp/ephemeral.pem"
cat << EOF >${CWD}/kernel-vars
#!/bin/sh
export KERNEL_VERSION=${KERNEL_VERSION}
export KERNEL_SUFFIX=${KERNEL_SUFFIX}
export KERNEL_DIR=${CWD}/${KERNEL_SRC}
+export EPHEMERAL_KEY=${EPHEMERAL_KEY}
+export EPHEMERAL_CERT=${EPHEMERAL_PEM}
EOF
echo "I: Build Debian Kernel package"
touch .scmversion
make bindeb-pkg BUILD_TOOLS=1 LOCALVERSION=${KERNEL_SUFFIX} KDEB_PKGVERSION=${KERNEL_VERSION}-1 -j $(getconf _NPROCESSORS_ONLN)
+# Back to the old Kernel build-scripts directory
cd $CWD
-if [[ $? == 0 ]]; then
- for package in $(ls linux-*.deb)
- do
- ln -sf linux-kernel/$package ..
- done
+EPHEMERAL_KERNEL_KEY=$(grep -E "^CONFIG_MODULE_SIG_KEY=" ${KERNEL_SRC}/$KERNEL_CONFIG | awk -F= '{print $2}' | tr -d \")
+if test -f "${EPHEMERAL_KEY}"; then
+ rm -f ${EPHEMERAL_KEY}
+fi
+if test -f "${EPHEMERAL_PEM}"; then
+ rm -f ${EPHEMERAL_PEM}
+fi
+if test -f "${KERNEL_SRC}/${EPHEMERAL_KERNEL_KEY}"; then
+ openssl rsa -in ${KERNEL_SRC}/${EPHEMERAL_KERNEL_KEY} -out ${EPHEMERAL_KEY}
+ openssl x509 -in ${KERNEL_SRC}/${EPHEMERAL_KERNEL_KEY} -out ${EPHEMERAL_PEM}
fi
diff --git a/packages/linux-kernel/build-mellanox-ofed.sh b/packages/linux-kernel/build-mellanox-ofed.sh
index a157ee61..3f8a50f0 100755
--- a/packages/linux-kernel/build-mellanox-ofed.sh
+++ b/packages/linux-kernel/build-mellanox-ofed.sh
@@ -117,6 +117,18 @@ cp $(find $CWD/$DRIVER_DIR/DEBS/$DEB_DISTRO -type f | grep '\.deb$') "$CWD/"
echo "I: Cleanup ${DRIVER_NAME} source"
cd ${CWD}
+
+# Sign modules
+DEB_NAME=$(ls mlnx-ofed-kernel-modules_*)
+TMP_DIR="tmp-ofed-sign"
+dpkg-deb --raw-extract ${DEB_NAME} ${TMP_DIR}
+# Sign generated Kernel modules
+${CWD}/sign-modules.sh ${TMP_DIR}
+# Cleanup and repack DEB
+rm -f ${DEB_NAME}
+dpkg-deb --build ${TMP_DIR} ${DEB_NAME}
+rm -rf ${TMP_DIR}
+
if [ -f ${DRIVER_FILE} ]; then
rm -f ${DRIVER_FILE}
fi
diff --git a/packages/linux-kernel/build-nat-rtsp.sh b/packages/linux-kernel/build-nat-rtsp.sh
index 40018cfb..33f1311d 100755
--- a/packages/linux-kernel/build-nat-rtsp.sh
+++ b/packages/linux-kernel/build-nat-rtsp.sh
@@ -15,7 +15,10 @@ fi
. ${KERNEL_VAR_FILE}
-cd ${SRC} && make KERNELDIR=$KERNEL_DIR
+cd ${SRC}
+git reset --hard HEAD
+git clean --force -d -x
+make KERNELDIR=$KERNEL_DIR
# Copy binary to package directory
DEBIAN_DIR=tmp/lib/modules/${KERNEL_VERSION}${KERNEL_SUFFIX}/extra
@@ -26,6 +29,9 @@ DEBIAN_POSTINST="${CWD}/vyos-nat-rtsp.postinst"
echo "#!/bin/sh" > ${DEBIAN_POSTINST}
echo "/sbin/depmod -a ${KERNEL_VERSION}${KERNEL_SUFFIX}" >> ${DEBIAN_POSTINST}
+# Sign generated Kernel modules
+${CWD}/sign-modules.sh ${DEBIAN_DIR}
+
# Build Debian Package
fpm --input-type dir --output-type deb --name nat-rtsp \
--version $(git describe --tags --always) --deb-compression gz \
diff --git a/packages/linux-kernel/build-openvpn-dco.sh b/packages/linux-kernel/build-openvpn-dco.sh
index fd427825..518729ee 100755
--- a/packages/linux-kernel/build-openvpn-dco.sh
+++ b/packages/linux-kernel/build-openvpn-dco.sh
@@ -15,13 +15,19 @@ fi
. ${KERNEL_VAR_FILE}
-cd ${SRC} && make KERNEL_SRC=$KERNEL_DIR
+cd ${SRC}
+git reset --hard HEAD
+git clean --force -d -x
+make KERNEL_SRC=$KERNEL_DIR
# Copy binary to package directory
DEBIAN_DIR=tmp/lib/modules/${KERNEL_VERSION}${KERNEL_SUFFIX}/extra
mkdir -p ${DEBIAN_DIR}
cp drivers/net/ovpn-dco/ovpn-dco-v2.ko ${DEBIAN_DIR}
+# Sign generated Kernel modules
+${CWD}/sign-modules.sh ${DEBIAN_DIR}
+
# Build Debian Package
fpm --input-type dir --output-type deb --name openvpn-dco \
--version $(git describe | sed s/^v//) --deb-compression gz \
diff --git a/packages/linux-kernel/sign-modules.sh b/packages/linux-kernel/sign-modules.sh
new file mode 100755
index 00000000..cfb368eb
--- /dev/null
+++ b/packages/linux-kernel/sign-modules.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+BASE_DIR=$(dirname $0)
+MODULE_DIR=$1
+. ${BASE_DIR}/kernel-vars
+
+SIGN_FILE="${KERNEL_DIR}/scripts/sign-file"
+
+if [ -f ${EPHEMERAL_KEY} ] && [ -f ${EPHEMERAL_CERT} ]; then
+ find ${MODULE_DIR} -type f -name \*.ko | while read MODULE; do
+ echo "I: Signing ${MODULE} ..."
+ ${SIGN_FILE} sha512 ${EPHEMERAL_KEY} ${EPHEMERAL_CERT} ${MODULE}
+ done
+fi
+
diff --git a/scripts/check-qemu-install b/scripts/check-qemu-install
index dfb772d8..21084655 100755
--- a/scripts/check-qemu-install
+++ b/scripts/check-qemu-install
@@ -544,6 +544,11 @@ try:
c.sendline('systemd-detect-virt')
c.expect('kvm')
c.expect(op_mode_prompt)
+ # Ensure ephemeral key is loaded
+ vyos_kernel_key = 'VyOS build time autogenerated kernel key'
+ c.sendline(f'show log kernel | match "{vyos_kernel_key}"')
+ c.expect(f'.*{vyos_kernel_key}.*')
+ c.expect(op_mode_prompt)
#################################################
# Executing test-suite
diff --git a/scripts/package-build/linux-kernel/build-accel-ppp.sh b/scripts/package-build/linux-kernel/build-accel-ppp.sh
index 1685ff8d..a2f8df52 100755
--- a/scripts/package-build/linux-kernel/build-accel-ppp.sh
+++ b/scripts/package-build/linux-kernel/build-accel-ppp.sh
@@ -13,6 +13,10 @@ if [ ! -f ${KERNEL_VAR_FILE} ]; then
exit 1
fi
+cd ${ACCEL_SRC}
+git reset --hard HEAD
+git clean --force -d -x
+
PATCH_DIR=${CWD}/patches/accel-ppp
if [ -d $PATCH_DIR ]; then
cd ${ACCEL_SRC}
@@ -36,6 +40,10 @@ cmake -DBUILD_IPOE_DRIVER=TRUE \
-DMODULES_KDIR=${KERNEL_VERSION}${KERNEL_SUFFIX} \
-DCPACK_TYPE=Debian12 ..
make
+
+# Sign generated Kernel modules
+${CWD}/sign-modules.sh .
+
cpack -G DEB
# rename resulting Debian package according git description
diff --git a/scripts/package-build/linux-kernel/build-intel-ixgbe.sh b/scripts/package-build/linux-kernel/build-intel-ixgbe.sh
index 5f45c62a..797ed60f 100755
--- a/scripts/package-build/linux-kernel/build-intel-ixgbe.sh
+++ b/scripts/package-build/linux-kernel/build-intel-ixgbe.sh
@@ -80,6 +80,9 @@ fi
echo "I: Building Debian package vyos-intel-${DRIVER_NAME}"
cd ${CWD}
+# Sign generated Kernel modules
+${CWD}/sign-modules.sh ${DEBIAN_DIR}
+
# delete non required files which are also present in the kernel package
# und thus lead to duplicated files
find ${DEBIAN_DIR} -name "modules.*" | xargs rm -f
@@ -105,3 +108,6 @@ fi
if [ -d ${DEBIAN_DIR} ]; then
rm -rf ${DEBIAN_DIR}
fi
+if [ -f ${DEBIAN_POSTINST} ]; then
+ rm -f ${DEBIAN_POSTINST}
+fi
diff --git a/scripts/package-build/linux-kernel/build-intel-ixgbevf.sh b/scripts/package-build/linux-kernel/build-intel-ixgbevf.sh
index a965e0de..7d389832 100755
--- a/scripts/package-build/linux-kernel/build-intel-ixgbevf.sh
+++ b/scripts/package-build/linux-kernel/build-intel-ixgbevf.sh
@@ -72,6 +72,9 @@ fi
echo "I: Building Debian package vyos-intel-${DRIVER_NAME}"
cd ${CWD}
+# Sign generated Kernel modules
+${CWD}/sign-modules.sh ${DEBIAN_DIR}
+
# delete non required files which are also present in the kernel package
# und thus lead to duplicated files
find ${DEBIAN_DIR} -name "modules.*" | xargs rm -f
@@ -97,4 +100,6 @@ fi
if [ -d ${DEBIAN_DIR} ]; then
rm -rf ${DEBIAN_DIR}
fi
-
+if [ -f ${DEBIAN_POSTINST} ]; then
+ rm -f ${DEBIAN_POSTINST}
+fi
diff --git a/scripts/package-build/linux-kernel/build-intel-qat.sh b/scripts/package-build/linux-kernel/build-intel-qat.sh
index 765cea3f..708398d7 100755
--- a/scripts/package-build/linux-kernel/build-intel-qat.sh
+++ b/scripts/package-build/linux-kernel/build-intel-qat.sh
@@ -84,6 +84,9 @@ fi
echo "I: Building Debian package vyos-intel-${DRIVER_NAME}"
cd ${CWD}
+# Sign generated Kernel modules
+${CWD}/sign-modules.sh ${DEBIAN_DIR}
+
# delete non required files which are also present in the kernel package
# und thus lead to duplicated files
find ${DEBIAN_DIR} -name "modules.*" | xargs rm -f
@@ -109,3 +112,6 @@ fi
if [ -d ${DEBIAN_DIR} ]; then
rm -rf ${DEBIAN_DIR}
fi
+if [ -f ${DEBIAN_POSTINST} ]; then
+ rm -f ${DEBIAN_POSTINST}
+fi
diff --git a/scripts/package-build/linux-kernel/build-jool.py b/scripts/package-build/linux-kernel/build-jool.py
index 570293f5..3d2c3d6a 100755
--- a/scripts/package-build/linux-kernel/build-jool.py
+++ b/scripts/package-build/linux-kernel/build-jool.py
@@ -29,9 +29,8 @@ def add_depends(package_dir: str, package_name: str,
# find kernel version and source path
arch: str = find_arch()
defaults_file: str = Path('../../../data/defaults.toml').read_text()
-architecture_file: str = Path(f'../../../data/architectures/{arch}.toml').read_text()
KERNEL_VER: str = toml_loads(defaults_file).get('kernel_version')
-KERNEL_FLAVOR: str = toml_loads(architecture_file).get('kernel_flavor')
+KERNEL_FLAVOR: str = toml_loads(defaults_file).get('kernel_flavor')
KERNEL_SRC: str = Path.cwd().as_posix() + '/linux'
# define variables
@@ -66,7 +65,7 @@ MODULES_DIR := extra
# main packaging script based on dh7 syntax
%:
- dh $@
+ dh $@
override_dh_clean:
dh_clean --exclude=debian/{PACKAGE_NAME}.substvars
@@ -88,7 +87,7 @@ override_dh_auto_install:
install -D -m 644 src/mod/common/jool_common.ko ${{PACKAGE_BUILD_DIR}}/lib/modules/${{KVER}}/${{MODULES_DIR}}/jool_common.ko
install -D -m 644 src/mod/nat64/jool.ko ${{PACKAGE_BUILD_DIR}}/lib/modules/${{KVER}}/${{MODULES_DIR}}/jool.ko
install -D -m 644 src/mod/siit/jool_siit.ko ${{PACKAGE_BUILD_DIR}}/lib/modules/${{KVER}}/${{MODULES_DIR}}/jool_siit.ko
-
+ ${{KERNEL_DIR}}/../sign-modules.sh ${{PACKAGE_BUILD_DIR}}/lib
'''
bild_rules = Path(f'{PACKAGE_DIR}/debian/rules')
bild_rules.write_text(build_rules_text)
diff --git a/scripts/package-build/linux-kernel/build-kernel.sh b/scripts/package-build/linux-kernel/build-kernel.sh
index 2c02f5c3..6f3b94ec 100755
--- a/scripts/package-build/linux-kernel/build-kernel.sh
+++ b/scripts/package-build/linux-kernel/build-kernel.sh
@@ -9,16 +9,20 @@ if [ ! -d ${KERNEL_SRC} ]; then
exit 1
fi
-echo "I: Copy Kernel config (x86_64_vyos_defconfig) to Kernel Source"
-cp -rv arch/ ${KERNEL_SRC}/
-
cd ${KERNEL_SRC}
-echo "I: clean modified files"
-git reset --hard HEAD
+if [ -d .git ]; then
+ echo "I: Clean modified files - reset Git repo"
+ git reset --hard HEAD
+ git clean --force -d -x
+fi
+
+echo "I: Copy Kernel config (x86_64_vyos_defconfig) to Kernel Source"
+cp -rv ${CWD}/arch/ .
KERNEL_VERSION=$(make kernelversion)
-KERNEL_SUFFIX=-$(dpkg --print-architecture)-vyos
+KERNEL_SUFFIX=-$(awk -F "= " '/kernel_flavor/ {print $2}' ../../../data/defaults.toml | tr -d \")
+KERNEL_CONFIG=arch/x86/configs/vyos_defconfig
# VyOS requires some small Kernel Patches - apply them here
# It's easier to habe them here and make use of the upstream
@@ -31,26 +35,53 @@ do
patch -p1 < ${PATCH_DIR}/${patch}
done
+# Change name of Signing Cert
+sed -i -e "s/CN =.*/CN=VyOS build time autogenerated kernel key/" certs/default_x509.genkey
+
+TRUSTED_KEYS_FILE=trusted_keys.pem
+# start with empty key file
+echo -n "" > $TRUSTED_KEYS_FILE
+CERTS=$(find ../../../data/live-build-config/includes.chroot/var/lib/shim-signed/mok -name "*.pem" -type f || true)
+if [ ! -z "${CERTS}" ]; then
+ # add known public keys to Kernel certificate chain
+ for file in $CERTS; do
+ cat $file >> $TRUSTED_KEYS_FILE
+ done
+ # Force Kernel module signing and embed public keys
+ echo "CONFIG_SYSTEM_TRUSTED_KEYRING" >> $KERNEL_CONFIG
+ echo "CONFIG_SYSTEM_TRUSTED_KEYS=\"$TRUSTED_KEYS_FILE\"" >> $KERNEL_CONFIG
+fi
+
echo "I: make vyos_defconfig"
# Select Kernel configuration - currently there is only one
make vyos_defconfig
echo "I: Generate environment file containing Kernel variable"
+EPHEMERAL_KEY="/tmp/ephemeral.key"
+EPHEMERAL_PEM="/tmp/ephemeral.pem"
cat << EOF >${CWD}/kernel-vars
#!/bin/sh
export KERNEL_VERSION=${KERNEL_VERSION}
export KERNEL_SUFFIX=${KERNEL_SUFFIX}
export KERNEL_DIR=${CWD}/${KERNEL_SRC}
+export EPHEMERAL_KEY=${EPHEMERAL_KEY}
+export EPHEMERAL_CERT=${EPHEMERAL_PEM}
EOF
echo "I: Build Debian Kernel package"
touch .scmversion
make bindeb-pkg BUILD_TOOLS=1 LOCALVERSION=${KERNEL_SUFFIX} KDEB_PKGVERSION=${KERNEL_VERSION}-1 -j $(getconf _NPROCESSORS_ONLN)
+# Back to the old Kernel build-scripts directory
cd $CWD
-if [[ $? == 0 ]]; then
- for package in $(ls linux-*.deb)
- do
- ln -sf linux-kernel/$package ..
- done
+EPHEMERAL_KERNEL_KEY=$(grep -E "^CONFIG_MODULE_SIG_KEY=" ${KERNEL_SRC}/$KERNEL_CONFIG | awk -F= '{print $2}' | tr -d \")
+if test -f "${EPHEMERAL_KEY}"; then
+ rm -f ${EPHEMERAL_KEY}
+fi
+if test -f "${EPHEMERAL_PEM}"; then
+ rm -f ${EPHEMERAL_PEM}
+fi
+if test -f "${KERNEL_SRC}/${EPHEMERAL_KERNEL_KEY}"; then
+ openssl rsa -in ${KERNEL_SRC}/${EPHEMERAL_KERNEL_KEY} -out ${EPHEMERAL_KEY}
+ openssl x509 -in ${KERNEL_SRC}/${EPHEMERAL_KERNEL_KEY} -out ${EPHEMERAL_PEM}
fi
diff --git a/scripts/package-build/linux-kernel/build-mellanox-ofed.sh b/scripts/package-build/linux-kernel/build-mellanox-ofed.sh
new file mode 100755
index 00000000..3f8a50f0
--- /dev/null
+++ b/scripts/package-build/linux-kernel/build-mellanox-ofed.sh
@@ -0,0 +1,140 @@
+#!/bin/sh
+DROP_DEV_DBG_DEBS=1
+DEB_DISTRO='debian12.1'
+CWD=$(pwd)
+KERNEL_VAR_FILE=${CWD}/kernel-vars
+
+if [ $(id -u) -ne 0 ]; then
+ echo "Mellanox OFED script needs to be run as root"
+ exit
+fi
+
+if ! dpkg-architecture -iamd64; then
+ echo "Mellanox OFED is only buildable on amd64 platforms"
+ exit 0
+fi
+
+if [ ! -f ${KERNEL_VAR_FILE} ]; then
+ echo "Kernel variable file '${KERNEL_VAR_FILE}' does not exist, run ./build_kernel.sh first"
+ exit 1
+fi
+
+. ${KERNEL_VAR_FILE}
+
+mlxver="24.07-0.6.1.0"
+url="https://www.mellanox.com/downloads/ofed/MLNX_OFED-${mlxver}/MLNX_OFED_SRC-debian-${mlxver}.tgz"
+
+cd ${CWD}
+
+DRIVER_FILE=$(basename ${url} | sed -e s/tar_0/tar/)
+DRIVER_SHA1="c64defa8fb38dcbce153adc09834ab5cdcecd791"
+
+DRIVER_DIR="${DRIVER_FILE%.tgz}"
+DRIVER_NAME="ofed"
+DRIVER_PRFX="MLNX_OFED"
+DRIVER_VERSION=$(echo ${DRIVER_DIR} | awk -F${DRIVER_PRFX} '{print $2}' | sed 's/^-//;s|_SRC-debian-||')
+DRIVER_VERSION_EXTRA=""
+
+# Build up Debian related variables required for packaging
+DEBIAN_ARCH=$(dpkg --print-architecture)
+DEBIAN_DIR="${CWD}/vyos-mellanox-${DRIVER_NAME}_${DRIVER_VERSION}_${DEBIAN_ARCH}"
+DEBIAN_CONTROL="${DEBIAN_DIR}/DEBIAN/control"
+DEBIAN_POSTINST="${CWD}/vyos-mellanox-ofed.postinst"
+
+# Fetch OFED driver source from Nvidia
+if [ -e ${DRIVER_FILE} ]; then
+ rm -f ${DRIVER_FILE}
+fi
+curl -L -o ${DRIVER_FILE} ${url}
+if [ "$?" -ne "0" ]; then
+ exit 1
+fi
+
+# Verify integrity
+echo "${DRIVER_SHA1} ${DRIVER_FILE}" | sha1sum -c -
+if [ $? != 0 ]; then
+ echo SHA1 checksum missmatch
+ exit 1
+fi
+
+# Unpack archive
+if [ -d ${DRIVER_DIR} ]; then
+ rm -rf ${DRIVER_DIR}
+fi
+mkdir -p ${DRIVER_DIR}
+tar -C ${DRIVER_DIR} --strip-components=1 -xf ${DRIVER_FILE}
+
+# Build/install debs
+cd ${DRIVER_DIR}
+if [ -z $KERNEL_DIR ]; then
+ echo "KERNEL_DIR not defined"
+ exit 1
+fi
+
+rm -f SOURCES/ibarr_*.tar.gz
+rm -f SOURCES/ibdump_*.tar.gz
+rm -f SOURCES/ibsim_*.tar.gz
+rm -f SOURCES/iser_*.tar.gz
+rm -f SOURCES/isert_*.tar.gz
+rm -f SOURCES/kernel-mft_*.tar.gz
+rm -f SOURCES/knem_*.tar.gz
+rm -f SOURCES/libvma_*.tar.gz
+rm -f SOURCES/libxlio_*.tar.gz
+rm -f SOURCES/mlnx-ethtool_*.tar.gz
+rm -f SOURCES/mlnx-iproute2_*.tar.gz
+rm -f SOURCES/mlnx-nfsrdma_*.tar.gz
+rm -f SOURCES/mlnx-nvme_*.tar.gz
+rm -f SOURCES/mlx-steering-dump_*.tar.gz
+rm -f SOURCES/mpitests_*.tar.gz
+rm -f SOURCES/mstflint_*.tar.gz
+rm -f SOURCES/ofed-scripts_*.tar.gz
+rm -f SOURCES/openmpi_*.tar.gz
+rm -f SOURCES/openvswitch_*.tar.gz
+rm -f SOURCES/perftest_*.tar.gz
+rm -f SOURCES/rdma-core_*.tar.gz
+rm -f SOURCES/rshim_*.tar.gz
+rm -f SOURCES/sockperf_*.tar.gz
+rm -f SOURCES/srp_*.tar.gz
+rm -f SOURCES/ucx_*.tar.gz
+
+./install.pl \
+ --basic --dpdk \
+ --without-dkms \
+ --without-mlnx-nvme-modules \
+ --with-vma --vma-vpi --vma-eth \
+ --guest --hypervisor \
+ --builddir ${DEBIAN_DIR}/mlx \
+ --distro ${DEB_DISTRO} \
+ --kernel-sources ${KERNEL_DIR} \
+ --kernel ${KERNEL_VERSION}${KERNEL_SUFFIX}
+
+if [ $DROP_DEV_DBG_DEBS -eq 1 ]; then
+ echo "I: Removing development and debug packages"
+ rm -f $(find $CWD/$DRIVER_DIR/DEBS/$DEB_DISTRO -type f | grep -E '\-dev|\-dbg')
+fi
+
+cp $(find $CWD/$DRIVER_DIR/DEBS/$DEB_DISTRO -type f | grep '\.deb$') "$CWD/"
+
+echo "I: Cleanup ${DRIVER_NAME} source"
+cd ${CWD}
+
+# Sign modules
+DEB_NAME=$(ls mlnx-ofed-kernel-modules_*)
+TMP_DIR="tmp-ofed-sign"
+dpkg-deb --raw-extract ${DEB_NAME} ${TMP_DIR}
+# Sign generated Kernel modules
+${CWD}/sign-modules.sh ${TMP_DIR}
+# Cleanup and repack DEB
+rm -f ${DEB_NAME}
+dpkg-deb --build ${TMP_DIR} ${DEB_NAME}
+rm -rf ${TMP_DIR}
+
+if [ -f ${DRIVER_FILE} ]; then
+ rm -f ${DRIVER_FILE}
+fi
+if [ -d ${DRIVER_DIR} ]; then
+ rm -rf ${DRIVER_DIR}
+fi
+if [ -d ${DEBIAN_DIR} ]; then
+ rm -rf ${DEBIAN_DIR}
+fi
diff --git a/scripts/package-build/linux-kernel/build-nat-rtsp.sh b/scripts/package-build/linux-kernel/build-nat-rtsp.sh
index ec7d19a6..33f1311d 100755
--- a/scripts/package-build/linux-kernel/build-nat-rtsp.sh
+++ b/scripts/package-build/linux-kernel/build-nat-rtsp.sh
@@ -15,7 +15,10 @@ fi
. ${KERNEL_VAR_FILE}
-cd ${SRC} && make KERNELDIR=$KERNEL_DIR
+cd ${SRC}
+git reset --hard HEAD
+git clean --force -d -x
+make KERNELDIR=$KERNEL_DIR
# Copy binary to package directory
DEBIAN_DIR=tmp/lib/modules/${KERNEL_VERSION}${KERNEL_SUFFIX}/extra
@@ -26,6 +29,9 @@ DEBIAN_POSTINST="${CWD}/vyos-nat-rtsp.postinst"
echo "#!/bin/sh" > ${DEBIAN_POSTINST}
echo "/sbin/depmod -a ${KERNEL_VERSION}${KERNEL_SUFFIX}" >> ${DEBIAN_POSTINST}
+# Sign generated Kernel modules
+${CWD}/sign-modules.sh ${DEBIAN_DIR}
+
# Build Debian Package
fpm --input-type dir --output-type deb --name nat-rtsp \
--version $(git describe --tags --always) --deb-compression gz \
@@ -36,3 +42,7 @@ fpm --input-type dir --output-type deb --name nat-rtsp \
--license "GPL2" --chdir tmp
mv *.deb ..
+
+if [ -f ${DEBIAN_POSTINST} ]; then
+ rm -f ${DEBIAN_POSTINST}
+fi
diff --git a/scripts/package-build/linux-kernel/build-openvpn-dco.sh b/scripts/package-build/linux-kernel/build-openvpn-dco.sh
index fd427825..518729ee 100755
--- a/scripts/package-build/linux-kernel/build-openvpn-dco.sh
+++ b/scripts/package-build/linux-kernel/build-openvpn-dco.sh
@@ -15,13 +15,19 @@ fi
. ${KERNEL_VAR_FILE}
-cd ${SRC} && make KERNEL_SRC=$KERNEL_DIR
+cd ${SRC}
+git reset --hard HEAD
+git clean --force -d -x
+make KERNEL_SRC=$KERNEL_DIR
# Copy binary to package directory
DEBIAN_DIR=tmp/lib/modules/${KERNEL_VERSION}${KERNEL_SUFFIX}/extra
mkdir -p ${DEBIAN_DIR}
cp drivers/net/ovpn-dco/ovpn-dco-v2.ko ${DEBIAN_DIR}
+# Sign generated Kernel modules
+${CWD}/sign-modules.sh ${DEBIAN_DIR}
+
# Build Debian Package
fpm --input-type dir --output-type deb --name openvpn-dco \
--version $(git describe | sed s/^v//) --deb-compression gz \
diff --git a/scripts/package-build/linux-kernel/build.py b/scripts/package-build/linux-kernel/build.py
index 1bcab686..3aacc7f1 100755
--- a/scripts/package-build/linux-kernel/build.py
+++ b/scripts/package-build/linux-kernel/build.py
@@ -98,6 +98,8 @@ def build_package(package: dict, dependencies: list) -> None:
build_intel_ixgbe()
elif package['build_cmd'] == 'build_intel_ixgbevf':
build_intel_ixgbevf()
+ elif package['build_cmd'] == 'build_mellanox_ofed':
+ build_mellanox_ofed()
elif package['build_cmd'] == 'build_jool':
build_jool()
elif package['build_cmd'] == 'build_openvpn_dco':
@@ -183,6 +185,11 @@ def build_intel_ixgbevf():
run(['./build-intel-ixgbevf.sh'], check=True)
+def build_mellanox_ofed():
+ """Build Mellanox OFED"""
+ run(['sudo ./build-mellanox-ofed.sh'], check=True)
+
+
def build_jool():
"""Build Jool"""
run(['echo y | ./build-jool.py'], check=True, shell=True)
diff --git a/scripts/package-build/linux-kernel/package.toml b/scripts/package-build/linux-kernel/package.toml
index 8b030da0..44102298 100644
--- a/scripts/package-build/linux-kernel/package.toml
+++ b/scripts/package-build/linux-kernel/package.toml
@@ -60,3 +60,8 @@ commit_id = ""
scm_url = ""
build_cmd = "build_jool"
+[[packages]]
+name = "mlnx"
+commit_id = ""
+scm_url = ""
+build_cmd = "build_mellanox_ofed"
diff --git a/scripts/package-build/linux-kernel/sign-modules.sh b/scripts/package-build/linux-kernel/sign-modules.sh
new file mode 100755
index 00000000..cfb368eb
--- /dev/null
+++ b/scripts/package-build/linux-kernel/sign-modules.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+BASE_DIR=$(dirname $0)
+MODULE_DIR=$1
+. ${BASE_DIR}/kernel-vars
+
+SIGN_FILE="${KERNEL_DIR}/scripts/sign-file"
+
+if [ -f ${EPHEMERAL_KEY} ] && [ -f ${EPHEMERAL_CERT} ]; then
+ find ${MODULE_DIR} -type f -name \*.ko | while read MODULE; do
+ echo "I: Signing ${MODULE} ..."
+ ${SIGN_FILE} sha512 ${EPHEMERAL_KEY} ${EPHEMERAL_CERT} ${MODULE}
+ done
+fi
+