diff options
Diffstat (limited to 'scripts/package-build/linux-kernel')
| -rw-r--r-- | scripts/package-build/linux-kernel/.gitignore | 32 | ||||
| -rw-r--r-- | scripts/package-build/linux-kernel/README.md | 41 | ||||
| l--------- | scripts/package-build/linux-kernel/arch | 1 | ||||
| -rwxr-xr-x | scripts/package-build/linux-kernel/build-accel-ppp.sh | 42 | ||||
| -rwxr-xr-x | scripts/package-build/linux-kernel/build-intel-ixgbe.sh | 107 | ||||
| -rwxr-xr-x | scripts/package-build/linux-kernel/build-intel-ixgbevf.sh | 100 | ||||
| -rwxr-xr-x | scripts/package-build/linux-kernel/build-intel-qat.sh | 111 | ||||
| -rwxr-xr-x | scripts/package-build/linux-kernel/build-jool.py | 98 | ||||
| -rwxr-xr-x | scripts/package-build/linux-kernel/build-kernel.sh | 56 | ||||
| -rwxr-xr-x | scripts/package-build/linux-kernel/build-linux-firmware.sh | 98 | ||||
| -rwxr-xr-x | scripts/package-build/linux-kernel/build-nat-rtsp.sh | 38 | ||||
| -rwxr-xr-x | scripts/package-build/linux-kernel/build-openvpn-dco.sh | 33 | ||||
| -rwxr-xr-x | scripts/package-build/linux-kernel/build.py | 239 | ||||
| -rw-r--r-- | scripts/package-build/linux-kernel/package.toml | 62 | ||||
| l--------- | scripts/package-build/linux-kernel/patches | 1 |
15 files changed, 1059 insertions, 0 deletions
diff --git a/scripts/package-build/linux-kernel/.gitignore b/scripts/package-build/linux-kernel/.gitignore new file mode 100644 index 00000000..0a18ea8c --- /dev/null +++ b/scripts/package-build/linux-kernel/.gitignore @@ -0,0 +1,32 @@ +/linux* +/wireguard +/wireguard-linux-compat +/accel-ppp +/intel-qat +/linux-firmware +/vyos-drivers-intel* +/vyos-drivers-realtek* +/ovpn-dco +/nat-rtsp* +/jool* +/qat* +/QAT* +*.tar.gz +*.tar.xz +/*.postinst + +# Intel Driver source +i40e-*/ +igb-*/ +ixgbe-*/ +ixgbevf-*/ +vyos-intel-*/ +vyos-linux-firmware*/ +kernel-vars +r8152-*.tar.bz2 + +*.buildinfo +*.build +*.changes +*.deb +*.dsc diff --git a/scripts/package-build/linux-kernel/README.md b/scripts/package-build/linux-kernel/README.md new file mode 100644 index 00000000..56954e5a --- /dev/null +++ b/scripts/package-build/linux-kernel/README.md @@ -0,0 +1,41 @@ +# Build +``` +./build.py --config package.toml --packages linux-kernel accel-ppp xxx +``` + +# About + +VyOS runs on a custom Linux Kernel (which is 4.19) at the time of this writing. +This repository holds a Jenkins Pipeline which is used to build the Custom +Kernel (x86_64/amd64 at the moment) and all required out-of tree modules. + +VyOS does not utilize the build in Intel Kernel drivers for its NICs as those +Kernels sometimes lack features e.g. configurable receive-side-scaling queues. +On the other hand we ship additional not mainlined features as WireGuard VPN. + +## Kernel + +The Kernel is build from the vanilla repositories hosted at https://git.kernel.org. +VyOS requires two additional patches to work which are stored in the patches/kernel +folder. + +### Config + +The Kernel configuration used is [x86_64_vyos_defconfig](x86_64_vyos_defconfig) +which will be copied on demand during the Pipeline run into the `arch/x86/configs`i +direcotry of the Kernel source tree. + +Other configurations can be added in the future easily. + +### Modules + +VyOS utilizes several Out-of-Tree modules (e.g. WireGuard, Accel-PPP and Intel +network interface card drivers). Module source code is retrieved from the +upstream repository and - when needed - patched so it can be build using this +pipeline. + +In the past VyOS maintainers had a fork of the Linux Kernel, WireGuard and +Accel-PPP. This is fine but increases maintenance effort. By utilizing vanilla +repositories upgrading to new versions is very easy - only the branch/commit/tag +used when cloning the repository via [Jenkinsfile](Jenkinsfile) needs to be +adjusted. diff --git a/scripts/package-build/linux-kernel/arch b/scripts/package-build/linux-kernel/arch new file mode 120000 index 00000000..f5f81fdc --- /dev/null +++ b/scripts/package-build/linux-kernel/arch @@ -0,0 +1 @@ +../../../packages/linux-kernel/arch
\ No newline at end of file diff --git a/scripts/package-build/linux-kernel/build-accel-ppp.sh b/scripts/package-build/linux-kernel/build-accel-ppp.sh new file mode 100755 index 00000000..1685ff8d --- /dev/null +++ b/scripts/package-build/linux-kernel/build-accel-ppp.sh @@ -0,0 +1,42 @@ +#!/bin/sh +CWD=$(pwd) +KERNEL_VAR_FILE=${CWD}/kernel-vars + +ACCEL_SRC=${CWD}/accel-ppp +if [ ! -d ${ACCEL_SRC} ]; then + echo "Accel-PPP source not found" + exit 1 +fi + +if [ ! -f ${KERNEL_VAR_FILE} ]; then + echo "Kernel variable file '${KERNEL_VAR_FILE}' does not exist, run ./build_kernel.sh first" + exit 1 +fi + +PATCH_DIR=${CWD}/patches/accel-ppp +if [ -d $PATCH_DIR ]; then + cd ${ACCEL_SRC} + for patch in $(ls ${PATCH_DIR}) + do + echo "I: Apply patch: ${PATCH_DIR}/${patch}" + patch -p1 < ${PATCH_DIR}/${patch} + done +fi + +. ${KERNEL_VAR_FILE} +mkdir -p ${ACCEL_SRC}/build +cd ${ACCEL_SRC}/build + +echo "I: Build Accel-PPP Debian package" +cmake -DBUILD_IPOE_DRIVER=TRUE \ + -DBUILD_VLAN_MON_DRIVER=TRUE \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DKDIR=${KERNEL_DIR} \ + -DLUA=5.3 \ + -DMODULES_KDIR=${KERNEL_VERSION}${KERNEL_SUFFIX} \ + -DCPACK_TYPE=Debian12 .. +make +cpack -G DEB + +# rename resulting Debian package according git description +mv accel-ppp*.deb ${CWD}/accel-ppp_$(git describe --always --tags)_$(dpkg --print-architecture).deb diff --git a/scripts/package-build/linux-kernel/build-intel-ixgbe.sh b/scripts/package-build/linux-kernel/build-intel-ixgbe.sh new file mode 100755 index 00000000..5f45c62a --- /dev/null +++ b/scripts/package-build/linux-kernel/build-intel-ixgbe.sh @@ -0,0 +1,107 @@ +#!/bin/sh +CWD=$(pwd) +KERNEL_VAR_FILE=${CWD}/kernel-vars + +if ! dpkg-architecture -iamd64; then + echo "Intel ixgbe is only buildable on amd64 platforms" + exit 0 +fi + +if [ ! -f ${KERNEL_VAR_FILE} ]; then + echo "Kernel variable file '${KERNEL_VAR_FILE}' does not exist, run ./build_kernel.sh first" + exit 1 +fi + +. ${KERNEL_VAR_FILE} + +url="https://sourceforge.net/projects/e1000/files/ixgbe%20stable/5.20.3/ixgbe-5.20.3.tar.gz" + +cd ${CWD} + +DRIVER_FILE=$(basename ${url} | sed -e s/tar_0/tar/) +DRIVER_DIR="${DRIVER_FILE%.tar.gz}" +DRIVER_NAME="ixgbe" +DRIVER_VERSION=$(echo ${DRIVER_DIR} | awk -F${DRIVER_NAME} '{print $2}' | sed 's/^-//') +DRIVER_VERSION_EXTRA="" + +# Build up Debian related variables required for packaging +DEBIAN_ARCH=$(dpkg --print-architecture) +DEBIAN_DIR="${CWD}/vyos-intel-${DRIVER_NAME}_${DRIVER_VERSION}_${DEBIAN_ARCH}" +DEBIAN_CONTROL="${DEBIAN_DIR}/DEBIAN/control" +DEBIAN_POSTINST="${CWD}/vyos-intel-ixgbe.postinst" + +# Fetch Intel driver source from SourceForge +if [ -e ${DRIVER_FILE} ]; then + rm -f ${DRIVER_FILE} +fi +curl -L -o ${DRIVER_FILE} ${url} +if [ "$?" -ne "0" ]; then + exit 1 +fi + +# Unpack archive +if [ -d ${DRIVER_DIR} ]; then + rm -rf ${DRIVER_DIR} +fi +mkdir -p ${DRIVER_DIR} +tar -C ${DRIVER_DIR} --strip-components=1 -xf ${DRIVER_FILE} + +cd ${DRIVER_DIR}/src +if [ -z $KERNEL_DIR ]; then + echo "KERNEL_DIR not defined" + exit 1 +fi + +# See https://lore.kernel.org/lkml/f90837d0-810e-5772-7841-28d47c44d260@intel.com/ +echo "I: remove pci_enable_pcie_error_reporting() code no longer present in Kernel" +sed -i '/.*pci_disable_pcie_error_reporting(pdev);/d' ixgbe_main.c +sed -i '/.*pci_enable_pcie_error_reporting(pdev);/d' ixgbe_main.c + +# See https://vyos.dev/T6155 +echo "I: always enable allow_unsupported_sfp for all NICs by default" +patch -l -p1 < ../../patches/ixgbe/allow_unsupported_sfp.patch + +# See https://vyos.dev/T6162 +echo "I: add 1000BASE-BX support" +patch -l -p1 < ../../patches/ixgbe/add_1000base-bx_support.patch + +echo "I: Compile Kernel module for Intel ${DRIVER_NAME} driver" +make KSRC=${KERNEL_DIR} INSTALL_MOD_PATH=${DEBIAN_DIR} INSTALL_FW_PATH=${DEBIAN_DIR} -j $(getconf _NPROCESSORS_ONLN) install + +if [ "x$?" != "x0" ]; then + exit 1 +fi + +if [ -f ${DEBIAN_DIR}.deb ]; then + rm ${DEBIAN_DIR}.deb +fi + +# build Debian package +echo "I: Building Debian package vyos-intel-${DRIVER_NAME}" +cd ${CWD} + +# delete non required files which are also present in the kernel package +# und thus lead to duplicated files +find ${DEBIAN_DIR} -name "modules.*" | xargs rm -f + +echo "#!/bin/sh" > ${DEBIAN_POSTINST} +echo "/sbin/depmod -a ${KERNEL_VERSION}${KERNEL_SUFFIX}" >> ${DEBIAN_POSTINST} + +fpm --input-type dir --output-type deb --name vyos-intel-${DRIVER_NAME} \ + --version ${DRIVER_VERSION} --deb-compression gz \ + --maintainer "VyOS Package Maintainers <maintainers@vyos.net>" \ + --description "Vendor based driver for Intel ${DRIVER_NAME}" \ + --depends linux-image-${KERNEL_VERSION}${KERNEL_SUFFIX} \ + --license "GPL2" -C ${DEBIAN_DIR} --after-install ${DEBIAN_POSTINST} + +echo "I: Cleanup ${DRIVER_NAME} source" +cd ${CWD} +if [ -e ${DRIVER_FILE} ]; then + rm -f ${DRIVER_FILE} +fi +if [ -d ${DRIVER_DIR} ]; then + rm -rf ${DRIVER_DIR} +fi +if [ -d ${DEBIAN_DIR} ]; then + rm -rf ${DEBIAN_DIR} +fi diff --git a/scripts/package-build/linux-kernel/build-intel-ixgbevf.sh b/scripts/package-build/linux-kernel/build-intel-ixgbevf.sh new file mode 100755 index 00000000..a965e0de --- /dev/null +++ b/scripts/package-build/linux-kernel/build-intel-ixgbevf.sh @@ -0,0 +1,100 @@ +#!/bin/sh +CWD=$(pwd) +KERNEL_VAR_FILE=${CWD}/kernel-vars + +if ! dpkg-architecture -iamd64; then + echo "Intel ixgbevf is only buildable on amd64 platforms" + exit 0 +fi + +if [ ! -f ${KERNEL_VAR_FILE} ]; then + echo "Kernel variable file '${KERNEL_VAR_FILE}' does not exist, run ./build_kernel.sh first" + exit 1 +fi + +. ${KERNEL_VAR_FILE} + +url="https://sourceforge.net/projects/e1000/files/ixgbevf%20stable/4.18.9/ixgbevf-4.18.9.tar.gz" + +cd ${CWD} + +DRIVER_FILE=$(basename ${url} | sed -e s/tar_0/tar/) +DRIVER_DIR="${DRIVER_FILE%.tar.gz}" +DRIVER_NAME="ixgbevf" +DRIVER_VERSION=$(echo ${DRIVER_DIR} | awk -F${DRIVER_NAME} '{print $2}' | sed 's/^-//') +DRIVER_VERSION_EXTRA="" + +# Build up Debian related variables required for packaging +DEBIAN_ARCH=$(dpkg --print-architecture) +DEBIAN_DIR="${CWD}/vyos-intel-${DRIVER_NAME}_${DRIVER_VERSION}_${DEBIAN_ARCH}" +DEBIAN_CONTROL="${DEBIAN_DIR}/DEBIAN/control" +DEBIAN_POSTINST="${CWD}/vyos-intel-ixgbevf.postinst" + +# Fetch Intel driver source from SourceForge +if [ -e ${DRIVER_FILE} ]; then + rm -f ${DRIVER_FILE} +fi +curl -L -o ${DRIVER_FILE} ${url} +if [ "$?" -ne "0" ]; then + exit 1 +fi + +# Unpack archive +if [ -d ${DRIVER_DIR} ]; then + rm -rf ${DRIVER_DIR} +fi +mkdir -p ${DRIVER_DIR} +tar -C ${DRIVER_DIR} --strip-components=1 -xf ${DRIVER_FILE} + +cd ${DRIVER_DIR}/src +if [ -z $KERNEL_DIR ]; then + echo "KERNEL_DIR not defined" + exit 1 +fi + +# See https://lore.kernel.org/lkml/f90837d0-810e-5772-7841-28d47c44d260@intel.com/ +echo "I: remove pci_enable_pcie_error_reporting() code no longer present in Kernel" +sed -i '/.*pci_disable_pcie_error_reporting(pdev);/d' ixgbevf_main.c +sed -i '/.*pci_enable_pcie_error_reporting(pdev);/d' ixgbevf_main.c + +echo "I: Compile Kernel module for Intel ${DRIVER_NAME} driver" +make KSRC=${KERNEL_DIR} INSTALL_MOD_PATH=${DEBIAN_DIR} INSTALL_FW_PATH=${DEBIAN_DIR} -j $(getconf _NPROCESSORS_ONLN) install + +if [ "x$?" != "x0" ]; then + exit 1 +fi + +if [ -f ${DEBIAN_DIR}.deb ]; then + rm ${DEBIAN_DIR}.deb +fi + +# build Debian package +echo "I: Building Debian package vyos-intel-${DRIVER_NAME}" +cd ${CWD} + +# delete non required files which are also present in the kernel package +# und thus lead to duplicated files +find ${DEBIAN_DIR} -name "modules.*" | xargs rm -f + +echo "#!/bin/sh" > ${DEBIAN_POSTINST} +echo "/sbin/depmod -a ${KERNEL_VERSION}${KERNEL_SUFFIX}" >> ${DEBIAN_POSTINST} + +fpm --input-type dir --output-type deb --name vyos-intel-${DRIVER_NAME} \ + --version ${DRIVER_VERSION} --deb-compression gz \ + --maintainer "VyOS Package Maintainers <maintainers@vyos.net>" \ + --description "Vendor based driver for Intel ${DRIVER_NAME}" \ + --depends linux-image-${KERNEL_VERSION}${KERNEL_SUFFIX} \ + --license "GPL2" -C ${DEBIAN_DIR} --after-install ${DEBIAN_POSTINST} + +echo "I: Cleanup ${DRIVER_NAME} source" +cd ${CWD} +if [ -e ${DRIVER_FILE} ]; then + rm -f ${DRIVER_FILE} +fi +if [ -d ${DRIVER_DIR} ]; then + rm -rf ${DRIVER_DIR} +fi +if [ -d ${DEBIAN_DIR} ]; then + rm -rf ${DEBIAN_DIR} +fi + diff --git a/scripts/package-build/linux-kernel/build-intel-qat.sh b/scripts/package-build/linux-kernel/build-intel-qat.sh new file mode 100755 index 00000000..765cea3f --- /dev/null +++ b/scripts/package-build/linux-kernel/build-intel-qat.sh @@ -0,0 +1,111 @@ +#!/bin/sh +CWD=$(pwd) +KERNEL_VAR_FILE=${CWD}/kernel-vars + +if ! dpkg-architecture -iamd64; then + echo "Intel-QAT is only buildable on amd64 platforms" + exit 0 +fi + +if [ ! -f ${KERNEL_VAR_FILE} ]; then + echo "Kernel variable file '${KERNEL_VAR_FILE}' does not exist, run ./build_kernel.sh first" + exit 1 +fi + +. ${KERNEL_VAR_FILE} + +url="https://dev.packages.vyos.net/source-mirror/QAT.L.4.24.0-00005.tar.gz" + +cd ${CWD} + +DRIVER_FILE=$(basename ${url} | sed -e s/tar_0/tar/) +DRIVER_DIR="${DRIVER_FILE%.tar.gz}" +DRIVER_NAME="QAT" +DRIVER_NAME_EXTRA="L." +DRIVER_VERSION=$(echo ${DRIVER_DIR} | awk -F${DRIVER_NAME} '{print $2}' | awk -F${DRIVER_NAME_EXTRA} '{print $2}') +DRIVER_VERSION_EXTRA="-0" + +# Build up Debian related variables required for packaging +DEBIAN_ARCH=$(dpkg --print-architecture) +DEBIAN_DIR="${CWD}/vyos-intel-${DRIVER_NAME}_${DRIVER_VERSION}${DRIVER_VERSION_EXTRA}_${DEBIAN_ARCH}" +DEBIAN_CONTROL="${DEBIAN_DIR}/DEBIAN/control" +DEBIAN_POSTINST="${CWD}/vyos-intel-qat.postinst" + +# Fetch Intel driver source from SourceForge +if [ -e ${DRIVER_FILE} ]; then + rm -f ${DRIVER_FILE} +fi +curl -L -o ${DRIVER_FILE} ${url} +if [ "$?" -ne "0" ]; then + exit 1 +fi + +# Unpack archive +if [ -d ${DRIVER_DIR} ]; then + rm -rf ${DRIVER_DIR} +fi +mkdir -p ${DRIVER_DIR} +tar -C ${DRIVER_DIR} -xf ${DRIVER_FILE} + +cd ${DRIVER_DIR} +if [ -z $KERNEL_DIR ]; then + echo "KERNEL_DIR not defined" + exit 1 +fi + +echo "I: Compile Kernel module for Intel ${DRIVER_NAME} driver" +mkdir -p \ + ${DEBIAN_DIR}/lib/firmware \ + ${DEBIAN_DIR}/usr/sbin \ + ${DEBIAN_DIR}/usr/lib/x86_64-linux-gnu \ + ${DEBIAN_DIR}/etc/init.d +KERNEL_SOURCE_ROOT=${KERNEL_DIR} ./configure --enable-kapi --enable-qat-lkcf +make -j $(getconf _NPROCESSORS_ONLN) all +make INSTALL_MOD_PATH=${DEBIAN_DIR} INSTALL_FW_PATH=${DEBIAN_DIR} \ + qat-driver-install adf-ctl-all + +if [ "x$?" != "x0" ]; then + exit 1 +fi + +cp quickassist/qat/fw/*.bin ${DEBIAN_DIR}/lib/firmware +cp build/*.so ${DEBIAN_DIR}/usr/lib/x86_64-linux-gnu +cp build/adf_ctl ${DEBIAN_DIR}/usr/sbin +cp quickassist/build_system/build_files/qat_service ${DEBIAN_DIR}/etc/init.d +cp build/usdm_drv.ko ${DEBIAN_DIR}/lib/modules/${KERNEL_VERSION}${KERNEL_SUFFIX}/updates/drivers +chmod 644 ${DEBIAN_DIR}/lib/firmware/* +chmod 755 ${DEBIAN_DIR}/etc/init.d/* ${DEBIAN_DIR}/usr/local/bin/* + +if [ -f ${DEBIAN_DIR}.deb ]; then + rm ${DEBIAN_DIR}.deb +fi + +# build Debian package +echo "I: Building Debian package vyos-intel-${DRIVER_NAME}" +cd ${CWD} + +# delete non required files which are also present in the kernel package +# und thus lead to duplicated files +find ${DEBIAN_DIR} -name "modules.*" | xargs rm -f + +echo "#!/bin/sh" > ${DEBIAN_POSTINST} +echo "/sbin/depmod -a ${KERNEL_VERSION}${KERNEL_SUFFIX}" >> ${DEBIAN_POSTINST} + +fpm --input-type dir --output-type deb --name vyos-intel-${DRIVER_NAME} \ + --version ${DRIVER_VERSION}${DRIVER_VERSION_EXTRA} --deb-compression gz \ + --maintainer "VyOS Package Maintainers <maintainers@vyos.net>" \ + --description "Vendor based driver for Intel ${DRIVER_NAME}" \ + --depends linux-image-${KERNEL_VERSION}${KERNEL_SUFFIX} \ + --license "GPL2" -C ${DEBIAN_DIR} --after-install ${DEBIAN_POSTINST} + +echo "I: Cleanup ${DRIVER_NAME} source" +cd ${CWD} +if [ -e ${DRIVER_FILE} ]; then + rm -f ${DRIVER_FILE} +fi +if [ -d ${DRIVER_DIR} ]; then + rm -rf ${DRIVER_DIR} +fi +if [ -d ${DEBIAN_DIR} ]; then + rm -rf ${DEBIAN_DIR} +fi diff --git a/scripts/package-build/linux-kernel/build-jool.py b/scripts/package-build/linux-kernel/build-jool.py new file mode 100755 index 00000000..570293f5 --- /dev/null +++ b/scripts/package-build/linux-kernel/build-jool.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 + +from tomllib import loads as toml_loads +from requests import get +from pathlib import Path +from subprocess import run + +def find_arch() -> str: + tmp=run(['dpkg-architecture', '-q', 'DEB_HOST_ARCH'], capture_output=True) + return tmp.stdout.decode().strip() + +# dependency modifier +def add_depends(package_dir: str, package_name: str, + depends: list[str]) -> None: + """Add dependencies to a package + + Args: + package_dir (str): a directory where package sources are located + package_name (str): a name of package + depends (list[str]): a list of dependencies to add + """ + depends_list: str = ', '.join(depends) + depends_line: str = f'misc:Depends={depends_list}\n' + + substvars_file = Path(f'{package_dir}/debian/{package_name}.substvars') + substvars_file.write_text(depends_line) + + +# find kernel version and source path +arch: str = find_arch() +defaults_file: str = Path('../../../data/defaults.toml').read_text() +architecture_file: str = Path(f'../../../data/architectures/{arch}.toml').read_text() +KERNEL_VER: str = toml_loads(defaults_file).get('kernel_version') +KERNEL_FLAVOR: str = toml_loads(architecture_file).get('kernel_flavor') +KERNEL_SRC: str = Path.cwd().as_posix() + '/linux' + +# define variables +PACKAGE_NAME: str = 'jool' +PACKAGE_VERSION: str = '4.1.9+bf4c7e3669' +PACKAGE_DIR: str = f'{PACKAGE_NAME}-{PACKAGE_VERSION}' +SOURCES_ARCHIVE: str = 'jool-4.1.9+bf4c7e3669.tar.gz' +SOURCES_URL: str = f'https://github.com/NICMx/Jool/archive/7f08c42c615ed63cf0fdc1522d91aa0809f6d990.tar.gz' + +# download sources +sources_archive = Path(SOURCES_ARCHIVE) +sources_archive.write_bytes(get(SOURCES_URL).content) + +# prepare sources +debmake_cmd: list[str] = [ + 'debmake', '-e', 'support@vyos.io', '-f', 'VyOS Support', '-p', + PACKAGE_NAME, '-u', PACKAGE_VERSION, '-a', SOURCES_ARCHIVE +] +run(debmake_cmd) + +# add kernel to dependencies +add_depends(PACKAGE_DIR, PACKAGE_NAME, + [f'linux-image-{KERNEL_VER}-{KERNEL_FLAVOR}']) + +# configure build rules +build_rules_text: str = f'''#!/usr/bin/make -f +# config +export KERNEL_DIR := {KERNEL_SRC} +PACKAGE_BUILD_DIR := debian/{PACKAGE_NAME} +KVER := {KERNEL_VER}-{KERNEL_FLAVOR} +MODULES_DIR := extra + +# main packaging script based on dh7 syntax +%: + dh $@ + +override_dh_clean: + dh_clean --exclude=debian/{PACKAGE_NAME}.substvars + +override_dh_prep: + dh_prep --exclude=debian/{PACKAGE_NAME}.substvars + +# override_dh_auto_clean: +# make -C src/mod clean + +override_dh_auto_build: + dh_auto_build $@ + make -C ${{KERNEL_DIR}} M=$$PWD/src/mod/common modules + make -C ${{KERNEL_DIR}} M=$$PWD/src/mod/nat64 modules + make -C ${{KERNEL_DIR}} M=$$PWD/src/mod/siit modules + +override_dh_auto_install: + dh_auto_install $@ + install -D -m 644 src/mod/common/jool_common.ko ${{PACKAGE_BUILD_DIR}}/lib/modules/${{KVER}}/${{MODULES_DIR}}/jool_common.ko + install -D -m 644 src/mod/nat64/jool.ko ${{PACKAGE_BUILD_DIR}}/lib/modules/${{KVER}}/${{MODULES_DIR}}/jool.ko + install -D -m 644 src/mod/siit/jool_siit.ko ${{PACKAGE_BUILD_DIR}}/lib/modules/${{KVER}}/${{MODULES_DIR}}/jool_siit.ko + +''' +bild_rules = Path(f'{PACKAGE_DIR}/debian/rules') +bild_rules.write_text(build_rules_text) + +# build a package +debuild_cmd: list[str] = ['debuild'] +run(debuild_cmd, cwd=PACKAGE_DIR) diff --git a/scripts/package-build/linux-kernel/build-kernel.sh b/scripts/package-build/linux-kernel/build-kernel.sh new file mode 100755 index 00000000..2c02f5c3 --- /dev/null +++ b/scripts/package-build/linux-kernel/build-kernel.sh @@ -0,0 +1,56 @@ +#!/bin/bash +CWD=$(pwd) +KERNEL_SRC=linux + +set -e + +if [ ! -d ${KERNEL_SRC} ]; then + echo "Linux Kernel source directory does not exists, please 'git clone'" + exit 1 +fi + +echo "I: Copy Kernel config (x86_64_vyos_defconfig) to Kernel Source" +cp -rv arch/ ${KERNEL_SRC}/ + +cd ${KERNEL_SRC} + +echo "I: clean modified files" +git reset --hard HEAD + +KERNEL_VERSION=$(make kernelversion) +KERNEL_SUFFIX=-$(dpkg --print-architecture)-vyos + +# VyOS requires some small Kernel Patches - apply them here +# It's easier to habe them here and make use of the upstream +# repository instead of maintaining a full Kernel Fork. +# Saving time/resources is essential :-) +PATCH_DIR=${CWD}/patches/kernel +for patch in $(ls ${PATCH_DIR}) +do + echo "I: Apply Kernel patch: ${PATCH_DIR}/${patch}" + patch -p1 < ${PATCH_DIR}/${patch} +done + +echo "I: make vyos_defconfig" +# Select Kernel configuration - currently there is only one +make vyos_defconfig + +echo "I: Generate environment file containing Kernel variable" +cat << EOF >${CWD}/kernel-vars +#!/bin/sh +export KERNEL_VERSION=${KERNEL_VERSION} +export KERNEL_SUFFIX=${KERNEL_SUFFIX} +export KERNEL_DIR=${CWD}/${KERNEL_SRC} +EOF + +echo "I: Build Debian Kernel package" +touch .scmversion +make bindeb-pkg BUILD_TOOLS=1 LOCALVERSION=${KERNEL_SUFFIX} KDEB_PKGVERSION=${KERNEL_VERSION}-1 -j $(getconf _NPROCESSORS_ONLN) + +cd $CWD +if [[ $? == 0 ]]; then + for package in $(ls linux-*.deb) + do + ln -sf linux-kernel/$package .. + done +fi diff --git a/scripts/package-build/linux-kernel/build-linux-firmware.sh b/scripts/package-build/linux-kernel/build-linux-firmware.sh new file mode 100755 index 00000000..2b1fa7b7 --- /dev/null +++ b/scripts/package-build/linux-kernel/build-linux-firmware.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# All selected drivers are then precomfiled "make drivers/foo/bar.i" and we grep for +# the magic word "UNIQUE_ID_firmware" which identifies firmware files. + +CWD=$(pwd) +LINUX_SRC="linux" +LINUX_FIRMWARE="linux-firmware" +KERNEL_VAR_FILE=${CWD}/kernel-vars + +if [ ! -d ${LINUX_SRC} ]; then + echo "Kernel source missing" + exit 1 +fi + +if [ ! -d ${LINUX_FIRMWARE} ]; then + echo "Linux firmware repository missing" + exit 1 +fi + +. ${KERNEL_VAR_FILE} + +result=() +# Retrieve firmware blobs from source files +FW_FILES=$(find ${LINUX_SRC}/debian/linux-image/lib/modules/${KERNEL_VERSION}${KERNEL_SUFFIX}/kernel/drivers/net -name *.ko | xargs modinfo | grep "^firmware:" | awk '{print $2}') + +# Debian package will use the descriptive Git commit as version +GIT_COMMIT=$(cd ${CWD}/${LINUX_FIRMWARE}; git describe --always) +VYOS_FIRMWARE_NAME="vyos-linux-firmware" +VYOS_FIRMWARE_DIR="${VYOS_FIRMWARE_NAME}_${GIT_COMMIT}-0_all" +if [ -d ${VYOS_FIRMWARE_DIR} ]; then + # remove Debian package folder and deb file from previous runs + rm -rf ${VYOS_FIRMWARE_DIR}* +fi +mkdir -p ${VYOS_FIRMWARE_DIR} + +# Install firmware files to build directory +LINUX_FIRMWARE_BUILD_DIR="${LINUX_FIRMWARE}_${GIT_COMMIT}" + +if [ -d ${LINUX_FIRMWARE_BUILD_DIR} ]; then + rm -rf "${LINUX_FIRMWARE_BUILD_DIR}" +fi + +mkdir -p "${LINUX_FIRMWARE_BUILD_DIR}" + +( + cd ${LINUX_FIRMWARE} + ./copy-firmware.sh "${CWD}/${LINUX_FIRMWARE_BUILD_DIR}" +) + +# Copy firmware file from linux firmware build directory into +# assembly folder for the vyos-firmware package +SED_REPLACE="s@${CWD}/${LINUX_FIRMWARE}/@@" +for FILE_PATTERN in ${FW_FILES}; do + find "${LINUX_FIRMWARE_BUILD_DIR}" -path "*/${FILE_PATTERN}" -print0 | while IFS= read -r -d $'\0' FILE; do + TARGET="$(echo "${FILE}" | sed "s/${LINUX_FIRMWARE_BUILD_DIR}\///g")" + TARGET_DIR="${VYOS_FIRMWARE_DIR}/lib/firmware/$(dirname "${TARGET}")" + # If file is a symlink install the symlink target as well + if [ -h "${FILE}" ]; then + if [ ! -f "${TARGET_DIR}/$(basename "${TARGET}")" ]; then + if [ -f "${LINUX_FIRMWARE_BUILD_DIR}/${TARGET}" ]; then + mkdir -p "${TARGET_DIR}" + + echo "I: install firmware: ${TARGET}" + cp "${CWD}/${LINUX_FIRMWARE_BUILD_DIR}/${TARGET}" "${TARGET_DIR}" + # If file links to other folder which this script not cover. Create folder and copy together. + if [ -L "${LINUX_FIRMWARE_BUILD_DIR}/${TARGET}" ]; then + REALPATH_TARGET=$(realpath --relative-to="${CWD}/${LINUX_FIRMWARE_BUILD_DIR}" "${CWD}/${LINUX_FIRMWARE_BUILD_DIR}/${TARGET}") + REALPATH_TARGET_DIR="${VYOS_FIRMWARE_DIR}/lib/firmware/$(dirname "${REALPATH_TARGET}")" + mkdir -p "${REALPATH_TARGET_DIR}" + echo "I: install firmware: ${REALPATH_TARGET}" + cp "${CWD}/${LINUX_FIRMWARE_BUILD_DIR}/${REALPATH_TARGET}" "${REALPATH_TARGET_DIR}" + fi + else + echo "I: firmware file not found: ${TARGET}" + fi + fi + fi + + if [ -f "${FILE}" ]; then + mkdir -p "${TARGET_DIR}" + echo "I: install firmware: ${TARGET}" + cp -P "${CWD}/${LINUX_FIRMWARE_BUILD_DIR}/${TARGET}" "${TARGET_DIR}" + else + echo "I: firmware file not found: ${TARGET}" + fi + done +done + +echo "I: Create linux-firmware package" +rm -f ${VYOS_FIRMWARE_NAME}_*.deb +fpm --input-type dir --output-type deb --name ${VYOS_FIRMWARE_NAME} \ + --maintainer "VyOS Package Maintainers <maintainers@vyos.net>" \ + --description "Binary firmware for various drivers in the Linux kernel" \ + --architecture all --version ${GIT_COMMIT} --deb-compression gz -C ${VYOS_FIRMWARE_DIR} + +rm -rf "${LINUX_FIRMWARE_BUILD_DIR}" +rm -rf ${VYOS_FIRMWARE_DIR} diff --git a/scripts/package-build/linux-kernel/build-nat-rtsp.sh b/scripts/package-build/linux-kernel/build-nat-rtsp.sh new file mode 100755 index 00000000..ec7d19a6 --- /dev/null +++ b/scripts/package-build/linux-kernel/build-nat-rtsp.sh @@ -0,0 +1,38 @@ +#!/bin/sh +CWD=$(pwd) +KERNEL_VAR_FILE=${CWD}/kernel-vars + +SRC=${CWD}/nat-rtsp +if [ ! -d ${SRC} ]; then + echo "nat-rtsp source not found" + exit 1 +fi + +if [ ! -f ${KERNEL_VAR_FILE} ]; then + echo "Kernel variable file '${KERNEL_VAR_FILE}' does not exist, run ./build_kernel.sh first" + exit 1 +fi + +. ${KERNEL_VAR_FILE} + +cd ${SRC} && make KERNELDIR=$KERNEL_DIR + +# Copy binary to package directory +DEBIAN_DIR=tmp/lib/modules/${KERNEL_VERSION}${KERNEL_SUFFIX}/extra +mkdir -p ${DEBIAN_DIR} +cp nf_conntrack_rtsp.ko nf_nat_rtsp.ko ${DEBIAN_DIR} + +DEBIAN_POSTINST="${CWD}/vyos-nat-rtsp.postinst" +echo "#!/bin/sh" > ${DEBIAN_POSTINST} +echo "/sbin/depmod -a ${KERNEL_VERSION}${KERNEL_SUFFIX}" >> ${DEBIAN_POSTINST} + +# Build Debian Package +fpm --input-type dir --output-type deb --name nat-rtsp \ + --version $(git describe --tags --always) --deb-compression gz \ + --maintainer "VyOS Package Maintainers <maintainers@vyos.net>" \ + --description "Connection tracking and NAT support for RTSP" \ + --depends linux-image-${KERNEL_VERSION}${KERNEL_SUFFIX} \ + --after-install ${DEBIAN_POSTINST} \ + --license "GPL2" --chdir tmp + +mv *.deb .. diff --git a/scripts/package-build/linux-kernel/build-openvpn-dco.sh b/scripts/package-build/linux-kernel/build-openvpn-dco.sh new file mode 100755 index 00000000..fd427825 --- /dev/null +++ b/scripts/package-build/linux-kernel/build-openvpn-dco.sh @@ -0,0 +1,33 @@ +#!/bin/sh +CWD=$(pwd) +KERNEL_VAR_FILE=${CWD}/kernel-vars + +SRC=${CWD}/ovpn-dco +if [ ! -d ${SRC} ]; then + echo "OpenVPN DCO source not found" + exit 1 +fi + +if [ ! -f ${KERNEL_VAR_FILE} ]; then + echo "Kernel variable file '${KERNEL_VAR_FILE}' does not exist, run ./build_kernel.sh first" + exit 1 +fi + +. ${KERNEL_VAR_FILE} + +cd ${SRC} && make KERNEL_SRC=$KERNEL_DIR + +# Copy binary to package directory +DEBIAN_DIR=tmp/lib/modules/${KERNEL_VERSION}${KERNEL_SUFFIX}/extra +mkdir -p ${DEBIAN_DIR} +cp drivers/net/ovpn-dco/ovpn-dco-v2.ko ${DEBIAN_DIR} + +# Build Debian Package +fpm --input-type dir --output-type deb --name openvpn-dco \ + --version $(git describe | sed s/^v//) --deb-compression gz \ + --maintainer "VyOS Package Maintainers <maintainers@vyos.net>" \ + --description "OpenVPN Data Channel Offload" \ + --depends linux-image-${KERNEL_VERSION}${KERNEL_SUFFIX} \ + --license "GPL2" --chdir tmp + +mv *.deb .. diff --git a/scripts/package-build/linux-kernel/build.py b/scripts/package-build/linux-kernel/build.py new file mode 100755 index 00000000..1bcab686 --- /dev/null +++ b/scripts/package-build/linux-kernel/build.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +import datetime +import glob +import shutil +import toml +import os +import subprocess + +from argparse import ArgumentParser +from pathlib import Path +from subprocess import run, CalledProcessError + +# Relative path to defaults.toml +defaults_path = "../../../data/defaults.toml" + + +def ensure_dependencies(dependencies: list) -> None: + """Ensure Debian build dependencies are met""" + if not dependencies: + print("I: No additional dependencies to install") + return + + print("I: Ensure Debian build dependencies are met") + run(['sudo', 'apt-get', 'install', '-y'] + dependencies, check=True) + + +def prepare_package(repo_dir: Path, install_data: str) -> None: + """Prepare a package""" + if not install_data: + print("I: No install data provided, skipping package preparation") + return + + install_file = repo_dir / 'debian/install' + install_file.parent.mkdir(parents=True, exist_ok=True) + install_file.write_text(install_data) + print("I: Prepared package") + + +def clone_or_update_repo(repo_dir: Path, scm_url: str, commit_id: str) -> None: + """Clone the repository if it does not exist, otherwise update it""" + if repo_dir.exists(): + #run(['git', 'fetch'], cwd=repo_dir, check=True) + run(['git', 'checkout', commit_id], cwd=repo_dir, check=True) + #run(['git', 'pull'], cwd=repo_dir, check=True) + else: + run(['git', 'clone', scm_url, str(repo_dir)], check=True) + run(['git', 'checkout', commit_id], cwd=repo_dir, check=True) + + +def build_package(package: dict, dependencies: list) -> None: + """Build a package from the repository + + Args: + package (dict): Package information + dependencies (list): List of additional dependencies + """ + timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + repo_name = package['name'] + repo_dir = Path(repo_name) + + try: + # Clone or update the repository + #clone_or_update_repo(repo_dir, package['scm_url'], package['commit_id']) + + # Ensure dependencies + #ensure_dependencies(dependencies) + + # Prepare the package if required + #if package.get('prepare_package', False): + # prepare_package(repo_dir, package.get('install_data', '')) + + # Execute the build command + if package['build_cmd'] == 'build_kernel': + build_kernel(package['kernel_version']) + elif package['build_cmd'] == 'build_linux_firmware': + build_linux_firmware(package['commit_id'], package['scm_url']) + elif package['build_cmd'] == 'build_accel_ppp': + build_accel_ppp(package['commit_id'], package['scm_url']) + elif package['build_cmd'] == 'build_intel_qat': + build_intel_qat() + elif package['build_cmd'] == 'build_intel_ixgbe': + build_intel_ixgbe() + elif package['build_cmd'] == 'build_intel_ixgbevf': + build_intel_ixgbevf() + elif package['build_cmd'] == 'build_jool': + build_jool() + elif package['build_cmd'] == 'build_openvpn_dco': + build_openvpn_dco(package['commit_id'], package['scm_url']) + elif package['build_cmd'] == 'build_nat_rtsp': + build_nat_rtsp(package['commit_id'], package['scm_url']) + else: + run(package['build_cmd'], cwd=repo_dir, check=True, shell=True) + + except CalledProcessError as e: + print(f"Failed to build package {repo_name}: {e}") + finally: + # Clean up repository directory + # shutil.rmtree(repo_dir, ignore_errors=True) + pass + + +def cleanup_build_deps(repo_dir: Path) -> None: + """Clean up build dependency packages""" + try: + if repo_dir.exists(): + for file in glob.glob(str(repo_dir / '*build-deps*.deb')): + os.remove(file) + print("Cleaned up build dependency packages") + except Exception as e: + print(f"Error cleaning up build dependencies: {e}") + + +def copy_packages(repo_dir: Path) -> None: + """Copy generated .deb packages to the parent directory""" + try: + deb_files = glob.glob(str(repo_dir / '*.deb')) + for deb_file in deb_files: + shutil.copy(deb_file, repo_dir.parent) + print("Copied generated .deb packages") + except Exception as e: + print(f"Error copying packages: {e}") + + +def merge_dicts(defaults, package): + return {**defaults, **package} + + +def build_kernel(kernel_version): + """Build the Linux kernel""" + run(['gpg2', '--locate-keys', 'torvalds@kernel.org', 'gregkh@kernel.org'], check=True) + run(['curl', '-OL', f'https://www.kernel.org/pub/linux/kernel/v6.x/linux-{kernel_version}.tar.xz'], check=True) + run(['curl', '-OL', f'https://www.kernel.org/pub/linux/kernel/v6.x/linux-{kernel_version}.tar.sign'], check=True) + # Using pipes to handle decompression and verification + with subprocess.Popen(['xz', '-cd', f'linux-{kernel_version}.tar.xz'], stdout=subprocess.PIPE) as proc_xz: + run(['gpg2', '--verify', f'linux-{kernel_version}.tar.sign', '-'], stdin=proc_xz.stdout, check=True) + run(['tar', 'xf', f'linux-{kernel_version}.tar.xz'], check=True) + os.symlink(f'linux-{kernel_version}', 'linux') + run(['./build-kernel.sh'], check=True) + + +def build_linux_firmware(commit_id, scm_url): + """Build Linux firmware""" + repo_dir = Path('linux-firmware') + clone_or_update_repo(repo_dir, scm_url, commit_id) + run(['./build-linux-firmware.sh'], check=True) + + +def build_accel_ppp(commit_id, scm_url): + """Build accel-ppp""" + repo_dir = Path('accel-ppp') + clone_or_update_repo(repo_dir, scm_url, commit_id) + run(['./build-accel-ppp.sh'], check=True) + + +def build_intel_qat(): + """Build Intel QAT""" + run(['./build-intel-qat.sh'], check=True) + + +def build_intel_ixgbe(): + """Build Intel IXGBE""" + run(['./build-intel-ixgbe.sh'], check=True) + + +def build_intel_ixgbevf(): + """Build Intel IXGBEVF""" + run(['./build-intel-ixgbevf.sh'], check=True) + + +def build_jool(): + """Build Jool""" + run(['echo y | ./build-jool.py'], check=True, shell=True) + + +def build_openvpn_dco(commit_id, scm_url): + """Build OpenVPN DCO""" + repo_dir = Path('ovpn-dco') + clone_or_update_repo(repo_dir, scm_url, commit_id) + run(['./build-openvpn-dco.sh'], check=True) + + +def build_nat_rtsp(commit_id, scm_url): + """Build RTSP netfilter helper""" + repo_dir = Path('nat-rtsp') + clone_or_update_repo(repo_dir, scm_url, commit_id) + run(['./build-nat-rtsp.sh'], check=True) + + +if __name__ == '__main__': + # Prepare argument parser + arg_parser = ArgumentParser() + arg_parser.add_argument('--config', default='package.toml', help='Path to the package configuration file') + arg_parser.add_argument('--packages', nargs='+', help='Names of packages to build (default: all)', default=[]) + args = arg_parser.parse_args() + + # Load package configuration + with open(args.config, 'r') as file: + config = toml.load(file) + + # Extract defaults and packages + with open(defaults_path, 'r') as file: + defaults = toml.load(file) + + packages = config['packages'] + + # Filter packages if specific packages are specified in the arguments + if args.packages: + packages = [pkg for pkg in packages if pkg['name'] in args.packages] + + # Merge defaults into each package + packages = [merge_dicts(defaults, pkg) for pkg in packages] + + for package in packages: + dependencies = package.get('dependencies', {}).get('packages', []) + + # Build the package + build_package(package, dependencies) + + # Clean up build dependency packages after build + cleanup_build_deps(Path(package['name'])) + + # Copy generated .deb packages to parent directory + copy_packages(Path(package['name'])) diff --git a/scripts/package-build/linux-kernel/package.toml b/scripts/package-build/linux-kernel/package.toml new file mode 100644 index 00000000..8b030da0 --- /dev/null +++ b/scripts/package-build/linux-kernel/package.toml @@ -0,0 +1,62 @@ +# [defaults] +# We get the kernel_version from vyos-build/data/defaults.toml +# kernel_version = "6.6.47" +# kernel_flavor = "amd64-vyos" + + +[[packages]] +name = "linux-kernel" +commit_id = "" # Uses defaults.kernel_version +scm_url = "" +build_cmd = "build_kernel" + +[[packages]] +name = "linux-firmware" +commit_id = "20240610" +scm_url = "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git" +build_cmd = "build_linux_firmware" + +[[packages]] +name = "accel-ppp" +commit_id = "1.13.0" +scm_url = "https://github.com/accel-ppp/accel-ppp.git" +build_cmd = "build_accel_ppp" + + +[[packages]] +name = "ovpn-dco" +commit_id = "v0.2.20231117" +scm_url = "https://github.com/OpenVPN/ovpn-dco" +build_cmd = "build_openvpn_dco" + +[[packages]] +name = "nat-rtsp" +commit_id = "475af0a" +scm_url = "https://github.com/maru-sama/rtsp-linux.git" +build_cmd = "build_nat_rtsp" + + +[[packages]] +name = "qat" +commit_id = "" +scm_url = "" +build_cmd = "build_intel_qat" + +[[packages]] +name = "ixgbe" +commit_id = "" +scm_url = "" +build_cmd = "build_intel_ixgbe" + +[[packages]] +name = "ixgbevf" +commit_id = "" +scm_url = "" +build_cmd = "build_intel_ixgbevf" + +[[packages]] +name = "jool" +commit_id = "" +scm_url = "" +build_cmd = "build_jool" + diff --git a/scripts/package-build/linux-kernel/patches b/scripts/package-build/linux-kernel/patches new file mode 120000 index 00000000..fd016d35 --- /dev/null +++ b/scripts/package-build/linux-kernel/patches @@ -0,0 +1 @@ +../../../packages/linux-kernel/patches
\ No newline at end of file |