Merge remote-tracking branch 'origin/current' into equuleus

fixed merge errors in:
    * scripts/live-build-config
      - kernel parameters

    * scripts/build-packages
      - fixed list of packages to compile

7 files changed, 49 insertions, 64 deletions

diff --git a/scripts/build-config b/scripts/build-config
index a12ecdc4..dd74a50e 100755
--- a/scripts/build-config
+++ b/scripts/build-config
@@ -67,7 +67,7 @@ options = {
    'architecture': ('Image target architecture (amd64 or i386 or armhf)', lambda: build_defaults['architecture'], lambda x: x in ['amd64', 'i386', 'armhf']),
    'build-by': ('Builder identifier (e.g. jrandomhacker@example.net)', get_default_build_by, None),
    'debian-mirror': ('Debian repository mirror for ISO build', lambda: build_defaults['debian_mirror'], None),
-   'debian-security-mirror': ('Debian security updated mirror', lambda: build_defaults['debian_security_mirror'], None),
+   'debian-security-mirror': ('Debian security updates mirror', lambda: build_defaults['debian_security_mirror'], None),
    'pbuilder-debian-mirror': ('Debian repository mirror for pbuilder env bootstrap', lambda: build_defaults['debian_mirror'], None),
    'vyos-mirror': ('VyOS package mirror', lambda: build_defaults["vyos_mirror"], None),
    'build-type': ('Build type, release or development', lambda: 'development', lambda x: x in ['release', 'development']),
diff --git a/scripts/build-kernel b/scripts/build-kernel
deleted file mode 100755
index e58d2d5a..00000000
--- a/scripts/build-kernel
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-if [ ! -f "Makefile" ]; then
-  echo "Makefile missing, This script needs to be executed inside the vyos-kernel package directory"
-  exit 1
-fi
-
-if grep -Fxq "KBUILD_OUTPUT" Makefile; then
-  echo "Wrong Makefile?, This script needs to be executed inside the vyos-kernel package directory"
-  exit 1
-fi
-
-
-# newer Kernel versions come with a "# SPDX-License-Identifier: GPL-2.0" identifier
-# as the first line in the file - use compatible approach with grep rather then sed
-VERSION=$(grep "^VERSION" Makefile | grep -Eo '[0-9]{1,4}')
-PATCHLEVEL=$(grep "^PATCHLEVEL" Makefile | grep -Eo '[0-9]{1,4}')
-SUBLEVEL=$(grep "^SUBLEVEL" Makefile | grep -Eo '[0-9]{1,4}')
-ARCH=$(dpkg --print-architecture)
-
-case "$ARCH" in
-    amd64)
-        make x86_64_vyos_defconfig
-        TARGETS="kernel_headers kernel_image"
-        # the following targets are not supported for Linux Kernels > 4.14 as
-        # they have been removed from the Makefile (commits 18afab8c1d3c2 &
-        # 22cba31bae9dc).
-        if [ ${PATCHLEVEL} -lt 14 ]; then
-            TARGETS+=" kernel_manual kernel_doc"
-        fi
-        CONCURRENCY_LEVEL=4 LOCALVERSION="" make-kpkg --rootcmd fakeroot --initrd --append_to_version -amd64-vyos --revision=$VERSION.$PATCHLEVEL.$SUBLEVEL-1+vyos1+current1 -j$(cat /proc/cpuinfo | grep processor | wc -l) ${TARGETS}
-    ;;
-
-    armhf)
-        make armhf_vyos_defconfig
-    ;;
-esac
diff --git a/scripts/build-oracle-image b/scripts/build-oracle-image
index 61f8ddee..e2066fc7 100755
--- a/scripts/build-oracle-image
+++ b/scripts/build-oracle-image
@@ -57,7 +57,7 @@ dateymd=$(date +%Y%m%d)
 PARTED=/sbin/parted
 OUTPUT=disk.raw
 OUTPUTQCOW2=VyOS-"$dateymd".qcow2
-IMAGE_SIZE=2
+IMAGE_SIZE=10
 qemu-img create -f raw ${OUTPUT} ${IMAGE_SIZE}G
 
 ${PARTED} -s ${OUTPUT} mktable msdos
diff --git a/scripts/build-packages b/scripts/build-packages
index c57dd822..5abd1ea6 100755
--- a/scripts/build-packages
+++ b/scripts/build-packages
@@ -98,7 +98,10 @@ def clone_package(pkg, log):
     First cleanup any possible leftovers from previous builds
     """
 
-    if args.clean:
+    if args.keep:
+        log.debug("Keep possibly modified package '{}'".format(pkg['path']))
+        return False
+    elif args.clean:
         # delete repository from disk
         if os.path.isdir(pkg['path']):
             log.debug("Cleaning '{}'".format(pkg['path']))
@@ -186,7 +189,12 @@ vyos_packages = ['vyatta-bash',
                  'vyatta-conntrack-sync',
                  'vyos-xe-guest-utilities',
                  'vyos-netplug',
-                 'eventwatchd']
+                 'pmacct',
+                 'ddclient',
+                 'igmpproxy',
+                 'eventwatchd',
+                 'live-boot',
+                 'conntrack-tools']
 
 # Special packages mean packages which are located no in the VyOS namespace
 # or require fancy build instructions
@@ -239,7 +247,6 @@ pkg_special.append( add_package('net-snmp', custombuild_cmd=net_snmp_build_cmd)
 # Linux (VyOS) Kernel
 #
 kernel_build_cmd = "make x86_64_vyos_defconfig && " \
-                   "echo $(make kernelversion)-amd64-vyos > " + repo_root + "/data/kernel_version && " \
                    "sed -i 's/\"kernel_version\": \"[0-9].[0-9][0-9].[0-9]*\"/\"kernel_version\": \"'$(make kernelversion)'\"/' " + repo_root + "/data/defaults.json && " \
                    "make bindeb-pkg LOCALVERSION='-amd64-vyos' KDEB_PKGVERSION=$(make kernelversion)-1 -j $(getconf _NPROCESSORS_ONLN)"
 pkg_special.append( add_package('vyos-kernel', branch='linux-vyos-4.19.y', custombuild_cmd=kernel_build_cmd) )
@@ -249,7 +256,7 @@ pkg_special.append( add_package('vyos-kernel', branch='linux-vyos-4.19.y', custo
 #
 # WireGuard Kernel Module
 #
-wireguard_build_cmd = "echo 'src/wireguard.ko /lib/modules/'$(cat " + repo_root + "/data/defaults.json | jq '.kernel_version' | tr -d \\\") > debian/wireguard-modules.install && " \
+wireguard_build_cmd = "echo 'src/wireguard.ko /lib/modules/'$(cat " + repo_root + "/data/defaults.json | jq '.kernel_version' | tr -d \\\")-amd64-vyos/extra > debian/wireguard-modules.install && " \
                       "KERNELDIR=" + repo_root + "/packages/vyos-kernel dpkg-buildpackage -b -us -uc -tc -j" + str(os.cpu_count())
 pkg_special.append( add_package('vyos-wireguard', custombuild_cmd=wireguard_build_cmd) )
 
@@ -258,8 +265,8 @@ pkg_special.append( add_package('vyos-wireguard', custombuild_cmd=wireguard_buil
 #
 # Accell-PPP Package and Kernel Module
 #
-accel_ppp_build_cmd = "echo 'lib/modules/'$(cat " + repo_root + "/data/defaults.json | jq '.kernel_version' | tr -d \\\")'/extra/*.ko' > debian/vyos-accel-ppp-ipoe-kmod.install && " \
-                      "sed -i 's#[0-9].[0-9][0-9].[0-9]*-amd64-vyos#'$(cat " + repo_root + "/data/defaults.json | jq '.kernel_version' | tr -d \\\")'#g' debian/rules && " \
+accel_ppp_build_cmd = "echo 'lib/modules/'$(cat " + repo_root + "/data/defaults.json | jq '.kernel_version' | tr -d \\\")-amd64-vyos/extra/*.ko > debian/vyos-accel-ppp-ipoe-kmod.install && " \
+                      "sed -i 's#[0-9].[0-9][0-9].[0-9]*-amd64-vyos#'$(cat " + repo_root + "/data/defaults.json | jq '.kernel_version' | tr -d \\\")'-amd64-vyos#g' debian/rules && " \
                       "KERNELDIR=" + repo_root + "/packages/vyos-kernel dpkg-buildpackage -b -us -uc -tc -j" + str(os.cpu_count())
 pkg_special.append( add_package('vyos-accel-ppp', custombuild_cmd=accel_ppp_build_cmd) )
 
@@ -272,11 +279,14 @@ pkg_build = []
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
+    exclusive = parser.add_mutually_exclusive_group(required=False)
+    exclusive.add_argument('-c', '--clean', action='store_true', help='Re-clone required Git repositories')
+    exclusive.add_argument('-k', '--keep', action='store_true', help='Keep modified Git repositories')
+    exclusive.add_argument('-f', '--fetch', action='store_true', help='Fetch sources only, no build')
+
     parser.add_argument('-v', '--verbose', action='count', default=0, help='Increase logging verbosity for each occurance')
-    parser.add_argument('-c', '--clean', action='store_true', help='Re-clone required Git repositories')
     parser.add_argument('-l', '--list-packages', action='store_true', help='List all packages to build')
     parser.add_argument('-b', '--build', nargs='+', help='Whitespace separated list of packages to build')
-    parser.add_argument('-f', '--fetch', action='store_true', help='Fetch sources only, no build')
     parser.add_argument('-p', '--parallel', action='store_true', help='Build on all CPUs')
     parser.add_argument('--blacklist', nargs='+', help='Do not build/report packages when calling --list')
 
diff --git a/scripts/live-build-config b/scripts/live-build-config
index 7141df00..9855467d 100755
--- a/scripts/live-build-config
+++ b/scripts/live-build-config
@@ -35,7 +35,8 @@ util.check_build_config()
 lb_config_tmpl = """
 lb config noauto \
         --architectures {{architecture}} \
-        --bootappend-live "boot=live components hostname=vyos username=live nopersistence noautologin nonetworking union=overlay console=tty0 console=ttyS0,115200n8 net.ifnames=0 biosdevname=0" \
+        --bootappend-live "boot=live components hostname=vyos username=live nopersistence noautologin nonetworking union=overlay console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \
+        --bootappend-live-failsafe "live components memtest noapic noapm nodma nomce nolapic nomodeset nosmp nosplash vga=normal console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \
         --linux-flavours {{kernel_flavor}} \
         --linux-packages linux-image-{{kernel_version}} \
         --bootloader syslinux,grub-efi \
diff --git a/scripts/make-version-file b/scripts/make-version-file
index 95574722..4df2cb6c 100755
--- a/scripts/make-version-file
+++ b/scripts/make-version-file
@@ -43,7 +43,16 @@ build_timestamp = now.strftime("%Y%m%d%H%M")
 build_date = now.strftime("%a %d %b %Y %H:%M UTC")
 
 # Assign a (hopefully) unique identifier to the build (UUID)
-build_id = str(uuid.uuid4())
+build_uuid = str(uuid.uuid4())
+
+# Initialize Git object from our repository
+repo = git.Repo('.')
+
+# Retrieve the Git commit ID of the repository, 14 charaters will be sufficient
+build_git = repo.head.object.hexsha[:14]
+# If somone played around with the source tree and the build is "dirty", mark it
+if repo.is_dirty():
+    build_git += "-dirty"
 
 # Create a build version
 if build_config['build_type'] == 'development':
@@ -52,7 +61,6 @@ if build_config['build_type'] == 'development':
         with open('data/versions') as f:
             version_mapping = json.load(f)
 
-        repo = git.Repo('.')
         git_branch = repo.active_branch.name
         branch_version = version_mapping[git_branch]
 
@@ -68,10 +76,10 @@ version_data = {
     'version': version,
     'built_by': build_config['build_by'],
     'built_on': build_date,
-    'build_id': build_id
+    'build_uuid': build_uuid,
+    'build_git': build_git
 }
 
-
 os.makedirs(os.path.join(defaults.CHROOT_INCLUDES_DIR, 'usr/share/vyos'), exist_ok=True)
 with open(os.path.join(defaults.CHROOT_INCLUDES_DIR, 'usr/share/vyos/version.json'), 'w') as f:
     json.dump(version_data, f)
diff --git a/scripts/template.ovf b/scripts/template.ovf
index 87237bf7..88ea5562 100644
--- a/scripts/template.ovf
+++ b/scripts/template.ovf
@@ -1,5 +1,5 @@
-<?xml version='1.0' encoding='UTF-8'?>
-<ovf:Envelope xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1" xmlns="http://schemas.dmtf.org/ovf/envelope/1" xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData" xmlns:vssd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:vmw="http://www.vmware.com/schema/ovf">
+<?xml version="1.0" encoding="UTF-8"?>
+<ovf:Envelope xmlns="http://schemas.dmtf.org/ovf/envelope/1" xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1" xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData" xmlns:vmw="http://www.vmware.com/schema/ovf" xmlns:vssd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
   <References>
     <File ovf:href="vyos_vmware_image.vmdk" ovf:id="file1" ovf:size="{{vmdk_file_size}}"/>
   </References>
@@ -39,22 +39,22 @@
     <ProductSection ovf:required="false">
       <Info>VyOS is a Linux-based network operating system that provides software-based network routing, firewall, and VPN functionality.</Info>
       <Product>VyOS</Product>
-      <Vendor>VyOS maintainers and contributors</Vendor>
+      <Vendor>Sentrium S.L.</Vendor>
       <Version>{{version}}</Version>
       <ProductUrl>https://www.vyos.io</ProductUrl>
       <VendorUrl>https://sentrium.io/</VendorUrl>
       <AppUrl/>
       <Category>Appliance user Settings</Category>
-      <Property ovf:key="password" ovf:type="string" ovf:userConfigurable="true" ovf:value="" ovf:qualifiers="MinLen(8)" ovf:password="true">
+      <Property ovf:key="password" ovf:password="true" ovf:qualifiers="MinLen(8)" ovf:type="string" ovf:userConfigurable="true" ovf:value="">
         <Label ovf:msgid="Password.label">Password</Label>
-        <Description ovf:msgid="Password.description">The password for the appliance 'vyos' account. Passwords must be at least 8 characters in length.</Description>
+        <Description ovf:msgid="Password.description">The password for the appliance &apos;vyos&apos; account. Passwords must be at least 8 characters in length.</Description>
       </Property>
-      <Property ovf:key="public-keys" ovf:type="string" ovf:value="" ovf:userConfigurable="true">
+      <Property ovf:key="public-keys" ovf:type="string" ovf:userConfigurable="true" ovf:value="">
         <Label>Public key</Label>
-        <Description>The public ssh key for the appliance 'vyos' account.</Description>
+        <Description>The public ssh key for the appliance &apos;vyos&apos; account.</Description>
       </Property>
       <Category>Appliance IPv4 Network Settings</Category>
-      <Property ovf:key="local-hostname" ovf:type="string" ovf:userConfigurable="true" ovf:value="" ovf:qualifiers="MinLen(0),MaxLen(65535)">
+      <Property ovf:key="local-hostname" ovf:qualifiers="MinLen(0),MaxLen(65535)" ovf:type="string" ovf:userConfigurable="true" ovf:value="">
         <Label>Hostname</Label>
         <Description>The host name for this virtual machine.</Description>
       </Property>
@@ -87,11 +87,11 @@
       <Info/>
       <Annotation>VyOS</Annotation>
     </AnnotationSection>
-    <OperatingSystemSection ovf:id="100" ovf:version="1" vmw:osType="debian8_64Guest">
-      <Info>Guest Operating System</Info>
-      <Description>VyOS</Description>
+    <OperatingSystemSection ovf:id="96" ovf:version="6" vmw:osType="debian8_64Guest">
+      <Info>The operating system installed</Info>
+      <Description>Debian GNU/Linux 8 (64-bit)</Description>
     </OperatingSystemSection>
-    <VirtualHardwareSection ovf:transport="com.vmware.guestInfo" ovf:required="false">
+    <VirtualHardwareSection ovf:required="false" ovf:transport="com.vmware.guestInfo">
       <Info>Virtual Hardware Requirements</Info>
       <System>
         <vssd:ElementName>Virtual Hardware Family</vssd:ElementName>
@@ -148,7 +148,7 @@
       </Item>
       <Item>
         <rasd:Address xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData">0</rasd:Address>
-        <rasd:ElementName xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData">SCSI Controller 0</rasd:ElementName>
+        <rasd:ElementName xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData">SCSI Controller 0 - VMware Paravirtual SCSI</rasd:ElementName>
         <rasd:InstanceID xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData">3</rasd:InstanceID>
         <rasd:ResourceSubType>VirtualSCSI</rasd:ResourceSubType>
         <rasd:ResourceType>6</rasd:ResourceType>
@@ -195,6 +195,8 @@
         <rasd:ResourceSubType>vmxnet3</rasd:ResourceSubType>
         <rasd:ResourceType>10</rasd:ResourceType>
       </ovf:Item>
+      <vmw:Config ovf:required="false" vmw:key="cpuHotAddEnabled" vmw:value="true"/>
+      <vmw:Config ovf:required="false" vmw:key="memoryHotAddEnabled" vmw:value="true"/>
     </VirtualHardwareSection>
   </VirtualSystem>
 </ovf:Envelope>