From a0b3251dba33a9a2e59aecd742746fc22f96fd8d Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sat, 30 Mar 2019 11:18:33 +0100 Subject: Remove outdated/unmaintained submodules All registered Git submodules were out of date. Even worse some of the modules are no longer in use and have been replaced by upstream packages. Keeping invalid information is even worse then no information. All required VyOS packages can be found in the future build-packages script. --- .gitignore | 2 +- .gitmodules | 252 -------------------------------------- packages/.gitignore | 5 - packages/conntrack-tools | 1 - packages/ddclient | 1 - packages/eventwatchd | 1 - packages/frr | 1 - packages/hvinfo | 1 - packages/igmpproxy | 1 - packages/ipaddrcheck | 1 - packages/libvyosconfig | 1 - packages/libyang | 1 - packages/lldpd | 1 - packages/mdns-repeater | 1 - packages/pdns-recursor | 1 - packages/pmacct | 1 - packages/rtrlib | 1 - packages/udp-broadcast-relay | 1 - packages/vyatta-bash | 1 - packages/vyatta-biosdevname | 1 - packages/vyatta-cfg | 1 - packages/vyatta-cfg-firewall | 1 - packages/vyatta-cfg-op-pppoe | 1 - packages/vyatta-cfg-qos | 1 - packages/vyatta-cfg-quagga | 1 - packages/vyatta-cfg-system | 1 - packages/vyatta-cfg-vpn | 1 - packages/vyatta-cluster | 1 - packages/vyatta-config-mgmt | 1 - packages/vyatta-config-migrate | 1 - packages/vyatta-conntrack | 1 - packages/vyatta-conntrack-sync | 1 - packages/vyatta-eventwatch | 1 - packages/vyatta-iproute | 1 - packages/vyatta-ipv6-rtradv | 1 - packages/vyatta-lldp | 1 - packages/vyatta-nat | 1 - packages/vyatta-netflow | 1 - packages/vyatta-op | 1 - packages/vyatta-op-dhcp-server | 1 - packages/vyatta-op-firewall | 1 - packages/vyatta-op-qos | 1 - packages/vyatta-op-quagga | 1 - packages/vyatta-op-vpn | 1 - packages/vyatta-openvpn | 1 - packages/vyatta-quagga | 1 - packages/vyatta-ravpn | 1 - packages/vyatta-util | 1 - packages/vyatta-wanloadbalance | 1 - packages/vyatta-webproxy | 1 - packages/vyatta-wireless | 1 - packages/vyatta-wirelessmodem | 1 - packages/vyatta-zone | 1 - packages/vyconf | 1 - packages/vyos-1x | 1 - packages/vyos-accel-ppp | 1 - packages/vyos-keepalived | 1 - packages/vyos-kernel | 1 - packages/vyos-netplug | 1 - packages/vyos-nhrp | 1 - packages/vyos-opennhrp | 1 - packages/vyos-salt-minion | 1 - packages/vyos-strongswan | 1 - packages/vyos-vmwaretools-scripts | 1 - packages/vyos-wireguard | 1 - packages/vyos-world | 1 - packages/vyos-xe-guest-utilities | 1 - 67 files changed, 1 insertion(+), 322 deletions(-) delete mode 100644 packages/.gitignore delete mode 160000 packages/conntrack-tools delete mode 160000 packages/ddclient delete mode 160000 packages/eventwatchd delete mode 160000 packages/frr delete mode 160000 packages/hvinfo delete mode 160000 packages/igmpproxy delete mode 160000 packages/ipaddrcheck delete mode 160000 packages/libvyosconfig delete mode 160000 packages/libyang delete mode 160000 packages/lldpd delete mode 160000 packages/mdns-repeater delete mode 160000 packages/pdns-recursor delete mode 160000 packages/pmacct delete mode 160000 packages/rtrlib delete mode 160000 packages/udp-broadcast-relay delete mode 160000 packages/vyatta-bash delete mode 160000 packages/vyatta-biosdevname delete mode 160000 packages/vyatta-cfg delete mode 160000 packages/vyatta-cfg-firewall delete mode 160000 packages/vyatta-cfg-op-pppoe delete mode 160000 packages/vyatta-cfg-qos delete mode 160000 packages/vyatta-cfg-quagga delete mode 160000 packages/vyatta-cfg-system delete mode 160000 packages/vyatta-cfg-vpn delete mode 160000 packages/vyatta-cluster delete mode 160000 packages/vyatta-config-mgmt delete mode 160000 packages/vyatta-config-migrate delete mode 160000 packages/vyatta-conntrack delete mode 160000 packages/vyatta-conntrack-sync delete mode 160000 packages/vyatta-eventwatch delete mode 160000 packages/vyatta-iproute delete mode 160000 packages/vyatta-ipv6-rtradv delete mode 160000 packages/vyatta-lldp delete mode 160000 packages/vyatta-nat delete mode 160000 packages/vyatta-netflow delete mode 160000 packages/vyatta-op delete mode 160000 packages/vyatta-op-dhcp-server delete mode 160000 packages/vyatta-op-firewall delete mode 160000 packages/vyatta-op-qos delete mode 160000 packages/vyatta-op-quagga delete mode 160000 packages/vyatta-op-vpn delete mode 160000 packages/vyatta-openvpn delete mode 160000 packages/vyatta-quagga delete mode 160000 packages/vyatta-ravpn delete mode 160000 packages/vyatta-util delete mode 160000 packages/vyatta-wanloadbalance delete mode 160000 packages/vyatta-webproxy delete mode 160000 packages/vyatta-wireless delete mode 160000 packages/vyatta-wirelessmodem delete mode 160000 packages/vyatta-zone delete mode 160000 packages/vyconf delete mode 160000 packages/vyos-1x delete mode 160000 packages/vyos-accel-ppp delete mode 160000 packages/vyos-keepalived delete mode 160000 packages/vyos-kernel delete mode 160000 packages/vyos-netplug delete mode 160000 packages/vyos-nhrp delete mode 160000 packages/vyos-opennhrp delete mode 160000 packages/vyos-salt-minion delete mode 160000 packages/vyos-strongswan delete mode 160000 packages/vyos-vmwaretools-scripts delete mode 160000 packages/vyos-wireguard delete mode 160000 packages/vyos-world delete mode 160000 packages/vyos-xe-guest-utilities diff --git a/.gitignore b/.gitignore index fee2d5ac..820df937 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ build/* packer_build/* packer_cache/* key/* -packages/*.buildlog +packages/* diff --git a/.gitmodules b/.gitmodules index c470fb3b..e69de29b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,252 +0,0 @@ -[submodule "packages/vyatta-bash"] - path = packages/vyatta-bash - url = https://github.com/vyos/vyatta-bash.git - branch = current -[submodule "packages/vyatta-cfg"] - path = packages/vyatta-cfg - url = https://github.com/vyos/vyatta-cfg.git - branch = current -[submodule "packages/vyatta-op"] - path = packages/vyatta-op - url = https://github.com/vyos/vyatta-op.git - branch = current -[submodule "packages/vyatta-cfg-system"] - path = packages/vyatta-cfg-system - url = https://github.com/vyos/vyatta-cfg-system.git - branch = current -[submodule "packages/vyatta-cfg-firewall"] - path = packages/vyatta-cfg-firewall - url = https://github.com/vyos/vyatta-cfg-firewall.git - branch = current -[submodule "packages/vyatta-op-firewall"] - path = packages/vyatta-op-firewall - url = https://github.com/vyos/vyatta-op-firewall.git - branch = current -[submodule "packages/vyatta-cfg-vpn"] - path = packages/vyatta-cfg-vpn - url = https://github.com/vyos/vyatta-cfg-vpn.git - branch = current -[submodule "packages/vyatta-op-vpn"] - path = packages/vyatta-op-vpn - url = https://github.com/vyos/vyatta-op-vpn.git - branch = current -[submodule "packages/vyatta-cfg-qos"] - path = packages/vyatta-cfg-qos - url = https://github.com/vyos/vyatta-cfg-qos.git - branch = current -[submodule "packages/vyatta-op-qos"] - path = packages/vyatta-op-qos - url = https://github.com/vyos/vyatta-op-qos.git - branch = current -[submodule "packages/vyatta-cfg-op-pppoe"] - path = packages/vyatta-cfg-op-pppoe - url = https://github.com/vyos/vyatta-cfg-op-pppoe.git - branch = current -[submodule "packages/vyatta-openvpn"] - path = packages/vyatta-openvpn - url = https://github.com/vyos/vyatta-openvpn.git - branch = current -[submodule "packages/vyatta-conntrack"] - path = packages/vyatta-conntrack - url = https://github.com/vyos/vyatta-conntrack.git - branch = current -[submodule "packages/vyatta-conntrack-sync"] - path = packages/vyatta-conntrack-sync - url = https://github.com/vyos/vyatta-conntrack-sync.git - branch = current -[submodule "packages/vyatta-nat"] - path = packages/vyatta-nat - url = https://github.com/vyos/vyatta-nat.git - branch = current -[submodule "packages/vyatta-config-mgmt"] - path = packages/vyatta-config-mgmt - url = https://github.com/vyos/vyatta-config-mgmt.git - branch = current -[submodule "packages/vyatta-config-migrate"] - path = packages/vyatta-config-migrate - url = https://github.com/vyos/vyatta-config-migrate.git - branch = current -[submodule "packages/vyatta-zone"] - path = packages/vyatta-zone - url = https://github.com/vyos/vyatta-zone.git - branch = current -[submodule "packages/vyatta-cluster"] - path = packages/vyatta-cluster - url = https://github.com/vyos/vyatta-cluster.git - branch = current -[submodule "packages/vyatta-eventwatch"] - path = packages/vyatta-eventwatch - url = https://github.com/vyos/vyatta-eventwatch.git - branch = current -[submodule "packages/vyatta-webproxy"] - path = packages/vyatta-webproxy - url = https://github.com/vyos/vyatta-webproxy.git - branch = current -[submodule "packages/vyatta-cfg-quagga"] - path = packages/vyatta-cfg-quagga - url = https://github.com/vyos/vyatta-cfg-quagga.git - branch = current -[submodule "packages/vyatta-op-quagga"] - path = packages/vyatta-op-quagga - url = https://github.com/vyos/vyatta-op-quagga.git - branch = current -[submodule "packages/vyatta-op-dhcp-server"] - path = packages/vyatta-op-dhcp-server - url = https://github.com/vyos/vyatta-op-dhcp-server.git - branch = current -[submodule "packages/vyatta-wireless"] - path = packages/vyatta-wireless - url = https://github.com/vyos/vyatta-wireless.git - branch = current -[submodule "packages/vyatta-wirelessmodem"] - path = packages/vyatta-wirelessmodem - url = https://github.com/vyos/vyatta-wirelessmodem.git - branch = current -[submodule "packages/vyatta-wanloadbalance"] - path = packages/vyatta-wanloadbalance - url = https://github.com/vyos/vyatta-wanloadbalance.git - branch = current -[submodule "packages/vyatta-netflow"] - path = packages/vyatta-netflow - url = https://github.com/vyos/vyatta-netflow.git - branch = current -[submodule "packages/vyatta-lldp"] - path = packages/vyatta-lldp - url = https://github.com/vyos/vyatta-lldp.git - branch = current -[submodule "packages/vyatta-ipv6-rtradv"] - path = packages/vyatta-ipv6-rtradv - url = https://github.com/vyos/vyatta-ipv6-rtradv.git - branch = current -[submodule "packages/vyatta-ravpn"] - path = packages/vyatta-ravpn - url = https://github.com/vyos/vyatta-ravpn.git - branch = current -[submodule "packages/vyos-nhrp"] - path = packages/vyos-nhrp - url = https://github.com/vyos/vyos-nhrp.git - branch = current -[submodule "packages/vyatta-util"] - path = packages/vyatta-util - url = https://github.com/vyos/vyatta-util.git - branch = current -[submodule "packages/vyos-world"] - path = packages/vyos-world - url = https://github.com/vyos/vyos-world.git - branch = current -[submodule "packages/vyos-kernel"] - path = packages/vyos-kernel - url = https://github.com/vyos/vyos-kernel.git - branch = linux-vyos-4.19.y -[submodule "packages/vyos-1x"] - path = packages/vyos-1x - url = https://github.com/vyos/vyos-1x.git - branch = current -[submodule "packages/mdns-repeater"] - path = packages/mdns-repeater - url = https://github.com/vyos/mdns-repeater.git - branch = current -[submodule "packages/udp-broadcast-relay"] - path = packages/udp-broadcast-relay - url = https://github.com/vyos/udp-broadcast-relay.git - branch = current -[submodule "packages/vyos-keepalived"] - path = packages/vyos-keepalived - url = https://github.com/vyos/vyos-keepalived.git - branch = current -[submodule "packages/vyatta-iproute"] - path = packages/vyatta-iproute - url = https://github.com/vyos/vyatta-iproute.git - branch = iproute2-vyos-4.18.y -[submodule "packages/vyatta-quagga"] - path = packages/vyatta-quagga - url = https://github.com/vyos/vyatta-quagga.git - branch = current -[submodule "packages/vyos-wireguard"] - path = packages/vyos-wireguard - url = https://github.com/vyos/vyos-wireguard.git - branch = current -[submodule "packages/pmacct"] - path = packages/pmacct - url = https://github.com/vyos/pmacct.git - branch = current -[submodule "packages/vyos-strongswan"] - path = packages/vyos-strongswan - url = https://github.com/vyos/vyos-strongswan.git - branch = current -[submodule "packages/vyos-accel-ppp"] - path = packages/vyos-accel-ppp - url = https://github.com/vyos/vyos-accel-ppp.git - branch = current -[submodule "packages/frr"] - path = packages/frr - url = https://github.com/FRRouting/frr.git - branch = master -[submodule "packages/conntrack-tools"] - path = packages/conntrack-tools - url = https://github.com/vyos/conntrack-tools.git - branch = current -[submodule "packages/eventwatchd"] - path = packages/eventwatchd - url = https://github.com/vyos/eventwatchd.git - branch = current -[submodule "packages/ddclient"] - path = packages/ddclient - url = https://github.com/vyos/ddclient.git - branch = current -[submodule "packages/vyatta-biosdevname"] - path = packages/vyatta-biosdevname - url = https://github.com/vyos/vyatta-biosdevname.git - branch = current -[submodule "packages/vyos-opennhrp"] - path = packages/vyos-opennhrp - url = https://github.com/vyos/vyos-opennhrp.git - branch = current -[submodule "packages/vyos-salt-minion"] - path = packages/vyos-salt-minion - url = https://github.com/vyos/vyos-salt-minion.git - branch = current -[submodule "packages/rtrlib"] - path = packages/rtrlib - url = https://github.com/rtrlib/rtrlib.git -[submodule "packages/libvyosconfig"] - path = packages/libvyosconfig - url = https://github.com/vyos/libvyosconfig.git - branch = master -[submodule "packages/vyos-vmwaretools-scripts"] - path = packages/vyos-vmwaretools-scripts - url = https://github.com/vyos/vyos-vmwaretools-scripts.git - branch = current -[submodule "packages/vyconf"] - path = packages/vyconf - url = https://github.com/vyos/vyconf.git - branch = master -[submodule "packages/vyos-netplug"] - path = packages/vyos-netplug - url = https://github.com/vyos/vyos-netplug.git - branch = current -[submodule "packages/vyos-xe-guest-utilities"] - path = packages/vyos-xe-guest-utilities - url = http://github.com/vyos/vyos-xe-guest-utilities.git - branch = current -[submodule "packages/igmpproxy"] - path = packages/igmpproxy - url = https://github.com/vyos/igmpproxy.git - branch = current -[submodule "packages/hvinfo"] - path = packages/hvinfo - url = https://github.com/dmbaturin/hvinfo.git - branch = master -[submodule "packages/libyang"] - path = packages/libyang - url = https://github.com/opensourcerouting/libyang.git - branch = debian -[submodule "packages/ipaddrcheck"] - path = packages/ipaddrcheck - url = https://github.com/vyos/ipaddrcheck.git -[submodule "packages/lldpd"] - path = packages/lldpd - url = https://github.com/vincentbernat/lldpd.git -[submodule "packages/pdns-recursor"] - path = packages/pdns-recursor - url = https://salsa.debian.org/dns-team/pdns-recursor.git diff --git a/packages/.gitignore b/packages/.gitignore deleted file mode 100644 index ed11cabe..00000000 --- a/packages/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -*.dsc -*.tar.gz -*.deb -*.build -*.changes diff --git a/packages/conntrack-tools b/packages/conntrack-tools deleted file mode 160000 index 77cf2ae1..00000000 --- a/packages/conntrack-tools +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 77cf2ae1cfc281ab11d3fa5ddc73c11bcb60ccd9 diff --git a/packages/ddclient b/packages/ddclient deleted file mode 160000 index 869a613c..00000000 --- a/packages/ddclient +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 869a613c81245678f1261034ff941fc047c2c15d diff --git a/packages/eventwatchd b/packages/eventwatchd deleted file mode 160000 index c3d8ae73..00000000 --- a/packages/eventwatchd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c3d8ae7352f3e9c65b0f05dda245d144252574ca diff --git a/packages/frr b/packages/frr deleted file mode 160000 index c7262232..00000000 --- a/packages/frr +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c72622322df02e02cb9b6d01379052c1ddda2b53 diff --git a/packages/hvinfo b/packages/hvinfo deleted file mode 160000 index 6d15dff8..00000000 --- a/packages/hvinfo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6d15dff8eb5418a23035d18078f367bc799b702c diff --git a/packages/igmpproxy b/packages/igmpproxy deleted file mode 160000 index 1f87c675..00000000 --- a/packages/igmpproxy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1f87c675725bff8f380bf7364c3961ee5a5a9e4f diff --git a/packages/ipaddrcheck b/packages/ipaddrcheck deleted file mode 160000 index 8f0d655c..00000000 --- a/packages/ipaddrcheck +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8f0d655cc68995a0ef88dc905dc62db2af23ec90 diff --git a/packages/libvyosconfig b/packages/libvyosconfig deleted file mode 160000 index e75e4ae6..00000000 --- a/packages/libvyosconfig +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e75e4ae638c49e512a3013d1806e59a67978c5f9 diff --git a/packages/libyang b/packages/libyang deleted file mode 160000 index 179da47f..00000000 --- a/packages/libyang +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 179da47f2e8def1907372408a9df97a0dcbc1347 diff --git a/packages/lldpd b/packages/lldpd deleted file mode 160000 index adbb26bc..00000000 --- a/packages/lldpd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit adbb26bc0a49d083472d24762255ff48bd001e45 diff --git a/packages/mdns-repeater b/packages/mdns-repeater deleted file mode 160000 index 05197745..00000000 --- a/packages/mdns-repeater +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 051977458f4078a06008bb6c6fd338cd994cf56c diff --git a/packages/pdns-recursor b/packages/pdns-recursor deleted file mode 160000 index 28af51a3..00000000 --- a/packages/pdns-recursor +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 28af51a3942875f0896fab2ea5945ded4a191975 diff --git a/packages/pmacct b/packages/pmacct deleted file mode 160000 index 313a010b..00000000 --- a/packages/pmacct +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 313a010b72817a34348ad405ff911b10b9230035 diff --git a/packages/rtrlib b/packages/rtrlib deleted file mode 160000 index db2d254d..00000000 --- a/packages/rtrlib +++ /dev/null @@ -1 +0,0 @@ -Subproject commit db2d254d75b835abb169f2f9e4a8cfbe74a3c365 diff --git a/packages/udp-broadcast-relay b/packages/udp-broadcast-relay deleted file mode 160000 index 13127cbd..00000000 --- a/packages/udp-broadcast-relay +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 13127cbd62e69be2d186c9d76d112729ea95a1ef diff --git a/packages/vyatta-bash b/packages/vyatta-bash deleted file mode 160000 index 001f9b1a..00000000 --- a/packages/vyatta-bash +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 001f9b1a9bd2b307bca94510ce05bc067e34f84d diff --git a/packages/vyatta-biosdevname b/packages/vyatta-biosdevname deleted file mode 160000 index 28ccb143..00000000 --- a/packages/vyatta-biosdevname +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 28ccb143a09e8a1fb8363757c9a82c3e732a5525 diff --git a/packages/vyatta-cfg b/packages/vyatta-cfg deleted file mode 160000 index c56e20b9..00000000 --- a/packages/vyatta-cfg +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c56e20b98438e33822244315249e3edf0e356593 diff --git a/packages/vyatta-cfg-firewall b/packages/vyatta-cfg-firewall deleted file mode 160000 index 5248dbcc..00000000 --- a/packages/vyatta-cfg-firewall +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5248dbcc34eb6b041af928a0451ea866fc77488d diff --git a/packages/vyatta-cfg-op-pppoe b/packages/vyatta-cfg-op-pppoe deleted file mode 160000 index 2de6ecd6..00000000 --- a/packages/vyatta-cfg-op-pppoe +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2de6ecd6068e6a9debb032a074395398f2dc4df1 diff --git a/packages/vyatta-cfg-qos b/packages/vyatta-cfg-qos deleted file mode 160000 index 0a5049b3..00000000 --- a/packages/vyatta-cfg-qos +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0a5049b38b0830b289d0052a1fa2ce64785bc6ba diff --git a/packages/vyatta-cfg-quagga b/packages/vyatta-cfg-quagga deleted file mode 160000 index 82dd077c..00000000 --- a/packages/vyatta-cfg-quagga +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 82dd077c98131a8a4b3383139e782f534bc310a1 diff --git a/packages/vyatta-cfg-system b/packages/vyatta-cfg-system deleted file mode 160000 index 2f9b0eca..00000000 --- a/packages/vyatta-cfg-system +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2f9b0ecaf33c8598dc1a74ac0fb4f34abaad97cd diff --git a/packages/vyatta-cfg-vpn b/packages/vyatta-cfg-vpn deleted file mode 160000 index 5c167234..00000000 --- a/packages/vyatta-cfg-vpn +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5c1672341b33dc726da5d7845725bd74e3cc7cb6 diff --git a/packages/vyatta-cluster b/packages/vyatta-cluster deleted file mode 160000 index 1881e2ef..00000000 --- a/packages/vyatta-cluster +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1881e2ef2fb041e7836db90b12450d63c1628473 diff --git a/packages/vyatta-config-mgmt b/packages/vyatta-config-mgmt deleted file mode 160000 index a47cd02b..00000000 --- a/packages/vyatta-config-mgmt +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a47cd02bca9cb25125e14fe1b731029fb985d89c diff --git a/packages/vyatta-config-migrate b/packages/vyatta-config-migrate deleted file mode 160000 index 28f7ce91..00000000 --- a/packages/vyatta-config-migrate +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 28f7ce91358975ba9b36ac2874f6b47482815aab diff --git a/packages/vyatta-conntrack b/packages/vyatta-conntrack deleted file mode 160000 index 36213527..00000000 --- a/packages/vyatta-conntrack +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 362135271f8bd647d8fa772f775d26c1edcac7c6 diff --git a/packages/vyatta-conntrack-sync b/packages/vyatta-conntrack-sync deleted file mode 160000 index 4f6ae976..00000000 --- a/packages/vyatta-conntrack-sync +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4f6ae97660e512d90f1d4ac9822cdc1172f0fdf3 diff --git a/packages/vyatta-eventwatch b/packages/vyatta-eventwatch deleted file mode 160000 index ab07c835..00000000 --- a/packages/vyatta-eventwatch +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ab07c835d1b09ab5424190dd240dfa65e77b81f2 diff --git a/packages/vyatta-iproute b/packages/vyatta-iproute deleted file mode 160000 index 90dd151e..00000000 --- a/packages/vyatta-iproute +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 90dd151e2a23aa9ca3d6a5ab4c9c97349e65f4a0 diff --git a/packages/vyatta-ipv6-rtradv b/packages/vyatta-ipv6-rtradv deleted file mode 160000 index 809508f4..00000000 --- a/packages/vyatta-ipv6-rtradv +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 809508f4b3248e8dc4bfbcb527084d3bac4c78f8 diff --git a/packages/vyatta-lldp b/packages/vyatta-lldp deleted file mode 160000 index b7c70732..00000000 --- a/packages/vyatta-lldp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b7c70732793bfd88c23ffb424bf44fd65b0bf8bc diff --git a/packages/vyatta-nat b/packages/vyatta-nat deleted file mode 160000 index 52fb9ab3..00000000 --- a/packages/vyatta-nat +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 52fb9ab3f8a1d1aeda0149946b1e1b77617bc6cd diff --git a/packages/vyatta-netflow b/packages/vyatta-netflow deleted file mode 160000 index 22b60a0e..00000000 --- a/packages/vyatta-netflow +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 22b60a0e4dc9ff7d4001b55bf1f17e55e5267389 diff --git a/packages/vyatta-op b/packages/vyatta-op deleted file mode 160000 index 6152bc3a..00000000 --- a/packages/vyatta-op +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6152bc3abd0cb1dc6c7717a653558a1c3393fe31 diff --git a/packages/vyatta-op-dhcp-server b/packages/vyatta-op-dhcp-server deleted file mode 160000 index 2c8c4ae2..00000000 --- a/packages/vyatta-op-dhcp-server +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2c8c4ae26ea671cd48c80335b127bbac7ad0783a diff --git a/packages/vyatta-op-firewall b/packages/vyatta-op-firewall deleted file mode 160000 index 7ad3f22b..00000000 --- a/packages/vyatta-op-firewall +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7ad3f22bedd1ce169b076fd3fabb32761082b66e diff --git a/packages/vyatta-op-qos b/packages/vyatta-op-qos deleted file mode 160000 index ec1c87ec..00000000 --- a/packages/vyatta-op-qos +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ec1c87ec516d0814d1f4b6115dd93b4287f3055d diff --git a/packages/vyatta-op-quagga b/packages/vyatta-op-quagga deleted file mode 160000 index 24189c45..00000000 --- a/packages/vyatta-op-quagga +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 24189c4533bc9147ac3c780ce80bf49194beb67c diff --git a/packages/vyatta-op-vpn b/packages/vyatta-op-vpn deleted file mode 160000 index 0ec07e89..00000000 --- a/packages/vyatta-op-vpn +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0ec07e891b0191b24185ea09c6f33f6502597df6 diff --git a/packages/vyatta-openvpn b/packages/vyatta-openvpn deleted file mode 160000 index 5f754cd2..00000000 --- a/packages/vyatta-openvpn +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5f754cd2c2660f3fd58210c92d5847225fc2b6b5 diff --git a/packages/vyatta-quagga b/packages/vyatta-quagga deleted file mode 160000 index b721e84e..00000000 --- a/packages/vyatta-quagga +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b721e84e0915406cb5458fea7f444b45b5d1196a diff --git a/packages/vyatta-ravpn b/packages/vyatta-ravpn deleted file mode 160000 index 6fb662b7..00000000 --- a/packages/vyatta-ravpn +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6fb662b78e2b688d2a00d0221821904ef841b391 diff --git a/packages/vyatta-util b/packages/vyatta-util deleted file mode 160000 index 22ee60e3..00000000 --- a/packages/vyatta-util +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 22ee60e33a9e5fc4dc372ae8798b4f05d1b520c3 diff --git a/packages/vyatta-wanloadbalance b/packages/vyatta-wanloadbalance deleted file mode 160000 index c4ad578e..00000000 --- a/packages/vyatta-wanloadbalance +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c4ad578e0d82b688d5ed60b649604982eae8350a diff --git a/packages/vyatta-webproxy b/packages/vyatta-webproxy deleted file mode 160000 index 4a86d4a3..00000000 --- a/packages/vyatta-webproxy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4a86d4a3f886b8b42311dabd0ddceabdf8cdd519 diff --git a/packages/vyatta-wireless b/packages/vyatta-wireless deleted file mode 160000 index 5b92a6e5..00000000 --- a/packages/vyatta-wireless +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5b92a6e5a446b3b74ad78832ea82f977d6bba325 diff --git a/packages/vyatta-wirelessmodem b/packages/vyatta-wirelessmodem deleted file mode 160000 index 8022e5ca..00000000 --- a/packages/vyatta-wirelessmodem +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8022e5ca50591afa2db948fd9454ce4f4cfb406f diff --git a/packages/vyatta-zone b/packages/vyatta-zone deleted file mode 160000 index f8f40ba6..00000000 --- a/packages/vyatta-zone +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f8f40ba6bf5a95a5b147f9cd0e7e425cdb9c015a diff --git a/packages/vyconf b/packages/vyconf deleted file mode 160000 index 1ebea2e6..00000000 --- a/packages/vyconf +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1ebea2e6914e5d2c4f07654bdbd489aa2c1ce165 diff --git a/packages/vyos-1x b/packages/vyos-1x deleted file mode 160000 index 81e1cf8d..00000000 --- a/packages/vyos-1x +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 81e1cf8dbf02b95a89caa180f483a66655591d1a diff --git a/packages/vyos-accel-ppp b/packages/vyos-accel-ppp deleted file mode 160000 index df95d6b7..00000000 --- a/packages/vyos-accel-ppp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit df95d6b78be568cbce40fe7b2632dfb9697d2076 diff --git a/packages/vyos-keepalived b/packages/vyos-keepalived deleted file mode 160000 index dc8ade02..00000000 --- a/packages/vyos-keepalived +++ /dev/null @@ -1 +0,0 @@ -Subproject commit dc8ade02407f836873e78887f4b3c94b6540a578 diff --git a/packages/vyos-kernel b/packages/vyos-kernel deleted file mode 160000 index 615786a2..00000000 --- a/packages/vyos-kernel +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 615786a28e0ea6234ba8b02e457bcac48ab0718b diff --git a/packages/vyos-netplug b/packages/vyos-netplug deleted file mode 160000 index 2aa80a57..00000000 --- a/packages/vyos-netplug +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2aa80a57a1ee2dfa8955b520d3aaec7543158669 diff --git a/packages/vyos-nhrp b/packages/vyos-nhrp deleted file mode 160000 index aae14502..00000000 --- a/packages/vyos-nhrp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit aae14502e55b41e822d890ba08e3460609454cd0 diff --git a/packages/vyos-opennhrp b/packages/vyos-opennhrp deleted file mode 160000 index 41f0852d..00000000 --- a/packages/vyos-opennhrp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 41f0852dd0b8d4900a1a743334d88afe490ee587 diff --git a/packages/vyos-salt-minion b/packages/vyos-salt-minion deleted file mode 160000 index c64e526a..00000000 --- a/packages/vyos-salt-minion +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c64e526a98d4ea8ee3a300bf36cd4348cd46a56c diff --git a/packages/vyos-strongswan b/packages/vyos-strongswan deleted file mode 160000 index 0d4aee54..00000000 --- a/packages/vyos-strongswan +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0d4aee5440ebef1042bb8d9e65ee3dc127257df8 diff --git a/packages/vyos-vmwaretools-scripts b/packages/vyos-vmwaretools-scripts deleted file mode 160000 index b41f886f..00000000 --- a/packages/vyos-vmwaretools-scripts +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b41f886f59997f28a19e541e6a40b9d39ac43f60 diff --git a/packages/vyos-wireguard b/packages/vyos-wireguard deleted file mode 160000 index 9c6555a2..00000000 --- a/packages/vyos-wireguard +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9c6555a20b4b8ff5d92fae2f843439b136448c8e diff --git a/packages/vyos-world b/packages/vyos-world deleted file mode 160000 index 69570ca9..00000000 --- a/packages/vyos-world +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 69570ca927a1dc87f0f1eb778aaaa33df7541711 diff --git a/packages/vyos-xe-guest-utilities b/packages/vyos-xe-guest-utilities deleted file mode 160000 index a31c0926..00000000 --- a/packages/vyos-xe-guest-utilities +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a31c092661c65e177967535e8c53dc5b7a04b5cd -- cgit v1.2.3 From 63493afa4c9cb05cdf7318b841edf52ad0779014 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Tue, 9 Apr 2019 20:15:57 +0200 Subject: Replace build-submodules with Python based package builder Support building individual VyOS packages by this modules. Call scripts/build-packages -l to list all available packages which will be build when invoking scripts/build-packages. --- Jenkinsfile | 130 +++++++++++-- README.md | 102 +++------- scripts/build-intel-drivers | 79 ++++++++ scripts/build-packages | 332 ++++++++++++++++++++++++++++++++ scripts/build-submodules | 452 -------------------------------------------- 5 files changed, 554 insertions(+), 541 deletions(-) create mode 100755 scripts/build-intel-drivers create mode 100755 scripts/build-packages delete mode 100755 scripts/build-submodules diff --git a/Jenkinsfile b/Jenkinsfile index 4bc64b26..169a52f3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,22 +1,76 @@ #!/usr/bin/env groovy +// Copyright (C) 2018 VyOS maintainers and contributors +// +// This program is free software; you can redistribute it and/or modify +// in order to easy exprort images built to "external" world +// it under the terms of the GNU General Public License version 2 or later as +// published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + @NonCPS + +def getGitBranchName() { + return scm.branches[0].name +} + +def getGitRepoURL() { + return scm.userRemoteConfigs[0].url +} + +// Returns true if this is a custom build launched on any project fork, +// returns false if this is build from git@github.com:vyos/vyos-build.git +def isCustomBuild() { + def gitURI = "git@github.com:vyos/vyos-build.git" + def httpURI = "https://github.com/vyos/vyos-build.git" + + return ! ((getGitRepoURL() == gitURI) || (getGitRepoURL() == httpURI)) +} + def setDescription() { def item = Jenkins.instance.getItemByFullName(env.JOB_NAME) - item.setDescription("VyOS image build using a\nPipeline build inside Docker container.") + + // build up the main description text + def description = "" + description += "

Build VyOS ISO image

" + description += "All required Vyatta/VyOS packages are build from source prior to assembling the ISO." + + if (isCustomBuild()) { + description += "

" + description += "Build not started from official Git repository!
" + description += "
" + description += "Repository: " + getGitRepoURL() + "
" + description += "Branch: " + getGitBranchName() + "
" + description += "

" + } else { + description += "Sources taken from Git branch: " + getGitBranchName() + "
" + } + + item.setDescription(description) item.save() } -setDescription() - /* Only keep the 10 most recent builds. */ def projectProperties = [ - [$class: 'BuildDiscarderProperty',strategy: [$class: 'LogRotator', numToKeepStr: '5']], + [$class: 'BuildDiscarderProperty',strategy: [$class: 'LogRotator', numToKeepStr: '1']], ] properties(projectProperties) +setDescription() pipeline { + options { + disableConcurrentBuilds() + timeout(time: 4, unit: 'HOURS') + parallelsAlwaysFailFast() + } agent { dockerfile { filename 'Dockerfile' @@ -27,23 +81,71 @@ pipeline { } stages { - stage('Submodule Init') { + stage('VyOS Packages') { steps { - sh ''' - git submodule update --init --recursive --remote - ''' + script { + def build = [:] + // get a list of available package from scripts/build-packages + packageList = sh( + script: "scripts/build-packages -l | grep '*' | sed -e 's/ \\* //'", + returnStdout: true + ).split("\r?\n") + + packageList.each { pkg -> + skipList = ['vyos-kernel', 'vyos-wireguard', 'vyos-accel-ppp'] + if (pkg in skipList) { + return + } + + // add each object from 'packageList' to the 'build' array + build[pkg] = { + // we're already in the script{} block, so do our advanced stuff here + sh( + script: "scripts/build-packages -vvv -b ${pkg}", + returnStdout: true + ) + } + } + // Still within the 'Script' block, run the parallel array object + parallel build + } } } - stage('Build Packages') { + + stage('Kernel') { steps { - sh ''' - #!/bin/sh - scripts/build-submodules --verbose - ''' + sh "scripts/build-packages -vvv -b vyos-kernel" + } + } + + stage('Kernel Modules') { + steps { + script { + def build = [:] + kernelModules = ['vyos-wireguard', 'vyos-accel-ppp'] + kernelModules.each { pkg -> + // add each object from 'packageList' to the 'build' array + build[pkg] = { + // we're already in the script{} block, so do our advanced stuff here + sh( + script: "scripts/build-packages -vvv -b ${pkg}", + returnStdout: true + ) + } + } + // Still within the 'Script' block, run the parallel array object + parallel build + } + } + } + + stage('Intel Drivers') { + steps { + sh "KSRC=\$(pwd)/packages/vyos-kernel scripts/build-intel-drivers" } } - stage('Build ISO') { + stage('ISO Image') { steps { sh ''' #!/bin/sh diff --git a/README.md b/README.md index d4cd6c97..655994c8 100644 --- a/README.md +++ b/README.md @@ -106,47 +106,18 @@ Run following command after building the QEMU image. $ make vmware ``` -## Building the ISO image inside a docker container +# Building ISO or individual packages via Docker Using our [Dockerfile](docker/Dockerfile) you create your own Docker container that is used to build a VyOS ISO image or other required VyOS packages. The [Dockerfile](docker/Dockerfile) contains some of the most used packages needed -to build a VyOS ISO, a qemu image, and several of the submodules. Please note -that this is not complete and only gives you a brief overview! +to build a VyOS ISO, a qemu image, and several of the submodules. -``` -squashfs-tools # Required for squashfs file system -git # Required, for cloning the source -autoconf # Required, for generating build scripts -dpkg-dev # Required, used in build scripts -live-build # Required, for ISO build -syslinux # Required, for ISO build -genisoimage # Required, for ISO build -make # Required, for ISO build -lsb-release # Required, used by configure script -fakechroot # Required, for ISO build -devscripts # Optional, for building submodules (kernel etc) -kernel-package # Optional, for building the kernel -libtool # Optional, for building certain packages (vyatta-op-vpn) -libglib2.0-dev # Optional, for building vyatta-cfg -libboost-filesystem-dev # Optional, for building vyatta-cfg -libapt-pkg-dev # Optional, for building vyatta-cfg -flex # Optional, for building vyatta-cfg -bison # Optional, for building vyatta-cfg -libperl-dev # Optional, for building vyatta-cfg -libnfnetlink-dev # Optional, for building vyatta-cfg-vpn -vim # Optional, vim, vi, nano or other text editor -jq # Optional, for qemu build -qemu-system-x86 # Optional, for qemu build -qemu-utils # Optional, for qemu build -packer # Optional, for qemu build -quilt # Optional, for building vyos-1x -python3-lxml # Optional, for building vyos-1x -python3-setuptools # Optional, for building vyos-1x -python3-nose # Optional, for building vyos-1x -python3-coverage # Optional, for building vyos-1x -... -``` +If you are interested which individual packages are required please check our +[Dockerfile](docker/Dockerfile) which holds the most complete documentation +of required packages. Listing individual packages here tend to be always +outdated. We try to list required packages in groups through their inheritance +in the [Dockerfile](docker/Dockerfile). To build the docker image ensure you have a working [Docker](https://www.docker.com) environment and then run the following commands: @@ -155,7 +126,7 @@ environment and then run the following commands: $ docker build -t vyos-builder docker ``` -Run the newly built container: +Run newly built container: ```bash $ docker run --rm -it --privileged -v $(pwd):/vyos -w /vyos vyos-builder bash ``` @@ -183,32 +154,27 @@ order to build the VyOS ISO image. ## Building subpackages inside Docker -Prior to building packages you need to checkout and update the submodules you want to compile - -```bash -$ git submodule update --init packages/PACKAGENAME -$ cd packages/PACKAGENAME -$ git checkout BRANCH -``` - -`PACKAGENAME` is the name of the package you want to compile -`BRANCH` is `crux` for VyOS 1.2.x, latest rolling releases use `current` - -Fetching all submodules at once and update them to the recent remote branches -`HEAD` is done by calling: +[scripts/build-packages](scripts/build-packages) provides an easy interface +for automate the process of building all VyOS related packages that are not part +of the upstream Debian version. Execute it in the root of the `vyos-build` +directory to start compilation. ```bash -$ git submodule update --init --recursive -$ git submodule update --remote +$ scripts/build-packages -h +usage: build-packages [-h] [-v] [-c] [-l] [-b BUILD [BUILD ...]] [-f] + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Increase logging verbosity for each occurance + -c, --clean Re-clone required Git repositories + -l, --list-packages List all packages to build + -b BUILD [BUILD ...], --build BUILD [BUILD ...] + Whitespace separated list of packages to build + -f, --fetch Fetch sources only, no build ``` -### Building packages - -The [scripts/build-submodules](scripts/build-submodules) script is used to -automate the process of building (in the future) all VyOS related packages that -are not part of the upstream Debian version. Execute it in the root of the -`vyos-build` directory to start compilation on all supported packages that are -checked out. +Git repositoriers are automatically fetched and build on demand. If you wan't to +work offline you can fetch all source code first with the `-f` option. The easiest way to compile is with the above mentioned [Docker](docker/Dockerfile) container, it includes all dependencies for compiling supported packages. @@ -217,7 +183,7 @@ container, it includes all dependencies for compiling supported packages. $ docker run --rm -it -v $(pwd):/vyos -w /vyos \ --sysctl net.ipv6.conf.lo.disable_ipv6=0 \ vyos-builder \ - ./scripts/build-submodules + ./scripts/build-packages ``` **NOTE:** `--sysctl net.ipv6.conf.lo.disable_ipv6=0` is required to build the @@ -237,29 +203,15 @@ Executed from the root of `vyos-build` $ docker run --rm -it -v $(pwd):/vyos -w /vyos/packages/PACKAGENAME \ --sysctl net.ipv6.conf.lo.disable_ipv6=0 \ vyos-builder \ - dpkg-buildpackage -uc -us -tc -b + ./scripts/build-packages -b ``` **NOTE:** `--sysctl net.ipv6.conf.lo.disable_ipv6=0` is only needed when building `vyos-strongswan` and can be ignored on other packages. -**NOTE:** Prior to executing this you need to checkout and update the submodules -you want to recompile! - **NOTE:** `vyos-strongswan` will only compile on a Linux system, running on macOS or Windows might result in a unittest deadlock (it never exits). -Packages that are known to not build using this procedure (as of now): - -``` -vyatta-util - Not needed anymore -vyatta-quagga - Not needed anymore -vyos-1x - Unmet build dependencies: whois libvyosconfig0 -vyos-frr - A lot of requirements, scary stuff... -vyos-kernel - Need special build instructions -vyos-wireguard - Needs special build instructions -``` - # Development process ## Git branches diff --git a/scripts/build-intel-drivers b/scripts/build-intel-drivers new file mode 100755 index 00000000..07c71f88 --- /dev/null +++ b/scripts/build-intel-drivers @@ -0,0 +1,79 @@ +#!/bin/bash -x + +basedir=$(pwd) +if [ -z "$KSRC" ]; then + echo "Kernel source KSRC= not specified" + exit 1 +fi + +VERSION=$(grep "^VERSION" $KSRC/Makefile | grep -Eo '[0-9]{1,4}') +PATCHLEVEL=$(grep "^PATCHLEVEL" $KSRC/Makefile | grep -Eo '[0-9]{1,4}') +SUBLEVEL=$(grep "^SUBLEVEL" $KSRC/Makefile | grep -Eo '[0-9]{1,4}') +KERNEL_VER="$VERSION.$PATCHLEVEL.$SUBLEVEL-amd64-vyos" + +pkgdir="$basedir/packages" +if [ ! -d "$basedir/packages" ]; then + mkdir $basedir/packages +fi + +URLS=" \ + https://downloadmirror.intel.com/14687/eng/ixgbe-5.5.5.tar.gz \ + https://downloadmirror.intel.com/13663/eng/igb-5.3.5.22.tar.gz \ + https://downloadmirror.intel.com/15817/eng/e1000e-3.4.2.1.tar.gz \ + https://downloadmirror.intel.com/24411/eng/i40e-2.7.29.tar.gz \ + https://downloadmirror.intel.com/18700/eng/ixgbevf-4.5.2.tar.gz \ + https://downloadmirror.intel.com/24693/eng/i40evf-3.6.15.tar.gz \ +" + +# The intel IGBVF driver can't be compiled with recent Kernel versions +# due to interlanl API changes. Driver is from 10.09.2015 +# Source code is here: https://downloadmirror.intel.com/18298/eng/igbvf-2.3.9.6.tar.gz +for URL in $URLS +do + cd $pkgdir + filename=${URL##*/} + dirname_full=$(echo $filename | awk -F".tar.gz" '{print $1}') + dirname=$(echo $dirname_full | awk -F- '{print $1}') + version="$(echo $dirname_full | awk -F- '{print $2}')-0" + deb_pkg=${dirname}_${version}_amd64 + deb_pkg_dir=$pkgdir/vyos-intel-${deb_pkg} + + if [ -d "$dirname_full" ]; then + rm -rf $dirname_full + fi + if [ -d "$deb_pkg_dir" ]; then + rm -rf $deb_pkg_dir + fi + if [ ! -f "$filename" ]; then + wget ${URL} + ret=$? + if [ "$ret" != "0" ]; then + echo "Download of ${URL} failed!" + exit $ret + fi + fi + + tar xf $filename + cd $dirname_full/src + + KSRC=$KSRC INSTALL_MOD_PATH=$deb_pkg_dir \ + make -j $(cat /proc/cpuinfo | grep processor | wc -l) install + + mkdir -p $deb_pkg_dir/DEBIAN + echo "Package: vyos-intel-$dirname" >$deb_pkg_dir/DEBIAN/control + echo "Version: $version" >>$deb_pkg_dir/DEBIAN/control + echo "Section: kernel" >>$deb_pkg_dir/DEBIAN/control + echo "Priority: extra" >>$deb_pkg_dir/DEBIAN/control + echo "Architecture: amd64" >>$deb_pkg_dir/DEBIAN/control + echo "Maintainer: VyOS Package Maintainers " >>$deb_pkg_dir/DEBIAN/control + echo "Installed-Size: 9" >>$deb_pkg_dir/DEBIAN/control + echo "Depends: linux-image" >>$deb_pkg_dir/DEBIAN/control + echo "Description: Intel Vendor driver for $dirname" >>$deb_pkg_dir/DEBIAN/control + echo " Replacement for the in Kernel drivers" >>$deb_pkg_dir/DEBIAN/control + + # Cleanup files which might also be present in linux-image-4.19.20-amd64-vyos + rm -rf $deb_pkg_dir/usr $deb_pkg_dir/lib/modules/$KERNEL_VER/modules.* + + cd $pkgdir + dpkg-deb --build $(basename $deb_pkg_dir) +done diff --git a/scripts/build-packages b/scripts/build-packages new file mode 100755 index 00000000..e800b80a --- /dev/null +++ b/scripts/build-packages @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import sys +import shutil +import argparse +import logging + +current_working_directory = os.getcwd() +repo_root = subprocess.check_output('git rev-parse --show-toplevel', shell=True, universal_newlines=True).rstrip('\n') +repo_sha = subprocess.check_output('git rev-parse --short=12 HEAD', shell=True, universal_newlines=True).rstrip('\n') + +def add_package(name, url=None, commit='HEAD', branch='current', tag=None, custombuild_cmd=None): + """ + Build up source package with URL and build commands executed during the later + called build_package step. + + If no additional information is passed we will use the latest commit from current + branch + + If no URL is passed we assume it's a regular VyOS package from the VyOS Github + namespace at https://github.com/vyos + """ + + if not url: + url = 'https://github.com/vyos/' + name + '.git' + + package = { + 'name': name, + 'url': url, + 'commit': commit, + 'tag': tag, + 'branch': branch, + 'path': repo_root + '/packages/' + name, + 'custombuild_cmd': custombuild_cmd + } + + return package + + + +def call(bashCommand, log, **kwargs): + """ + Run command with arguments. Wait for command to complete. + + Send output to logging module passed as 'log'. + """ + + from subprocess import Popen, PIPE, STDOUT, check_output, CalledProcessError + from tempfile import TemporaryFile + from time import sleep + + log.debug("Executing '{}'".format(bashCommand)) + + # code borrowsed from: + # https://stackoverflow.com/questions/38374063/python-can-we-use-tempfile-with-subprocess-to-get-non-buffering-live-output-in-p + # the temp file will be automatically cleaned up + output = TemporaryFile() + error = TemporaryFile() + + kwargs['stdout'] = output + kwargs['stderr'] = error + kwargs['shell'] = True + kwargs['universal_newlines'] = True + + sub = Popen(bashCommand, **kwargs) + while sub.poll() is None: + where = output.tell() + lines = output.readline() + if not lines: + sleep(0.3) + output.seek(where) + else: + log.debug(lines.decode().rstrip('\n')) + + where = error.tell() + lines = error.readline() + if not lines: + sleep(0.3) + error.seek(where) + else: + log.info(lines.decode().rstrip('\n')) + + error.close() + output.close() + return sub.returncode + +def clone_package(pkg, log): + """ + Clone Git repository from URL embedded in pkg to local disk + + First cleanup any possible leftovers from previous builds + """ + + if args.clean: + # delete repository from disk + if os.path.isdir(pkg['path']): + log.debug("Cleaning '{}'".format(pkg['path'])) + shutil.rmtree(pkg['path']) + else: + if os.path.isdir(pkg['path']): + # Change current directory into Git repo for this package + os.chdir(pkg['path']) + + bashCommand = 'git clean -d -x --force && git reset --hard ' + pkg['commit'] + return call(bashCommand, log) + + # resolve given tag to commit id to use shallow clone + bashCommand = 'git clone ' + pkg['url'] + if pkg['tag']: + bashCommand += ' --branch ' + pkg['tag'] + elif pkg['branch']: + bashCommand += ' --depth 1 --branch ' + pkg['branch'] + + bashCommand += ' ' + pkg['path'] + return call(bashCommand, log) + + +def build_package(pkg, log=None): + """ + Generate Debian package from passed 'pkg' + """ + + # Change current directory into Git repo for this package + os.chdir(pkg['path']) + + # Overwrite custom build command if required, e.g. libyang + bashCommand = '' + if pkg['custombuild_cmd']: + bashCommand = pkg['custombuild_cmd'] + else: + # Build package + bashCommand = 'dpkg-buildpackage -uc -us -tc -b' + if args.parallel: + bashCommand += ' -j' + str(os.cpu_count()) + + return call(bashCommand, log) + +# a List of all Vyatta/VyOS based packages +vyos_packages = ['vyatta-bash', + 'vyatta-cfg', + 'vyatta-op', + 'vyatta-cfg-system', + 'vyatta-cfg-firewall', + 'vyatta-op-firewall', + 'vyatta-cfg-vpn', + 'vyatta-op-vpn', + 'vyatta-cfg-qos', + 'vyatta-op-qos', + 'vyatta-cfg-op-pppoe', + 'vyatta-openvpn', + 'vyatta-conntrack', + 'vyatta-conntrack-sync', + 'vyatta-nat', + 'vyatta-config-mgmt', + 'vyatta-config-migrate', + 'vyatta-zone', + 'vyatta-cluster', + 'vyatta-eventwatch', + 'vyatta-webproxy', + 'vyatta-cfg-quagga', + 'vyatta-op-quagga', + 'vyatta-op-dhcp-server', + 'vyatta-wireless', + 'vyatta-wirelessmodem', + 'vyatta-wanloadbalance', + 'vyatta-netflow', + 'vyatta-lldp', + 'vyatta-ipv6-rtradv', + 'vyatta-ravpn', + 'vyos-nhrp', + 'vyos-world', + 'vyos-1x', + 'vyatta-iproute', + 'vyatta-quagga', + 'vyos-strongswan', + 'vyos-vmwaretools-scripts', + 'vyos-netplug', + 'vyos-xe-guest-utilities', + 'vyatta-biosdevname', + 'vyos-opennhrp', + 'vyos-salt-minion', + 'xl2tpd', + 'mdns-repeater', + 'udp-broadcast-relay', + 'pmacct', + 'ddclient', + 'igmpproxy', + 'eventwatchd', + 'conntrack-tools'] + +# Special packages mean packages which are located no in the VyOS namespace +# or require fancy build instructions +pkg_special = [] + +# libvyosconfig/ipaddrcheck uses a different default branch +libvyosconfig_build_cmd = "eval $(opam env --root=/opt/opam --set-root) && " \ + "dpkg-buildpackage -b -us -uc -tc" +pkg_special.append( add_package('libvyosconfig', branch='master', custombuild_cmd=libvyosconfig_build_cmd)) +pkg_special.append( add_package('ipaddrcheck', branch='master')) + +# Packages where we directly build the upstream source +pkg_special.append( add_package('hvinfo', url='https://github.com/dmbaturin/hvinfo.git', branch='master') ) +pkg_special.append( add_package('lldpd', url='https://github.com/vincentbernat/lldpd.git', branch='master', tag='1.0.3') ) + +# +# FreeRangeRouting (FRR) packages +# +pkg_special.append( add_package('rtrlib', url='https://github.com/rtrlib/rtrlib.git', branch='master', tag='v0.6.3') ) + +frr_build_cmd = './tools/tarsource.sh -V && dpkg-buildpackage -us -uc -Ppkg.frr.rtrlib -d' +pkg_special.append( add_package('frr', url='https://github.com/FRRouting/frr.git', branch='master', tag='frr-7.0', custombuild_cmd=frr_build_cmd) ) +#libyang_build_cmd = 'mkdir build && cd build && cmake .. && make build-deb && mv debs/* ' + repo_root + '/packages' +#pkg_special.append( add_package('libyang', url='https://github.com/opensourcerouting/libyang.git', commit='179da47', branch='master', custombuild_cmd=libyang_build_cmd) ) + + +# +# Linux (VyOS) Kernel +# +kernel_build_cmd = "export VERSION=$(grep '^VERSION' Makefile | grep -Eo '[0-9]{1,4}') && " \ + "export PATCHLEVEL=$(grep '^PATCHLEVEL' Makefile | grep -Eo '[0-9]{1,4}') && " \ + "export SUBLEVEL=$(grep '^SUBLEVEL' Makefile | grep -Eo '[0-9]{1,4}') && " \ + "echo ${VERSION}.${PATCHLEVEL}.${SUBLEVEL}-amd64-vyos > " + repo_root + "/data/kernel_version && " \ + "make x86_64_vyos_defconfig && " \ + "LOCALVERSION='' make-kpkg --rootcmd fakeroot --initrd --append_to_version -amd64-vyos " \ + " --revision=${VERSION}.${PATCHLEVEL}.${SUBLEVEL}-0 kernel_headers kernel_image -j" + str(os.cpu_count()) +pkg_special.append( add_package('vyos-kernel', branch='linux-vyos-4.19.y', custombuild_cmd=kernel_build_cmd) ) + + + +# +# WireGuard Kernel Module +# +wireguard_build_cmd = "echo 'src/wireguard.ko /lib/modules/'$(cat " + repo_root + "/data/kernel_version) > debian/wireguard-modules.install && " \ + "KERNELDIR=" + repo_root + "/packages/vyos-kernel dpkg-buildpackage -b -us -uc -tc -j" + str(os.cpu_count()) +pkg_special.append( add_package('vyos-wireguard', custombuild_cmd=wireguard_build_cmd) ) + + + +# +# Accell-PPP Package and Kernel Module +# +accel_ppp_build_cmd = "echo 'lib/modules/'$(cat " + repo_root + "/data/kernel_version)'/extra/*.ko' > debian/vyos-accel-ppp-ipoe-kmod.install && " \ + "sed -i 's#[0-9].[0-9][0-9].[0-9]*-amd64-vyos#'$(cat " + repo_root + "/data/kernel_version)'#g' debian/rules && " \ + "KERNELDIR=" + repo_root + "/packages/vyos-kernel dpkg-buildpackage -b -us -uc -tc -j" + str(os.cpu_count()) +pkg_special.append( add_package('vyos-accel-ppp', custombuild_cmd=accel_ppp_build_cmd) ) + + +# A list of all packages we will build in the end +pkg_build = [] + + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-v', '--verbose', action='count', default=0, help='Increase logging verbosity for each occurance') + parser.add_argument('-c', '--clean', action='store_true', help='Re-clone required Git repositories') + parser.add_argument('-l', '--list-packages', action='store_true', help='List all packages to build') + parser.add_argument('-b', '--build', nargs='+', help='Whitespace separated list of packages to build') + parser.add_argument('-f', '--fetch', action='store_true', help='Fetch sources only, no build') + parser.add_argument('-p', '--parallel', action='store_true', help='Build on all CPUs') + + + args = parser.parse_args() + + levels = [ logging.INFO, logging.WARNING, logging.DEBUG ] + level = levels[min(len(levels)-1,args.verbose)] # capped to number of levels + logging.basicConfig(level=level, format="%(asctime)s %(name)s %(message)s") + + print("Using vyos-build repository ('{}') commit '{}'\n".format(repo_root, repo_sha)) + + if args.list_packages: + print("Individual packages available for build:") + for pkg in vyos_packages: + print(' * ' + pkg) + for pkg in pkg_special: + print(' * ' + pkg['name']) + + sys.exit(0) + + # + # Only add selective packages to the build list + # + if args.build: + # NOTE: remove double added packages from list + for target in args.build: + if target in vyos_packages: + pkg_build.append(add_package( target )) + continue + + found = False + for pkg in pkg_special: + if target == pkg['name']: + found = True + # package already formed + pkg_build.append( pkg ) + break + + if not found: + print("Invalid choice '" + target + "', for -b/--build use --list-packages for complete list!") + sys.exit(1) + + else: + # Add all VyOS packages to the package list + for pkg in vyos_packages: + pkg_build.append(add_package( pkg )) + + # We also wan't to build all of our special packages + for pkg in pkg_special: + pkg_build.append( pkg ) + + # Build all VyOS packages (packages found on https://github.com/vyos + # and referenced in vyos_packages) + for pkg in pkg_build: + # Create a logging instance per package + log = logging.getLogger(pkg['name']) + + ret = clone_package(pkg, log) + if ret: + log.error("ERROR cloning source") + sys.exit(1) + else: + # only build packages if fetch flag is not set + if not args.fetch: + ret = build_package(pkg, log) + if ret: + log.error("ERROR building source") + sys.exit(1) + + sys.exit(0) diff --git a/scripts/build-submodules b/scripts/build-submodules deleted file mode 100755 index fc8fdf83..00000000 --- a/scripts/build-submodules +++ /dev/null @@ -1,452 +0,0 @@ -#!/bin/bash - -if [ ! -d "packages" ]; then - echo "This script needs to be executed inside the top root of vyos-build" - exit 1 -fi - -print_help() { - echo "Script for building all subpackages to vyos" - echo "Execute this sctipt from the root of the vyos-build directory" - echo "" - echo "This script could be executed from a Debian Jessie installation with all dependencies" - echo "or from the vyos-builder docker container." - echo "" - echo "Build the container:" - echo " docker build -t vyos-builder ." - echo "Compile packages:" - echo " docker run --rm -it -v $(pwd):/vyos -w /vyos --sysctl net.ipv6.conf.lo.disable_ipv6=0 vyos-builder scripts/build-submodules" - echo "" - echo "Parameters:" - echo " --init-packages - Initiate all subpackages before building" - echo " --verbose - Enable verbose output" - echo " --build-[packagename] - build only selected packages" -} - -BUILDLIST="" -VERBOSE=0 - -while test $# -gt 0 -do - case "$1" in - -h|-?|--help) - print_help - exit 0 - ;; - --init-packages) - INIT_PACKAGES=1 - ;; - --verbose) - VERBOSE=1 - ;; - --build-*) - VAL=$(sed s/^--build-//g <<< $1) - BUILDLIST="$BUILDLIST $VAL" - ;; - *) - (>&2 echo "Error: Argument $1 is not valid") - echo "" - print_help - exit 1 - ;; - esac - shift -done - -status_start() { - echo -ne "[ ] $1" -} -status_ok() { - echo -ne "\r[\e[32m OK \e[39m]\n" -} - -status_fail() { - echo -ne "\r[\e[31mFAIL\e[39m]\n" -} - -status_skip() { - echo -ne "\r[SKIP] $1\033[K\n" -} - -error_msg() { - echo -ne " $1\n" -} - -verbose_msg() { - if [ $VERBOSE -ne 0 ]; then - echo "Current Environment:" - env - - if [ ! -z "$1" ]; then - echo "Logfile:" - cat $1 - fi - fi -} - -ROOTDIR="$(pwd)" -PKGDIR="$ROOTDIR/packages" -SCRIPTDIR="$ROOTDIR/scripts" - -# Source OPAM environment if not already set -if [ -z "$OPAMROOT" ]; then - if [ -x "$(command -v opam)" ]; then - eval $(opam env --root=/opt/opam --set-root) - else - echo "WARNING: 'opam' not installed, can't build VyConf and libvyosconfig" - fi -fi - -package_in_buildlist() { - # Return true if buildlist is not set - if [ -z "$BUILDLIST" ]; then - return 0 - fi - - if [[ $BUILDLIST =~ (^|[[:space:]])$1($|[[:space:]]) ]]; then - return 0 - fi - - return 1 -} - -build_package() { - PKG=$1 - COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD) - if ! package_in_buildlist $1; then - return 0 - fi - status_start "Building package: $PKG Commit id: $COMMITID" - if [ ! -d "$PKGDIR/$PKG/debian" ]; then - status_skip "No source for: $PKG" - return 1 - fi - ( set -e; set -x - cd $PKGDIR/$PKG - dpkg-buildpackage -uc -us -tc -b -j$(cat /proc/cpuinfo | grep processor | wc -l) - ) >>$PKGDIR/$PKG.buildlog 2>&1 - if [ $? -ne 0 ]; then - status_fail - verbose_msg "$PKGDIR/$PKG.buildlog" - error_msg "Failed to build package $PKG, look in $PKG.buildlog to examine the fault\n" - return 2 - fi - status_ok -} - -echo "Cleaning up buildfiles..." -rm -rf $PKGDIR/*.deb -rm -rf $PKGDIR/*.changes -rm -rf $PKGDIR/*.buildlog -COMMITID=$(cd $PKGDIR; git rev-parse --short=10 HEAD) -echo "-----------------------------------------------------" -echo "Starting build process for all packages" -echo "vyos-build Commit ID: $COMMITID" - -initialize_packages() { - status_start "Initializing packages" - ( - set -x - git submodule update --init --recursive - git submodule update --remote - ) >>$PKGDIR/init-packages.buildlog 2>&1 - if [ $? -ne 0 ]; then - status_fail - verbose_msg "$PKGDIR/init-packages.buildlog" - error_msg "Failed to update all package, look in init-packages.buildlog to examine the fault\n" - return 1 - fi - status_ok -} -if [ $INIT_PACKAGES ]; then - initialize_packages -fi - -build_libyang() { - PKG=libyang - COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD) - if ! package_in_buildlist $1; then - return 0 - fi - status_start "Building package: $PKG Commit id: $COMMITID" - if [ ! -f "$PKGDIR/$PKG/README.md" ]; then - status_skip "No source for: $PKG" - return 1 - fi - - ( set -e; set -x - cd $PKGDIR/$PKG - git checkout 179da47f2e8de - - git clean -dxf - git reset --hard - - mkdir build - cd build - - cmake .. - make build-deb - - cp debs/* $PKGDIR - - ) >>$PKGDIR/$PKG.buildlog 2>&1 - - if [ $? -ne 0 ]; then - status_fail - error_msg "Failed to build package $PKG, look in $PKG.buildlog to examine the fault\n" - return 2 - fi - status_ok -} -build_libyang - -build_frr() { - PKG=frr - COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD) - if ! package_in_buildlist $1; then - return 0 - fi - status_start "Building package: $PKG Commit id: $COMMITID" - if [ ! -f "$PKGDIR/$PKG/README.md" ]; then - status_skip "No source for: $PKG" - return 1 - fi - - ( set -e; set -x - cd $PKGDIR/$PKG - git clean -dxf - git reset --hard - - # Run bootstrap.sh and make a dist tarball - # http://docs.frrouting.org/projects/dev-guide/en/latest/packaging-debian.html - ./bootstrap.sh - ./configure --with-pkg-extra-version=-vyos - make dist - - # Create backports debian sources - mv debianpkg debian - make -f debian/rules backports - - # Create a new directory to build the package and populate with package source. - mkdir frrpkg - cd frrpkg - tar xf ../frr_*.orig.tar.gz - cd frr* - source /etc/os-release - tar xf ../../frr_*${ID}${VERSION_ID}*.debian.tar.xz - - # Build Debian Package - debuild --no-lintian --set-envvar=WANT_SNMP=1 --set-envvar=WANT_RPKI=1 \ - --set-envvar=WANT_CUMULUS_MODE=0 -b -uc -us \ - -j$(cat /proc/cpuinfo | grep processor | wc -l) - - mv ../frr_*.deb $PKGDIR - ) >>$PKGDIR/$PKG.buildlog 2>&1 - if [ $? -ne 0 ]; then - status_fail - error_msg "Failed to build package $PKG, look in $PKG.buildlog to examine the fault\n" - return 2 - fi - status_ok -} -build_frr - -for PKG in mdns-repeater \ - pmacct \ - udp-broadcast-relay \ - conntrack-tools \ - eventwatchd \ - ddclient \ - rtrlib \ - hvinfo \ - igmpproxy \ - ipaddrcheck \ - lldpd \ - libvyosconfig \ - vyatta-bash \ - vyatta-biosdevname \ - vyatta-cfg \ - vyatta-cfg-firewall \ - vyatta-cfg-op-pppoe \ - vyatta-cfg-qos \ - vyatta-cfg-quagga \ - vyatta-cfg-system \ - vyatta-cfg-vpn \ - vyatta-cluster \ - vyatta-config-mgmt \ - vyatta-config-migrate \ - vyatta-conntrack \ - vyatta-conntrack-sync \ - vyatta-eventwatch \ - vyatta-iproute \ - vyatta-ipv6-rtradv \ - vyatta-lldp \ - vyatta-nat \ - vyatta-netflow \ - vyatta-op \ - vyatta-op-dhcp-server \ - vyatta-op-firewall \ - vyatta-op-qos \ - vyatta-op-quagga \ - vyatta-op-vpn \ - vyatta-openvpn \ - vyatta-ravpn \ - vyatta-wanloadbalance \ - vyatta-webgui \ - vyatta-webproxy \ - vyatta-wireless \ - vyatta-wirelessmodem \ - vyatta-zone \ - vyos-xe-guest-utilities \ - vyos-keepalived \ - vyos-netplug \ - vyos-nhrp \ - vyos-opennhrp \ - vyos-salt-minion \ - vyos-strongswan \ - vyos-vmwaretools-scripts \ - vyos-world \ - vyos-1x \ - ; do - build_package "$PKG" - ERRCODE=$? - if [ "$ERRCODE" -ne "0" ]; then - exit $ERRCODE - fi -done - -# KERNEL -build_kernel() { - PKG="vyos-kernel" - if ! package_in_buildlist "vyos-kernel"; then - return 0 - fi - COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD) - status_start "Building-package: vyos-kernel Commit ID:$COMMITID" - if [ ! -f "$PKGDIR/vyos-kernel/Makefile" ]; then - status_skip "No source for: vyos-kernel" - return 0 - fi - - ( set -e; set -x - cd $PKGDIR/vyos-kernel > /dev/null - bash -c "$SCRIPTDIR/build-kernel" - ) >>$PKGDIR/vyos-kernel.buildlog 2>&1 - if [ $? -ne 0 ]; then - status_fail - verbose_msg "$PKGDIR/vyos-kernel.buildlog" - error_msg "Failed to build package vyos-kernel, look in vyos-kernel.buildlog to examine the fault\n" - return 1 - fi - - VERSION=$(grep "^VERSION" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}') - PATCHLEVEL=$(grep "^PATCHLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}') - SUBLEVEL=$(grep "^SUBLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}') - ARCH=$(dpkg --print-architecture) - status_ok -} -build_kernel -ERRCODE=$? -if [ "$ERRCODE" -ne "0" ]; then - exit $ERRCODE -fi - - -# WIREGUARD -build_wireguard() { - PKG="vyos-wireguard" - if ! package_in_buildlist "vyos-wireguard"; then - return 0 - fi - COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD) - status_start "Building package: vyos-wireguard Commit ID:$COMMITID" - - if [ ! -d "$PKGDIR/vyos-wireguard/debian" ]; then - status_skip "No source for: vyos-wireguard" - return 0 - fi - - if [ ! -f "$PKGDIR/vyos-kernel/Makefile" ]; then - status_fail - error_msg "No Makefile found in kernel package" - return 1 - fi - - if ! grep -q "KBUILD_OUTPUT" $PKGDIR/vyos-kernel/Makefile; then - status_fail - error_msg "Failed to build package vyos-wireguard, no kernel source found\n" - return 1 - fi - - VERSION=$(grep "^VERSION" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}') - PATCHLEVEL=$(grep "^PATCHLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}') - SUBLEVEL=$(grep "^SUBLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}') - ARCH=$(dpkg --print-architecture) - # Collect kernel information - ( set -e; set -x - cd $PKGDIR/vyos-wireguard - echo "src/wireguard.ko /lib/modules/$VERSION.$PATCHLEVEL.$SUBLEVEL-$ARCH-vyos/extra" | tee debian/wireguard-modules.install - KERNELDIR=$PKGDIR/vyos-kernel dpkg-buildpackage -uc -us -tc -b -j$(cat /proc/cpuinfo | grep processor | wc -l) - ) >>$PKGDIR/vyos-wireguard.buildlog 2>&1 - if [ $? -ne 0 ]; then - status_fail - verbose_msg "$PKGDIR/vyos-wireguard.buildlog" - error_msg "Failed to build package vyos-wireguard, look in vyos-wireguard.buildlog to examine the fault\n" - return 2 - fi - status_ok -} -build_wireguard -ERRCODE=$? -if [ "$ERRCODE" -ne "0" ]; then - exit $ERRCODE -fi - -# ACCEL-PPP -build_accel-ppp() { - PKG="vyos-accel-ppp" - if ! package_in_buildlist "accel-ppp"; then - return 0 - fi - COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD) - status_start "Building package: vyos-accel-ppp Commit ID: $COMMITID" - if [ ! -d "$PKGDIR/vyos-accel-ppp/debian" ]; then - status_skip "No source for: vyos-accel-ppp" - return 0 - fi - - if [ ! -f "$PKGDIR/vyos-kernel/Makefile" ]; then - status_fail - error_msg "No Makefile found in kernel package" - return 1 - fi - - if ! grep -q "KBUILD_OUTPUT" $PKGDIR/vyos-kernel/Makefile; then - status_fail - error_msg "Failed to build package vyos-accel-ppp, no kernel source found\n" - fi - # Collect kernel information - VERSION=$(grep "^VERSION" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}') - PATCHLEVEL=$(grep "^PATCHLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}') - SUBLEVEL=$(grep "^SUBLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}') - ARCH=$(dpkg --print-architecture) - - ( set -e; set -x - cd $PKGDIR/vyos-accel-ppp - echo "lib/modules/$VERSION.$PATCHLEVEL.$SUBLEVEL-$ARCH-vyos/extra/*.ko" | tee debian/vyos-accel-ppp-ipoe-kmod.install - sed -i "s#[0-9].[0-9][0-9].[0-9]-amd64-vyos#$VERSION.$PATCHLEVEL.$SUBLEVEL-$ARCH-vyos#g" debian/rules - KERNELDIR=$PKGDIR/vyos-kernel dpkg-buildpackage -uc -us -tc -b -j$(cat /proc/cpuinfo | grep processor | wc -l) - ) >>$PKGDIR/vyos-accel-ppp.buildlog 2>&1 - if [ $? -ne 0 ]; then - status_fail - verbose_msg "$PKGDIR/vyos-accel-ppp.buildlog" - error_msg "Failed to build package vyos-accel-ppp, look in vyos-accel-ppp.buildlog to examine the fault\n" - return 1 - fi - status_ok -} -build_accel-ppp -ERRCODE=$? -if [ "$ERRCODE" -ne "0" ]; then - exit $ERRCODE -fi -- cgit v1.2.3 From 21a09ff13dadffd85e6d46d38ecf6b3e9dc8ca2e Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sun, 14 Apr 2019 18:03:50 +0200 Subject: intel: adjust URL for ixgbevf 4.5.2 --- scripts/build-intel-drivers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build-intel-drivers b/scripts/build-intel-drivers index 07c71f88..ef039e4c 100755 --- a/scripts/build-intel-drivers +++ b/scripts/build-intel-drivers @@ -21,7 +21,7 @@ URLS=" \ https://downloadmirror.intel.com/13663/eng/igb-5.3.5.22.tar.gz \ https://downloadmirror.intel.com/15817/eng/e1000e-3.4.2.1.tar.gz \ https://downloadmirror.intel.com/24411/eng/i40e-2.7.29.tar.gz \ - https://downloadmirror.intel.com/18700/eng/ixgbevf-4.5.2.tar.gz \ + https://downloadmirror.intel.com/28725/eng/ixgbevf-4.5.2.tar.gz https://downloadmirror.intel.com/24693/eng/i40evf-3.6.15.tar.gz \ " -- cgit v1.2.3 From 9e234232bf1ac1d139dde6da7c74f82a8b5b4f41 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Tue, 16 Apr 2019 20:12:54 +0200 Subject: Packages: define custom command for vyos-strongswan This is required to get a python3-vici package which is required by vyos-1x --- docker/Dockerfile | 4 +++- scripts/build-packages | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 36895335..c4351555 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -259,7 +259,9 @@ RUN apt-get update && apt-get install -y \ libsqlite3-dev \ dh-apparmor \ gperf \ - libsystemd-dev + libsystemd-dev \ + python3-stdeb \ + python-setuptools # Packages needed for vyos-opennhrp RUN apt-get update && apt-get install -y \ diff --git a/scripts/build-packages b/scripts/build-packages index e800b80a..07baa040 100755 --- a/scripts/build-packages +++ b/scripts/build-packages @@ -174,7 +174,6 @@ vyos_packages = ['vyatta-bash', 'vyos-1x', 'vyatta-iproute', 'vyatta-quagga', - 'vyos-strongswan', 'vyos-vmwaretools-scripts', 'vyos-netplug', 'vyos-xe-guest-utilities', @@ -204,6 +203,14 @@ pkg_special.append( add_package('ipaddrcheck', branch='master')) pkg_special.append( add_package('hvinfo', url='https://github.com/dmbaturin/hvinfo.git', branch='master') ) pkg_special.append( add_package('lldpd', url='https://github.com/vincentbernat/lldpd.git', branch='master', tag='1.0.3') ) +# VyOS strongswan ships additional python3-vici packages required by vyos-1x and this is not build by default +vyos_strongswan_build_cmd = "dpkg-buildpackage -b -us -uc -tc && " \ + "autoreconf -i && ./configure --enable-python-eggs && " \ + "cd src/libcharon/plugins/vici/python && make && " \ + "python3 setup.py --command-packages=stdeb.command bdist_deb && " \ + "mv ./deb_dist/*.deb " + repo_root + "/packages" +pkg_special.append( add_package('vyos-strongswan', custombuild_cmd=vyos_strongswan_build_cmd) ) + # # FreeRangeRouting (FRR) packages # -- cgit v1.2.3 From de68bd05fef1db2b0363352afb84d77b393e7fe7 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sun, 21 Apr 2019 13:54:53 +0200 Subject: Packages: add ethtool (debian/1%4.19-1) package to build procedure --- scripts/build-packages | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/build-packages b/scripts/build-packages index 07baa040..08c6ee22 100755 --- a/scripts/build-packages +++ b/scripts/build-packages @@ -200,8 +200,9 @@ pkg_special.append( add_package('libvyosconfig', branch='master', custombuild_cm pkg_special.append( add_package('ipaddrcheck', branch='master')) # Packages where we directly build the upstream source -pkg_special.append( add_package('hvinfo', url='https://github.com/dmbaturin/hvinfo.git', branch='master') ) -pkg_special.append( add_package('lldpd', url='https://github.com/vincentbernat/lldpd.git', branch='master', tag='1.0.3') ) +pkg_special.append( add_package('hvinfo', url='https://github.com/dmbaturin/hvinfo.git', branch='master') ) +pkg_special.append( add_package('lldpd', url='https://github.com/vincentbernat/lldpd.git', branch='master', tag='1.0.3') ) +pkg_special.append( add_package('ethtool', url='https://salsa.debian.org/kernel-team/ethtool.git', branch='master', tag='debian/1%4.19-1') ) # VyOS strongswan ships additional python3-vici packages required by vyos-1x and this is not build by default vyos_strongswan_build_cmd = "dpkg-buildpackage -b -us -uc -tc && " \ -- cgit v1.2.3 From 89a0214f24f43492319caf5b4b74ec641a591294 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Mon, 22 Apr 2019 13:42:45 +0200 Subject: Packages: remove vyatta-quagga --- scripts/build-packages | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build-packages b/scripts/build-packages index 08c6ee22..7e23665b 100755 --- a/scripts/build-packages +++ b/scripts/build-packages @@ -173,7 +173,6 @@ vyos_packages = ['vyatta-bash', 'vyos-world', 'vyos-1x', 'vyatta-iproute', - 'vyatta-quagga', 'vyos-vmwaretools-scripts', 'vyos-netplug', 'vyos-xe-guest-utilities', -- cgit v1.2.3 From 2ffff339c5515280a1926d2ee134d5d2c7cdbcfc Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Tue, 23 Apr 2019 08:49:46 +0200 Subject: Makefile: proper handle 'make iso' error codes --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 3bfa9597..628574e2 100644 --- a/Makefile +++ b/Makefile @@ -28,9 +28,13 @@ prepare: .ONESHELL: iso: check_build_config clean prepare @set -e + @set -o pipefail @echo "It's not like I'm building this specially for you or anything!" cd $(build_dir) lb build 2>&1 | tee build.log + if [ "$?" != "0" ]; then + exit 1 + fi cd .. @scripts/copy-image -- cgit v1.2.3 From a9c5df1f42b59b3c0b61474d74ceeed7e65c9df2 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Tue, 23 Apr 2019 08:50:00 +0200 Subject: Jenkins: archive ISO and build.log artifacts --- Jenkinsfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 169a52f3..90c38d1c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -157,9 +157,6 @@ pipeline { # Configure the ISO ./configure --build-by="autobuild@vyos.net" --debian-mirror="http://ftp.us.debian.org/debian/" - # Debug to see which Debian packages we have so far - ls -al packages/*.deb - # Finally build our ISO sudo make iso ''' @@ -168,6 +165,12 @@ pipeline { } post { + always { + archiveArtifacts artifacts: 'build/build.log', fingerprint: true + } + success { + archiveArtifacts artifacts: 'build/vyos*.iso', fingerprint: true + } cleanup { echo 'One way or another, I have finished' // the 'build' directory got elevated permissions during the build -- cgit v1.2.3 From f4df31d37e9192dfcbe0a6d97ae8634fe20dae08 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Tue, 23 Apr 2019 15:06:48 +0200 Subject: Packages: add bgpq3 (debian/0.1.33-1) package to build procedure --- scripts/build-packages | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build-packages b/scripts/build-packages index 7e23665b..93618fbc 100755 --- a/scripts/build-packages +++ b/scripts/build-packages @@ -202,6 +202,7 @@ pkg_special.append( add_package('ipaddrcheck', branch='master')) pkg_special.append( add_package('hvinfo', url='https://github.com/dmbaturin/hvinfo.git', branch='master') ) pkg_special.append( add_package('lldpd', url='https://github.com/vincentbernat/lldpd.git', branch='master', tag='1.0.3') ) pkg_special.append( add_package('ethtool', url='https://salsa.debian.org/kernel-team/ethtool.git', branch='master', tag='debian/1%4.19-1') ) +pkg_special.append( add_package('bgpq3', url='https://salsa.debian.org/debian/bgpq3.git', branch='master', tag='debian/0.1.33-1') ) # VyOS strongswan ships additional python3-vici packages required by vyos-1x and this is not build by default vyos_strongswan_build_cmd = "dpkg-buildpackage -b -us -uc -tc && " \ -- cgit v1.2.3 From dee033f97cfc9818bcf2143d79ff18726beb7c22 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Fri, 26 Apr 2019 09:56:57 +0200 Subject: Packages: use 'make kernelversion' instead of grep magic Exract version by using the included "make kernelversion" instead of writing a custom grep script that was used before. VERSION=$(grep '^VERSION' Makefile | grep -Eo '[0-9]{1,4}') PATCHLEVEL=$(grep '^PATCHLEVEL' Makefile | grep -Eo '[0-9]{1,4}') SUBLEVEL=$(grep '^SUBLEVEL' Makefile | grep -Eo '[0-9]{1,4}') $ make kernelversion 4.19.36 --- scripts/build-packages | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/build-packages b/scripts/build-packages index 93618fbc..6d4b1c76 100755 --- a/scripts/build-packages +++ b/scripts/build-packages @@ -226,13 +226,10 @@ pkg_special.append( add_package('frr', url='https://github.com/FRRouting/frr.git # # Linux (VyOS) Kernel # -kernel_build_cmd = "export VERSION=$(grep '^VERSION' Makefile | grep -Eo '[0-9]{1,4}') && " \ - "export PATCHLEVEL=$(grep '^PATCHLEVEL' Makefile | grep -Eo '[0-9]{1,4}') && " \ - "export SUBLEVEL=$(grep '^SUBLEVEL' Makefile | grep -Eo '[0-9]{1,4}') && " \ - "echo ${VERSION}.${PATCHLEVEL}.${SUBLEVEL}-amd64-vyos > " + repo_root + "/data/kernel_version && " \ - "make x86_64_vyos_defconfig && " \ +kernel_build_cmd = "make x86_64_vyos_defconfig && " \ + "echo $(make kernelversion)-amd64-vyos > " + repo_root + "/data/kernel_version && " \ "LOCALVERSION='' make-kpkg --rootcmd fakeroot --initrd --append_to_version -amd64-vyos " \ - " --revision=${VERSION}.${PATCHLEVEL}.${SUBLEVEL}-0 kernel_headers kernel_image -j" + str(os.cpu_count()) + " --revision=$(make kernelversion)-1 kernel_headers kernel_image -j" + str(os.cpu_count()) pkg_special.append( add_package('vyos-kernel', branch='linux-vyos-4.19.y', custombuild_cmd=kernel_build_cmd) ) -- cgit v1.2.3 From b1c495b72be765f616cd681a7ff0c9e194c67479 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Fri, 26 Apr 2019 10:27:35 +0200 Subject: Packages: extract kernel version from defaults.json This requires 'jq' as part of the Docker container. $ cat data/defaults.json | jq '.kernel_version' | tr -d \" --- data/kernel_version | 2 +- docker/Dockerfile | 5 ++--- scripts/build-packages | 7 ++++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/data/kernel_version b/data/kernel_version index 1a6cc42f..ab416d6c 100644 --- a/data/kernel_version +++ b/data/kernel_version @@ -1 +1 @@ -4.19.32-amd64-vyos +4.19.36-amd64-vyos diff --git a/docker/Dockerfile b/docker/Dockerfile index c4351555..21b38417 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -70,7 +70,8 @@ RUN apt-get update && apt-get install -y \ python3-sphinx \ python3-pystache \ pkg-config \ - debhelper + debhelper \ + jq RUN apt-get update && apt-get install -y -t jessie-backports \ python3-git \ @@ -280,8 +281,6 @@ RUN echo 'deb http://ftp.debian.org/debian stretch main' | tee -a /etc/apt/sourc rm -rf /var/lib/apt/lists/* # Install packer -RUN apt-get update && apt-get install -y \ - jq RUN export LATEST="$(curl -s https://checkpoint-api.hashicorp.com/v1/check/packer | \ jq -r -M '.current_version')"; \ echo "url https://releases.hashicorp.com/packer/"$LATEST"/packer_"$LATEST"_linux_amd64.zip" |\ diff --git a/scripts/build-packages b/scripts/build-packages index 6d4b1c76..0677e9f7 100755 --- a/scripts/build-packages +++ b/scripts/build-packages @@ -228,6 +228,7 @@ pkg_special.append( add_package('frr', url='https://github.com/FRRouting/frr.git # kernel_build_cmd = "make x86_64_vyos_defconfig && " \ "echo $(make kernelversion)-amd64-vyos > " + repo_root + "/data/kernel_version && " \ + "sed -i 's/\"kernel_version\": \"[0-9].[0-9][0-9].[0-9]*\"/\"kernel_version\": \"'$(make kernelversion)'\"/' " + repo_root + "/data/defaults.json && " \ "LOCALVERSION='' make-kpkg --rootcmd fakeroot --initrd --append_to_version -amd64-vyos " \ " --revision=$(make kernelversion)-1 kernel_headers kernel_image -j" + str(os.cpu_count()) pkg_special.append( add_package('vyos-kernel', branch='linux-vyos-4.19.y', custombuild_cmd=kernel_build_cmd) ) @@ -237,7 +238,7 @@ pkg_special.append( add_package('vyos-kernel', branch='linux-vyos-4.19.y', custo # # WireGuard Kernel Module # -wireguard_build_cmd = "echo 'src/wireguard.ko /lib/modules/'$(cat " + repo_root + "/data/kernel_version) > debian/wireguard-modules.install && " \ +wireguard_build_cmd = "echo 'src/wireguard.ko /lib/modules/'$(cat " + repo_root + "/data/defaults.json | jq '.kernel_version' | tr -d \\\") > debian/wireguard-modules.install && " \ "KERNELDIR=" + repo_root + "/packages/vyos-kernel dpkg-buildpackage -b -us -uc -tc -j" + str(os.cpu_count()) pkg_special.append( add_package('vyos-wireguard', custombuild_cmd=wireguard_build_cmd) ) @@ -246,8 +247,8 @@ pkg_special.append( add_package('vyos-wireguard', custombuild_cmd=wireguard_buil # # Accell-PPP Package and Kernel Module # -accel_ppp_build_cmd = "echo 'lib/modules/'$(cat " + repo_root + "/data/kernel_version)'/extra/*.ko' > debian/vyos-accel-ppp-ipoe-kmod.install && " \ - "sed -i 's#[0-9].[0-9][0-9].[0-9]*-amd64-vyos#'$(cat " + repo_root + "/data/kernel_version)'#g' debian/rules && " \ +accel_ppp_build_cmd = "echo 'lib/modules/'$(cat " + repo_root + "/data/defaults.json | jq '.kernel_version' | tr -d \\\")'/extra/*.ko' > debian/vyos-accel-ppp-ipoe-kmod.install && " \ + "sed -i 's#[0-9].[0-9][0-9].[0-9]*-amd64-vyos#'$(cat " + repo_root + "/data/defaults.json | jq '.kernel_version' | tr -d \\\")'#g' debian/rules && " \ "KERNELDIR=" + repo_root + "/packages/vyos-kernel dpkg-buildpackage -b -us -uc -tc -j" + str(os.cpu_count()) pkg_special.append( add_package('vyos-accel-ppp', custombuild_cmd=accel_ppp_build_cmd) ) -- cgit v1.2.3 From c4808dbfe570154e3ed649e2e7fe567d18b559e6 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Fri, 26 Apr 2019 11:52:05 +0200 Subject: README: adjust chapters to new VyOS package build script --- README.md | 177 ++++++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 108 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index 655994c8..885f2fba 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ VyOS is not new. It is a fork of Vyatta Core that was created when the open source version of it was discontinued. If you are a Vyatta Core user, you can upgrade your installation to VyOS. -# What is this repository? +# About this repository VyOS is a GNU/Linux distribution based on Debian. Just like any other distribution, it consists of multiple packages. @@ -28,23 +28,34 @@ This is the top level repository that contains links to repositories with VyOS specific packages (organized as Git submodules) and scripts and data that are used for building those packages and the installation image. -# Structure of this repository +# Repository Structure There are several directories with their own purpose: - build/ Used for temporary files used for the build and for build artifacts - scripts/ Scripts that are used for the build process - data/ Data required for building the ISO (such as boot splash) - tools/ Scripts that are used for maintainer's tasks automation and other - purposes, but not during ISO build process + * `build/` Used for temporary files used for the build and for build artifacts + * `scripts/` Scripts that are used for the build process + * `data/` Data required for building the ISO (e.g. boot splash/configs) + * `tools/` Scripts that are used for maintainer's tasks automation and other + purposes, but not during ISO build process -# Building VyOS installation images +# Building installation images ## Prerequisites To build a VyOS 1.2.0 image, you need Debian 8 "Jessie" environment (with jessie-backports repository). +If you are working on a Debian Jessie machine, no special preparation is needed, +you only need to enable jessie-backports and install build dependencies. + +If you are interested which individual packages are required please check our +[Dockerfile](docker/Dockerfile) which holds the most complete documentation +of required packages. Listing individual packages here tend to be always +outdated. We try to list required packages in groups through their inheritance +in the [Dockerfile](docker/Dockerfile). + +### Debootstrap + If you do not have a Debian Jessie machine, you may create a chroot environment with the [debootstrap](https://wiki.debian.org/Debootstrap) tool. @@ -56,69 +67,19 @@ $ sudo apt-get install debootstrap $ sudo debootstrap jessie vyos-chroot $ sudo chroot vyos-chroot -$ echo "deb http://deb.debian.org/debian jessie-backports main" >> /etc/apt/sources.list +$ echo "deb http://archive.debian.org/debian/ jessie-backports main" >> /etc/apt/sources.list $ apt-get update ``` -If you are working on a Debian Jessie machine, no special preparation is needed, -you only need to enable jessie-backports and install build dependencies. An -up-to-date dependency list can be found in our [Dockerfile](docker/Dockerfile). - -Several packages are required for building the ISO: -* `python3` -* `live-build` -* `pbuilder` -* `python3-pystache` - -The `./configure` script will warn you if any dependencies are missing. Individual -packages may have other build dependencies. If some dependencies are missing, -package build scripts will tell you. - -## Building the ISO image - -Before you can build an image, you need to configure your build. - -To build an image, use the following commands: - -```bash -$ ./configure -$ make iso -``` - -The `./configure` script has a number of options that you can see by calling it -with `--help` - -## Building the images for virtualization platforms - -### QEMU - -Run following command after building the ISO image. - -```bash -$ make qemu -``` - -### VMware - -Run following command after building the QEMU image. - -```bash -$ make vmware -``` +**NOTE:** We recommend to use the Docker build method -# Building ISO or individual packages via Docker +### Docker Using our [Dockerfile](docker/Dockerfile) you create your own Docker container that is used to build a VyOS ISO image or other required VyOS packages. The [Dockerfile](docker/Dockerfile) contains some of the most used packages needed to build a VyOS ISO, a qemu image, and several of the submodules. -If you are interested which individual packages are required please check our -[Dockerfile](docker/Dockerfile) which holds the most complete documentation -of required packages. Listing individual packages here tend to be always -outdated. We try to list required packages in groups through their inheritance -in the [Dockerfile](docker/Dockerfile). - To build the docker image ensure you have a working [Docker](https://www.docker.com) environment and then run the following commands: @@ -152,16 +113,95 @@ Linux 948a2be7c52c 3.16.0-7-amd64 #1 SMP Debian 3.16.59-1 (2018-10-03) x86_64 GN After the Docker container is running you can follow up the instructions below in order to build the VyOS ISO image. +## Building the ISO image + +The `./configure` script will warn you if any dependencies are missing. Individual +packages may have other build dependencies. If some dependencies are missing, +package build scripts will tell you. + +```bash +$ ./configure --help +usage: configure [-h] [--build-by BUILD_BY] [--version VERSION] + [--pbuilder-debian-mirror PBUILDER_DEBIAN_MIRROR] + [--debian-security-mirror DEBIAN_SECURITY_MIRROR] + [--architecture ARCHITECTURE] [--vyos-mirror VYOS_MIRROR] + [--build-type BUILD_TYPE] [--debian-mirror DEBIAN_MIRROR] + [--debug] [--custom-apt-entry CUSTOM_APT_ENTRY] + [--custom-apt-key CUSTOM_APT_KEY] + [--custom-package CUSTOM_PACKAGE] + +optional arguments: + -h, --help show this help message and exit + --build-by BUILD_BY Builder identifier (e.g. jrandomhacker@example.net) + --version VERSION Version number (release builds only) + --pbuilder-debian-mirror PBUILDER_DEBIAN_MIRROR + Debian repository mirror for pbuilder env bootstrap + --debian-security-mirror DEBIAN_SECURITY_MIRROR + Debian security updated mirror + --architecture ARCHITECTURE + Image target architecture (amd64 or i386 or armhf) + --vyos-mirror VYOS_MIRROR + VyOS package mirror + --build-type BUILD_TYPE + Build type, release or development + --debian-mirror DEBIAN_MIRROR + Debian repository mirror for ISO build + --debug Enable debug output + --custom-apt-entry CUSTOM_APT_ENTRY + Custom APT entry + --custom-apt-key CUSTOM_APT_KEY + Custom APT key file + --custom-package CUSTOM_PACKAGE + Custom package to install from repositories +``` + +Before you can build an image, you need to configure your build. + +Each build needs to run the `./configure` step first where you can extend your +ISO by additional packages (`--custom-package`) or specify who build this nice +ISO image (`--build-by`). If you have local Debian mirrors, you can select them +by `--debian-mirror` or `--debian-security-mirror`. + +```bash +$ ./configure --custom-package vim --build-by jrandom@hacker.com +$ sudo make iso +``` + +After some time you will find the resulting ISO image in the `build` folder. + +### Building images for virtualization platforms + +#### QEMU + +Run following command after building the ISO image. + +```bash +$ make qemu +``` + +#### VMware + +Run following command after building the QEMU image. + +```bash +$ make vmware +``` + ## Building subpackages inside Docker +VyOS requires a bunch of packages which are VyOS specific and thus can not be +found in any Debian Upstream mirrror. Those packages can be found at the VyOS +GitHub project (https://github.com/vyos) and there is a nice helper script +available to build and list those individual packages. + [scripts/build-packages](scripts/build-packages) provides an easy interface -for automate the process of building all VyOS related packages that are not part +to automate the process of building all VyOS related packages that are not part of the upstream Debian version. Execute it in the root of the `vyos-build` directory to start compilation. ```bash -$ scripts/build-packages -h -usage: build-packages [-h] [-v] [-c] [-l] [-b BUILD [BUILD ...]] [-f] +$ scripts/build-packages -h +usage: build-packages [-h] [-v] [-c] [-l] [-b BUILD [BUILD ...]] [-f] [-p] optional arguments: -h, --help show this help message and exit @@ -171,6 +211,7 @@ optional arguments: -b BUILD [BUILD ...], --build BUILD [BUILD ...] Whitespace separated list of packages to build -f, --fetch Fetch sources only, no build + -p, --parallel Build on all CPUs ``` Git repositoriers are automatically fetched and build on demand. If you wan't to @@ -182,8 +223,7 @@ container, it includes all dependencies for compiling supported packages. ```bash $ docker run --rm -it -v $(pwd):/vyos -w /vyos \ --sysctl net.ipv6.conf.lo.disable_ipv6=0 \ - vyos-builder \ - ./scripts/build-packages + vyos-builder scripts/build-packages ``` **NOTE:** `--sysctl net.ipv6.conf.lo.disable_ipv6=0` is required to build the @@ -192,7 +232,7 @@ $ docker run --rm -it -v $(pwd):/vyos -w /vyos \ **NOTE:** Prior to executing this script you need to create or build the Docker container and checkout all packages you want to compile. -### Building a single package +### Building single package(s) The script above runs all package build inside the Docker container, this is also possible to do by hand using: @@ -202,8 +242,7 @@ Executed from the root of `vyos-build` ```bash $ docker run --rm -it -v $(pwd):/vyos -w /vyos/packages/PACKAGENAME \ --sysctl net.ipv6.conf.lo.disable_ipv6=0 \ - vyos-builder \ - ./scripts/build-packages -b + vyos-builder scripts/build-packages -b ``` **NOTE:** `--sysctl net.ipv6.conf.lo.disable_ipv6=0` is only needed when -- cgit v1.2.3 From bda5855952a59fa654e54c77b468375e35a898ed Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Fri, 26 Apr 2019 11:54:44 +0200 Subject: Docker: Use close sections when working with jessie-backports Making the jessie-backports repository globally available might cause problems when installing packages which have mixed versions. This was observed with the following packages which could not be installed when jessie-backports was listed always in /etc/apt/sources.list * libnl-3-200 * libnl-3-dev * libnl-genl-3-200 * libnl-genl-3-dev --- docker/Dockerfile | 40 ++++++++++++++++++++++++++++++---------- docker/sources.list | 4 ---- 2 files changed, 30 insertions(+), 14 deletions(-) delete mode 100644 docker/sources.list diff --git a/docker/Dockerfile b/docker/Dockerfile index 21b38417..ebb998c1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,12 +21,6 @@ LABEL authors="VyOS Maintainers " ENV DEBIAN_FRONTEND noninteractive -COPY sources.list /etc/apt/sources.list -COPY no--check-valid-until /etc/apt/apt.conf.d/ -COPY vyos-dev.key /tmp/vyos-dev.key - -RUN apt-key add /tmp/vyos-dev.key - # Standard shell should be bash not dash RUN echo "dash dash/sh boolean false" | debconf-set-selections && \ dpkg-reconfigure dash @@ -73,11 +67,28 @@ RUN apt-get update && apt-get install -y \ debhelper \ jq +# +# Setup Debian Jessie Backports repository +# +COPY no--check-valid-until /etc/apt/apt.conf.d/ +RUN echo "deb http://archive.debian.org/debian/ jessie-backports main" \ + > /etc/apt/sources.list.d/jessie-backports.list + RUN apt-get update && apt-get install -y -t jessie-backports \ python3-git \ python3-pip \ gosu +# Package needed for mdns-repeater +RUN apt-get update && apt-get install -y -t jessie-backports \ + dh-systemd + +# +# Remove Debian Jessie Backports repository +# +RUN rm -f /etc/apt/sources.list.d/jessie-backports.list \ + /etc/apt/apt.conf.d/no--check-valid-until + # # Building libvyosconf requires a full configured OPAM/OCaml setup # @@ -124,10 +135,6 @@ RUN eval $(opam env --root=/opt/opam --set-root) && \ dpkg-buildpackage -uc -us -tc -b && \ dpkg -i ../libvyosconfig0_*_amd64.deb -# Package needed for mdns-repeater -RUN apt-get update && apt-get install -y -t jessie-backports \ - dh-systemd - # Packages needed for vyatta-cfg RUN apt-get update && apt-get install -y \ libglib2.0-dev \ @@ -188,6 +195,14 @@ RUN export RTRLIB_VERSION="0.6.3" && \ cd /tmp/rtrlib-${RTRLIB_VERSION} && dpkg-buildpackage -uc -us -tc -b && \ dpkg -i ../librtr*_amd64.deb ../librtr*_all.deb +# +# Setup VyOS Debian repository +# +COPY vyos-dev.key /tmp/vyos-dev.key +RUN apt-key add /tmp/vyos-dev.key +RUN echo "deb http://dev.packages.vyos.net/repositories/current/debian/ current main" \ + > /etc/apt/sources.list.d/vyos.list + # Packages needed to build frr itself # libyang-dev packages are hsoted on dev.packages.vyos.net see # https://github.com/FRRouting/frr/blob/master/doc/developer/building-libyang.rst @@ -203,6 +218,11 @@ RUN apt-get update && apt-get install -y \ python3-pytest \ texinfo +# +# Cleanup VyOS Debian Repository +# +RUN rm -f /etc/apt/sources.list.d/vyos.list + # Packages needed for conntrack-tools RUN apt-get update && apt-get install -y \ libnetfilter-conntrack-dev \ diff --git a/docker/sources.list b/docker/sources.list deleted file mode 100644 index ec827988..00000000 --- a/docker/sources.list +++ /dev/null @@ -1,4 +0,0 @@ -deb http://deb.debian.org/debian/ jessie main -deb http://deb.debian.org/debian-security/ jessie/updates main -deb http://archive.debian.org/debian/ jessie-backports main -deb http://dev.packages.vyos.net/repositories/current/debian/ current main -- cgit v1.2.3 From 226a25dfdf9c7c21da405948e9de955388898d9c Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Fri, 26 Apr 2019 11:57:24 +0200 Subject: Docker: add package requirements to build keepalived --- docker/Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index ebb998c1..16a9bd62 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -292,6 +292,13 @@ RUN apt-get update && apt-get install -y \ RUN apt-get update && apt-get install -y \ xmlto +# Packages needed for keepalived +RUN apt-get update && apt-get install -y \ + libnl-3-200 \ + libnl-3-dev \ + libnl-genl-3-200 \ + libnl-genl-3-dev + # Update live-build RUN echo 'deb http://ftp.debian.org/debian stretch main' | tee -a /etc/apt/sources.list.d/stretch.list && \ apt-get update && apt-get install -y -t stretch \ -- cgit v1.2.3 From 04334e26fbd693b4081048ccf5a244d282d4bc10 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Fri, 26 Apr 2019 12:03:19 +0200 Subject: Packages: fix missing bytes stuck in the output buffer A last read() was missing to the temporary files when reporting errors after a command was executed. --- scripts/build-packages | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/build-packages b/scripts/build-packages index 0677e9f7..98b7ab48 100755 --- a/scripts/build-packages +++ b/scripts/build-packages @@ -82,6 +82,11 @@ def call(bashCommand, log, **kwargs): else: log.info(lines.decode().rstrip('\n')) + + # A last write needed after subprocess ends + log.debug(output.read().decode().rstrip('\n')) + log.info(error.read().decode().rstrip('\n')) + error.close() output.close() return sub.returncode -- cgit v1.2.3 From 6732344471534de53519d1dd35122d9b17d85011 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Fri, 26 Apr 2019 12:14:52 +0200 Subject: Packages: support --blacklist option to not build individual packages --- scripts/build-packages | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/scripts/build-packages b/scripts/build-packages index 98b7ab48..3917e84d 100755 --- a/scripts/build-packages +++ b/scripts/build-packages @@ -272,6 +272,7 @@ if __name__ == '__main__': parser.add_argument('-b', '--build', nargs='+', help='Whitespace separated list of packages to build') parser.add_argument('-f', '--fetch', action='store_true', help='Fetch sources only, no build') parser.add_argument('-p', '--parallel', action='store_true', help='Build on all CPUs') + parser.add_argument('--blacklist', nargs='+', help='Do not build/report packages when calling --list') args = parser.parse_args() @@ -282,6 +283,31 @@ if __name__ == '__main__': print("Using vyos-build repository ('{}') commit '{}'\n".format(repo_root, repo_sha)) + # + # Exclude packages from build process, + # also do not list them on --list + # + if args.blacklist: + for exclude in args.blacklist: + if exclude in vyos_packages: + vyos_packages.remove(exclude) + continue + + found = False + for pkg in pkg_special: + if exclude == pkg['name']: + found = True + # package already formed + pkg_special.remove(pkg) + break + + if not found: + print("Invalid choice '" + exclude + "', --list-packages for complete list!") + sys.exit(1) + + # + # List all available (to be build) packages + # if args.list_packages: print("Individual packages available for build:") for pkg in vyos_packages: -- cgit v1.2.3 From 9c7483c79f1d364d6f7666eb5af5da761be126e0 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Fri, 26 Apr 2019 12:50:50 +0200 Subject: Docker: fix pmacct build dependencies Debian Jessie Backports ships default-libmysqlclient-dev but Debian Jessie only provides libmysqlclient-dev. --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 16a9bd62..beb88053 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -156,7 +156,7 @@ RUN apt-get update && apt-get install -y \ RUN apt-get update && apt-get install -y \ libpcap-dev \ libpq-dev \ - default-libmysqlclient-dev \ + libmysqlclient-dev \ libgeoip-dev \ librabbitmq-dev \ libjansson-dev \ -- cgit v1.2.3