Added the option to include files before and after package installation

Moved includes.chroot to includes.chroot_after_packages and added
includes.chroot_before_packages. includes.chroot does still work as before.

We also now use rsync for copying files if it is installed.
This improves runtime and space consumption for large includes.

Gbp-Dch: Short
Closes: #927128

9 files changed, 121 insertions, 17 deletions

diff --git a/debian/control b/debian/control
index ba835523d..1a7fd33b9 100644
--- a/debian/control
+++ b/debian/control
@@ -27,6 +27,7 @@ Recommends:
  live-boot-doc,
  live-config-doc,
  live-manual-html | live-manual,
+ rsync,
  systemd-container,
  wget,
  xz-utils,
diff --git a/functions/chroot.sh b/functions/chroot.sh
index 76733e50e..d62a4fad0 100755
--- a/functions/chroot.sh
+++ b/functions/chroot.sh
@@ -56,3 +56,23 @@ Chroot_package_list() {
 
 	dpkg-query --admindir=${CHROOT}/var/lib/dpkg -W -f'${Package}\n'
 }
+
+Chroot_copy_dir() {
+	local DIR="${1}"
+	local NAME="${2:-$(basename ${DIR})}"
+
+	Check_installed host /usr/bin/rsync rsync
+	if [ "${INSTALL_STATUS}" -eq "0" ]
+	then
+		Echo_message "Copying ${NAME} into chroot using rsync..."
+		rsync -Klrv --chown=0:0 "${DIR}" chroot/
+	else
+		cd "${DIR}"
+		Echo_message "Creating a tarball with files from ${NAME}..."
+		tar cf "${OLDPWD}"/chroot/"${NAME}".tar .
+		cd "${OLDPWD}"
+		Echo_message "Extracting the tarball in the chroot..."
+		Chroot chroot "tar -xvf ${NAME}.tar --no-same-owner --keep-directory-symlink --overwrite"
+		rm chroot/"${NAME}".tar
+	fi
+}
diff --git a/functions/compatibility.sh b/functions/compatibility.sh
new file mode 100755
index 000000000..f0ae08346
--- /dev/null
+++ b/functions/compatibility.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+## live-build(7) - System Build Scripts
+## Copyright (C) 2016-2020 The Debian Live team
+## Copyright (C) 2006-2015 Daniel Baumann <mail@daniel-baumann.ch>
+##
+## This program comes with ABSOLUTELY NO WARRANTY; for details see COPYING.
+## This is free software, and you are welcome to redistribute it
+## under certain conditions; see COPYING for details.
+
+# We've added the option to also include files before packages renamed the old directory.
+# This function auto detects which version should be used.
+select_includes_chroot(){
+    local OLD_DIR="includes.chroot"
+    local NEW_DIR="includes.chroot_after_packages"
+
+    if Find_files "config/${NEW_DIR}/" && Find_files "config/${OLD_DIR}/"
+    then
+        Echo_error "You have files in ${OLD_DIR} and ${NEW_DIR}. Only one directory is allowed."
+        exit 1
+    fi
+
+    if Find_files "config/${NEW_DIR}/"
+    then
+        echo -n "${NEW_DIR}"
+    elif Find_files "config/${OLD_DIR}/"
+    then
+        echo -n "${OLD_DIR}"
+    fi
+}
diff --git a/scripts/build/chroot b/scripts/build/chroot
index 497911800..a0aa10be0 100755
--- a/scripts/build/chroot
+++ b/scripts/build/chroot
@@ -37,6 +37,7 @@ lb chroot_prep install all mode-archives-chroot "${@}"
 lb chroot_linux-image "${@}"
 lb chroot_firmware "${@}"
 lb chroot_preseed "${@}"
+lb chroot_includes_before_packages "${@}"
 
 for _PASS in install live; do
 	lb chroot_package-lists ${_PASS} "${@}"
@@ -47,7 +48,7 @@ for _PASS in install live; do
 	fi
 done
 
-lb chroot_includes "${@}"
+lb chroot_includes_after_packages "${@}"
 lb chroot_hooks "${@}"
 lb chroot_hacks "${@}"
 lb chroot_interactive "${@}"
diff --git a/scripts/build/chroot_hostname b/scripts/build/chroot_hostname
index 2b7f3920f..d2faeca8b 100755
--- a/scripts/build/chroot_hostname
+++ b/scripts/build/chroot_hostname
@@ -67,9 +67,15 @@ EOF
 		Acquire_lockfile
 
 		# Copying hostname from includes, if existing
-		if [ -e config/includes.chroot/etc/hostname ]
+		INCLUDE_DIR=$(select_includes_chroot)
+		# includes.chroot_before_packages takes the highest precedence
+		if [ -e config/includes.chroot_before_packages/etc/hostname ]
 		then
-			cp -a config/includes.chroot/etc/hostname chroot/etc/hostname
+			INCLUDE_DIR="includes.chroot_before_packages"
+		fi
+		if [ -n "${INCLUDE_DIR}" ] && [ -e config/"${INCLUDE_DIR}"/etc/hostname ]
+		then
+			cp -a config/"${INCLUDE_DIR}"/etc/hostname chroot/etc/hostname
 		fi
 
 		Echo_message "Deconfiguring file /bin/hostname"
diff --git a/scripts/build/chroot_includes b/scripts/build/chroot_includes_after_packages
index dc4bb6993..f4ca0bb2c 100755
--- a/scripts/build/chroot_includes
+++ b/scripts/build/chroot_includes_after_packages
@@ -15,13 +15,13 @@ set -e
 [ -e "${LIVE_BUILD}/scripts/build.sh" ] && . "${LIVE_BUILD}/scripts/build.sh" || . /usr/lib/live/build.sh
 
 # Setting static variables
-DESCRIPTION="Copy files into chroot"
+DESCRIPTION="Copy files into chroot after packages have been installed"
 USAGE="${PROGRAM} [--force]"
 
 # Processing arguments and configuration files
 Init_config_data "${@}"
 
-Echo_message "Begin copying chroot includes..."
+Echo_message "Begin copying chroot includes after packages have been installed..."
 
 # Requiring stage file
 Require_stagefiles config bootstrap
@@ -32,16 +32,13 @@ Check_stagefile
 # Acquire lock file
 Acquire_lockfile
 
-if Find_files config/includes.chroot/
+# Handleling backwards compatibility
+INCLUDE_DIR=$(select_includes_chroot)
+
+if [ -n "${INCLUDE_DIR}" ]
 then
-	# Copying includes
-	cd config/includes.chroot
-	Echo_message "Creating a tarball with files from includes.chroot..."
-	tar cf "${OLDPWD}"/chroot/includes.chroot.tar .
-	cd "${OLDPWD}"
-	Echo_message "Extracting the tarball in the chroot..."
-	Chroot chroot "tar -xvf includes.chroot.tar --no-same-owner --keep-directory-symlink --overwrite"
-	rm chroot/includes.chroot.tar
+	# Copying includes into chroot
+	Chroot_copy_dir config/"${INCLUDE_DIR}"/
 fi
 
 # Creating stage file
diff --git a/scripts/build/chroot_includes_before_packages b/scripts/build/chroot_includes_before_packages
new file mode 100755
index 000000000..5a2a592f3
--- /dev/null
+++ b/scripts/build/chroot_includes_before_packages
@@ -0,0 +1,42 @@
+#!/bin/sh
+
+## live-build(7) - System Build Scripts
+## Copyright (C) 2016-2020 The Debian Live team
+## Copyright (C) 2006-2015 Daniel Baumann <mail@daniel-baumann.ch>
+##
+## This program comes with ABSOLUTELY NO WARRANTY; for details see COPYING.
+## This is free software, and you are welcome to redistribute it
+## under certain conditions; see COPYING for details.
+
+
+set -e
+
+# Including common functions
+[ -e "${LIVE_BUILD}/scripts/build.sh" ] && . "${LIVE_BUILD}/scripts/build.sh" || . /usr/lib/live/build.sh
+
+# Setting static variables
+DESCRIPTION="Copy files into chroot before packages have been installed"
+USAGE="${PROGRAM} [--force]"
+
+# Processing arguments and configuration files
+Init_config_data "${@}"
+
+Echo_message "Begin copying chroot includes before packages have been installed..."
+
+# Requiring stage file
+Require_stagefiles config bootstrap
+
+# Checking stage file
+Check_stagefile
+
+# Acquire lock file
+Acquire_lockfile
+
+if Find_files config/includes.chroot_before_packages/
+then
+	# Copying includes into chroot
+	Chroot_copy_dir config/includes.chroot_before_packages/
+fi
+
+# Creating stage file
+Create_stagefile
diff --git a/scripts/build/chroot_resolv b/scripts/build/chroot_resolv
index c943f259d..49cb978ee 100755
--- a/scripts/build/chroot_resolv
+++ b/scripts/build/chroot_resolv
@@ -75,10 +75,16 @@ case "${_ACTION}" in
 		# Acquire lock file
 		Acquire_lockfile
 
-		if [ -e config/includes.chroot/etc/resolv.conf ]
+		INCLUDE_DIR=$(select_includes_chroot)
+		# includes.chroot_before_packages takes the highest precedence
+		if [ -e config/includes.chroot_before_packages/etc/resolv.conf ]
+		then
+			INCLUDE_DIR="includes.chroot_before_packages"
+		fi
+		if [ -n "${INCLUDE_DIR}" ] && [ -e config/"${INCLUDE_DIR}"/etc/resolv.conf ]
 		then
 			# Copying local resolv.conf
-			cp config/includes.chroot/etc/resolv.conf chroot/etc/resolv.conf
+			cp config/"${INCLUDE_DIR}"/etc/resolv.conf chroot/etc/resolv.conf
 			chmod 644 chroot/etc/resolv.conf
 			Chroot chroot "chown root:root /etc/resolv.conf"
 			rm -f chroot/etc/resolv.conf.orig
diff --git a/scripts/build/config b/scripts/build/config
index 4f2a53538..a5e578c3e 100755
--- a/scripts/build/config
+++ b/scripts/build/config
@@ -1312,7 +1312,8 @@ mkdir -p config/hooks/normal
 mkdir -p config/includes
 mkdir -p config/includes.binary
 mkdir -p config/includes.bootstrap
-mkdir -p config/includes.chroot
+mkdir -p config/includes.chroot_after_packages
+mkdir -p config/includes.chroot_before_packages
 mkdir -p config/includes.installer
 mkdir -p config/includes.source
 mkdir -p config/package-lists