From 7867641fd0e23c290a5ae2b8f04f685948b917e3 Mon Sep 17 00:00:00 2001 From: Lyndon Brown Date: Wed, 12 Feb 2020 05:00:08 +0000 Subject: firmware: save the compressed contents file to disk instead of decompressed the existing logic was to decompress the contents file from the downloaded archive to disk, then process it to obtain a package list. the largest one by far is for 'main'; 'non-free' and 'contrib' are tiny in comparison. for sid-amd64 currently, the archive file is 37 MB, while the decompressed file it contains is 592.3 MB. we always delete the files and download afresh (currently), and a previous commit optimised by deleting the files once we're done with them to avoid wasting disk space leaving them behind. here we switch to storing the downloaded compressed file to disk instead, reducing disk space usage (and IO) by hundreds of megabytes; piping the decompression directly into awk instead of having awk read from the stored file. this moves the appending of new items into the list back within the archive area loop, which is fine since we're replacing the file for each loop now so the previous issue relating to appending is of no concern. Gbp-Dch: Short Closes: #952910 --- functions/firmwarelists.sh | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/functions/firmwarelists.sh b/functions/firmwarelists.sh index 5c552fc98..3c1f9a936 100755 --- a/functions/firmwarelists.sh +++ b/functions/firmwarelists.sh @@ -18,20 +18,21 @@ Firmware_List_From_Contents () { local DISTRO_CHROOT="${2}" local ARCHIVE_AREAS="${3}" - local CONTENTS_FILE="cache/contents.chroot/contents.${DISTRO_CHROOT}.${LB_ARCHITECTURES}" - - # Ensure fresh - rm -f "${CONTENTS_FILE}" - for _ARCHIVE_AREA in ${ARCHIVE_AREAS} do + local CONTENTS_FILE="cache/contents.chroot/contents.${DISTRO_CHROOT}.${LB_ARCHITECTURES}.gz" local CONTENTS_URL="${MIRROR_CHROOT}/dists/${DISTRO_CHROOT}/${_ARCHIVE_AREA}/Contents-${LB_ARCHITECTURES}.gz" - wget ${WGET_OPTIONS} "${CONTENTS_URL}" -O - | gunzip -c >> "${CONTENTS_FILE}" - done + # Ensure fresh + rm -f "${CONTENTS_FILE}" - FIRMWARE_PACKAGES="${FIRMWARE_PACKAGES} $(awk '/^lib\/firmware/ { print $2 }' "${CONTENTS_FILE}" | sort -u)" + wget ${WGET_OPTIONS} "${CONTENTS_URL}" -O "${CONTENTS_FILE}" - # Don't waste disk space preserving since always getting fresh - rm -f "${CONTENTS_FILE}" + local PACKAGES + PACKAGES="$(gunzip -c "${CONTENTS_FILE}" | awk '/^lib\/firmware/ { print $2 }' | sort -u )" + FIRMWARE_PACKAGES="${FIRMWARE_PACKAGES} ${PACKAGES}" + + # Don't waste disk space preserving since always getting fresh + rm -f "${CONTENTS_FILE}" + done } -- cgit v1.2.3