linux-firmware: T2594: upgrade firmware extraction script

The logic of generating the required firmware file name has been moved from
build-linux-firmware.sh directly into list-required-firmware.py which makes
much more sense. That move was made by Daniil Baturin - I only did the
integration part. Thanks!

2 files changed, 80 insertions, 51 deletions

diff --git a/packages/linux-kernel/build-linux-firmware.sh b/packages/linux-kernel/build-linux-firmware.sh
index 3194aa2c..37a3768a 100755
--- a/packages/linux-kernel/build-linux-firmware.sh
+++ b/packages/linux-kernel/build-linux-firmware.sh
@@ -30,15 +30,8 @@ fi
 
 result=()
 # Retrieve firmware blobs from source files
-for FILE in $(${CWD}/list-required-firmware.py -k ${LINUX_SRC} -c ${CWD}/x86_64_vyos_defconfig -s drivers/net -s drivers/usb); do
-    cd ${CWD}/${LINUX_SRC}
-    echo "I: determine required firmware blobs for: ${FILE}"
-    make LOCALVERSION=${KERNEL_SUFFIX} ${FILE/.c/.i} > /dev/null 2>&1
-
-    if [ "$?" == "0" ]; then
-        result+=( $(grep UNIQUE_ID_firmware ${FILE/.c/.i} | cut -d" " -f12- | xargs printf "%s" | sed -e "s/;/ /g") )
-    fi
-done
+cd ${LINUX_SRC}
+FW_FILES=$(../list-required-firmware.py -c ../x86_64_vyos_defconfig -s drivers/net 2>/dev/null)
 
 # Debian package will use the descriptive Git commit as version
 GIT_COMMIT=$(cd ${CWD}/${LINUX_FIRMWARE}; git describe --always)
@@ -53,11 +46,10 @@ mkdir -p ${VYOS_FIRMWARE_DIR}
 # Copy firmware file from linux firmware repository into
 # assembly folder for the vyos-firmware package
 SED_REPLACE="s@${CWD}/${LINUX_FIRMWARE}/@@"
-for FW in ${result[@]}; do
-    FW_FILE=$(basename $FW)
-
+for FW_PATH in ${FW_FILES}; do
+    FW_FILE=$(basename $FW_PATH)
     res=()
-    for tmp in $(find ${CWD}/linux-firmware -type f -name ${FW_FILE} | sed -e ${SED_REPLACE} )
+    for tmp in $(find ${CWD}/linux-firmware -type f -name ${FW_FILE} | sed -e ${SED_REPLACE})
     do
         res+=( "$tmp" )
     done
diff --git a/packages/linux-kernel/list-required-firmware.py b/packages/linux-kernel/list-required-firmware.py
index bbb8c7d7..1cbf7b8e 100755
--- a/packages/linux-kernel/list-required-firmware.py
+++ b/packages/linux-kernel/list-required-firmware.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
-# Copyright (C) 2020 VyOS maintainers and contributors
+#
+# Copyright (C) 2020 Daniil Baturin
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -14,83 +15,119 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-import argparse
 import re
 import os
 import sys
 import glob
+import argparse
+import subprocess
 
+# Loads the kernel config -- only options set to y or m
 def load_config(path):
     with open(path, 'r') as f:
         config = f.read()
     targets = re.findall(r'(.*)=(?:y|m)', config)
     return targets
 
-def find_subdirs(config, path):
+# Finds subdir targets from the Makefile
+# that are enabled by the kernel build config
+def find_enabled_subdirs(config, makefile_path):
     try:
-        with open(os.path.join(path, 'Makefile'), 'r') as f:
+        with open(makefile_path, 'r') as f:
             makefile = f.read()
     except OSError:
-        # No Makefile
+        # Shouldn't happen due to the way collect_source_files()
+        # calls this function.
         return []
 
-    dir_stmts = re.findall(r'obj-\$\((.*)\)\s+\+=\s+(.*)(?:\n|$)', makefile)
+    dir_stmts = re.findall(r'obj-\$\((.*)\)\s+\+=\s+(.*)/(?:\n|$)', makefile)
     subdirs = []
 
     for ds in dir_stmts:
+        config_key, src_dir = ds
+
         if args.debug:
             print("Processing make targets from {0} ({1})".format(ds[1], ds[0]), file=sys.stderr)
-        if ds[0] in config:
-            dirname = os.path.dirname(ds[1])
-            if dirname:
-                subdirs.append(dirname)
+        if config_key in config:
+            subdirs.append(src_dir)
         elif args.debug:
             print("{0} is disabled in the config, ignoring {1}".format(ds[0], ds[1]), file=sys.stderr)
 
     return subdirs
 
-
-def find_firmware(file):
+# For filtering
+def file_loads_firmware(file):
     with open(file, 'r') as f:
         source = f.read()
-    fws = re.findall(r'MODULE_FIRMWARE\((.*)\)', source)
-    return fws
+    if re.search(r'MODULE_FIRMWARE\((.*)\)', source):
+        return True
 
-def walk_dir(config, path):
-    subdirs = find_subdirs(config, path)
+# Find all source files that reference firmware
+def collect_source_files(config, path):
+    files = []
 
-    if args.debug:
-        print("Looking for C files in {0}".format(path), file=sys.stderr)
-    c_files = glob.glob("{0}/*.c".format(path))
+    makefile = os.path.join(path, "Makefile")
 
-    for cf in c_files:
-        fws = find_firmware(cf)
-        if fws:
-            print(cf)
-            if args.debug:
-                print("Referenced firmware: {0}".format(fws))
+    # Find and process all C files in this directory
+    # This is a compromise: sometimes there are single-file modules,
+    # that in fact may be disabled in the config,
+    # so this approach can create occasional false positives.
+    c_files = glob.glob("{0}/*.c".format(path))
+    files = list(filter(file_loads_firmware, c_files))
 
+    # Now walk the subdirectories
+    enabled_subdirs = find_enabled_subdirs(config, makefile)
+    subdirs = glob.glob("{0}/*/".format(path))
     for d in subdirs:
-        d = os.path.join(path, d)
-        walk_dir(config, d)
+        dir_name = d.rstrip("/")
+
+        if os.path.exists(os.path.join(d, "Makefile")):
+            # If there's a makefile, it's an independent module
+            # or a high level dir
+            if os.path.basename(dir_name) in enabled_subdirs:
+                files = files + collect_source_files(config, d)
+        else:
+            # It's simply a subdirectory of the current module
+            # Some modules, like iwlwifi, keep their firmware-loading files
+            # in subdirs, so we have to handle this case
+            c_files = glob.iglob("{0}/**/*.c".format(d), recursive=True)
+            files += list(filter(file_loads_firmware, c_files))
+
+    return files
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument("-s", "--source-dir", action="append", help="Kernel source directory")
-    parser.add_argument("-k", "--kernel-dir", action="store", help="Kernel source directory")
+    parser.add_argument("-s", "--source-dir", action="append", help="Kernel source directory to process", required=True)
     parser.add_argument("-c", "--kernel-config", action="store", help="Kernel configuration")
     parser.add_argument("-d", "--debug", action="store_true", help="Enable Debug output")
+    parser.add_argument("-f", "--list-source-files", action="store_true", help="List source files that reference firmware and exit")
     args = parser.parse_args()
 
-    if args.source_dir and args.kernel_dir and args.kernel_config:
-        config = load_config(args.kernel_config)
+    if not args.kernel_config:
+        args.kernel_config = ".config"
 
-        cwd = os.getcwd()
-        os.chdir(f'{cwd}/{args.kernel_dir}')
-        for directory in args.source_dir:
-            walk_dir(config, directory)
+    config = load_config(args.kernel_config)
 
-    else:
-        parser.print_help()
-        sys.exit(1)
+    # Collect source files that reference firmware
+    for directory in args.source_dir:
+        source_files = collect_source_files(config, directory)
 
+    if args.list_source_files:
+        for sf in source_files:
+            print(sf)
+    else:
+        fw_files = []
+        for sf in source_files:
+            i_file = re.sub(r'\.c', r'.i', sf)
+            res = subprocess.run(["make {0} 2>&1".format(i_file)], shell=True, capture_output=True)
+            if res.returncode != 0:
+                print("Failed to preprocess file {0}".format(sf), file=sys.stderr)
+                print(res.stdout.decode(), file=sys.stderr)
+            else:
+                with open(i_file, 'r') as f:
+                    source = f.read()
+                    fw_statements = re.findall(r'__UNIQUE_ID_firmware.*"firmware"\s+"="\s+(.*);', source)
+                    fw_files += list(map(lambda s: re.sub(r'(\s|")', r'', s), fw_statements))
+
+        for fw in fw_files:
+            print(fw)