added private gnupg keyring for circinus build and also added persistent data directory

7 files changed, 87 insertions, 57 deletions

diff --git a/new/data/.gitignore b/new/data/.gitignore
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/new/data/.gitignore
diff --git a/new/image_builder.py b/new/image_builder.py
index cb5f879..4914093 100755
--- a/new/image_builder.py
+++ b/new/image_builder.py
@@ -16,7 +16,7 @@ import pendulum
 from lib.debranding import Debranding
 from lib.docker import Docker
 from lib.git import Git
-from lib.helpers import setup_logging, refuse_root, project_dir, get_my_log_file
+from lib.helpers import setup_logging, refuse_root, get_my_log_file, apt_dir, build_dir
 
 
 class ImageBuilder:
@@ -53,7 +53,7 @@ class ImageBuilder:
             vyos_mirror = self.vyos_mirror
             logging.info("Using supplied APT repository at %s" % vyos_mirror)
 
-        self.vyos_build_repo = os.path.join(project_dir, "build", "%s-image-build" % self.branch)
+        self.vyos_build_repo = os.path.join(build_dir, "%s-image-build" % self.branch)
 
         logging.info("Pulling vyos-build docker image")
         self.docker = Docker(self.vyos_build_docker, self.branch, self.vyos_build_repo)
@@ -108,7 +108,7 @@ class ImageBuilder:
 
         extra_mounts = []
         if self.vyos_mirror == "local":
-            apt_key_path = os.path.join(project_dir, "apt", "apt.gpg.key")
+            apt_key_path = os.path.join(apt_dir, "apt.gpg.key")
             extra_mounts.append((apt_key_path, "/opt/apt.gpg.key"))
 
         self.docker.run(
@@ -119,21 +119,21 @@ class ImageBuilder:
         )
 
         image_path = None
-        build_dir = os.path.join(self.vyos_build_repo, "build")
-        if os.path.exists(build_dir):
-            for entry in os.scandir(build_dir):
+        my_build_dir = os.path.join(self.vyos_build_repo, "build")
+        if os.path.exists(my_build_dir):
+            for entry in os.scandir(my_build_dir):
                 if version in entry.name and entry.name.endswith(".iso"):
                     image_path = entry.path
                     break
 
         if image_path is None:
-            image_path = os.path.join(build_dir, "live-image-amd64.hybrid.iso")
+            image_path = os.path.join(my_build_dir, "live-image-amd64.hybrid.iso")
 
         if not os.path.exists(image_path):
             logging.error(
                 "Build failed (image not found), see log above for reason why"
                 ", inspect build here: %s"
-                ", log file: %s" % (build_dir, get_my_log_file())
+                ", log file: %s" % (my_build_dir, get_my_log_file())
             )
             exit(1)
 
@@ -194,7 +194,7 @@ class ImageBuilder:
 
 class AptWebServerHandler(SimpleHTTPRequestHandler):
     def __init__(self, *args, **kwargs):
-        super().__init__(*args, directory=os.path.join(project_dir, "apt"), **kwargs)
+        super().__init__(*args, directory=os.path.join(apt_dir), **kwargs)
 
     def log_message(self, format, *args):
         pass
diff --git a/new/lib/apt.py b/new/lib/apt.py
index 5e2e60d..33bacb8 100644
--- a/new/lib/apt.py
+++ b/new/lib/apt.py
@@ -1,19 +1,18 @@
 import logging
 import os
-from pathlib import Path
 import re
 from shlex import quote
 
-from lib.helpers import quote_all, execute
+from lib.helpers import quote_all, execute, data_dir, resources_dir, apt_dir
 
 
 class Apt:
     _repo_dir = None
 
-    def __init__(self, project_dir, branch, directory):
-        self.project_dir: str = project_dir
+    def __init__(self, branch, directory):
         self.branch = branch
         self.directory = directory
+        self.gpg_keyring_path = os.path.join(data_dir, ".gnupg")
 
     def scan_for_dist_files(self, directory):
         dsc_files = []
@@ -46,20 +45,20 @@ class Apt:
         return dsc_files, binary_files
 
     def initialize_repository(self):
-        gpg_keyring_path = os.path.join(Path.home(), ".gnupg/pubring.kbx")
-        if not os.path.exists(gpg_keyring_path):
+        pub_keyring_path = os.path.join(self.gpg_keyring_path, "pubring.kbx")
+        if not os.path.exists(pub_keyring_path):
             logging.info("Generating GPG singing key")
-            material_path = os.path.join(self.project_dir, "resources", "gpg-gen-key.txt")
-            execute("gpg --batch --gen-key < %s" % quote_all(material_path))
+            material_path = os.path.join(resources_dir, "gpg-gen-key.txt")
+            execute("gpg --homedir %s --batch --gen-key < %s" % quote_all(self.gpg_keyring_path, material_path))
 
-        conf_dir = os.path.join(self.project_dir, "apt", self.branch, "conf")
+        conf_dir = os.path.join(apt_dir, self.branch, "conf")
         if not os.path.exists(conf_dir):
             logging.info("Initializing APT repository")
             os.makedirs(conf_dir)
 
         dist_path = os.path.join(conf_dir, "distributions")
         if not os.path.exists(dist_path):
-            material_path = os.path.join(self.project_dir, "resources", "apt-distributions.txt")
+            material_path = os.path.join(resources_dir, "apt-distributions.txt")
             with open(material_path, "r") as file:
                 contents = file.read()
                 contents = contents.replace("%branch%", self.branch)
@@ -70,7 +69,7 @@ class Apt:
 
         options_path = os.path.join(conf_dir, "options")
         if not os.path.exists(options_path):
-            material_path = os.path.join(self.project_dir, "resources", "apt-options.txt")
+            material_path = os.path.join(resources_dir, "apt-options.txt")
             with open(material_path, "r") as file:
                 contents = file.read()
 
@@ -82,8 +81,8 @@ class Apt:
 
         pub_key_path = os.path.join(root_dir, "apt.gpg.key")
         if not os.path.exists(pub_key_path):
-            execute("gpg --armor --output %s --export-options export-minimal --export %s" % (
-                pub_key_path, self.get_key_id()
+            execute("gpg --homedir %s --armor --output %s --export-options export-minimal --export %s" % (
+                self.gpg_keyring_path, pub_key_path, self.get_key_id()
             ))
 
         return repo_dir
@@ -94,7 +93,7 @@ class Apt:
         return self._repo_dir
 
     def get_key_id(self):
-        output = execute("gpg --list-keys --keyid-format=long signing@not-vyos")
+        output = execute("gpg --homedir %s --list-keys --keyid-format=long signing@not-vyos" % self.gpg_keyring_path)
         we_in_pub = False
         key_id = None
         for line in output.split("\n"):
@@ -119,36 +118,64 @@ class Apt:
         prefix_len = len(self.directory)
 
         for dsc_file in dsc_files:
-            logging.info("Pushing %s to the APT repository" % dsc_file[prefix_len:])
-
             with open(dsc_file, "r") as file:
                 fields = self.parse_package_info(file.read(), dsc_file, ["Source"])
 
             package = fields["Source"]
-            execute("reprepro -v -b %s removesrc %s %s" % quote_all(repo_dir, self.branch, package))
-            execute("reprepro -v -b %s includedsc %s %s" % quote_all(repo_dir, self.branch, dsc_file))
 
-        for binary_file in binary_files:
-            logging.info("Pushing %s to the APT repository" % binary_file[prefix_len:])
+            logging.info("Removing sources of %s from the APT repository" % package)
+
+            execute("reprepro --gnupghome %s  -v -b %s removesrc %s %s" % quote_all(
+                self.gpg_keyring_path, repo_dir, self.branch, package
+            ))
 
+        for binary_file in binary_files:
             output = execute("dpkg-deb -f %s" % quote_all(binary_file))
             fields = self.parse_package_info(output, binary_file, ["Package", "Architecture"])
 
             package = fields["Package"]
             architecture = fields["Architecture"]
 
-            additional_params = []
-            if architecture != "all":
-                additional_params.extend(["-A", quote(architecture)])
+            logging.info("Removing binaries of %s from the APT repository" % package)
+
+            extra = self.construct_reprepro_bin_extra(architecture)
+            execute("reprepro --gnupghome %s  -v -b %s%s remove %s %s" % (
+                self.gpg_keyring_path, repo_dir, extra, self.branch, package
+            ))
+
+        execute("reprepro --gnupghome %s -v -b %s deleteunreferenced" % (
+            self.gpg_keyring_path, repo_dir
+        ))
 
-            extra = " ".join(additional_params)
-            if extra:
-                extra = " " + extra
+        for dsc_file in dsc_files:
+            logging.info("Pushing %s to the APT repository" % dsc_file[prefix_len:])
+
+            execute("reprepro --gnupghome %s -v -b %s includedsc %s %s" % quote_all(
+                self.gpg_keyring_path, repo_dir, self.branch, dsc_file
+            ))
+
+        for binary_file in binary_files:
+            logging.info("Pushing %s to the APT repository" % binary_file[prefix_len:])
+
+            output = execute("dpkg-deb -f %s" % quote_all(binary_file))
+            fields = self.parse_package_info(output, binary_file, ["Architecture"])
+
+            architecture = fields["Architecture"]
+
+            extra = self.construct_reprepro_bin_extra(architecture)
+            execute("reprepro --gnupghome %s -v -b %s%s includedeb %s %s" % (
+                self.gpg_keyring_path, repo_dir, extra, self.branch, binary_file
+            ))
 
-            execute("reprepro -v -b %s%s remove %s %s" % (repo_dir, extra, self.branch, package))
-            execute("reprepro -v -b %s%s includedeb %s %s" % (repo_dir, extra, self.branch, binary_file))
+    def construct_reprepro_bin_extra(self, architecture):
+        additional_params = []
+        if architecture != "all":
+            additional_params.extend(["-A", quote(architecture)])
 
-            execute("reprepro -v -b %s deleteunreferenced" % repo_dir)
+        extra = " ".join(additional_params)
+        if extra:
+            extra = " " + extra
+        return extra
 
     def parse_package_info(self, contents, subject, required_keys: list):
         fields = {}
diff --git a/new/lib/debranding.py b/new/lib/debranding.py
index 7b0fef3..b7cdb8d 100644
--- a/new/lib/debranding.py
+++ b/new/lib/debranding.py
@@ -8,7 +8,7 @@ import shutil
 import tomlkit
 
 from lib.cache import Cache
-from lib.helpers import project_dir
+from lib.helpers import resources_dir, data_dir
 
 
 class Debranding:
@@ -19,7 +19,7 @@ class Debranding:
     alternative_name = None
 
     def __init__(self):
-        self.cache = Cache(os.path.join(project_dir, "build", "debranding-cache.json"), dict, {})
+        self.cache = Cache(os.path.join(data_dir, "debranding-cache.json"), dict, {})
 
     def populate_cli_parser(self, parser: argparse.ArgumentParser):
         parser.add_argument("--keep-branding", action="store_true", help="Keep VyOS branding as opposite to debranding")
@@ -100,7 +100,7 @@ class Debranding:
 
         logging.info("Applying debranding...")
 
-        new_splash = os.path.join(project_dir, "resources/not-vyos/splash.png")
+        new_splash = os.path.join(resources_dir, "not-vyos/splash.png")
         target_splash = os.path.join(root_dir, "data/live-build-config/includes.binary/isolinux/splash.png")
         shutil.copy2(new_splash, target_splash)
 
diff --git a/new/lib/github.py b/new/lib/github.py
index b53f49b..960f9ee 100644
--- a/new/lib/github.py
+++ b/new/lib/github.py
@@ -13,7 +13,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(inspect.getfi
 
 from helpers import setup_logging
 from lib.cache import Cache
-from lib.helpers import refuse_root
+from lib.helpers import refuse_root, data_dir
 
 
 class GitHub:
@@ -184,8 +184,7 @@ if __name__ == "__main__":
 
             github = GitHub()
 
-            project_dir = os.path.realpath(os.path.dirname(__file__))
-            cache = Cache(os.path.join(project_dir, "build", "github-vyos-cache.json"), dict, {})
+            cache = Cache(os.path.join(data_dir, "github-vyos-cache.json"), dict, {})
             repositories = cache.callback("repos", callback=lambda: github.find_org_repositories("vyos"))
 
             pprint(github.analyze_repositories_workflow("vyos", repositories, "circinus"))
diff --git a/new/lib/helpers.py b/new/lib/helpers.py
index bcbfffb..cc3ee4f 100644
--- a/new/lib/helpers.py
+++ b/new/lib/helpers.py
@@ -9,6 +9,11 @@ import sys
 from time import monotonic
 
 project_dir: str = os.path.realpath(os.path.join(os.path.dirname(__file__), ".."))
+apt_dir: str = os.path.join(project_dir, "apt")
+build_dir: str = os.path.join(project_dir, "build")
+data_dir: str = os.path.join(project_dir, "data")
+resources_dir: str = os.path.join(project_dir, "resources")
+scripts_dir: str = os.path.join(project_dir, "scripts")
 
 
 def quote_all(*args):
@@ -143,7 +148,7 @@ def setup_logging(name="test"):
     stderr_handler.setFormatter(formatter)
     logger.addHandler(stderr_handler)
 
-    log_file = os.path.join(project_dir, "build", "%s.log" % name)
+    log_file = os.path.join(build_dir, "%s.log" % name)
     if os.path.exists(log_file):
         previous_log_file = "%s.2" % log_file
         if os.path.exists(previous_log_file):
diff --git a/new/package_builder.py b/new/package_builder.py
index b48fdbe..b4d738a 100755
--- a/new/package_builder.py
+++ b/new/package_builder.py
@@ -13,11 +13,11 @@ from lib.debranding import Debranding
 from lib.docker import Docker
 from lib.git import Git
 from lib.github import GitHub
-from lib.helpers import setup_logging, ProcessException, refuse_root, project_dir, get_my_log_file
+from lib.helpers import setup_logging, ProcessException, refuse_root, get_my_log_file, data_dir, build_dir, scripts_dir
 
 
 class PackageBuilder:
-    build_dir = None
+    my_build_dir = None
     docker_image = None
     updated_repos = None
     apt = None
@@ -36,7 +36,7 @@ class PackageBuilder:
         self.debranding = debranding
 
         self.github = GitHub()
-        self.cache = Cache(os.path.join(project_dir, "build", "builder-cache-%s.json" % self.branch), dict, {})
+        self.cache = Cache(os.path.join(data_dir, "builder-cache-%s.json" % self.branch), dict, {})
 
     def build(self):
         begin = monotonic()
@@ -46,14 +46,14 @@ class PackageBuilder:
         logging.info("Building packages for %s" % self.branch)
         packages = self.get_packages_metadata()
 
-        self.build_dir = os.path.join(project_dir, "build", self.branch)
-        if not os.path.exists(self.build_dir):
-            os.makedirs(self.build_dir)
+        self.my_build_dir = os.path.join(build_dir, self.branch)
+        if not os.path.exists(self.my_build_dir):
+            os.makedirs(self.my_build_dir)
 
-        self.apt = Apt(project_dir, self.branch, self.build_dir)
+        self.apt = Apt(self.branch, self.my_build_dir)
 
         logging.info("Pulling vyos-build docker image")
-        vyos_build_repo = os.path.join(os.path.join(self.build_dir, "vyos-build"))
+        vyos_build_repo = os.path.join(os.path.join(self.my_build_dir, "vyos-build"))
         self.docker = Docker(self.vyos_build_docker, self.branch, vyos_build_repo)
         self.docker.pull()
 
@@ -89,7 +89,7 @@ class PackageBuilder:
         if "hash" not in my_state:
             my_state["hash"] = None
 
-        repo_path = os.path.join(self.build_dir, repo_name)
+        repo_path = os.path.join(self.my_build_dir, repo_name)
         parent_path = repo_path
 
         if package["build_type"] == "dpkg-buildpackage":
@@ -130,17 +130,16 @@ class PackageBuilder:
         self.debranding.remove_package_branding(repo_path, package["package_name"])
 
         if package["build_type"] == "build.py":
-            my_directory = os.path.join(self.build_dir, "vyos-build", package["path"])
+            my_directory = os.path.join(self.my_build_dir, "vyos-build", package["path"])
             if not self.skip_build or new:
                 # It's important to run bash in interactive mode, non-interactive shell breaks dependency on .bashrc.
                 # It's also required to call python explicitly since some scripts don't have correct shebang.
                 self.docker.run("bash -i -c 'python3 ./build.py'", work_dir="/vyos/%s" % package["path"])
 
         elif package["build_type"] == "dpkg-buildpackage":
-            my_directory = os.path.join(self.build_dir, repo_name)
+            my_directory = os.path.join(self.my_build_dir, repo_name)
             virtual_dir = "/vyos-%s" % package["package_name"]
 
-            scripts_dir = os.path.join(project_dir, "scripts")
             virtual_scripts = "%s-scripts" % virtual_dir
 
             build_script = "generic-build-script.sh"