#!/usr/bin/env python3 import argparse from datetime import datetime import logging import os.path from shlex import quote from time import time, monotonic import tomlkit from lib.apt import Apt from lib.debranding import Debranding from lib.docker import Docker from lib.git import Git from lib.helpers import setup_logging, ProcessException, refuse_root, get_my_log_file, data_dir, build_dir, scripts_dir, \ quote_all, TerminalTitle, ensure_directories, replace_github_repo_org from lib.objectstorage import ObjectStorage from lib.packagedefinitions import PackageDefinitions from lib.scripting import Scripting class PackageBuilder: my_build_dir = None docker_image = None updated_repos = None apt = None docker = None def __init__(self, branch, analyze_org, clone_org, single_package, dirty_build, ignore_missing_binaries, skip_build, skip_apt, force_build, vyos_build_docker, rescan_packages, pre_build_hook, debranding: Debranding): self.branch = branch self.analyze_org = analyze_org self.clone_org = clone_org self.single_package = single_package self.dirty_build = dirty_build self.ignore_missing_binaries = ignore_missing_binaries self.skip_build = skip_build self.skip_apt = skip_apt self.force_build = force_build self.vyos_build_docker = vyos_build_docker self.rescan_packages = rescan_packages self.pre_build_hook = pre_build_hook self.debranding = debranding self.vyos_stream_mode = self.clone_org != "vyos" self.package_definitions = PackageDefinitions(self.vyos_stream_mode) self.build_data = ObjectStorage( os.path.join(data_dir, "builder-data-%s.json" % self.branch), dict, {} ) self.package_cache = ObjectStorage( os.path.join(data_dir, "package-metadata-cache-%s.json" % self.branch), dict, {} ) self.scripting = Scripting() self.terminal_title = TerminalTitle("Package builder: ") ensure_directories() def build(self): self.terminal_title.set("Preparation...") begin = monotonic() if self.single_package is not None: logging.info("Executing single package build of %s" % self.single_package) logging.info("Building packages for %s" % self.branch) packages = self.get_packages_metadata() self.my_build_dir = os.path.join(build_dir, self.branch) if not os.path.exists(self.my_build_dir): os.makedirs(self.my_build_dir) self.apt = Apt(self.branch, self.my_build_dir) logging.info("Pulling vyos-build docker image") vyos_build_repo = os.path.join(os.path.join(self.my_build_dir, "vyos-build")) self.docker = Docker(self.vyos_build_docker, self.branch, vyos_build_repo, self.vyos_stream_mode) self.docker.pull() self.updated_repos = [] found = 0 built = 0 counter = 0 total = 1 if self.single_package is not None else len(packages) for package in packages.values(): found += 1 if self.single_package is not None and self.single_package != package["package_name"]: continue counter += 1 message = "Processing package: %s (%s of %s)" % (package["package_name"], counter, total) self.terminal_title.set(message) logging.info(message) self.build_package(package) built += 1 if built == 0: self.terminal_title.set("ERROR") if self.single_package is not None: logging.error("Specified --single-package=%s was not found" % self.single_package) else: if found == 0: logging.error("Something's wrong, no packages were found!") else: logging.error("Something's wrong, no packages were built but these were found: %s" % packages) exit(1) elapsed = round(monotonic() - begin, 3) message = "Done in %s seconds" % elapsed self.terminal_title.set(message) logging.info(message) def build_package(self, package): repo_name = package["repo_name"] my_state = self.build_data.get(package["package_name"], default={}, data_type=dict) if "hash" not in my_state: my_state["hash"] = None repo_path = os.path.join(self.my_build_dir, repo_name) parent_path = repo_path if package["build_type"] == "dpkg-buildpackage": if not os.path.exists(repo_path): os.makedirs(repo_path) repo_path = os.path.join(repo_path, "sources") git_url = replace_github_repo_org(package["git_url"], self.clone_org) git = Git(repo_path) if git.exists() and self.clone_org not in git.get_remote_url("origin"): git.set_remote_url("origin", git_url) git.fetch() try: changed = git.resolve_changes(package["change_patterns"], my_state["hash"]) if not changed and not self.force_build: logging.info("Package is up to date, skipping build") return except ProcessException as e: if "not a git repository" in str(e): self.docker.rmtree(parent_path) else: raise new = False if repo_name not in self.updated_repos: self.updated_repos.append(repo_name) if os.path.exists(parent_path) and not self.dirty_build: # We want to delete original repo and do fresh clone to clean cached build files. self.docker.rmtree(parent_path) if not os.path.exists(repo_path): logging.info("Cloning repository %s" % git_url) git.clone(git_url, package["branch"]) new = True else: git_url = git.get_remote_url("origin") logging.info("Pulling repository %s" % git_url) git.pull() else: git_url = git.get_remote_url("origin") logging.info("Using shared repository %s" % git_url) self.debranding.remove_package_branding(repo_path, package["package_name"]) if self.pre_build_hook: self.scripting.run(self.pre_build_hook, repo_path, vars={ "BRANCH": self.branch, "PACKAGE_NAME": package["package_name"], }) virtual_scripts = "/my-build-scripts" if package["build_type"] == "build.py": my_directory = os.path.join(self.my_build_dir, "vyos-build", package["path"]) if not self.skip_build or new: package_toml_path = os.path.join(my_directory, "package.toml") if os.path.exists(package_toml_path): self.modify_package_toml(package_toml_path) # It's important to run bash in interactive mode, non-interactive shell breaks dependency on .bashrc. build_script = os.path.join(virtual_scripts, "build_py.sh") vyos_dir = "/vyos/%s" % package["path"] self.docker.run( "bash -i -c '%s %s'" % quote_all(build_script, package["package_name"]), work_dir=vyos_dir, extra_mounts=[ (scripts_dir, virtual_scripts) ], ) elif package["build_type"] == "dpkg-buildpackage": my_directory = os.path.join(self.my_build_dir, repo_name) virtual_dir = "/vyos-%s" % package["package_name"] build_script = "generic-build-script.sh" custom_build_script = os.path.join(scripts_dir, "%s.sh" % package["package_name"]) if os.path.exists(custom_build_script): build_script = os.path.basename(custom_build_script) sources_dir = os.path.join(virtual_dir, "sources") if not self.skip_build or new: # Again, interactive shell is essential. virtual_build_script = os.path.join(virtual_scripts, build_script) self.docker.run("bash -i %s" % quote(virtual_build_script), work_dir=sources_dir, extra_mounts=[ (my_directory, virtual_dir), (scripts_dir, virtual_scripts), ]) else: logging.error("Unknown build_type: %s" % package) return dsc_files, binary_files = self.apt.scan_for_dist_files(my_directory) if len(binary_files) == 0: message = "%s: something is wrong, no binary files found" % package["package_name"] message += ", build dir: %s," % my_directory message += ", log file: %s" % get_my_log_file() if self.ignore_missing_binaries: logging.error(message) else: raise Exception(message) else: my_state["hash"] = git.get_last_commit_hash() if not self.skip_apt or new: self.apt.fill_apt_repository(dsc_files, binary_files) self.build_data.set(package["package_name"], my_state) def modify_package_toml(self, path): with open(path, "r") as file: payload = tomlkit.load(file) changed = False if "packages" in payload: for package in payload["packages"]: if "scm_url" in package: scm_url = replace_github_repo_org(package["scm_url"], self.clone_org, carefully_only_this_org="vyos") if scm_url != package["scm_url"]: changed = True logging.info("Updating package.toml GIT url from %s to %s" % (package["scm_url"], scm_url)) package["scm_url"] = scm_url if "commit_id" in package and package["commit_id"] in ["master", "main"]: package["commit_id"] = self.branch if changed: with open(path, "w") as file: tomlkit.dump(payload, file) def get_packages_metadata(self): if self.package_definitions.is_static(self.branch): packages = self.package_definitions.get_definitions(self.analyze_org, self.branch) else: packages_timestamp = self.package_cache.get("packages_timestamp") packages = self.package_cache.get("packages") if not packages_timestamp or not packages or packages_timestamp <= time() - 3600 * 24 or self.rescan_packages: packages = self.package_definitions.get_definitions(self.analyze_org, self.branch) self.package_cache.set("packages_timestamp", time()) self.package_cache.set("packages", packages) else: date = datetime.fromtimestamp(float(packages_timestamp)).astimezone().strftime("%Y-%m-%d %H:%M:%S") logging.info("Using previously generated package metadata (%s)" % date) return packages if __name__ == "__main__": setup_logging(name="package_builder") try: refuse_root() debranding = Debranding() parser = argparse.ArgumentParser() parser.add_argument("branch", help="VyOS branch (current, circinus)") parser.add_argument("--analyze-org", default="vyos", help="What GitHub organization to use for analysis (used only for current)") parser.add_argument("--clone-org", default="NOTvyos", help="What GitHub organization to use for sources") parser.add_argument("--single-package", help="Build only this package") parser.add_argument("--force-build", action="store_true", help="Force build even if package is up to date") parser.add_argument("--rescan-packages", action="store_true", help="Force package metadata scan even if last scan was run recently") parser.add_argument("--vyos-build-docker", default="vyos/vyos-build", help="Default option uses vyos/vyos-build from dockerhub") scripting_info = "the current working directory is the repo of given package" scripting_info += ", available environment variables: VYOS_BUILD_BRANCH, VYOS_BUILD_PACKAGE_NAME" parser.add_argument("--pre-build-hook", help="Script to execute before build, %s" % scripting_info) debranding.populate_cli_parser(parser) parser.add_argument("--dirty-build", action="store_true", help="DEV - Build with reused sources (don't clone fresh sources) for existing packages") parser.add_argument("--ignore-missing-binaries", action="store_true", help="DEV - Don't terminate when missing binaries are detected") parser.add_argument("--skip-build", action="store_true", help="DEV - Skip build stage for existing packages") parser.add_argument("--skip-apt", action="store_true", help="DEV - Skip reprepro stage for existing packages") args = parser.parse_args() values = vars(args) debranding.extract_cli_values(values) builder = PackageBuilder(debranding=debranding, **values) try: builder.build() except Exception: builder.terminal_title.set("ERROR") raise except KeyboardInterrupt: exit(1) except Exception as e: logging.exception(e) logging.error("Something went wrong, log file: %s" % get_my_log_file()) exit(1)