From 6e66715c45a77be01855f269e03a9e6be1e50bf4 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Thu, 24 Dec 2015 12:04:24 -0500 Subject: Add sudo to required packages, since we use it in scripts. Using sudo from scripts may be objectionable, but I don't see any alternatives for things that do chroot. --- scripts/check-build-env | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts/check-build-env') diff --git a/scripts/check-build-env b/scripts/check-build-env index 11e1c66b..168c7199 100755 --- a/scripts/check-build-env +++ b/scripts/check-build-env @@ -4,6 +4,7 @@ import os import sys required_packages = [ + 'sudo', 'make', 'live-build', 'pbuilder', -- cgit v1.2.3 From 90ca7062c19e7237d9e4f3fe9f442db2a99a7048 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Sat, 26 Dec 2015 20:14:08 -0500 Subject: Add license headers to scripts. Some people rightfully complained about their absense already. --- scripts/build-config | 24 ++++++++++++++++++++++++ scripts/check-build-env | 20 ++++++++++++++++++++ scripts/check-config | 21 +++++++++++++++++++++ scripts/defaults.py | 18 ++++++++++++++++++ scripts/live-build-config | 20 ++++++++++++++++++++ scripts/pbuilder-config | 19 +++++++++++++++++++ scripts/pbuilder-setup | 19 +++++++++++++++++++ scripts/util.py | 19 +++++++++++++++++++ tools/gpl-header-template | 17 +++++++++++++++++ 9 files changed, 177 insertions(+) create mode 100644 tools/gpl-header-template (limited to 'scripts/check-build-env') diff --git a/scripts/build-config b/scripts/build-config index e001ef3d..d0d2fdac 100755 --- a/scripts/build-config +++ b/scripts/build-config @@ -1,4 +1,26 @@ #!/usr/bin/env python +# Copyright (C) 2015 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# File: build-config +# Purpose: +# This script serves the same purpose as ./configure in traditional +# autoconf setups. +# It takes build configuration options from command line, checks them, +# builds a config dictionary, augments it with some default and/or +# computed values and saves it to build/build-config.json +# for other build scripts to read. import argparse import re @@ -21,6 +43,8 @@ def get_default_build_by(): return "{user}@{host}".format(user= getpass.getuser(), host=platform.node()) +# Options dict format: +# '$option_name_without_leading_dashes': { ('$help_string', $default_value_generator_thunk, $value_checker_thunk) } options = { 'architecture': ('Image target architecture (amd64 or i586)', lambda: 'amd64', lambda x: x in ['amd64', 'i586']), 'build-by': ('Builder identifier (e.g. jrandomhacker@example.net)', get_default_build_by, None), diff --git a/scripts/check-build-env b/scripts/check-build-env index 168c7199..6f08847f 100755 --- a/scripts/check-build-env +++ b/scripts/check-build-env @@ -1,4 +1,24 @@ #!/usr/bin/env python +# +# Copyright (C) 2015 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# File: check-build-env +# Purpose: +# Checks if packages required for package and ISO image build +# are installed. + import os import sys diff --git a/scripts/check-config b/scripts/check-config index 3949869b..58a1a3f1 100755 --- a/scripts/check-config +++ b/scripts/check-config @@ -1,4 +1,25 @@ #!/usr/bin/env python +# +# Copyright (C) 2015 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# File: check-config +# Purpose: +# Checks if the build config file (build/build-config.json) exists. +# This is to prevent accidental execution of plumbing make targets +# from going too far and failing with confusing errors. + import sys import os diff --git a/scripts/defaults.py b/scripts/defaults.py index 2dbaeb0c..beef4a7a 100644 --- a/scripts/defaults.py +++ b/scripts/defaults.py @@ -1,3 +1,21 @@ +# Copyright (C) 2015 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# File: defaults.py +# Purpose: Various default values for use in build scripts. + + import os BUILD_DIR = 'build' diff --git a/scripts/live-build-config b/scripts/live-build-config index 59f0bbb2..bd9a7767 100755 --- a/scripts/live-build-config +++ b/scripts/live-build-config @@ -1,4 +1,24 @@ #!/usr/bin/env python +# +# Copyright (C) 2015 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# File: live-build-config +# Purpose: +# Creates a live-build config command from template using the build config +# and executes it, to prepare the system for building the installation ISO. + import sys import os diff --git a/scripts/pbuilder-config b/scripts/pbuilder-config index 5cda7b13..06e14cbf 100755 --- a/scripts/pbuilder-config +++ b/scripts/pbuilder-config @@ -1,4 +1,23 @@ #!/usr/bin/env python +# +# Copyright (C) 2015 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# File: pbuilder-config +# Purpose: +# Generates a pbuilderrc file for use by package build scripts. + import sys import os diff --git a/scripts/pbuilder-setup b/scripts/pbuilder-setup index 264db0b2..fbd49a4f 100755 --- a/scripts/pbuilder-setup +++ b/scripts/pbuilder-setup @@ -1,4 +1,23 @@ #!/usr/bin/env python +# +# Copyright (C) 2015 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# File: pbuilder-setup +# Purpose: +# Bootstraps a Debian environment for use by pbuilder. + import sys import os diff --git a/scripts/util.py b/scripts/util.py index 740d4ccc..00377672 100644 --- a/scripts/util.py +++ b/scripts/util.py @@ -1,3 +1,22 @@ +# Copyright (C) 2015 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# File: util.py +# Purpose: +# Various common functions for use in build scripts. + + import sys import os diff --git a/tools/gpl-header-template b/tools/gpl-header-template new file mode 100644 index 00000000..f3f82192 --- /dev/null +++ b/tools/gpl-header-template @@ -0,0 +1,17 @@ +# Copyright (C) 2015 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# File: +# Purpose: + -- cgit v1.2.3 From 08a6856b3c8f3146343647f996fca9c48fb4e960 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Fri, 4 Mar 2016 15:05:03 -0500 Subject: Factor out dependency checking functions to its own class (ref T9). --- scripts/check-build-env | 49 ++++++++++++++++++++++------------------------ scripts/check-vm-build-env | 48 ++++++++++++++++++--------------------------- scripts/util.py | 36 ++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 55 deletions(-) (limited to 'scripts/check-build-env') diff --git a/scripts/check-build-env b/scripts/check-build-env index 6f08847f..6dcf885b 100755 --- a/scripts/check-build-env +++ b/scripts/check-build-env @@ -23,34 +23,31 @@ import os import sys -required_packages = [ - 'sudo', - 'make', - 'live-build', - 'pbuilder', - 'devscripts', - 'python-pystache' -] - - -def is_installed(name): - result = os.system("dpkg-query -W --showformat='${{Status}}\n' {name} 2>&1 | grep 'install ok installed' >/dev/null".format(name=name)) - return True if result == 0 else False - - -missing_packages = [] +import util + +deps = { + 'packages': [ + 'sudo', + 'make', + 'live-build', + 'pbuilder', + 'devscripts', + 'python-pystache' + ], + 'binaries': [] +} print("Checking if packages required for VyOS image build are installed") -for p in required_packages: - if not is_installed(p): - missing_packages.append(p) +checker = util.DependencyChecker(deps) -if missing_packages: - print("Your system does not have some of the required packages installed.") - print("Please install the following packages:") - print(" ".join(missing_packages)) - sys.exit(1) -else: - print("All required packages are installed") +missing = checker.get_missing_dependencies() +if not missing: + print("All dependencies are installed") sys.exit(0) +else: + checker.print_missing_deps() + sys.exit(1) + +sys.exit(0) + diff --git a/scripts/check-vm-build-env b/scripts/check-vm-build-env index 290b28c8..47ec5922 100755 --- a/scripts/check-vm-build-env +++ b/scripts/check-vm-build-env @@ -21,41 +21,31 @@ import os import sys -from distutils.spawn import find_executable -required_packages = [ - 'make', - 'qemu-system-x86', - 'qemu-utils' -] +import util +deps = { + 'packages': [ + 'make', + 'qemu-system-x86', + 'qemu-utils' + ], + 'binaries': ['packer'] +} -def is_installed(name): - result = os.system("dpkg-query -W --showformat='${{Status}}\n' {name} 2>&1 | grep 'install ok installed' >/dev/null".format(name=name)) - return True if result == 0 else False +print("Checking if packages required for VyOS image build are installed") +checker = util.DependencyChecker(deps) -missing_packages = [] - -print("Checking if packages required for VyOS VM image build are installed") - -for p in required_packages: - if not is_installed(p): - missing_packages.append(p) - -if missing_packages: - print("Your system does not have some of the required packages installed.") - print("Please install the following packages:") - print(" ".join(missing_packages)) - sys.exit(1) -else: - print("All required packages are installed") - -if find_executable("packer"): - print("Your system has Packer.") +missing = checker.get_missing_dependencies() +if not missing: + print("All dependencies are installed") + sys.exit(0) else: - print("Your system does not have Packer.") - print("Please install Packer from https://www.packer.io/downloads.html.") + checker.print_missing_deps() + if 'packer' in missing['binaries']: + print("Your system does not have Packer.") + print("Please install Packer from https://www.packer.io/downloads.html.") sys.exit(1) sys.exit(0) diff --git a/scripts/util.py b/scripts/util.py index 00377672..7cc33364 100644 --- a/scripts/util.py +++ b/scripts/util.py @@ -19,6 +19,7 @@ import sys import os +from distutils.spawn import find_executable import defaults @@ -27,3 +28,38 @@ def check_build_config(): print("Build config file ({file}) does not exist".format(file=defaults.BUILD_CONFIG)) print("If you are running this script by hand, you should better not. Run 'make iso' instead.") sys.exit(1) + + +class DependencyChecker(object): + def __init__(self, spec): + missing_packages = self._get_missing_packages(spec['packages']) + missing_binaries = self._get_missing_binaries(spec['binaries']) + self.__missing = {'packages': missing_packages, 'binaries': missing_binaries} + + + def _package_installed(self, name): + result = os.system("dpkg-query -W --showformat='${{Status}}\n' {name} 2>&1 | grep 'install ok installed' >/dev/null".format(name=name)) + return True if result == 0 else False + + def _get_missing_packages(self, packages): + missing_packages = [] + for p in packages: + if not self._package_installed(p): + missing_packages.append(p) + return missing_packages + + def _get_missing_binaries(self, binaries): + missing_binaries = [] + for b in binaries: + if not find_executable(b): + missing_binaries.append(b) + return missing_binaries + + def get_missing_dependencies(self): + if self.__missing['packages'] or self.__missing['binaries']: + return self.__missing + return None + + def print_missing_deps(self): + print("Missing packages: " + " ".join(self.__missing['packages'])) + print("Missing binaries: " + " ".join(self.__missing['binaries'])) -- cgit v1.2.3 From ea48b7bd747df4ae5c8ee64c72cdc67e31ddcd5a Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Wed, 25 Jan 2017 17:01:16 -0500 Subject: Switch all build scripts to python3. Since we only support jessie as build host, and jessie knowingly does have python3 (although not by default), we don't really need to worry about being both 2 and 3 compatible. --- README.md | 2 +- scripts/build-config | 3 ++- scripts/check-build-env | 4 ++-- scripts/check-config | 2 +- scripts/check-vm-build-env | 2 +- scripts/live-build-config | 2 +- scripts/make-version-file | 4 ++-- scripts/pbuilder-setup | 3 +-- scripts/query-json | 5 +++-- 9 files changed, 14 insertions(+), 13 deletions(-) (limited to 'scripts/check-build-env') diff --git a/README.md b/README.md index 11edb5fb..0d6e53c2 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ There are several directories with their own purpose: To build a VyOS image, you need a machine that runs Debian Jessie. Other build hosts are not supported. -Several packages are required for building the ISO and all packages, namely live-build, pbuilder, python-pystache and devscripts. +Several packages are required for building the ISO and all packages, namely python3, live-build, pbuilder, python3-pystache and devscripts. Individual packages may have other build dependencies. If some packages are missing, build scripts will tell you. ## Building the ISO image diff --git a/scripts/build-config b/scripts/build-config index d273717f..9ea92e1f 100755 --- a/scripts/build-config +++ b/scripts/build-config @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# # Copyright (C) 2015 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify diff --git a/scripts/check-build-env b/scripts/check-build-env index 6dcf885b..7f02c02a 100755 --- a/scripts/check-build-env +++ b/scripts/check-build-env @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright (C) 2015 VyOS maintainers and contributors # @@ -32,7 +32,7 @@ deps = { 'live-build', 'pbuilder', 'devscripts', - 'python-pystache' + 'python3-pystache' ], 'binaries': [] } diff --git a/scripts/check-config b/scripts/check-config index 55d5467a..d2236619 100755 --- a/scripts/check-config +++ b/scripts/check-config @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright (C) 2015 VyOS maintainers and contributors # diff --git a/scripts/check-vm-build-env b/scripts/check-vm-build-env index c32f2b13..8efab848 100755 --- a/scripts/check-vm-build-env +++ b/scripts/check-vm-build-env @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright (C) 2016 VyOS maintainers and contributors # diff --git a/scripts/live-build-config b/scripts/live-build-config index cb9e84dc..2e2d4704 100755 --- a/scripts/live-build-config +++ b/scripts/live-build-config @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright (C) 2015 VyOS maintainers and contributors # diff --git a/scripts/make-version-file b/scripts/make-version-file index 6c597090..3bb33319 100755 --- a/scripts/make-version-file +++ b/scripts/make-version-file @@ -1,4 +1,5 @@ -#!/usr/bin/python +#!/usr/bin/python3 +# # Copyright (C) 2016 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify @@ -19,7 +20,6 @@ # that is included in the image and used by 'show version' command # and install/upgrade scripts. -from __future__ import print_function import os import datetime import json diff --git a/scripts/pbuilder-setup b/scripts/pbuilder-setup index fbd49a4f..a89348b8 100755 --- a/scripts/pbuilder-setup +++ b/scripts/pbuilder-setup @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright (C) 2015 VyOS maintainers and contributors # @@ -50,4 +50,3 @@ result = os.system(pbuilder_create_command) if result > 0: print("pbuilder environment bootstrap failed") sys.exit(1) - diff --git a/scripts/query-json b/scripts/query-json index 23e64ef7..2f1ea32f 100755 --- a/scripts/query-json +++ b/scripts/query-json @@ -1,4 +1,5 @@ -#!/usr/bin/python +#!/usr/bin/python3 +# # Copyright (C) 2016 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify @@ -14,7 +15,7 @@ # along with this program. If not, see . # # File: query-config -# Purpose: Extracts field values a flat JSON file, +# Purpose: Extracts field values from a flat JSON file, # for use in languages that can't handle JSON easily, # (I'm looking at you, Bourne shell!) -- cgit v1.2.3 From d2e4f63f03ef5628ab94be34bf368e8c8458a2d0 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Thu, 5 Apr 2018 14:59:44 +0200 Subject: T596: use a more descriptive dev build version format. --- data/versions | 3 +++ scripts/check-build-env | 3 ++- scripts/make-version-file | 24 ++++++++++++++++++++---- 3 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 data/versions (limited to 'scripts/check-build-env') diff --git a/data/versions b/data/versions new file mode 100644 index 00000000..3d0c2871 --- /dev/null +++ b/data/versions @@ -0,0 +1,3 @@ +{ + "current": "1.2.0" +} diff --git a/scripts/check-build-env b/scripts/check-build-env index 7f02c02a..7377be64 100755 --- a/scripts/check-build-env +++ b/scripts/check-build-env @@ -32,7 +32,8 @@ deps = { 'live-build', 'pbuilder', 'devscripts', - 'python3-pystache' + 'python3-pystache', + 'python3-git' ], 'binaries': [] } diff --git a/scripts/make-version-file b/scripts/make-version-file index 3bb33319..0459b8bf 100755 --- a/scripts/make-version-file +++ b/scripts/make-version-file @@ -1,6 +1,6 @@ #!/usr/bin/python3 # -# Copyright (C) 2016 VyOS maintainers and contributors +# Copyright (C) 2018 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -25,16 +25,18 @@ import datetime import json import uuid +import git + import defaults import util +# Load the build config util.check_build_config() with open(defaults.BUILD_CONFIG, 'r') as f: build_config = json.load(f) - +# Create a build timestamp now = datetime.datetime.today() - build_timestamp = now.strftime("%Y%m%d%H%M") # FIXME: use aware rather than naive object @@ -43,9 +45,23 @@ build_date = now.strftime("%a %d %b %Y %H:%M UTC") # Assign a (hopefully) unique identifier to the build (UUID) build_id = str(uuid.uuid4()) +# Create a build version if build_config['build_type'] == 'development': - version = "999.{0}".format(build_timestamp) + try: + # Load the branch to version mapping file + with open('data/versions') as f: + version_mapping = json.load(f) + + repo = git.Repo('.') + git_branch = repo.active_branch.name + branch_version = version_mapping[git_branch] + + version = "{0}-rolling+{1}".format(branch_version, build_timestamp) + except Exception as e: + print("Could not build a version string specific to git branch, falling back to default: {0}".format(str(e))) + version = "999.{0}".format(build_timestamp) else: + # Release build, use the version from ./configure arguments version = build_config['version'] version_data = { -- cgit v1.2.3