summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChristian Breunig <christian@breunig.cc>2023-11-20 08:13:04 +0100
committerGitHub <noreply@github.com>2023-11-20 08:13:04 +0100
commit293f6b2ea882ebbf5b54e9ff19b1854e627a06c3 (patch)
treeebefd4ed971efeae29670071e17bdb2bf52557d0
parentcd19b9d6b0c21a5d07a9f5a98e5e90d09d8d4cc9 (diff)
parentd87dfa557c962b9ac1bb7483a3b419095ae5fccf (diff)
downloadvyos-1x-293f6b2ea882ebbf5b54e9ff19b1854e627a06c3.tar.gz
vyos-1x-293f6b2ea882ebbf5b54e9ff19b1854e627a06c3.zip
Merge pull request #2505 from vyos/mergify/bp/sagitta/pr-2241
T2405: add Git support to commit-archive (backport #2241)
-rw-r--r--debian/control1
-rw-r--r--interface-definitions/system-config-mgmt.xml.in29
-rw-r--r--python/vyos/config_mgmt.py20
-rw-r--r--python/vyos/remote.py131
-rwxr-xr-xscripts/build-command-templates2
5 files changed, 171 insertions, 12 deletions
diff --git a/debian/control b/debian/control
index f31c5a510..42c0b580b 100644
--- a/debian/control
+++ b/debian/control
@@ -62,6 +62,7 @@ Depends:
frr-snmp,
fuse-overlayfs,
libpam-google-authenticator,
+ git,
grc,
haproxy,
hostapd,
diff --git a/interface-definitions/system-config-mgmt.xml.in b/interface-definitions/system-config-mgmt.xml.in
index de5a8cc16..61089ce34 100644
--- a/interface-definitions/system-config-mgmt.xml.in
+++ b/interface-definitions/system-config-mgmt.xml.in
@@ -17,11 +17,36 @@
<properties>
<help>Commit archive location</help>
<valueHelp>
- <format>uri</format>
- <description>Uniform Resource Identifier</description>
+ <format>http://&lt;user&gt;:&lt;passwd&gt;@&lt;host&gt;/&lt;path&gt;</format>
+ <description/>
+ </valueHelp>
+ <valueHelp>
+ <format>https://&lt;user&gt;:&lt;passwd&gt;@&lt;host&gt;/&lt;path&gt;</format>
+ <description/>
+ </valueHelp>
+ <valueHelp>
+ <format>ftp://&lt;user&gt;:&lt;passwd&gt;@&lt;host&gt;/&lt;path&gt;</format>
+ <description/>
+ </valueHelp>
+ <valueHelp>
+ <format>sftp://&lt;user&gt;:&lt;passwd&gt;@&lt;host&gt;/&lt;path&gt;</format>
+ <description/>
+ </valueHelp>
+ <valueHelp>
+ <format>scp://&lt;user&gt;:&lt;passwd&gt;@&lt;host&gt;/&lt;path&gt;</format>
+ <description/>
+ </valueHelp>
+ <valueHelp>
+ <format>tftp://&lt;host&gt;/&lt;path&gt;</format>
+ <description/>
+ </valueHelp>
+ <valueHelp>
+ <format>git+https://&lt;user&gt;:&lt;passwd&gt;@&lt;host&gt;/&lt;path&gt;</format>
+ <description/>
</valueHelp>
<constraint>
<validator name="url --file-transport"/>
+ <regex>(ssh|git|git\+(\w+)):\/\/.*</regex>
</constraint>
<multi/>
</properties>
diff --git a/python/vyos/config_mgmt.py b/python/vyos/config_mgmt.py
index 654a8d698..df7240c88 100644
--- a/python/vyos/config_mgmt.py
+++ b/python/vyos/config_mgmt.py
@@ -22,10 +22,11 @@ import logging
from typing import Optional, Tuple, Union
from filecmp import cmp
from datetime import datetime
-from textwrap import dedent
+from textwrap import dedent, indent
from pathlib import Path
from tabulate import tabulate
from shutil import copy, chown
+from urllib.parse import urlsplit, urlunsplit
from vyos.config import Config
from vyos.configtree import ConfigTree, ConfigTreeError, show_diff
@@ -377,9 +378,22 @@ Proceed ?'''
remote_file = f'config.boot-{hostname}.{timestamp}'
source_address = self.source_address
+ if self.effective_locations:
+ print("Archiving config...")
for location in self.effective_locations:
- upload(archive_config_file, f'{location}/{remote_file}',
- source_host=source_address)
+ url = urlsplit(location)
+ _, _, netloc = url.netloc.rpartition("@")
+ redacted_location = urlunsplit(url._replace(netloc=netloc))
+ print(f" {redacted_location}", end=" ", flush=True)
+ try:
+ upload(archive_config_file, f'{location}/{remote_file}',
+ source_host=source_address, raise_error=True)
+ print("OK")
+ except Exception as e:
+ print("FAILED!")
+ print()
+ print(indent(str(e), " > "))
+ print()
# op-mode functions
#
diff --git a/python/vyos/remote.py b/python/vyos/remote.py
index 4be477d24..8b90e4530 100644
--- a/python/vyos/remote.py
+++ b/python/vyos/remote.py
@@ -14,6 +14,7 @@
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
import os
+import pwd
import shutil
import socket
import ssl
@@ -22,6 +23,9 @@ import sys
import tempfile
import urllib.parse
+from contextlib import contextmanager
+from pathlib import Path
+
from ftplib import FTP
from ftplib import FTP_TLS
@@ -37,11 +41,22 @@ from vyos.utils.io import ask_yes_no
from vyos.utils.io import is_interactive
from vyos.utils.io import print_error
from vyos.utils.misc import begin
-from vyos.utils.process import cmd
+from vyos.utils.process import cmd, rc_cmd
from vyos.version import get_version
CHUNK_SIZE = 8192
+@contextmanager
+def umask(mask: int):
+ """
+ Context manager that temporarily sets the process umask.
+ """
+ oldmask = os.umask(mask)
+ try:
+ yield
+ finally:
+ os.umask(oldmask)
+
class InteractivePolicy(MissingHostKeyPolicy):
"""
Paramiko policy for interactively querying the user on whether to proceed
@@ -310,35 +325,137 @@ class TftpC:
with open(location, 'rb') as f:
cmd(f'{self.command} -T - "{self.urlstring}"', input=f.read())
+class GitC:
+ def __init__(self,
+ url,
+ progressbar=False,
+ check_space=False,
+ source_host=None,
+ source_port=0,
+ timeout=10,
+ ):
+ self.command = 'git'
+ self.url = url
+ self.urlstring = urllib.parse.urlunsplit(url)
+ if self.urlstring.startswith("git+"):
+ self.urlstring = self.urlstring.replace("git+", "", 1)
+
+ def download(self, location: str):
+ raise NotImplementedError("not supported")
+
+ @umask(0o077)
+ def upload(self, location: str):
+ scheme = self.url.scheme
+ _, _, scheme = scheme.partition("+")
+ netloc = self.url.netloc
+ url = Path(self.url.path).parent
+ with tempfile.TemporaryDirectory(prefix="git-commit-archive-") as directory:
+ # Determine username, fullname, email for Git commit
+ pwd_entry = pwd.getpwuid(os.getuid())
+ user = pwd_entry.pw_name
+ name = pwd_entry.pw_gecos.split(",")[0] or user
+ fqdn = socket.getfqdn()
+ email = f"{user}@{fqdn}"
+
+ # environment vars for our git commands
+ env = {
+ "GIT_TERMINAL_PROMPT": "0",
+ "GIT_AUTHOR_NAME": name,
+ "GIT_AUTHOR_EMAIL": email,
+ "GIT_COMMITTER_NAME": name,
+ "GIT_COMMITTER_EMAIL": email,
+ }
+
+ # build ssh command for git
+ ssh_command = ["ssh"]
+
+ # if we are not interactive, we use StrictHostKeyChecking=yes to avoid any prompts
+ if not sys.stdout.isatty():
+ ssh_command += ["-o", "StrictHostKeyChecking=yes"]
+
+ env["GIT_SSH_COMMAND"] = " ".join(ssh_command)
+
+ # git clone
+ path_repository = Path(directory) / "repository"
+ scheme = f"{scheme}://" if scheme else ""
+ rc, out = rc_cmd(
+ [self.command, "clone", f"{scheme}{netloc}{url}", str(path_repository), "--depth=1"],
+ env=env,
+ shell=False,
+ )
+ if rc:
+ raise Exception(out)
+
+ # git add
+ filename = Path(Path(self.url.path).name).stem
+ dst = path_repository / filename
+ shutil.copy2(location, dst)
+ rc, out = rc_cmd(
+ [self.command, "-C", str(path_repository), "add", filename],
+ env=env,
+ shell=False,
+ )
+
+ # git commit -m
+ commit_message = os.environ.get("COMMIT_COMMENT", "commit")
+ rc, out = rc_cmd(
+ [self.command, "-C", str(path_repository), "commit", "-m", commit_message],
+ env=env,
+ shell=False,
+ )
+
+ # git push
+ rc, out = rc_cmd(
+ [self.command, "-C", str(path_repository), "push"],
+ env=env,
+ shell=False,
+ )
+ if rc:
+ raise Exception(out)
+
def urlc(urlstring, *args, **kwargs):
"""
Dynamically dispatch the appropriate protocol class.
"""
- url_classes = {'http': HttpC, 'https': HttpC, 'ftp': FtpC, 'ftps': FtpC, \
- 'sftp': SshC, 'ssh': SshC, 'scp': SshC, 'tftp': TftpC}
+ url_classes = {
+ "http": HttpC,
+ "https": HttpC,
+ "ftp": FtpC,
+ "ftps": FtpC,
+ "sftp": SshC,
+ "ssh": SshC,
+ "scp": SshC,
+ "tftp": TftpC,
+ "git": GitC,
+ }
url = urllib.parse.urlsplit(urlstring)
+ scheme, _, _ = url.scheme.partition("+")
try:
- return url_classes[url.scheme](url, *args, **kwargs)
+ return url_classes[scheme](url, *args, **kwargs)
except KeyError:
- raise ValueError(f'Unsupported URL scheme: "{url.scheme}"')
+ raise ValueError(f'Unsupported URL scheme: "{scheme}"')
-def download(local_path, urlstring, progressbar=False, check_space=False,
+def download(local_path, urlstring, progressbar=False, raise_error=False, check_space=False,
source_host='', source_port=0, timeout=10.0):
try:
progressbar = progressbar and is_interactive()
urlc(urlstring, progressbar, check_space, source_host, source_port, timeout).download(local_path)
except Exception as err:
+ if raise_error:
+ raise
print_error(f'Unable to download "{urlstring}": {err}')
except KeyboardInterrupt:
print_error('\nDownload aborted by user.')
-def upload(local_path, urlstring, progressbar=False,
+def upload(local_path, urlstring, progressbar=False, raise_error=False,
source_host='', source_port=0, timeout=10.0):
try:
progressbar = progressbar and is_interactive()
urlc(urlstring, progressbar, source_host, source_port, timeout).upload(local_path)
except Exception as err:
+ if raise_error:
+ raise
print_error(f'Unable to upload "{urlstring}": {err}')
except KeyboardInterrupt:
print_error('\nUpload aborted by user.')
diff --git a/scripts/build-command-templates b/scripts/build-command-templates
index c8ae83d9d..2e7f8b994 100755
--- a/scripts/build-command-templates
+++ b/scripts/build-command-templates
@@ -145,6 +145,8 @@ def get_properties(p, default=None):
description = v.find("description").text
if default != None and default.text == format:
description += f' (default)'
+ # Is no description was specified, keep it empty
+ if not description: description = ''
vh.append( (format, description) )
props["val_help"] = vh
except: