Merge pull request #2505 from vyos/mergify/bp/sagitta/pr-2241

T2405: add Git support to commit-archive (backport #2241)

5 files changed, 171 insertions, 12 deletions

diff --git a/debian/control b/debian/control
index f31c5a510..42c0b580b 100644
--- a/debian/control
+++ b/debian/control
@@ -62,6 +62,7 @@ Depends:
   frr-snmp,
   fuse-overlayfs,
   libpam-google-authenticator,
+  git,
   grc,
   haproxy,
   hostapd,
diff --git a/interface-definitions/system-config-mgmt.xml.in b/interface-definitions/system-config-mgmt.xml.in
index de5a8cc16..61089ce34 100644
--- a/interface-definitions/system-config-mgmt.xml.in
+++ b/interface-definitions/system-config-mgmt.xml.in
@@ -17,11 +17,36 @@
                 <properties>
                   <help>Commit archive location</help>
                   <valueHelp>
-                    <format>uri</format>
-                    <description>Uniform Resource Identifier</description>
+                    <format>http://&lt;user&gt;:&lt;passwd&gt;@&lt;host&gt;/&lt;path&gt;</format>
+                    <description/>
+                  </valueHelp>
+                  <valueHelp>
+                    <format>https://&lt;user&gt;:&lt;passwd&gt;@&lt;host&gt;/&lt;path&gt;</format>
+                    <description/>
+                  </valueHelp>
+                  <valueHelp>
+                    <format>ftp://&lt;user&gt;:&lt;passwd&gt;@&lt;host&gt;/&lt;path&gt;</format>
+                    <description/>
+                  </valueHelp>
+                  <valueHelp>
+                    <format>sftp://&lt;user&gt;:&lt;passwd&gt;@&lt;host&gt;/&lt;path&gt;</format>
+                    <description/>
+                  </valueHelp>
+                  <valueHelp>
+                    <format>scp://&lt;user&gt;:&lt;passwd&gt;@&lt;host&gt;/&lt;path&gt;</format>
+                    <description/>
+                  </valueHelp>
+                  <valueHelp>
+                    <format>tftp://&lt;host&gt;/&lt;path&gt;</format>
+                    <description/>
+                  </valueHelp>
+                  <valueHelp>
+                    <format>git+https://&lt;user&gt;:&lt;passwd&gt;@&lt;host&gt;/&lt;path&gt;</format>
+                    <description/>
                   </valueHelp>
                   <constraint>
                     <validator name="url --file-transport"/>
+                    <regex>(ssh|git|git\+(\w+)):\/\/.*</regex>
                   </constraint>
                   <multi/>
                 </properties>
diff --git a/python/vyos/config_mgmt.py b/python/vyos/config_mgmt.py
index 654a8d698..df7240c88 100644
--- a/python/vyos/config_mgmt.py
+++ b/python/vyos/config_mgmt.py
@@ -22,10 +22,11 @@ import logging
 from typing import Optional, Tuple, Union
 from filecmp import cmp
 from datetime import datetime
-from textwrap import dedent
+from textwrap import dedent, indent
 from pathlib import Path
 from tabulate import tabulate
 from shutil import copy, chown
+from urllib.parse import urlsplit, urlunsplit
 
 from vyos.config import Config
 from vyos.configtree import ConfigTree, ConfigTreeError, show_diff
@@ -377,9 +378,22 @@ Proceed ?'''
         remote_file = f'config.boot-{hostname}.{timestamp}'
         source_address = self.source_address
 
+        if self.effective_locations:
+            print("Archiving config...")
         for location in self.effective_locations:
-            upload(archive_config_file, f'{location}/{remote_file}',
-                   source_host=source_address)
+            url = urlsplit(location)
+            _, _, netloc = url.netloc.rpartition("@")
+            redacted_location = urlunsplit(url._replace(netloc=netloc))
+            print(f"  {redacted_location}", end=" ", flush=True)
+            try:
+                upload(archive_config_file, f'{location}/{remote_file}',
+                       source_host=source_address, raise_error=True)
+                print("OK")
+            except Exception as e:
+                print("FAILED!")
+                print()
+                print(indent(str(e), "   > "))
+                print()
 
     # op-mode functions
     #
diff --git a/python/vyos/remote.py b/python/vyos/remote.py
index 4be477d24..8b90e4530 100644
--- a/python/vyos/remote.py
+++ b/python/vyos/remote.py
@@ -14,6 +14,7 @@
 # License along with this library.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+import pwd
 import shutil
 import socket
 import ssl
@@ -22,6 +23,9 @@ import sys
 import tempfile
 import urllib.parse
 
+from contextlib import contextmanager
+from pathlib import Path
+
 from ftplib import FTP
 from ftplib import FTP_TLS
 
@@ -37,11 +41,22 @@ from vyos.utils.io import ask_yes_no
 from vyos.utils.io import is_interactive
 from vyos.utils.io import print_error
 from vyos.utils.misc import begin
-from vyos.utils.process import cmd
+from vyos.utils.process import cmd, rc_cmd
 from vyos.version import get_version
 
 CHUNK_SIZE = 8192
 
+@contextmanager
+def umask(mask: int):
+    """
+    Context manager that temporarily sets the process umask.
+    """
+    oldmask = os.umask(mask)
+    try:
+        yield
+    finally:
+        os.umask(oldmask)
+
 class InteractivePolicy(MissingHostKeyPolicy):
     """
     Paramiko policy for interactively querying the user on whether to proceed
@@ -310,35 +325,137 @@ class TftpC:
         with open(location, 'rb') as f:
             cmd(f'{self.command} -T - "{self.urlstring}"', input=f.read())
 
+class GitC:
+    def __init__(self,
+        url,
+        progressbar=False,
+        check_space=False,
+        source_host=None,
+        source_port=0,
+        timeout=10,
+    ):
+        self.command = 'git'
+        self.url = url
+        self.urlstring = urllib.parse.urlunsplit(url)
+        if self.urlstring.startswith("git+"):
+            self.urlstring = self.urlstring.replace("git+", "", 1)
+
+    def download(self, location: str):
+        raise NotImplementedError("not supported")
+
+    @umask(0o077)
+    def upload(self, location: str):
+        scheme = self.url.scheme
+        _, _, scheme = scheme.partition("+")
+        netloc = self.url.netloc
+        url = Path(self.url.path).parent
+        with tempfile.TemporaryDirectory(prefix="git-commit-archive-") as directory:
+            # Determine username, fullname, email for Git commit
+            pwd_entry = pwd.getpwuid(os.getuid())
+            user = pwd_entry.pw_name
+            name = pwd_entry.pw_gecos.split(",")[0] or user
+            fqdn = socket.getfqdn()
+            email = f"{user}@{fqdn}"
+
+            # environment vars for our git commands
+            env = {
+                "GIT_TERMINAL_PROMPT": "0",
+                "GIT_AUTHOR_NAME": name,
+                "GIT_AUTHOR_EMAIL": email,
+                "GIT_COMMITTER_NAME": name,
+                "GIT_COMMITTER_EMAIL": email,
+            }
+
+            # build ssh command for git
+            ssh_command = ["ssh"]
+
+            # if we are not interactive, we use StrictHostKeyChecking=yes to avoid any prompts
+            if not sys.stdout.isatty():
+                ssh_command += ["-o", "StrictHostKeyChecking=yes"]
+
+            env["GIT_SSH_COMMAND"] = " ".join(ssh_command)
+
+            # git clone
+            path_repository = Path(directory) / "repository"
+            scheme = f"{scheme}://" if scheme else ""
+            rc, out = rc_cmd(
+                [self.command, "clone", f"{scheme}{netloc}{url}", str(path_repository), "--depth=1"],
+                env=env,
+                shell=False,
+            )
+            if rc:
+                raise Exception(out)
+
+            # git add
+            filename = Path(Path(self.url.path).name).stem
+            dst = path_repository / filename
+            shutil.copy2(location, dst)
+            rc, out = rc_cmd(
+                [self.command, "-C", str(path_repository), "add", filename],
+                env=env,
+                shell=False,
+            )
+
+            # git commit -m
+            commit_message = os.environ.get("COMMIT_COMMENT", "commit")
+            rc, out = rc_cmd(
+                [self.command, "-C", str(path_repository), "commit", "-m", commit_message],
+                env=env,
+                shell=False,
+            )
+
+            # git push
+            rc, out = rc_cmd(
+                [self.command, "-C", str(path_repository), "push"],
+                env=env,
+                shell=False,
+            )
+            if rc:
+                raise Exception(out)
+
 
 def urlc(urlstring, *args, **kwargs):
     """
     Dynamically dispatch the appropriate protocol class.
     """
-    url_classes = {'http': HttpC, 'https': HttpC, 'ftp': FtpC, 'ftps': FtpC, \
-                   'sftp': SshC, 'ssh': SshC, 'scp': SshC, 'tftp': TftpC}
+    url_classes = {
+        "http": HttpC,
+        "https": HttpC,
+        "ftp": FtpC,
+        "ftps": FtpC,
+        "sftp": SshC,
+        "ssh": SshC,
+        "scp": SshC,
+        "tftp": TftpC,
+        "git": GitC,
+    }
     url = urllib.parse.urlsplit(urlstring)
+    scheme, _, _ = url.scheme.partition("+")
     try:
-        return url_classes[url.scheme](url, *args, **kwargs)
+        return url_classes[scheme](url, *args, **kwargs)
     except KeyError:
-        raise ValueError(f'Unsupported URL scheme: "{url.scheme}"')
+        raise ValueError(f'Unsupported URL scheme: "{scheme}"')
 
-def download(local_path, urlstring, progressbar=False, check_space=False,
+def download(local_path, urlstring, progressbar=False, raise_error=False, check_space=False,
              source_host='', source_port=0, timeout=10.0):
     try:
         progressbar = progressbar and is_interactive()
         urlc(urlstring, progressbar, check_space, source_host, source_port, timeout).download(local_path)
     except Exception as err:
+        if raise_error:
+            raise
         print_error(f'Unable to download "{urlstring}": {err}')
     except KeyboardInterrupt:
         print_error('\nDownload aborted by user.')
 
-def upload(local_path, urlstring, progressbar=False,
+def upload(local_path, urlstring, progressbar=False, raise_error=False,
            source_host='', source_port=0, timeout=10.0):
     try:
         progressbar = progressbar and is_interactive()
         urlc(urlstring, progressbar, source_host, source_port, timeout).upload(local_path)
     except Exception as err:
+        if raise_error:
+            raise
         print_error(f'Unable to upload "{urlstring}": {err}')
     except KeyboardInterrupt:
         print_error('\nUpload aborted by user.')
diff --git a/scripts/build-command-templates b/scripts/build-command-templates
index c8ae83d9d..2e7f8b994 100755
--- a/scripts/build-command-templates
+++ b/scripts/build-command-templates
@@ -145,6 +145,8 @@ def get_properties(p, default=None):
             description = v.find("description").text
             if default != None and default.text == format:
                 description += f' (default)'
+            # Is no description was specified, keep it empty
+            if not description: description = ''
             vh.append( (format, description) )
         props["val_help"] = vh
     except: