Merge branch 'current' into T5014-dnat

9 files changed, 193 insertions, 65 deletions

diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py
index d0cd87464..e18d9817d 100644
--- a/python/vyos/configtree.py
+++ b/python/vyos/configtree.py
@@ -418,10 +418,6 @@ class DiffTree:
         self.__diff_tree.argtypes = [c_char_p, c_void_p, c_void_p]
         self.__diff_tree.restype = c_void_p
 
-        self.__trim_tree = self.__lib.trim_tree
-        self.__trim_tree.argtypes = [c_void_p, c_void_p]
-        self.__trim_tree.restype = c_void_p
-
         check_path(path)
         path_str = " ".join(map(str, path)).encode()
 
@@ -435,11 +431,7 @@ class DiffTree:
         self.add = self.full.get_subtree(['add'])
         self.sub = self.full.get_subtree(['sub'])
         self.inter = self.full.get_subtree(['inter'])
-
-        # trim sub(-tract) tree to get delete tree for commands
-        ref = self.right.get_subtree(path, with_node=True) if path else self.right
-        res = self.__trim_tree(self.sub._get_config(), ref._get_config())
-        self.delete = ConfigTree(address=res)
+        self.delete = self.full.get_subtree(['del'])
 
     def to_commands(self):
         add = self.add.to_commands()
diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py
index 2793b201c..903cc8535 100644
--- a/python/vyos/firewall.py
+++ b/python/vyos/firewall.py
@@ -304,7 +304,7 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name):
     if 'ipsec' in rule_conf:
         if 'match_ipsec' in rule_conf['ipsec']:
             output.append('meta ipsec == 1')
-        if 'match_non_ipsec' in rule_conf['ipsec']:
+        if 'match_none' in rule_conf['ipsec']:
             output.append('meta ipsec == 0')
 
     if 'fragment' in rule_conf:
diff --git a/python/vyos/ifconfig/interface.py b/python/vyos/ifconfig/interface.py
index 120f2131b..99ddb2021 100644
--- a/python/vyos/ifconfig/interface.py
+++ b/python/vyos/ifconfig/interface.py
@@ -1288,9 +1288,11 @@ class Interface(Control):
 
         ifname = self.ifname
         config_file = f'/run/dhcp6c/dhcp6c.{ifname}.conf'
+        options_file = f'/run/dhcp6c/dhcp6c.{ifname}.options'
         systemd_service = f'dhcp6c@{ifname}.service'
 
         if enable and 'disable' not in self._config:
+            render(options_file, 'dhcp-client/dhcp6c_daemon-options.j2', self._config)
             render(config_file, 'dhcp-client/ipv6.j2', self._config)
 
             # We must ignore any return codes. This is required to enable
diff --git a/python/vyos/nat.py b/python/vyos/nat.py
index a56ca1ff3..418efe649 100644
--- a/python/vyos/nat.py
+++ b/python/vyos/nat.py
@@ -54,28 +54,32 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False):
         translation_str = 'return'
         log_suffix = '-EXCL'
     elif 'translation' in rule_conf:
-        translation_prefix = nat_type[:1]
-        translation_output = [f'{translation_prefix}nat']
         addr = dict_search_args(rule_conf, 'translation', 'address')
         port = dict_search_args(rule_conf, 'translation', 'port')
-
-        if addr and is_ip_network(addr):
-            if not ipv6:
-                map_addr =  dict_search_args(rule_conf, nat_type, 'address')
-                translation_output.append(f'{ip_prefix} prefix to {ip_prefix} {translation_prefix}addr map {{ {map_addr} : {addr} }}')
-                ignore_type_addr = True
-            else:
-                translation_output.append(f'prefix to {addr}')
-        elif addr == 'masquerade':
-            if port:
-                addr = f'{addr} to '
-            translation_output = [addr]
-            log_suffix = '-MASQ'
+        redirect_port = dict_search_args(rule_conf, 'translation', 'redirect', 'port')
+        if redirect_port:
+            translation_output = [f'redirect to {redirect_port}']
         else:
-            translation_output.append('to')
-            if addr:
-                addr = bracketize_ipv6(addr)
-                translation_output.append(addr)
+            translation_prefix = nat_type[:1]
+            translation_output = [f'{translation_prefix}nat']
+
+            if addr and is_ip_network(addr):
+                if not ipv6:
+                    map_addr =  dict_search_args(rule_conf, nat_type, 'address')
+                    translation_output.append(f'{ip_prefix} prefix to {ip_prefix} {translation_prefix}addr map {{ {map_addr} : {addr} }}')
+                    ignore_type_addr = True
+                else:
+                    translation_output.append(f'prefix to {addr}')
+            elif addr == 'masquerade':
+                if port:
+                    addr = f'{addr} to '
+                translation_output = [addr]
+                log_suffix = '-MASQ'
+            else:
+                translation_output.append('to')
+                if addr:
+                    addr = bracketize_ipv6(addr)
+                    translation_output.append(addr)
 
         options = []
         addr_mapping = dict_search_args(rule_conf, 'translation', 'options', 'address_mapping')
diff --git a/python/vyos/remote.py b/python/vyos/remote.py
index 16fe2b2c2..cf731c881 100644
--- a/python/vyos/remote.py
+++ b/python/vyos/remote.py
@@ -25,7 +25,7 @@ import urllib.parse
 from ftplib import FTP
 from ftplib import FTP_TLS
 
-from paramiko import SSHClient
+from paramiko import SSHClient, SSHException
 from paramiko import MissingHostKeyPolicy
 
 from requests import Session
@@ -50,7 +50,7 @@ class InteractivePolicy(MissingHostKeyPolicy):
     def missing_host_key(self, client, hostname, key):
         print_error(f"Host '{hostname}' not found in known hosts.")
         print_error('Fingerprint: ' + key.get_fingerprint().hex())
-        if ask_yes_no('Do you wish to continue?'):
+        if sys.stdout.isatty() and ask_yes_no('Do you wish to continue?'):
             if client._host_keys_filename\
                and ask_yes_no('Do you wish to permanently add this host/key pair to known hosts?'):
                 client._host_keys.add(hostname, key.get_name(), key)
@@ -96,7 +96,13 @@ def check_storage(path, size):
 
 
 class FtpC:
-    def __init__(self, url, progressbar=False, check_space=False, source_host='', source_port=0):
+    def __init__(self,
+                 url,
+                 progressbar=False,
+                 check_space=False,
+                 source_host='',
+                 source_port=0,
+                 timeout=10):
         self.secure = url.scheme == 'ftps'
         self.hostname = url.hostname
         self.path = url.path
@@ -106,12 +112,15 @@ class FtpC:
         self.source = (source_host, source_port)
         self.progressbar = progressbar
         self.check_space = check_space
+        self.timeout = timeout
 
     def _establish(self):
         if self.secure:
-            return FTP_TLS(source_address=self.source, context=ssl.create_default_context())
+            return FTP_TLS(source_address=self.source,
+                           context=ssl.create_default_context(),
+                           timeout=self.timeout)
         else:
-            return FTP(source_address=self.source)
+            return FTP(source_address=self.source, timeout=self.timeout)
 
     def download(self, location: str):
         # Open the file upfront before establishing connection.
@@ -150,7 +159,13 @@ class FtpC:
 
 class SshC:
     known_hosts = os.path.expanduser('~/.ssh/known_hosts')
-    def __init__(self, url, progressbar=False, check_space=False, source_host='', source_port=0):
+    def __init__(self,
+                 url,
+                 progressbar=False,
+                 check_space=False,
+                 source_host='',
+                 source_port=0,
+                 timeout=10.0):
         self.hostname = url.hostname
         self.path = url.path
         self.username = url.username or os.getenv('REMOTE_USERNAME')
@@ -159,6 +174,7 @@ class SshC:
         self.source = (source_host, source_port)
         self.progressbar = progressbar
         self.check_space = check_space
+        self.timeout = timeout
 
     def _establish(self):
         ssh = SSHClient()
@@ -169,7 +185,7 @@ class SshC:
         ssh.set_missing_host_key_policy(InteractivePolicy())
         # `socket.create_connection()` automatically picks a NIC and an IPv4/IPv6 address family
         #  for us on dual-stack systems.
-        sock = socket.create_connection((self.hostname, self.port), socket.getdefaulttimeout(), self.source)
+        sock = socket.create_connection((self.hostname, self.port), self.timeout, self.source)
         ssh.connect(self.hostname, self.port, self.username, self.password, sock=sock)
         return ssh
 
@@ -198,13 +214,20 @@ class SshC:
 
 
 class HttpC:
-    def __init__(self, url, progressbar=False, check_space=False, source_host='', source_port=0):
+    def __init__(self,
+                 url,
+                 progressbar=False,
+                 check_space=False,
+                 source_host='',
+                 source_port=0,
+                 timeout=10.0):
         self.urlstring = urllib.parse.urlunsplit(url)
         self.progressbar = progressbar
         self.check_space = check_space
         self.source_pair = (source_host, source_port)
         self.username = url.username or os.getenv('REMOTE_USERNAME')
         self.password = url.password or os.getenv('REMOTE_PASSWORD')
+        self.timeout = timeout
 
     def _establish(self):
         session = Session()
@@ -220,8 +243,11 @@ class HttpC:
             # Not only would it potentially mess up with the progress bar but
             # `shutil.copyfileobj(request.raw, file)` does not handle automatic decoding.
             s.headers.update({'Accept-Encoding': 'identity'})
-            with s.head(self.urlstring, allow_redirects=True) as r:
+            with s.head(self.urlstring,
+                        allow_redirects=True,
+                        timeout=self.timeout) as r:
                 # Abort early if the destination is inaccessible.
+                print('pre-3')
                 r.raise_for_status()
                 # If the request got redirected, keep the last URL we ended up with.
                 final_urlstring = r.url
@@ -235,7 +261,8 @@ class HttpC:
                     size = None
             if self.check_space:
                 check_storage(location, size)
-            with s.get(final_urlstring, stream=True) as r, open(location, 'wb') as f:
+            with s.get(final_urlstring, stream=True,
+                       timeout=self.timeout) as r, open(location, 'wb') as f:
                 if self.progressbar and size:
                     progress = make_incremental_progressbar(CHUNK_SIZE / size)
                     next(progress)
@@ -249,7 +276,10 @@ class HttpC:
     def upload(self, location: str):
         # Does not yet support progressbars.
         with self._establish() as s, open(location, 'rb') as f:
-            s.post(self.urlstring, data=f, allow_redirects=True)
+            s.post(self.urlstring,
+                   data=f,
+                   allow_redirects=True,
+                   timeout=self.timeout)
 
 
 class TftpC:
@@ -258,10 +288,16 @@ class TftpC:
     # 2. Since there's no concept authentication, we don't need to deal with keys/passwords.
     # 3. It would be a waste to import, audit and maintain a third-party library for TFTP.
     # 4. I'd rather not implement the entire protocol here, no matter how simple it is.
-    def __init__(self, url, progressbar=False, check_space=False, source_host=None, source_port=0):
+    def __init__(self,
+                 url,
+                 progressbar=False,
+                 check_space=False,
+                 source_host=None,
+                 source_port=0,
+                 timeout=10):
         source_option = f'--interface {source_host} --local-port {source_port}' if source_host else ''
         progress_flag = '--progress-bar' if progressbar else '-s'
-        self.command = f'curl {source_option} {progress_flag}'
+        self.command = f'curl {source_option} {progress_flag} --connect-timeout {timeout}'
         self.urlstring = urllib.parse.urlunsplit(url)
 
     def download(self, location: str):
@@ -286,10 +322,16 @@ def urlc(urlstring, *args, **kwargs):
         raise ValueError(f'Unsupported URL scheme: "{url.scheme}"')
 
 def download(local_path, urlstring, *args, **kwargs):
-    urlc(urlstring, *args, **kwargs).download(local_path)
+    try:
+        urlc(urlstring, *args, **kwargs).download(local_path)
+    except Exception as err:
+        print_error(f'Unable to download "{urlstring}": {err}')
 
 def upload(local_path, urlstring, *args, **kwargs):
-    urlc(urlstring, *args, **kwargs).upload(local_path)
+    try:
+        urlc(urlstring, *args, **kwargs).upload(local_path)
+    except Exception as err:
+        print_error(f'Unable to upload "{urlstring}": {err}')
 
 def get_remote_config(urlstring, source_host='', source_port=0):
     """
diff --git a/python/vyos/utils/kernel.py b/python/vyos/utils/kernel.py
index 0eb113174..1f3bbdffe 100644
--- a/python/vyos/utils/kernel.py
+++ b/python/vyos/utils/kernel.py
@@ -25,3 +25,14 @@ def check_kmod(k_mod):
         if not os.path.exists(f'/sys/module/{module}'):
             if call(f'modprobe {module}') != 0:
                 raise ConfigError(f'Loading Kernel module {module} failed')
+
+def unload_kmod(k_mod):
+    """ Common utility function to unload required kernel modules on demand """
+    from vyos import ConfigError
+    from vyos.utils.process import call
+    if isinstance(k_mod, str):
+        k_mod = k_mod.split()
+    for module in k_mod:
+        if os.path.exists(f'/sys/module/{module}'):
+            if call(f'rmmod {module}') != 0:
+                raise ConfigError(f'Unloading Kernel module {module} failed')
diff --git a/python/vyos/xml_ref/generate_cache.py b/python/vyos/xml_ref/generate_cache.py
index 792c6eea7..6a05d4608 100755
--- a/python/vyos/xml_ref/generate_cache.py
+++ b/python/vyos/xml_ref/generate_cache.py
@@ -18,10 +18,14 @@
 
 import sys
 import json
-import argparse
+from argparse import ArgumentParser
+from argparse import ArgumentTypeError
+from os import getcwd
+from os import makedirs
 from os.path import join
 from os.path import abspath
 from os.path import dirname
+from os.path import basename
 from xmltodict import parse
 
 _here = dirname(__file__)
@@ -29,9 +33,10 @@ _here = dirname(__file__)
 sys.path.append(join(_here, '..'))
 from configtree import reference_tree_to_json, ConfigTreeError
 
-xml_cache = abspath(join(_here, 'cache.py'))
 xml_cache_json = 'xml_cache.json'
 xml_tmp = join('/tmp', xml_cache_json)
+pkg_cache = abspath(join(_here, 'pkg_cache'))
+ref_cache = abspath(join(_here, 'cache.py'))
 
 node_data_fields = ("node_type", "multi", "valueless", "default_value")
 
@@ -45,16 +50,26 @@ def trim_node_data(cache: dict):
             if isinstance(cache[k], dict):
                 trim_node_data(cache[k])
 
+def non_trivial(s):
+    if not s:
+        raise ArgumentTypeError("Argument must be non empty string")
+    return s
+
 def main():
-    parser = argparse.ArgumentParser(description='generate and save dict from xml defintions')
+    parser = ArgumentParser(description='generate and save dict from xml defintions')
     parser.add_argument('--xml-dir', type=str, required=True,
                         help='transcluded xml interface-definition directory')
-    parser.add_argument('--save-json-dir', type=str,
-                        help='directory to save json cache if needed')
-    args = parser.parse_args()
-
-    xml_dir = abspath(args.xml_dir)
-    save_dir = abspath(args.save_json_dir) if args.save_json_dir else None
+    parser.add_argument('--package-name', type=non_trivial, default='vyos-1x',
+                        help='name of current package')
+    parser.add_argument('--output-path', help='path to generated cache')
+    args = vars(parser.parse_args())
+
+    xml_dir = abspath(args['xml_dir'])
+    pkg_name = args['package_name'].replace('-','_')
+    cache_name = pkg_name + '_cache.py'
+    out_path = args['output_path']
+    path = out_path if out_path is not None else pkg_cache
+    xml_cache = abspath(join(path, cache_name))
 
     try:
         reference_tree_to_json(xml_dir, xml_tmp)
@@ -67,21 +82,30 @@ def main():
 
     trim_node_data(d)
 
-    if save_dir is not None:
-        save_file = join(save_dir, xml_cache_json)
-        with open(save_file, 'w') as f:
-            f.write(json.dumps(d))
-
     syntax_version = join(xml_dir, 'xml-component-version.xml')
-    with open(syntax_version) as f:
-        content = f.read()
+    try:
+        with open(syntax_version) as f:
+            component = f.read()
+    except FileNotFoundError:
+        if pkg_name != 'vyos_1x':
+            component = ''
+        else:
+            print("\nWARNING: missing xml-component-version.xml\n")
+            sys.exit(1)
 
-    parsed = parse(content)
-    converted = parsed['interfaceDefinition']['syntaxVersion']
+    if component:
+        parsed = parse(component)
+    else:
+        parsed = None
     version = {}
-    for i in converted:
-        tmp = {i['@component']: i['@version']}
-        version |= tmp
+    # addon package definitions may have empty (== 0) version info
+    if parsed is not None and parsed['interfaceDefinition'] is not None:
+        converted = parsed['interfaceDefinition']['syntaxVersion']
+        if not isinstance(converted, list):
+            converted = [converted]
+        for i in converted:
+            tmp = {i['@component']: i['@version']}
+            version |= tmp
 
     version = {"component_version": version}
 
@@ -90,5 +114,7 @@ def main():
     with open(xml_cache, 'w') as f:
         f.write(f'reference = {str(d)}')
 
+    print(cache_name)
+
 if __name__ == '__main__':
     main()
diff --git a/python/vyos/xml_ref/pkg_cache/__init__.py b/python/vyos/xml_ref/pkg_cache/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/python/vyos/xml_ref/pkg_cache/__init__.py
diff --git a/python/vyos/xml_ref/update_cache.py b/python/vyos/xml_ref/update_cache.py
new file mode 100755
index 000000000..0842bcbe9
--- /dev/null
+++ b/python/vyos/xml_ref/update_cache.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2023 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+import os
+from copy import deepcopy
+from generate_cache import pkg_cache
+from generate_cache import ref_cache
+
+def dict_merge(source, destination):
+    dest = deepcopy(destination)
+
+    for key, value in source.items():
+        if key not in dest:
+            dest[key] = value
+        elif isinstance(source[key], dict):
+            dest[key] = dict_merge(source[key], dest[key])
+
+    return dest
+
+def main():
+    res = {}
+    cache_dir = os.path.basename(pkg_cache)
+    for mod in os.listdir(pkg_cache):
+        mod = os.path.splitext(mod)[0]
+        if not mod.endswith('_cache'):
+            continue
+        d = getattr(__import__(f'{cache_dir}.{mod}', fromlist=[mod]), 'reference')
+        if mod == 'vyos_1x_cache':
+            res = dict_merge(res, d)
+        else:
+            res = dict_merge(d, res)
+
+    with open(ref_cache, 'w') as f:
+        f.write(f'reference = {str(res)}')
+
+if __name__ == '__main__':
+    main()