Merge branch 't3255-rpki' of github.com:c-po/vyos-1x into current

* 't3255-rpki' of github.com:c-po/vyos-1x:
  rpki: T3255: provide full protocol support in XML and Python
  op-mode: ssh: support generating client keypair
  smoketest: openvpn: remove print() when generating keys

9 files changed, 429 insertions, 100 deletions

diff --git a/Makefile b/Makefile
index 2ff72cb21..66b1e8bb7 100644
--- a/Makefile
+++ b/Makefile
@@ -47,7 +47,6 @@ interface_definitions: $(config_xml_obj)
 	rm -f $(TMPL_DIR)/vpn/node.def
 	rm -f $(TMPL_DIR)/vpn/ipsec/node.def
 	rm -rf $(TMPL_DIR)/vpn/nipsec
-	rm -rf $(TMPL_DIR)/protocols/nrpki
 
 	# XXX: required until OSPF and RIP is migrated from vyatta-cfg-quagga to vyos-1x
 	mkdir $(TMPL_DIR)/interfaces/loopback/node.tag/ipv6
diff --git a/data/templates/frr/rpki.frr.tmpl b/data/templates/frr/rpki.frr.tmpl
new file mode 100644
index 000000000..bf72c7a9e
--- /dev/null
+++ b/data/templates/frr/rpki.frr.tmpl
@@ -0,0 +1,16 @@
+!
+rpki
+{% if cache is defined and cache is not none %}
+{%   for peer, peer_config in cache.items() %}
+{#     port is mandatory and preference uses a default value #}
+{%     if peer_config.ssh is defined and peer_config.ssh.username is defined and peer_config.ssh.username is not none %}
+ rpki cache {{ peer }} {{ peer_config.port }} {{ peer_config.ssh.username }} {{ peer_config.ssh.private_key_file }} {{ peer_config.ssh.public_key_file }} {{ peer_config.ssh.known_hosts_file }} preference {{ peer_config.preference }}
+{%     else %}
+ rpki cache {{ peer }} {{ peer_config.port }} preference {{ peer_config.preference }}
+{%     endif %}
+{%   endfor %}
+{% endif %}
+{% if polling_period is defined and polling_period is not none %}
+ rpki polling_period {{ polling_period }}
+{% endif %}
+!
diff --git a/interface-definitions/protocols-rpki.xml.in b/interface-definitions/protocols-rpki.xml.in
index b8db49e36..d96db0683 100644
--- a/interface-definitions/protocols-rpki.xml.in
+++ b/interface-definitions/protocols-rpki.xml.in
@@ -1,32 +1,44 @@
 <?xml version="1.0" encoding="utf-8"?>
-<!-- Protocol RPKI configuration -->
 <interfaceDefinition>
   <node name="protocols">
     <children>
-      <node name="nrpki" owner="${vyos_conf_scripts_dir}/protocols_rpki.py">
+      <node name="rpki" owner="${vyos_conf_scripts_dir}/protocols_rpki.py">
         <properties>
           <help>BGP prefix origin validation</help>
         </properties>
         <children>
           <tagNode name="cache">
             <properties>
-              <help>RPKI cache server instance</help>
+              <help>RPKI cache server address</help>
+              <valueHelp>
+                <format>ipv4</format>
+                <description>IP address of NTP server</description>
+              </valueHelp>
+              <valueHelp>
+                <format>ipv6</format>
+                <description>IPv6 address of NTP server</description>
+              </valueHelp>
+              <valueHelp>
+                <format>hostname</format>
+                <description>Fully qualified domain name of NTP server</description>
+              </valueHelp>
+              <constraint>
+                <validator name="ipv4-address"/>
+                <validator name="ipv6-address"/>
+                <validator name="fqdn"/>
+              </constraint>
             </properties>
             <children>
-              <leafNode name="address">
+              #include <include/port-number.xml.i>
+              <leafNode name="preference">
                 <properties>
-                  <help>RPKI cache server address</help>
-                </properties>
-              </leafNode>
-              <leafNode name="port">
-                <properties>
-                  <help>TCP port number</help>
+                  <help>Preference of the cache server</help>
                   <valueHelp>
-                    <format>u32:1-65535</format>
-                    <description>TCP port number</description>
+                    <format>u32:1-255</format>
+                    <description>Polling period</description>
                   </valueHelp>
                   <constraint>
-                    <validator name="numeric" argument="--range 1-65535"/>
+                    <validator name="numeric" argument="--range 1-255"/>
                   </constraint>
                 </properties>
               </leafNode>
@@ -68,55 +80,20 @@
               </node>
             </children>
           </tagNode>
-          <leafNode name="initial-synchronization-timeout">
-            <properties>
-              <help>Initial RPKI cache synchronization timeout</help>
-              <valueHelp>
-                <format>u32:0-4294967295</format>
-                <description>Initial RPKI cache synchronization timeout</description>
-              </valueHelp>
-              <constraint>
-                <validator name="numeric" argument="--range 0-4294967295"/>
-              </constraint>
-            </properties>
-          </leafNode>
           <leafNode name="polling-period">
             <properties>
-              <help>RPKI cache polling period</help>
-              <valueHelp>
-                <format>u32:1-1300</format>
-                <description>Polling period</description>
-              </valueHelp>
-              <constraint>
-                <validator name="numeric" argument="--range 1-1300"/>
-              </constraint>
-            </properties>
-          </leafNode>
-          <leafNode name="preference">
-            <properties>
-              <help>RPKI cache preference</help>
-              <valueHelp>
-                <format>u32:0-4294967295</format>
-                <description>RPKI cache preference</description>
-              </valueHelp>
-              <constraint>
-                <validator name="numeric" argument="--range 0-4294967295"/>
-              </constraint>
-            </properties>
-          </leafNode>
-          <leafNode name="timeout">
-            <properties>
-              <help>RPKI cache reply timeout</help>
+              <help>RPKI cache polling period (default: 600)</help>
               <valueHelp>
-                <format>u32:0-4294967295</format>
-                <description>RPKI cache reply timeout</description>
+                <format>u32:1-86400</format>
+                <description>Polling period in seconds</description>
               </valueHelp>
               <constraint>
-                <validator name="numeric" argument="--range 0-4294967295"/>
+                <validator name="numeric" argument="--range 1-86400"/>
               </constraint>
             </properties>
+            <defaultValue>600</defaultValue>
           </leafNode>
-        </children> 
+        </children>
       </node>
     </children>
   </node>
diff --git a/op-mode-definitions/generate-ssh-server-key.xml.in b/op-mode-definitions/generate-ssh-server-key.xml.in
index a6ebf1b78..86bb1b1bd 100644
--- a/op-mode-definitions/generate-ssh-server-key.xml.in
+++ b/op-mode-definitions/generate-ssh-server-key.xml.in
@@ -2,14 +2,30 @@
 <interfaceDefinition>
   <node name="generate">
     <properties>
-      <help>Generate an object</help>
+      <help>Generate an object/key</help>
     </properties>
     <children>
-      <node name="ssh-server-key">
+      <node name="ssh">
         <properties>
-          <help>Regenerate the host SSH keys and restart the SSH server</help>
+          <help>Generate SSH related keypairs</help>
         </properties>
-        <command>${vyos_op_scripts_dir}/generate_ssh_server_key.py</command>
+        <children>
+          <node name="server-key">
+            <properties>
+              <help>Re-generate SSH host keys and restart SSH server</help>
+            </properties>
+            <command>${vyos_op_scripts_dir}/generate_ssh_server_key.py</command>
+          </node>
+          <tagNode name="client-key">
+            <properties>
+              <help>Re-generate SSH client keypair</help>
+              <completionHelp>
+                <list>&lt;filename&gt;</list>
+              </completionHelp>
+            </properties>
+            <command>ssh-keygen -t rsa -f "$4" -N ""</command>
+          </tagNode>
+        </children>
       </node>
     </children>
   </node>
diff --git a/smoketest/scripts/cli/test_interfaces_openvpn.py b/smoketest/scripts/cli/test_interfaces_openvpn.py
index 00db3f667..a2a1a85ec 100755
--- a/smoketest/scripts/cli/test_interfaces_openvpn.py
+++ b/smoketest/scripts/cli/test_interfaces_openvpn.py
@@ -625,27 +625,27 @@ if __name__ == '__main__':
         # Generate mandatory SSL certificate
         tmp = f'openssl req -newkey rsa:4096 -new -nodes -x509 -days 3650 '\
               f'-keyout {ssl_key} -out {ssl_cert} -subj {subject}'
-        print(cmd(tmp))
+        cmd(tmp)
 
     if not os.path.isfile(ca_cert):
         # Generate "CA"
         tmp = f'openssl req -new -x509 -key {ssl_key} -out {ca_cert} -subj {subject}'
-        print(cmd(tmp))
+        cmd(tmp)
 
     if not os.path.isfile(dh_pem):
         # Generate "DH" key
         tmp = f'openssl dhparam -out {dh_pem} 2048'
-        print(cmd(tmp))
+        cmd(tmp)
 
     if not os.path.isfile(s2s_key):
         # Generate site-2-site key
         tmp = f'openvpn --genkey --secret {s2s_key}'
-        print(cmd(tmp))
+        cmd(tmp)
 
     if not os.path.isfile(auth_key):
         # Generate TLS auth key
         tmp = f'openvpn --genkey --secret {auth_key}'
-        print(cmd(tmp))
+        cmd(tmp)
 
     for file in [ca_cert, ssl_cert, ssl_key, dh_pem, s2s_key, auth_key]:
         cmd(f'sudo chown openvpn:openvpn {file}')
diff --git a/smoketest/scripts/cli/test_protocols_bgp.py b/smoketest/scripts/cli/test_protocols_bgp.py
index 87d10eb85..30d98976d 100755
--- a/smoketest/scripts/cli/test_protocols_bgp.py
+++ b/smoketest/scripts/cli/test_protocols_bgp.py
@@ -122,6 +122,9 @@ class TestProtocolsBGP(unittest.TestCase):
         self.session.commit()
         del self.session
 
+        # Check for running process
+        self.assertTrue(process_named_running(PROCESS_NAME))
+
     def verify_frr_config(self, peer, peer_config, frrconfig):
         # recurring patterns to verify for both a simple neighbor and a peer-group
         if 'cap_dynamic' in peer_config:
@@ -182,8 +185,6 @@ class TestProtocolsBGP(unittest.TestCase):
         self.assertIn(f' bgp default local-preference {local_pref}', frrconfig)
         self.assertIn(f' no bgp default ipv4-unicast', frrconfig)
 
-        # Check for running process
-        self.assertTrue(process_named_running(PROCESS_NAME))
 
     def test_bgp_02_neighbors(self):
         # Test out individual neighbor configuration items, not all of them are
@@ -419,39 +420,5 @@ class TestProtocolsBGP(unittest.TestCase):
         for prefix in listen_ranges:
             self.assertIn(f' bgp listen range {prefix} peer-group {peer_group}', frrconfig)
 
-
-    def test_bgp_07_rpki(self):
-        rpki_path = ['protocols', 'rpki']
-        init_tmo = '50'
-        polling = '400'
-        preference = '100'
-        timeout = '900'
-
-        cache = {
-            'foo' : { 'address' : '1.1.1.1', 'port' : '8080' },
-# T3253 only one peer supported
-#           'bar' : { 'address' : '2.2.2.2', 'port' : '9090' },
-        }
-
-        self.session.set(rpki_path + ['polling-period', polling])
-        self.session.set(rpki_path + ['preference', preference])
-
-        for name, config in cache.items():
-            self.session.set(rpki_path + ['cache', name, 'address', config['address']])
-            self.session.set(rpki_path + ['cache', name, 'port', config['port']])
-
-        # commit changes
-        self.session.commit()
-
-        # Verify FRR bgpd configuration
-        frrconfig = getFRRRPKIconfig()
-        self.assertIn(f'rpki polling_period {polling}', frrconfig)
-
-        for name, config in cache.items():
-            self.assertIn('rpki cache {address} {port} preference 1'.format(**config), frrconfig)
-
-        self.session.delete(rpki_path)
-
-
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/smoketest/scripts/cli/test_protocols_rpki.py b/smoketest/scripts/cli/test_protocols_rpki.py
new file mode 100755
index 000000000..7c65ca26f
--- /dev/null
+++ b/smoketest/scripts/cli/test_protocols_rpki.py
@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2021 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import unittest
+
+from vyos.configsession import ConfigSession
+from vyos.configsession import ConfigSessionError
+from vyos.util import cmd
+from vyos.util import process_named_running
+
+base_path = ['protocols', 'rpki']
+PROCESS_NAME = 'bgpd'
+
+rpki_known_hosts = '/config/auth/known_hosts'
+rpki_ssh_key = '/config/auth/id_rsa_rpki'
+rpki_ssh_pub = f'{rpki_ssh_key}.pub'
+
+def getFRRRPKIconfig():
+    return cmd(f'vtysh -c "show run" | sed -n "/rpki/,/^!/p"')
+
+class TestProtocolsRPKI(unittest.TestCase):
+    def setUp(self):
+        self.session = ConfigSession(os.getpid())
+
+    def tearDown(self):
+        self.session.delete(base_path)
+        self.session.commit()
+        del self.session
+
+        # Nothing RPKI specific should be left over in the config
+        frrconfig = getFRRRPKIconfig()
+        self.assertNotIn('rpki', frrconfig)
+
+        # Check for running process
+        self.assertTrue(process_named_running(PROCESS_NAME))
+
+    def test_rpki(self):
+        polling = '7200'
+        cache = {
+            '192.0.2.1' : {
+                'port' : '8080',
+                'preference' : '1'
+            },
+            '192.0.2.2' : {
+                'port' : '9090',
+                'preference' : '2'
+            },
+        }
+
+        self.session.set(base_path + ['polling-period', polling])
+        for peer, peer_config in cache.items():
+            self.session.set(base_path + ['cache', peer, 'port', peer_config['port']])
+            self.session.set(base_path + ['cache', peer, 'preference', peer_config['preference']])
+
+        # commit changes
+        self.session.commit()
+
+        # Verify FRR configuration
+        frrconfig = getFRRRPKIconfig()
+        self.assertIn(f'rpki polling_period {polling}', frrconfig)
+
+        for peer, peer_config in cache.items():
+            port = peer_config['port']
+            preference = peer_config['preference']
+            self.assertIn(f'rpki cache {peer} {port} preference {preference}', frrconfig)
+
+    def test_rpki_ssh(self):
+        polling = '7200'
+        cache = {
+            '192.0.2.3' : {
+                'port' : '1234',
+                'username' : 'foo',
+                'preference' : '10'
+            },
+            '192.0.2.4' : {
+                'port' : '5678',
+                'username' : 'bar',
+                'preference' : '20'
+            },
+        }
+
+        self.session.set(base_path + ['polling-period', polling])
+
+        for peer, peer_config in cache.items():
+            self.session.set(base_path + ['cache', peer, 'port', peer_config['port']])
+            self.session.set(base_path + ['cache', peer, 'preference', peer_config['preference']])
+            self.session.set(base_path + ['cache', peer, 'ssh', 'username', peer_config['username']])
+            self.session.set(base_path + ['cache', peer, 'ssh', 'public-key-file', rpki_ssh_pub])
+            self.session.set(base_path + ['cache', peer, 'ssh', 'private-key-file', rpki_ssh_key])
+            self.session.set(base_path + ['cache', peer, 'ssh', 'known-hosts-file', rpki_known_hosts])
+
+        # commit changes
+        self.session.commit()
+
+        # Verify FRR configuration
+        frrconfig = getFRRRPKIconfig()
+        self.assertIn(f'rpki polling_period {polling}', frrconfig)
+
+        for peer, peer_config in cache.items():
+            port = peer_config['port']
+            preference = peer_config['preference']
+            username = peer_config['username']
+            self.assertIn(f'rpki cache {peer} {port} {username} {rpki_ssh_key} {rpki_known_hosts} preference {preference}', frrconfig)
+
+
+    def test_rpki_verify_preference(self):
+        cache = {
+            '192.0.2.1' : {
+                'port' : '8080',
+                'preference' : '1'
+            },
+            '192.0.2.2' : {
+                'port' : '9090',
+                'preference' : '1'
+            },
+        }
+
+        for peer, peer_config in cache.items():
+            self.session.set(base_path + ['cache', peer, 'port', peer_config['port']])
+            self.session.set(base_path + ['cache', peer, 'preference', peer_config['preference']])
+
+        # check validate() - preferences must be unique
+        with self.assertRaises(ConfigSessionError):
+            self.session.commit()
+
+
+if __name__ == '__main__':
+    # Create OpenSSH keypair used in RPKI tests
+    if not os.path.isfile(rpki_ssh_key):
+        cmd(f'ssh-keygen -t rsa -f {rpki_ssh_key} -N ""')
+
+    if not os.path.isfile(rpki_known_hosts):
+        cmd(f'touch {rpki_known_hosts}')
+
+    unittest.main(verbosity=2, failfast=True)
diff --git a/src/conf_mode/protocols_rpki.py b/src/conf_mode/protocols_rpki.py
new file mode 100755
index 000000000..9ea066f7f
--- /dev/null
+++ b/src/conf_mode/protocols_rpki.py
@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2021 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from sys import exit
+
+from vyos.config import Config
+from vyos.configdict import dict_merge
+from vyos.configverify import verify_route_maps
+from vyos.template import render
+from vyos.template import render_to_string
+from vyos.util import call
+from vyos.util import dict_search
+from vyos.xml import defaults
+from vyos import ConfigError
+from vyos import frr
+from vyos import airbag
+airbag.enable()
+
+config_file = r'/tmp/rpki.frr'
+frr_daemon = 'bgpd'
+
+DEBUG = os.path.exists('/tmp/rpki.debug')
+if DEBUG:
+    import logging
+    lg = logging.getLogger("vyos.frr")
+    lg.setLevel(logging.DEBUG)
+    ch = logging.StreamHandler()
+    lg.addHandler(ch)
+
+def get_config(config=None):
+    if config:
+        conf = config
+    else:
+        conf = Config()
+    base = ['protocols', 'rpki']
+
+    rpki = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True)
+    if not conf.exists(base):
+        return rpki
+
+    # We have gathered the dict representation of the CLI, but there are default
+    # options which we need to update into the dictionary retrived.
+    default_values = defaults(base)
+    rpki = dict_merge(default_values, rpki)
+
+    return rpki
+
+def verify(rpki):
+    import pprint
+    pprint.pprint(rpki)
+
+    if not rpki:
+        return None
+
+    if 'cache' in rpki:
+        preferences = []
+        for peer, peer_config in rpki['cache'].items():
+            for mandatory in ['port', 'preference']:
+                if mandatory not in peer_config:
+                    raise ConfigError(f'RPKI cache "{peer}" {mandatory} must be defined!')
+
+            if 'preference' in peer_config:
+                preference = peer_config['preference']
+                if preference in preferences:
+                    raise ConfigError(f'RPKI cache with preference {preference} already configured!')
+                preferences.append(preference)
+
+            if 'ssh' in peer_config:
+                files = ['private_key_file', 'public_key_file', 'known_hosts_file']
+                for file in files:
+                    if file not in peer_config['ssh']:
+                        raise ConfigError('RPKI+SSH requires username, public/private ' \
+                                          'keys and known-hosts file to be defined!')
+
+                    filename = peer_config['ssh'][file]
+                    if not os.path.exists(filename):
+                        raise ConfigError(f'RPKI SSH {file.replace("-","-")} "{filename}" does not exist!')
+
+    return None
+
+def generate(rpki):
+    if not rpki:
+        rpki['new_frr_config'] = ''
+        return None
+
+    # render(config) not needed, its only for debug
+    render(config_file, 'frr/rpki.frr.tmpl', rpki)
+    rpki['new_frr_config'] = render_to_string('frr/rpki.frr.tmpl', rpki)
+
+    return None
+
+def apply(rpki):
+    # Save original configuration prior to starting any commit actions
+    frr_cfg = frr.FRRConfig()
+    frr_cfg.load_configuration(frr_daemon)
+    frr_cfg.modify_section('rpki', '')
+    frr_cfg.add_before(r'(ip prefix-list .*|route-map .*|line vty)', rpki['new_frr_config'])
+
+    # Debugging
+    if DEBUG:
+        from pprint import pprint
+        print('')
+        print('--------- DEBUGGING ----------')
+        pprint(dir(frr_cfg))
+        print('Existing config:\n')
+        for line in frr_cfg.original_config:
+            print(line)
+        print(f'Replacement config:\n')
+        print(f'{rpki["new_frr_config"]}')
+        print(f'Modified config:\n')
+        print(f'{frr_cfg}')
+
+    frr_cfg.commit_configuration(frr_daemon)
+
+    # If FRR config is blank, re-run the blank commit x times due to frr-reload
+    # behavior/bug not properly clearing out on one commit.
+    if rpki['new_frr_config'] == '':
+        for a in range(5):
+            frr_cfg.commit_configuration(frr_daemon)
+
+    raise ConfigError("sadf")
+    return None
+
+if __name__ == '__main__':
+    try:
+        c = get_config()
+        verify(c)
+        generate(c)
+        apply(c)
+    except ConfigError as e:
+        print(e)
+        exit(1)
diff --git a/src/migration-scripts/rpki/0-to-1 b/src/migration-scripts/rpki/0-to-1
new file mode 100755
index 000000000..9058af016
--- /dev/null
+++ b/src/migration-scripts/rpki/0-to-1
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2021 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from sys import exit
+from sys import argv
+from vyos.configtree import ConfigTree
+
+if (len(argv) < 1):
+    print("Must specify file name!")
+    exit(1)
+
+file_name = argv[1]
+
+with open(file_name, 'r') as f:
+    config_file = f.read()
+
+base = ['protocols', 'rpki']
+config = ConfigTree(config_file)
+
+# Nothing to do
+if not config.exists(base):
+    exit(0)
+
+if config.exists(base + ['cache']):
+    preference = 1
+    for cache in config.list_nodes(base + ['cache']):
+        address_node = base + ['cache', cache, 'address']
+        if config.exists(address_node):
+            address = config.return_value(address_node)
+            # We do not longer support the address leafNode, RPKI cache server
+            # IP address is now used from the tagNode
+            config.delete(address_node)
+            # VyOS 1.2 had no per instance preference, setting new defaults
+            config.set(base + ['cache', cache, 'preference'], value=preference)
+            # Increase preference for the next caching peer - actually VyOS 1.2
+            # supported only one but better save then sorry (T3253)
+            preference += 1
+            config.rename(base + ['cache', cache], address)
+
+try:
+    with open(file_name, 'w') as f:
+        f.write(config.to_string())
+except OSError as e:
+    print("Failed to save the modified config: {}".format(e))
+    exit(1)