vrrp: keepalived: T3847: migrate to get_config_dict()

3 files changed, 153 insertions, 267 deletions

diff --git a/src/conf_mode/vrrp.py b/src/conf_mode/vrrp.py
index f11dce879..8bb237102 100755
--- a/src/conf_mode/vrrp.py
+++ b/src/conf_mode/vrrp.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2018-2020 VyOS maintainers and contributors
+# Copyright (C) 2018-2021 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -17,252 +17,136 @@
 import os
 
 from sys import exit
-from ipaddress import ip_address
 from ipaddress import ip_interface
 from ipaddress import IPv4Interface
 from ipaddress import IPv6Interface
-from ipaddress import IPv4Address
-from ipaddress import IPv6Address
-from json import dumps
-from pathlib import Path
-
-import vyos.config
-
-from vyos import ConfigError
-from vyos.util import call
-from vyos.util import makedir
-from vyos.template import render
 
+from vyos.config import Config
+from vyos.configdict import dict_merge
 from vyos.ifconfig.vrrp import VRRP
-
+from vyos.template import render
+from vyos.template import is_ipv4
+from vyos.template import is_ipv6
+from vyos.util import call
+from vyos.xml import defaults
+from vyos import ConfigError
 from vyos import airbag
 airbag.enable()
 
 def get_config(config=None):
-    vrrp_groups = []
-    sync_groups = []
-
     if config:
-        config = config
+        conf = config
     else:
-        config = vyos.config.Config()
-
-    # Get the VRRP groups
-    for group_name in config.list_nodes("high-availability vrrp group"):
-        config.set_level("high-availability vrrp group {0}".format(group_name))
-
-        # Retrieve the values
-        group = {"preempt": True, "use_vmac": False, "disable": False}
-
-        if config.exists("disable"):
-            group["disable"] = True
-
-        group["name"] = group_name
-        group["vrid"] = config.return_value("vrid")
-        group["interface"] = config.return_value("interface")
-        group["description"] = config.return_value("description")
-        group["advertise_interval"] = config.return_value("advertise-interval")
-        group["priority"] = config.return_value("priority")
-        group["hello_source"] = config.return_value("hello-source-address")
-        group["peer_address"] = config.return_value("peer-address")
-        group["sync_group"] = config.return_value("sync-group")
-        group["preempt_delay"] = config.return_value("preempt-delay")
-        group["virtual_addresses"] = config.return_values("virtual-address")
-        group["virtual_addresses_excluded"] = config.return_values("virtual-address-excluded")
-
-        group["auth_password"] = config.return_value("authentication password")
-        group["auth_type"] = config.return_value("authentication type")
-
-        group["health_check_script"] = config.return_value("health-check script")
-        group["health_check_interval"] = config.return_value("health-check interval")
-        group["health_check_count"] = config.return_value("health-check failure-count")
-
-        group["master_script"] = config.return_value("transition-script master")
-        group["backup_script"] = config.return_value("transition-script backup")
-        group["fault_script"] = config.return_value("transition-script fault")
-        group["stop_script"] = config.return_value("transition-script stop")
-        group["script_mode_force"] = config.exists("transition-script mode-force")
-
-        if config.exists("no-preempt"):
-            group["preempt"] = False
-        if config.exists("rfc3768-compatibility"):
-            group["use_vmac"] = True
-
-        # Substitute defaults where applicable
-        if not group["advertise_interval"]:
-            group["advertise_interval"] = 1
-        if not group["priority"]:
-            group["priority"] = 100
-        if not group["preempt_delay"]:
-            group["preempt_delay"] = 0
-        if not group["health_check_interval"]:
-            group["health_check_interval"] = 60
-        if not group["health_check_count"]:
-            group["health_check_count"] = 3
-
-        # FIXUP: translate our option for auth type to keepalived's syntax
-        # for simplicity
-        if group["auth_type"]:
-            if group["auth_type"] == "plaintext-password":
-                group["auth_type"] = "PASS"
-            else:
-                group["auth_type"] = "AH"
-
-        vrrp_groups.append(group)
-
-    config.set_level("")
-
-    # Get the sync group used for conntrack-sync
-    conntrack_sync_group = None
-    if config.exists("service conntrack-sync failover-mechanism vrrp"):
-        conntrack_sync_group = config.return_value("service conntrack-sync failover-mechanism vrrp sync-group")
-
-    # Get the sync groups
-    for sync_group_name in config.list_nodes("high-availability vrrp sync-group"):
-        config.set_level("high-availability vrrp sync-group {0}".format(sync_group_name))
-
-        sync_group = {"conntrack_sync": False}
-        sync_group["name"] = sync_group_name
-        sync_group["members"] = config.return_values("member")
-        if conntrack_sync_group:
-            if conntrack_sync_group == sync_group_name:
-                sync_group["conntrack_sync"] = True
-
-        # add transition script configuration
-        sync_group["master_script"] = config.return_value("transition-script master")
-        sync_group["backup_script"] = config.return_value("transition-script backup")
-        sync_group["fault_script"] = config.return_value("transition-script fault")
-        sync_group["stop_script"] = config.return_value("transition-script stop")
-
-        sync_groups.append(sync_group)
-
-    # create a file with dict with proposed configuration
-    dirname = os.path.dirname(VRRP.location['vyos'])
-    makedir(dirname)
-    with open(VRRP.location['vyos'] + ".temp", 'w') as dict_file:
-        dict_file.write(dumps({'vrrp_groups': vrrp_groups, 'sync_groups': sync_groups}))
-
-    return (vrrp_groups, sync_groups)
-
-
-def verify(data):
-    vrrp_groups, sync_groups = data
-
-    for group in vrrp_groups:
-        # Check required fields
-        if not group["vrid"]:
-            raise ConfigError("vrid is required but not set in VRRP group {0}".format(group["name"]))
-        if not group["interface"]:
-            raise ConfigError("interface is required but not set in VRRP group {0}".format(group["name"]))
-        if not group["virtual_addresses"]:
-            raise ConfigError("virtual-address is required but not set in VRRP group {0}".format(group["name"]))
-
-        if group["auth_password"] and (not group["auth_type"]):
-            raise ConfigError("authentication type is required but not set in VRRP group {0}".format(group["name"]))
-
-        # Keepalived doesn't allow mixing IPv4 and IPv6 in one group, so we mirror that restriction
-
-        # XXX: filter on map object is destructive, so we force it to list.
-        # Additionally, filter objects always evaluate to True, empty or not,
-        # so we force them to lists as well.
-        vaddrs = list(map(lambda i: ip_interface(i), group["virtual_addresses"]))
-        vaddrs4 = list(filter(lambda x: isinstance(x, IPv4Interface), vaddrs))
-        vaddrs6 = list(filter(lambda x: isinstance(x, IPv6Interface), vaddrs))
-
-        if vaddrs4 and vaddrs6:
-            raise ConfigError("VRRP group {0} mixes IPv4 and IPv6 virtual addresses, this is not allowed. Create separate groups for IPv4 and IPv6".format(group["name"]))
-
-        if vaddrs4:
-            if group["hello_source"]:
-                hsa = ip_address(group["hello_source"])
-                if isinstance(hsa, IPv6Address):
-                    raise ConfigError("VRRP group {0} uses IPv4 but its hello-source-address is IPv6".format(group["name"]))
-            if group["peer_address"]:
-                pa = ip_address(group["peer_address"])
-                if isinstance(pa, IPv6Address):
-                    raise ConfigError("VRRP group {0} uses IPv4 but its peer-address is IPv6".format(group["name"]))
-
-        if vaddrs6:
-            if group["hello_source"]:
-                hsa = ip_address(group["hello_source"])
-                if isinstance(hsa, IPv4Address):
-                    raise ConfigError("VRRP group {0} uses IPv6 but its hello-source-address is IPv4".format(group["name"]))
-            if group["peer_address"]:
-                pa = ip_address(group["peer_address"])
-                if isinstance(pa, IPv4Address):
-                    raise ConfigError("VRRP group {0} uses IPv6 but its peer-address is IPv4".format(group["name"]))
-
-        # Warn the user about the deprecated mode-force option
-        if group['script_mode_force']:
-            print("""Warning: "transition-script mode-force" VRRP option is deprecated and will be removed in VyOS 1.4.""")
-            print("""It's no longer necessary, so you can safely remove it from your config now.""")
-
-    # Disallow same VRID on multiple interfaces
-    _groups = sorted(vrrp_groups, key=(lambda x: x["interface"]))
-    count = len(_groups) - 1
-    index = 0
-    while (index < count):
-        if (_groups[index]["vrid"] == _groups[index + 1]["vrid"]) and (_groups[index]["interface"] == _groups[index + 1]["interface"]):
-            raise ConfigError("VRID {0} is used in groups {1} and {2} that both use interface {3}. Groups on the same interface must use different VRIDs".format(
-              _groups[index]["vrid"], _groups[index]["name"], _groups[index + 1]["name"], _groups[index]["interface"]))
-        else:
-            index += 1
+        conf = Config()
+
+    base = ['high-availability', 'vrrp']
+    if not conf.exists(base):
+        return None
+
+    vrrp = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True)
+    # We have gathered the dict representation of the CLI, but there are default
+    # options which we need to update into the dictionary retrived.
+    if 'group' in vrrp:
+        default_values = defaults(base + ['group'])
+        for group in vrrp['group']:
+            vrrp['group'][group] = dict_merge(default_values, vrrp['group'][group])
+
+    ## Get the sync group used for conntrack-sync
+    conntrack_path = ['service', 'conntrack-sync', 'failover-mechanism', 'vrrp', 'sync-group']
+    if conf.exists(conntrack_path):
+        vrrp['conntrack_sync_group'] = conf.return_value(conntrack_path)
+
+    return vrrp
+
+def verify(vrrp):
+    if not vrrp:
+        return None
+
+    used_vrid_if = []
+    if 'group' in vrrp:
+        for group, group_config in vrrp['group'].items():
+            # Check required fields
+            if 'vrid' not in group_config:
+                raise ConfigError(f'VRID is required but not set in VRRP group "{group}"')
+
+            if 'interface' not in group_config:
+                raise ConfigError(f'Interface is required but not set in VRRP group "{group}"')
+
+            if 'virtual_address' not in group_config:
+                raise ConfigError(f'virtual-address is required but not set in VRRP group "{group}"')
+
+            if 'authentication' in group_config:
+                if not {'password', 'type'} <= set(group_config['authentication']):
+                    raise ConfigError(f'Authentication requires both type and passwortd to be set in VRRP group "{group}"')
+
+            # We can not use a VRID once per interface
+            interface = group_config['interface']
+            vrid = group_config['vrid']
+            tmp = {'interface': interface, 'vrid': vrid}
+            if tmp in used_vrid_if:
+                raise ConfigError(f'VRID "{vrid}" can only be used once on interface "{interface}"!')
+            used_vrid_if.append(tmp)
+
+            # Keepalived doesn't allow mixing IPv4 and IPv6 in one group, so we mirror that restriction
+
+            # XXX: filter on map object is destructive, so we force it to list.
+            # Additionally, filter objects always evaluate to True, empty or not,
+            # so we force them to lists as well.
+            vaddrs = list(map(lambda i: ip_interface(i), group_config['virtual_address']))
+            vaddrs4 = list(filter(lambda x: isinstance(x, IPv4Interface), vaddrs))
+            vaddrs6 = list(filter(lambda x: isinstance(x, IPv6Interface), vaddrs))
+
+            if vaddrs4 and vaddrs6:
+                raise ConfigError(f'VRRP group "{group}" mixes IPv4 and IPv6 virtual addresses, this is not allowed.\n' \
+                                  'Create individual groups for IPv4 and IPv6!')
+            if vaddrs4:
+                if 'hello_source_address' in group_config:
+                    if is_ipv6(group_config['hello_source_address']):
+                        raise ConfigError(f'VRRP group "{group}" uses IPv4 but hello-source-address is IPv6!')
+
+                if 'peer_address' in group_config:
+                    if is_ipv6(group_config['peer_address']):
+                        raise ConfigError(f'VRRP group "{group}" uses IPv4 but peer-address is IPv6!')
+
+            if vaddrs6:
+                if 'hello_source_address' in group_config:
+                    if is_ipv4(group_config['hello_source_address']):
+                        raise ConfigError(f'VRRP group "{group}" uses IPv6 but hello-source-address is IPv4!')
+
+                if 'peer_address' in group_config:
+                    if is_ipv4(group_config['peer_address']):
+                        raise ConfigError(f'VRRP group "{group}" uses IPv6 but peer-address is IPv4!')
+
+            # Warn the user about the deprecated mode-force option
+            if 'transition_script' in group_config and 'mode_force' in group_config['transition_script']:
+                print("""Warning: "transition-script mode-force" VRRP option is deprecated and will be removed in VyOS 1.4.""")
+                print("""It's no longer necessary, so you can safely remove it from your config now.""")
 
     # Check sync groups
-    vrrp_group_names = list(map(lambda x: x["name"], vrrp_groups))
-
-    for sync_group in sync_groups:
-        for m in sync_group["members"]:
-            if not (m in vrrp_group_names):
-                raise ConfigError("VRRP sync-group {0} refers to VRRP group {1}, but group {1} does not exist".format(sync_group["name"], m))
-
-
-def generate(data):
-    vrrp_groups, sync_groups = data
-
-    # Remove disabled groups from the sync group member lists
-    for sync_group in sync_groups:
-        for member in sync_group["members"]:
-            g = list(filter(lambda x: x["name"] == member, vrrp_groups))[0]
-            if g["disable"]:
-                print("Warning: ignoring disabled VRRP group {0} in sync-group {1}".format(g["name"], sync_group["name"]))
-    # Filter out disabled groups
-    vrrp_groups = list(filter(lambda x: x["disable"] is not True, vrrp_groups))
-
-    render(VRRP.location['config'], 'vrrp/keepalived.conf.tmpl',
-            {"groups": vrrp_groups, "sync_groups": sync_groups})
-    render(VRRP.location['daemon'], 'vrrp/daemon.tmpl', {})
+    if 'sync_group' in vrrp:
+        for sync_group, sync_config in vrrp['sync_group'].items():
+            if 'member' in sync_config:
+                for member in sync_config['member']:
+                    if member not in vrrp['group']:
+                        raise ConfigError(f'VRRP sync-group "{sync_group}" refers to VRRP group "{member}", '\
+                                          'but it does not exist!')
+
+def generate(vrrp):
+    if not vrrp:
+        return None
+
+    render(VRRP.location['config'], 'vrrp/keepalived.conf.tmpl', vrrp)
     return None
 
+def apply(vrrp):
+    service_name = 'keepalived.service'
+    if not vrrp:
+        call(f'systemctl stop {service_name}')
+        return None
 
-def apply(data):
-    vrrp_groups, sync_groups = data
-    if vrrp_groups:
-        # safely rename a temporary file with configuration dict
-        try:
-            dict_file = Path("{}.temp".format(VRRP.location['vyos']))
-            dict_file.rename(Path(VRRP.location['vyos']))
-        except Exception as err:
-            print("Unable to rename the file with keepalived config for FIFO pipe: {}".format(err))
-
-        if not VRRP.is_running():
-            print("Starting the VRRP process")
-            ret = call("systemctl restart keepalived.service")
-        else:
-            print("Reloading the VRRP process")
-            ret = call("systemctl reload keepalived.service")
-
-        if ret != 0:
-            raise ConfigError("keepalived failed to start")
-    else:
-        # VRRP is removed in the commit
-        print("Stopping the VRRP process")
-        call("systemctl stop keepalived.service")
-        os.unlink(VRRP.location['daemon'])
-
+    call(f'systemctl restart {service_name}')
     return None
 
-
 if __name__ == '__main__':
     try:
         c = get_config()
@@ -270,5 +154,5 @@ if __name__ == '__main__':
         generate(c)
         apply(c)
     except ConfigError as e:
-        print("VRRP error: {0}".format(str(e)))
+        print(e)
         exit(1)
diff --git a/src/etc/systemd/system/keepalived.service.d/override.conf b/src/etc/systemd/system/keepalived.service.d/override.conf
index e338b90a2..1c68913f2 100644
--- a/src/etc/systemd/system/keepalived.service.d/override.conf
+++ b/src/etc/systemd/system/keepalived.service.d/override.conf
@@ -6,7 +6,8 @@ After=vyos-router.service
 
 [Service]
 KillMode=process
+EnvironmentFile=
 ExecStart=
-ExecStart=/usr/sbin/keepalived --use-file /run/keepalived/keepalived.conf --pid /run/keepalived/keepalived.pid --dont-fork $DAEMON_ARGS
+ExecStart=/usr/sbin/keepalived --use-file /run/keepalived/keepalived.conf --pid /run/keepalived/keepalived.pid --dont-fork --snmp
 PIDFile=
 PIDFile=/run/keepalived/keepalived.pid
diff --git a/src/system/keepalived-fifo.py b/src/system/keepalived-fifo.py
index 159fd0f54..1fba0d75b 100755
--- a/src/system/keepalived-fifo.py
+++ b/src/system/keepalived-fifo.py
@@ -27,6 +27,7 @@ from queue import Queue
 from logging.handlers import SysLogHandler
 
 from vyos.ifconfig.vrrp import VRRP
+from vyos.configquery import ConfigTreeQuery
 from vyos.util import cmd
 
 # configure logging
@@ -44,12 +45,13 @@ mdns_update_command = 'sudo /usr/libexec/vyos/conf_mode/service_mdns-repeater.py
 class KeepalivedFifo:
     # init - read command arguments
     def __init__(self):
-        logger.info("Starting FIFO pipe for Keepalived")
+        logger.info('Starting FIFO pipe for Keepalived')
         # define program arguments
         cmd_args_parser = argparse.ArgumentParser(description='Create FIFO pipe for keepalived and process notify events', add_help=False)
         cmd_args_parser.add_argument('PIPE', help='path to the FIFO pipe')
         # parse arguments
         cmd_args = cmd_args_parser.parse_args()
+
         self._config_load()
         self.pipe_path = cmd_args.PIPE
 
@@ -61,33 +63,34 @@ class KeepalivedFifo:
     # load configuration
     def _config_load(self):
         try:
-            # read the dictionary file with configuration
-            with open(VRRP.location['vyos'], 'r') as dict_file:
-                vrrp_config_dict = json.load(dict_file)
+            base = ['high-availability', 'vrrp']
+            conf = ConfigTreeQuery()
+            if not conf.exists(base):
+                raise ValueError()
+
+            # Read VRRP configuration directly from CLI
+            vrrp_config_dict = conf.get_config_dict(base, key_mangling=('-', '_'),
+                                                    get_first_key=True)
             self.vrrp_config = {'vrrp_groups': {}, 'sync_groups': {}}
-            # save VRRP instances to the new dictionary
-            for vrrp_group in vrrp_config_dict['vrrp_groups']:
-                self.vrrp_config['vrrp_groups'][vrrp_group['name']] = {
-                    'STOP': vrrp_group.get('stop_script'),
-                    'FAULT': vrrp_group.get('fault_script'),
-                    'BACKUP': vrrp_group.get('backup_script'),
-                    'MASTER': vrrp_group.get('master_script')
-                }
-            # save VRRP sync groups to the new dictionary
-            for sync_group in vrrp_config_dict['sync_groups']:
-                self.vrrp_config['sync_groups'][sync_group['name']] = {
-                    'STOP': sync_group.get('stop_script'),
-                    'FAULT': sync_group.get('fault_script'),
-                    'BACKUP': sync_group.get('backup_script'),
-                    'MASTER': sync_group.get('master_script')
-                }
-            logger.debug("Loaded configuration: {}".format(self.vrrp_config))
+            for key in ['group', 'sync_group']:
+                if key not in vrrp_config_dict:
+                    continue
+                for group, group_config in vrrp_config_dict[key].items():
+                    if 'transition_script' not in group_config:
+                        continue
+                    self.vrrp_config['vrrp_groups'][group] = {
+                        'STOP': group_config['transition_script'].get('stop'),
+                        'FAULT': group_config['transition_script'].get('fault'),
+                        'BACKUP': group_config['transition_script'].get('backup'),
+                        'MASTER': group_config['transition_script'].get('master'),
+                    }
+            logger.info(f'Loaded configuration: {self.vrrp_config}')
         except Exception as err:
-            logger.error("Unable to load configuration: {}".format(err))
+            logger.error(f'Unable to load configuration: {err}')
 
     # run command
     def _run_command(self, command):
-        logger.debug("Running the command: {}".format(command))
+        logger.debug(f'Running the command: {command}')
         try:
             cmd(command)
         except OSError as err:
@@ -96,13 +99,13 @@ class KeepalivedFifo:
     # create FIFO pipe
     def pipe_create(self):
         if os.path.exists(self.pipe_path):
-            logger.info(f"PIPE already exist: {self.pipe_path}")
+            logger.info(f'PIPE already exist: {self.pipe_path}')
         else:
             os.mkfifo(self.pipe_path)
 
     # process message from pipe
     def pipe_process(self):
-        logger.debug("Message processing start")
+        logger.debug('Message processing start')
         regex_notify = re.compile(r'^(?P<type>\w+) "(?P<name>[\w-]+)" (?P<state>\w+) (?P<priority>\d+)$', re.MULTILINE)
         while self.stopme.is_set() is False:
             # wait for a new message event from pipe_wait
@@ -113,14 +116,14 @@ class KeepalivedFifo:
                 # get all messages from queue and try to process them
                 while self.message_queue.empty() is not True:
                     message = self.message_queue.get()
-                    logger.debug("Received message: {}".format(message))
+                    logger.debug(f'Received message: {message}')
                     notify_message = regex_notify.search(message)
                     # try to process a message if it looks valid
                     if notify_message:
                         n_type = notify_message.group('type')
                         n_name = notify_message.group('name')
                         n_state = notify_message.group('state')
-                        logger.info("{} {} changed state to {}".format(n_type, n_name, n_state))
+                        logger.info(f'{n_type} {n_name} changed state to {n_state}')
                         # check and run commands for VRRP instances
                         if n_type == 'INSTANCE':
                             if os.path.exists(mdns_running_file):
@@ -143,16 +146,16 @@ class KeepalivedFifo:
                     # mark task in queue as done
                     self.message_queue.task_done()
             except Exception as err:
-                logger.error("Error processing message: {}".format(err))
-        logger.debug("Terminating messages processing thread")
+                logger.error(f'Error processing message: {err}')
+        logger.debug('Terminating messages processing thread')
 
     # wait for messages
     def pipe_wait(self):
-        logger.debug("Message reading start")
+        logger.debug('Message reading start')
         self.pipe_read = os.open(self.pipe_path, os.O_RDONLY | os.O_NONBLOCK)
         while self.stopme.is_set() is False:
             # sleep a bit to not produce 100% CPU load
-            time.sleep(0.1)
+            time.sleep(0.250)
             try:
                 # try to read a message from PIPE
                 message = os.read(self.pipe_read, 500)
@@ -165,21 +168,19 @@ class KeepalivedFifo:
             except Exception as err:
                 # ignore the "Resource temporarily unavailable" error
                 if err.errno != 11:
-                    logger.error("Error receiving message: {}".format(err))
+                    logger.error(f'Error receiving message: {err}')
 
-        logger.debug("Closing FIFO pipe")
+        logger.debug('Closing FIFO pipe')
         os.close(self.pipe_read)
 
-
 # handle SIGTERM signal to allow finish all messages processing
 def sigterm_handle(signum, frame):
-    logger.info("Ending processing: Received SIGTERM signal")
+    logger.info('Ending processing: Received SIGTERM signal')
     fifo.stopme.set()
     thread_wait_message.join()
     fifo.message_event.set()
     thread_process_message.join()
 
-
 signal.signal(signal.SIGTERM, sigterm_handle)
 
 # init our class