dns: T5144: Modernize dynamic dns operation

Apply next round of configuration tree updates to 'service dns dynamic'
with the following changes:

- Migrate `service dns dynamic interface <interface> [use-web]`
       to `service dns dynamic address <interface>`
       or `service dns dynamic address web [web-options]`
  This communicates the intent that dynamic dns IP address is detected
  in only one way - using the `<interface>` or using an external web
  request, not both.
- When using external web request, (`service dns dynamic address web`),
  external url is optional (`web-options url`). Ddclient defaults are
  used when unspecified,
- Rename all config `login` to `username` for consistency and also to
  align better with alternative ddclient backends in consideration.
- Apply global 'ipv6-enable' to per service 'ip-version: ipv6'. Selecting
  usage of IPv4 or IPv6 (or both simultaneously) is now at per service
  (protocol) level instead of global level. This allows more control on
  the ability to select IPv4 in some cases and IPv6 in some other cases
  wherever supported by the underlying ddclient protocol.
- While the IP address (and by extension, the detection mechanism) is
  global, the way it is applied to a particular ddclient protocol depends
  on whether it supports IPv4 or IPv6 or both.
- Related to the above, this also prevents generating incorrect config
  file (`ddclient.conf`) with multiple global sections leading to an
  unpredictable behavior of ddclient.
- Implement provider (protocol) specific custom tweaks whenever possible
  (e.g., `zone`, `username`, `server` are not necessary in all cases).
- Move service name from a combination of 'protocol' (with protocol
  config autodetected) and custom (with protocol config specified) to a
  single 'service' key. This allows for consisent setup of multiple
  config for the same ddclient protocol (with different options and
  credentials). This also avoid ambiguity with usual networking term
  'protocol' and ddclient specific term 'protocol' (and can change with
  a move to a different backend).
- Apply upfront XML constraints and validations consistently wherever
  applicable.
- RFC2136 specific change: Rename rfc2136 config `record` to `host-name`
  for consistency.
- Cloudflare specific change: While ddclient still supports authenticating
  with email and global auth key, skipping `username` in config will
  indicate the intent to use API token authentication (with special
  'token' literal as `username`).

4 files changed, 204 insertions, 227 deletions

diff --git a/data/templates/dynamic-dns/ddclient.conf.j2 b/data/templates/dynamic-dns/ddclient.conf.j2
index e8ef5ac90..6d1eb03a7 100644
--- a/data/templates/dynamic-dns/ddclient.conf.j2
+++ b/data/templates/dynamic-dns/ddclient.conf.j2
@@ -1,50 +1,67 @@
+{% macro render_config(host, address, web_options, ip_suffixes=['']) %}
+{# Address: use=if, if=ethX, usev6=ifv6, ifv6=ethX, usev6=webv6, webv6=https://v6.example.com #}
+{% for ipv in ip_suffixes %}
+use{{ ipv }}={{ address if address == 'web' else 'if' }}{{ ipv }}, \
+{%     if address == 'web' %}
+{%         if web_options.url is vyos_defined %}
+web{{ ipv }}={{ web_options.url }}, \
+{%         endif %}
+{%         if web_options.skip is vyos_defined %}
+web-skip{{ ipv }}='{{ web_options.skip }}', \
+{%         endif %}
+{%     else %}
+if{{ ipv }}={{ address }}, \
+{%     endif %}
+{% endfor %}
+{# Other service options #}
+{% for k,v in kwargs.items() %}
+{%     if v is vyos_defined %}
+{{ k }}={{ v }}{{ ',' if not loop.last }} \
+{%     endif %}
+{% endfor %}
+{# Actual hostname for the service #}
+{{ host }}
+{% endmacro %}
 ### Autogenerated by dynamic_dns.py ###
 daemon=1m
 syslog=yes
 ssl=yes
 
-{% if interface is vyos_defined %}
-{%     for iface, iface_config in interface.items() %}
-# ddclient configuration for interface "{{ iface }}"
-{%         if iface_config.use_web is vyos_defined %}
-{%             set web_skip = ", web-skip='" ~ iface_config.use_web.skip ~ "'" if iface_config.use_web.skip is vyos_defined else '' %}
-use=web, web='{{ iface_config.use_web.url }}'{{ web_skip }}
-{%         else %}
-{{ 'usev6=ifv6' if iface_config.ipv6_enable is vyos_defined else 'use=if' }}, if={{ iface }}
-{%         endif %}
+{% if address is vyos_defined %}
+{%     for address, service_cfg in address.items() %}
+{%         if service_cfg.rfc2136 is vyos_defined %}
+{%             for name, config in service_cfg.rfc2136.items() %}
+{%                 if config.description is vyos_defined %}
+# {{ config.description }}
 
-{%         if iface_config.rfc2136 is vyos_defined %}
-{%             for rfc2136, config in iface_config.rfc2136.items() %}
-{%                 for dns_record in config.record if config.record is vyos_defined %}
-# RFC2136 dynamic DNS configuration for {{ rfc2136 }}, {{ config.zone }}, {{ dns_record }}
-server={{ config.server }}
-protocol=nsupdate
-password={{ config.key }}
-ttl={{ config.ttl }}
-zone={{ config.zone }}
-{{ dns_record }}
+{%                 endif %}
+{%                 for host in config.host_name if config.host_name is vyos_defined %}
+# RFC2136 dynamic DNS configuration for {{ name }}: [{{ config.zone }}, {{ host }}]
+{# Don't append 'new-style' compliant suffix ('usev4', 'usev6', 'ifv4', 'ifv6' etc.)
+   to the properties since 'nsupdate' doesn't support that yet. #}
+{{ render_config(host, address, service_cfg.web_options,
+                 protocol='nsupdate', server=config.server, zone=config.zone,
+                 password=config.key, ttl=config.ttl) }}
 
 {%                 endfor %}
 {%             endfor %}
 {%         endif %}
+{%         if service_cfg.service is vyos_defined %}
+{%             for name, config in service_cfg.service.items() %}
+{%                 if config.description is vyos_defined %}
+# {{ config.description }}
 
-{%         if iface_config.service is vyos_defined %}
-{%             for service, config in iface_config.service.items() %}
-{%                 for dns_record in config.host_name %}
-# DynDNS provider configuration for {{ service }}, {{ dns_record }}
-protocol={{ config.protocol }},
-max-interval=28d,
-{%                     if config.login is vyos_defined %}
-login={{ config.login }},
-{%                     endif %}
-password='{{ config.password }}',
-{%                     if config.server is vyos_defined %}
-server={{ config.server }},
-{%                     endif %}
-{%                     if config.zone is vyos_defined %}
-zone={{ config.zone }},
-{%                     endif %}
-{{ dns_record }}
+{%                 endif %}
+{%                 for host in config.host_name if config.host_name is vyos_defined %}
+{%                     set ip_suffixes = ['v4', 'v6'] if config.ip_version == 'both'
+                                                      else (['v6'] if config.ip_version == 'ipv6' else ['']) %}
+# Web service dynamic DNS configuration for {{ name }}: [{{ config.protocol }}, {{ host }}]
+{# For ipv4 only setup, don't append 'new-style' compliant suffix ('usev4', 'ifv4', 'webv4' etc.)
+   to the properties and instead live through the deprecation warnings for better compatibility
+   with most ddclient protocols. #}
+{{ render_config(host, address, service_cfg.web_options, ip_suffixes,
+                 protocol=config.protocol, server=config.server, zone=config.zone,
+                 login=config.username, password=config.password) }}
 
 {%                 endfor %}
 {%             endfor %}
diff --git a/interface-definitions/dns-dynamic.xml.in b/interface-definitions/dns-dynamic.xml.in
index 48c101d73..308dc5de0 100644
--- a/interface-definitions/dns-dynamic.xml.in
+++ b/interface-definitions/dns-dynamic.xml.in
@@ -12,141 +12,94 @@
               <help>Dynamic DNS</help>
             </properties>
             <children>
-              <tagNode name="interface">
+              <tagNode name="address">
                 <properties>
-                  <help>Interface to send Dynamic DNS updates for</help>
-                  <completionHelp>
-                    <script>${vyos_completion_dir}/list_interfaces</script>
-                  </completionHelp>
+                  <help>Obtain IP address to send Dynamic DNS update for</help>
                   <valueHelp>
                     <format>txt</format>
-                    <description>Interface name</description>
+                    <description>Use interface to obtain the IP address</description>
                   </valueHelp>
+                  <valueHelp>
+                    <format>web</format>
+                    <description>Use HTTP(S) web request to obtain the IP address</description>
+                  </valueHelp>
+                  <completionHelp>
+                    <script>${vyos_completion_dir}/list_interfaces</script>
+                    <list>web</list>
+                  </completionHelp>
                   <constraint>
                     #include <include/constraint/interface-name.xml.i>
+                    <regex>web</regex>
                   </constraint>
                 </properties>
                 <children>
-                  <tagNode name="rfc2136">
+                  <node name="web-options">
                     <properties>
-                      <help>RFC2136 Update name</help>
+                      <help>Options when using HTTP(S) web request to obtain the IP address</help>
                     </properties>
                     <children>
-                      <leafNode name="key">
+                      #include <include/url.xml.i>
+                      <leafNode name="skip">
                         <properties>
-                          <help>File containing the secret key shared with remote DNS server</help>
+                          <help>Pattern to skip from the HTTP(S) respose</help>
                           <valueHelp>
-                            <format>filename</format>
-                            <description>File in /config/auth directory</description>
+                            <format>txt</format>
+                            <description>Pattern to skip from the HTTP(S) respose to extract the external IP address</description>
                           </valueHelp>
                         </properties>
                       </leafNode>
-                      <leafNode name="record">
-                        <properties>
-                          <help>Record to be updated</help>
-                          <multi/>
-                        </properties>
-                      </leafNode>
-                      <leafNode name="server">
-                        <properties>
-                          <help>Server to be updated</help>
-                        </properties>
-                      </leafNode>
-                      <leafNode name="ttl">
+                    </children>
+                  </node>
+                  <tagNode name="rfc2136">
+                    <properties>
+                      <help>RFC2136 nsupdate configuration</help>
+                      <valueHelp>
+                        <format>txt</format>
+                        <description>RFC2136 nsupdate service name</description>
+                      </valueHelp>
+                    </properties>
+                    <children>
+                      #include <include/generic-description.xml.i>
+                      #include <include/dns/dynamic-service-host-name-server.xml.i>
+                      <leafNode name="key">
                         <properties>
-                          <help>Time To Live (default: 600)</help>
+                          <help>File containing the TSIG secret key shared with remote DNS server</help>
                           <valueHelp>
-                            <format>u32:1-86400</format>
-                            <description>DNS forwarding cache size</description>
+                            <format>filename</format>
+                            <description>File in /config/auth directory</description>
                           </valueHelp>
                           <constraint>
-                            <validator name="numeric" argument="--range 1-86400"/>
+                            <validator name="file-path" argument="--strict --parent-dir /config/auth"/>
                           </constraint>
                         </properties>
-                        <defaultValue>600</defaultValue>
                       </leafNode>
+                      #include <include/dns/time-to-live.xml.i>
                       <leafNode name="zone">
                         <properties>
-                          <help>Zone to be updated</help>
+                          <help>Forwarding zone to be updated</help>
+                          <valueHelp>
+                            <format>txt</format>
+                            <description>RFC2136 Zone to be updated</description>
+                          </valueHelp>
+                          <constraint>
+                            <validator name="fqdn"/>
+                          </constraint>
                         </properties>
                       </leafNode>
                     </children>
                   </tagNode>
                   <tagNode name="service">
                     <properties>
-                      <help>Service being used for Dynamic DNS</help>
-                      <completionHelp>
-                        <list>afraid changeip cloudflare dnspark dslreports dyndns easydns namecheap noip sitelutions zoneedit</list>
-                      </completionHelp>
+                      <help>Dynamic DNS configuration</help>
                       <valueHelp>
                         <format>txt</format>
-                        <description>Dynanmic DNS service with a custom name</description>
-                      </valueHelp>
-                      <valueHelp>
-                        <format>afraid</format>
-                        <description>afraid.org Services</description>
-                      </valueHelp>
-                      <valueHelp>
-                        <format>changeip</format>
-                        <description>changeip.com Services</description>
-                      </valueHelp>
-                      <valueHelp>
-                        <format>cloudflare</format>
-                        <description>cloudflare.com Services</description>
+                        <description>Dynamic DNS service name</description>
                       </valueHelp>
-                      <valueHelp>
-                        <format>dnspark</format>
-                        <description>dnspark.com Services</description>
-                      </valueHelp>
-                      <valueHelp>
-                        <format>dslreports</format>
-                        <description>dslreports.com Services</description>
-                      </valueHelp>
-                      <valueHelp>
-                        <format>dyndns</format>
-                        <description>dyndns.com Services</description>
-                      </valueHelp>
-                      <valueHelp>
-                        <format>easydns</format>
-                        <description>easydns.com Services</description>
-                      </valueHelp>
-                      <valueHelp>
-                        <format>namecheap</format>
-                        <description>namecheap.com Services</description>
-                      </valueHelp>
-                      <valueHelp>
-                        <format>noip</format>
-                        <description>noip.com Services</description>
-                      </valueHelp>
-                      <valueHelp>
-                        <format>sitelutions</format>
-                        <description>sitelutions.com Services</description>
-                      </valueHelp>
-                      <valueHelp>
-                        <format>zoneedit</format>
-                        <description>zoneedit.com Services</description>
-                      </valueHelp>
-                      <constraint>
-                        <regex>(custom|afraid|changeip|cloudflare|dnspark|dslreports|dyndns|easydns|namecheap|noip|sitelutions|zoneedit|\w+)</regex>
-                      </constraint>
-                      <constraintErrorMessage>You can use only predefined list of services or word characters (_, a-z, A-Z, 0-9) as service name</constraintErrorMessage>
                     </properties>
                     <children>
-                      <leafNode name="host-name">
-                        <properties>
-                          <help>Hostname to register with Dynamic DNS service</help>
-                          <constraint>
-                            #include <include/constraint/host-name.xml.i>
-                          </constraint>
-                          <constraintErrorMessage>Host-name must be alphanumeric and can contain hyphens</constraintErrorMessage>
-                          <multi/>
-                        </properties>
-                      </leafNode>
-                      <leafNode name="login">
-                        <properties>
-                          <help>Login/Username for Dynamic DNS service</help>
-                        </properties>
-                      </leafNode>
+                      #include <include/generic-description.xml.i>
+                      #include <include/dns/dynamic-service-host-name-server.xml.i>
+                      #include <include/generic-username.xml.i>
                       #include <include/generic-password.xml.i>
                       <leafNode name="protocol">
                         <properties>
@@ -159,7 +112,6 @@
                           </constraint>
                         </properties>
                       </leafNode>
-                      #include <include/server-ipv4-fqdn.xml.i>
                       <leafNode name="zone">
                         <properties>
                           <help>DNS zone to update (not used by all protocols)</help>
@@ -169,31 +121,33 @@
                           </valueHelp>
                         </properties>
                       </leafNode>
-                    </children>
-                  </tagNode>
-                  <node name="use-web">
-                    <properties>
-                      <help>Use HTTP(S) web request to obtain external IP address instead of the IP address associated with the interface</help>
-                    </properties>
-                    <children>
-                      <leafNode name="skip">
+                      <leafNode name="ip-version">
                         <properties>
-                          <help>Pattern to skip from the respose</help>
+                          <help>IP address version to use</help>
                           <valueHelp>
-                            <format>txt</format>
-                            <description>Pattern to skip from the respose of the given URL to extract the external IP address</description>
+                            <format>_ipv4</format>
+                            <description>Use only IPv4 address</description>
+                          </valueHelp>
+                          <valueHelp>
+                            <format>_ipv6</format>
+                            <description>Use only IPv6 address</description>
                           </valueHelp>
+                          <valueHelp>
+                            <format>both</format>
+                            <description>Use both IPv4 and IPv6 address</description>
+                          </valueHelp>
+                          <completionHelp>
+                            <list>ipv4 ipv6 both</list>
+                          </completionHelp>
+                          <constraint>
+                            <regex>(ipv[46]|both)</regex>
+                          </constraint>
+                          <constraintErrorMessage>IP Version must be literal 'ipv4', 'ipv6' or 'both'</constraintErrorMessage>
                         </properties>
+                        <defaultValue>ipv4</defaultValue>
                       </leafNode>
-                      #include <include/url.xml.i>
                     </children>
-                  </node>
-                  <leafNode name="ipv6-enable">
-                    <properties>
-                      <help>Explicitly use IPv6 address instead of IPv4 address to update the Dynamic DNS IP address</help>
-                      <valueless/>
-                    </properties>
-                  </leafNode>
+                  </tagNode>
                 </children>
               </tagNode>
             </children>
diff --git a/interface-definitions/include/dns/dynamic-service-host-name-server.xml.i b/interface-definitions/include/dns/dynamic-service-host-name-server.xml.i
new file mode 100644
index 000000000..ee1af2a36
--- /dev/null
+++ b/interface-definitions/include/dns/dynamic-service-host-name-server.xml.i
@@ -0,0 +1,34 @@
+<!-- include start from dns/dynamic-service-host-name-server.xml.i -->
+<leafNode name="host-name">
+  <properties>
+    <help>Hostname to register with Dynamic DNS service</help>
+    <constraint>
+        #include <include/constraint/host-name.xml.i>
+    </constraint>
+    <constraintErrorMessage>Host-name must be alphanumeric and can contain hyphens</constraintErrorMessage>
+    <multi/>
+  </properties>
+</leafNode>
+<leafNode name="server">
+  <properties>
+    <help>Remote Dynamic DNS server to send updates to</help>
+    <valueHelp>
+      <format>ipv4</format>
+      <description>IPv4 address of the remote server</description>
+    </valueHelp>
+    <valueHelp>
+      <format>ipv6</format>
+      <description>IPv6 address of the remote server</description>
+    </valueHelp>
+    <valueHelp>
+      <format>hostname</format>
+      <description>Fully qualified domain name of the remote server</description>
+    </valueHelp>
+    <constraint>
+      <validator name="ip-address"/>
+      <validator name="fqdn"/>
+    </constraint>
+    <constraintErrorMessage>Remote server must be IP address or fully qualified domain name</constraintErrorMessage>
+  </properties>
+</leafNode>
+<!-- include end -->
diff --git a/src/conf_mode/dynamic_dns.py b/src/conf_mode/dynamic_dns.py
index 426e3d693..879086cc4 100755
--- a/src/conf_mode/dynamic_dns.py
+++ b/src/conf_mode/dynamic_dns.py
@@ -29,20 +29,14 @@ airbag.enable()
 
 config_file = r'/run/ddclient/ddclient.conf'
 
-# Mapping of service name to service protocol
-default_service_protocol = {
-    'afraid': 'freedns',
-    'changeip': 'changeip',
-    'cloudflare': 'cloudflare',
-    'dnspark': 'dnspark',
-    'dslreports': 'dslreports1',
-    'dyndns': 'dyndns2',
-    'easydns': 'easydns',
-    'namecheap': 'namecheap',
-    'noip': 'noip',
-    'sitelutions': 'sitelutions',
-    'zoneedit': 'zoneedit1'
-}
+# Protocols that require zone
+zone_allowed = ['cloudflare', 'godaddy', 'hetzner', 'gandi', 'nfsn']
+
+# Protocols that do not require username
+username_unnecessary = ['1984', 'cloudflare', 'cloudns', 'duckdns', 'freemyip', 'hetzner', 'keysystems', 'njalla']
+
+# Protocols that support both IPv4 and IPv6
+dualstack_supported = ['cloudflare', 'dyndns2', 'freedns', 'njalla']
 
 def get_config(config=None):
     if config:
@@ -56,24 +50,13 @@ def get_config(config=None):
 
     dyndns = conf.get_config_dict(base_level, key_mangling=('-', '_'), get_first_key=True)
 
-    # We have gathered the dict representation of the CLI, but there are default
-    # options which we need to update into the dictionary retrived.
-    for interface in dyndns['interface']:
-        if 'service' in dyndns['interface'][interface]:
-            # 'Autodetect' protocol used by DynDNS service
-            for service in dyndns['interface'][interface]['service']:
-                if service in default_service_protocol:
-                    dyndns['interface'][interface]['service'][service].update(
-                        {'protocol' : default_service_protocol.get(service)})
-                else:
-                    dyndns['interface'][interface]['service'][service].update(
-                        {'custom': ''})
-
-        if 'rfc2136' in dyndns['interface'][interface]:
-            default_values = defaults(base_level + ['interface', 'rfc2136'])
-            for rfc2136 in dyndns['interface'][interface]['rfc2136']:
-                dyndns['interface'][interface]['rfc2136'][rfc2136] = dict_merge(
-                    default_values, dyndns['interface'][interface]['rfc2136'][rfc2136])
+    for address in dyndns['address']:
+        # Apply service specific defaults (stype = ['rfc2136', 'service'])
+        for svc_type in dyndns['address'][address]:
+            default_values = defaults(base_level + ['address', svc_type])
+            for svc_cfg in dyndns['address'][address][svc_type]:
+                dyndns['address'][address][svc_type][svc_cfg] = dict_merge(
+                    default_values, dyndns['address'][address][svc_type][svc_cfg])
 
     return dyndns
 
@@ -82,48 +65,37 @@ def verify(dyndns):
     if not dyndns:
         return None
 
-    # A 'node' corresponds to an interface
-    if 'interface' not in dyndns:
-        return None
-
-    for interface in dyndns['interface']:
+    for address in dyndns['address']:
         # RFC2136 - configuration validation
-        if 'rfc2136' in dyndns['interface'][interface]:
-            for rfc2136, config in dyndns['interface'][interface]['rfc2136'].items():
-
-                for tmp in ['record', 'zone', 'server', 'key']:
-                    if tmp not in config:
-                        raise ConfigError(f'"{tmp}" required for rfc2136 based '
-                                          f'DynDNS service on "{interface}"')
-
-                if not os.path.isfile(config['key']):
-                    raise ConfigError(f'"key"-file not found for rfc2136 based '
-                                      f'DynDNS service on "{interface}"')
-
-        # DynDNS service provider - configuration validation
-        if 'service' in dyndns['interface'][interface]:
-            for service, config in dyndns['interface'][interface]['service'].items():
-                error_msg = f'required for DynDNS service "{service}" on "{interface}"'
-                if 'host_name' not in config:
-                    raise ConfigError(f'"host-name" {error_msg}')
-
-                if 'login' not in config:
-                    if service != 'cloudflare' and ('protocol' not in config or config['protocol'] != 'cloudflare'):
-                        raise ConfigError(f'"login" (username) {error_msg}, unless using CloudFlare')
-
-                if 'password' not in config:
-                    raise ConfigError(f'"password" {error_msg}')
-
-                if 'zone' in config:
-                    if service != 'cloudflare' and ('protocol' not in config or config['protocol'] != 'cloudflare'):
-                        raise ConfigError(f'"zone" option only supported with CloudFlare')
-
-                if 'custom' in config:
-                    if 'protocol' not in config:
-                        raise ConfigError(f'"protocol" {error_msg}')
-
-                    if 'server' not in config:
-                        raise ConfigError(f'"server" {error_msg}')
+        if 'rfc2136' in dyndns['address'][address]:
+            for config in dyndns['address'][address]['rfc2136'].values():
+                for field in ['host_name', 'zone', 'server', 'key']:
+                    if field not in config:
+                        raise ConfigError(f'"{field.replace("_", "-")}" is required for RFC2136 '
+                                          f'based Dynamic DNS service on "{address}"')
+
+        # Dynamic DNS service provider - configuration validation
+        if 'service' in dyndns['address'][address]:
+            for service, config in dyndns['address'][address]['service'].items():
+                error_msg = f'is required for Dynamic DNS service "{service}" on "{address}"'
+
+                for field in ['host_name', 'password', 'protocol']:
+                    if field not in config:
+                        raise ConfigError(f'"{field.replace("_", "-")}" {error_msg}')
+
+                if config['protocol'] in zone_allowed and 'zone' not in config:
+                        raise ConfigError(f'"zone" {error_msg}')
+
+                if config['protocol'] not in zone_allowed and 'zone' in config:
+                        raise ConfigError(f'"{config["protocol"]}" does not support "zone"')
+
+                if config['protocol'] not in username_unnecessary:
+                    if 'username' not in config:
+                        raise ConfigError(f'"username" {error_msg}')
+
+                if (config['protocol'] not in dualstack_supported
+                        and config['ip_version'] == 'both'):
+                    raise ConfigError(f'"{config["protocol"]}" does not support IPv4 and IPv6 at the same time')
 
     return None