Merge branch 'pppoe-rewrite' of https://github.com/c-po/vyos-1x into current

* 'pppoe-rewrite' of https://github.com/c-po/vyos-1x: (23 commits)
  pppoe: T2055: do not try to start a deleted dialer interface
  pppoe: T1318: declutter name-server CLI nodes
  pppoe: T2055: remove router-advert node in client interface
  pppoe: T1318: migrate user-id and password nodes under an authentication node
  pppoe: T1318: rename link to source-interface
  pppoe: T1318: use include files for disable and descriptionx
  pppoe: T1318: rephrase help text on default-route
  interface-definitions: include: disable: rephrase help text
  pppoe: T1318: extend migrator for firewall, qos and ip routing nodes
  pppoe: T1318: proper delete old interfaces in migrator
  pppoe: T1318: increase priority so PPPoE is run after bond interfaces
  pppoe: T1318: fix migrator and add missing link statement
  pppoe: T1318: use lists rather then strings on Config()
  pppoe: T1318: support interface description
  pppoe: T1318: remove obsolete ipv6-up.d script
  pppoe: T1318: add op-mode commands for link information
  pppoe: T1318: use systemd to manage connection
  pppoe: T1318: remove process startup debug output
  pppoe: T1318: move process startup to apply()
  pppoe: T1318: "link" option is mandatory
  ...

6 files changed, 395 insertions, 11 deletions

diff --git a/src/completion/list_pppoe_peers.sh b/src/completion/list_pppoe_peers.sh
new file mode 100755
index 000000000..382a29264
--- /dev/null
+++ b/src/completion/list_pppoe_peers.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+if [ -d /etc/ppp/peers ]; then
+    cd /etc/ppp/peers
+    ls pppoe*
+fi
diff --git a/src/conf_mode/https.py b/src/conf_mode/https.py
index 5d90b2b53..d8e78c6cd 100755
--- a/src/conf_mode/https.py
+++ b/src/conf_mode/https.py
@@ -47,8 +47,8 @@ server {
         # SSL configuration
         #
 {% if server.address == '*' %}
-        listen 443 ssl;
-        listen [::]:443 ssl;
+        listen {{ server.port }} ssl;
+        listen [::]:{{ server.port }} ssl;
 {% else %}
         listen {{ server.address }}:{{ server.port }} ssl;
 {% endif %}
diff --git a/src/conf_mode/interfaces-pppoe.py b/src/conf_mode/interfaces-pppoe.py
new file mode 100755
index 000000000..245f5a4b0
--- /dev/null
+++ b/src/conf_mode/interfaces-pppoe.py
@@ -0,0 +1,253 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2019 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from sys import exit
+from copy import deepcopy
+from jinja2 import Template
+from subprocess import Popen, PIPE
+
+from vyos.config import Config
+from vyos import ConfigError
+from netifaces import interfaces
+
+# Please be careful if you edit the template.
+config_pppoe_tmpl = """
+### Autogenerated by interfaces-pppoe.py ###
+
+{% if description %}
+# {{ description }}
+{% endif %}
+
+# Require peer to provide the local IP address if it is not
+# specified explicitly in the config file.
+noipdefault
+
+# Don't show the password in logfiles:
+hide-password
+
+# Standard Link Control Protocol (LCP) parameters:
+lcp-echo-interval 20
+lcp-echo-failure 3
+
+# RFC 2516, paragraph 7 mandates that the following options MUST NOT be
+# requested and MUST be rejected if requested by the peer:
+# Address-and-Control-Field-Compression (ACFC)
+noaccomp
+
+# Asynchronous-Control-Character-Map (ACCM)
+default-asyncmap
+
+# Override any connect script that may have been set in /etc/ppp/options.
+connect /bin/true
+
+# Don't try to authenticate the remote node
+noauth
+
+# Don't try to proxy ARP for the remote endpoint. User can set proxy
+# arp entries up manually if they wish.  More importantly, having
+# the "proxyarp" parameter set disables the "defaultroute" option.
+noproxyarp
+
+plugin rp-pppoe.so
+{{ source_interface }}
+persist
+ifname {{ intf }}
+ipparam {{ intf }}
+debug
+logfile /var/log/vyatta/ppp_{{ intf }}.log
+{% if 'auto' in default_route -%}
+defaultroute
+{% elif 'force' in default_route -%}
+defaultroute
+replacedefaultroute
+{% endif %}
+mtu {{ mtu }}
+mru {{ mtu }}
+user "{{ auth_username }}"
+password "{{ auth_password }}"
+{% if name_server -%}
+usepeerdns
+{% endif %}
+{% if ipv6_enable -%}
++ipv6
+{% endif %}
+
+"""
+
+default_config_data = {
+    'access_concentrator': '',
+    'auth_username': '',
+    'auth_password': '',
+    'on_demand': False,
+    'default_route': 'auto',
+    'deleted': False,
+    'description': '',
+    'disable': False,
+    'intf': '',
+    'idle_timeout': '',
+    'ipv6_autoconf': False,
+    'ipv6_enable': False,
+    'local_address': '',
+    'mtu': '1492',
+    'name_server': True,
+    'remote_address': '',
+    'service_name': '',
+    'source_interface': ''
+}
+
+def subprocess_cmd(command):
+    p = Popen(command, stdout=PIPE, shell=True)
+    p.communicate()
+
+def get_config():
+    pppoe = deepcopy(default_config_data)
+    conf = Config()
+    base_path = ['interfaces', 'pppoe']
+
+    # determine tagNode instance
+    try:
+        pppoe['intf'] = os.environ['VYOS_TAGNODE_VALUE']
+    except KeyError as E:
+        print("Interface not specified")
+
+    # Check if interface has been removed
+    if not conf.exists(base_path + [pppoe['intf']]):
+        pppoe['deleted'] = True
+        return pppoe
+
+    # set new configuration level
+    conf.set_level(base_path + [pppoe['intf']])
+
+    # Access concentrator name (only connect to this concentrator)
+    if conf.exists(['access-concentrator']):
+        pppoe['access_concentrator'] = conf.return_values(['access-concentrator'])
+
+    # Authentication name supplied to PPPoE server
+    if conf.exists(['authentication', 'user']):
+        pppoe['auth_username'] = conf.return_value(['authentication', 'user'])
+
+    # Password for authenticating local machine to PPPoE server
+    if conf.exists(['authentication', 'password']):
+        pppoe['auth_password'] = conf.return_value(['authentication', 'password'])
+
+    # Access concentrator name (only connect to this concentrator)
+    if conf.exists(['connect-on-demand']):
+        pppoe['on_demand'] = True
+
+    # Enable/Disable default route to peer when link comes up
+    if conf.exists(['default-route']):
+        pppoe['default_route'] = conf.return_value(['default-route'])
+
+    # Retrieve interface description
+    if conf.exists(['description']):
+        pppoe['description'] = conf.return_value(['description'])
+
+    # Disable this interface
+    if conf.exists(['disable']):
+        pppoe['disable'] = True
+
+    # Delay before disconnecting idle session (in seconds)
+    if conf.exists(['idle-timeout']):
+        pppoe['idle_timeout'] = conf.return_value(['idle-timeout'])
+
+    # Enable Stateless Address Autoconfiguration (SLAAC)
+    if conf.exists(['ipv6', 'address', 'autoconf']):
+        pppoe['ipv6_autoconf'] = True
+
+    # Activate IPv6 support on this connection
+    if conf.exists(['ipv6', 'enable']):
+        pppoe['ipv6_enable'] = True
+
+    # IPv4 address of local end of PPPoE link
+    if conf.exists(['local-address']):
+        pppoe['local_address'] = conf.return_value(['local-address'])
+
+    # Physical Interface used for this PPPoE session
+    if conf.exists(['source-interface']):
+        pppoe['source_interface'] = conf.return_value('source-interface')
+
+    # Maximum Transmission Unit (MTU)
+    if conf.exists(['mtu']):
+        pppoe['mtu'] = conf.return_value(['mtu'])
+
+    # Do not use DNS servers provided by the peer
+    if conf.exists(['no-peer-dns']):
+        pppoe['name_server'] = False
+
+    # IPv4 address for remote end of PPPoE session
+    if conf.exists(['remote-address']):
+        pppoe['remote_address'] = conf.return_value(['remote-address'])
+
+    # Service name, only connect to access concentrators advertising this
+    if conf.exists(['service-name']):
+        pppoe['service_name'] = conf.return_value(['service-name'])
+
+    return pppoe
+
+def verify(pppoe):
+    if pppoe['deleted']:
+        # bail out early
+        return None
+
+    if not pppoe['source_interface']:
+        raise ConfigError('PPPoE source interface is missing')
+
+    return None
+
+def generate(pppoe):
+    config_file_pppoe = '/etc/ppp/peers/{}'.format(pppoe['intf'])
+
+    # Always hang-up PPPoE connection prior generating new configuration file
+    cmd = 'systemctl stop ppp@{}.service'.format(pppoe['intf'])
+    subprocess_cmd(cmd)
+
+    if pppoe['deleted']:
+        # Delete PPP configuration files
+        if os.path.exists(config_file_pppoe):
+            os.unlink(config_file_pppoe)
+
+    else:
+        # Create PPP configuration files
+        tmpl = Template(config_pppoe_tmpl)
+        config_text = tmpl.render(pppoe)
+        with open(config_file_pppoe, 'w') as f:
+            f.write(config_text)
+
+    return None
+
+def apply(pppoe):
+    if pppoe['deleted']:
+        # bail out early
+        return None
+
+    if not pppoe['disable']:
+        # Dial PPPoE connection
+        cmd = 'systemctl start ppp@{}.service'.format(pppoe['intf'])
+        subprocess_cmd(cmd)
+
+    return None
+
+if __name__ == '__main__':
+    try:
+        c = get_config()
+        verify(c)
+        generate(c)
+        apply(c)
+    except ConfigError as e:
+        print(e)
+        exit(1)
diff --git a/src/conf_mode/snmp.py b/src/conf_mode/snmp.py
index 7cffa5e04..ac94afb1a 100755
--- a/src/conf_mode/snmp.py
+++ b/src/conf_mode/snmp.py
@@ -710,18 +710,20 @@ def apply(snmp):
     # Passwords are not available immediately in the configuration file,
     # after daemon startup - we wait until they have been processed by
     # snmpd, which we see when a magic line appears in this file.
-    ready = False
-    while not ready:
+    while True:
         while not os.path.exists(config_file_user):
             sleep(0.5)
-            ready = True
 
-    with open(config_file_user, 'r') as f:
-        for line in f:
-            # Search for our magic string inside the file
-            if 'usmUser' in line:
-                ready = True
-                break
+        try:
+            with open(config_file_user, 'r') as f:
+                for line in f:
+                    # Search for our magic string inside the file
+                    if 'usmUser' in line:
+                        break
+        except IOError:
+            continue
+        else:
+            break
 
     # net-snmp is now regenerating the configuration file in the background
     # thus we need to re-open and re-read the file as the content changed.
diff --git a/src/etc/systemd/system/ppp@.service b/src/etc/systemd/system/ppp@.service
new file mode 100644
index 000000000..d271efb41
--- /dev/null
+++ b/src/etc/systemd/system/ppp@.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Dialing PPP connection %I
+After=network.target
+
+[Service]
+ExecStart=/usr/sbin/pppd call %I nodetach nolog
+Restart=on-failure
+RestartSec=5s
+
+[Install]
+WantedBy=multi-user.target
diff --git a/src/migration-scripts/interfaces/4-to-5 b/src/migration-scripts/interfaces/4-to-5
new file mode 100755
index 000000000..2a42c60ff
--- /dev/null
+++ b/src/migration-scripts/interfaces/4-to-5
@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+# De-nest PPPoE interfaces
+# Migrate boolean nodes to valueless
+
+import sys
+from vyos.configtree import ConfigTree
+
+def migrate_dialer(config, tree, intf):
+    for pppoe in config.list_nodes(tree):
+        # assemble string, 0 -> pppoe0
+        new_base = ['interfaces', 'pppoe']
+        pppoe_base = new_base + ['pppoe' + pppoe]
+        config.set(new_base)
+        # format as tag node to avoid loading problems
+        config.set_tag(new_base)
+
+        # Copy the entire old node to the new one before migrating individual
+        # parts
+        config.copy(tree + [pppoe], pppoe_base)
+
+        # Instead of letting the user choose between auto and none
+        # where auto is default, it makes more sesne to just offer
+        # an option to disable the default behavior (declutter CLI)
+        if config.exists(pppoe_base + ['name-server']):
+            tmp = config.return_value(pppoe_base + ['name-server'])
+            if tmp == "none":
+                config.set(pppoe_base + ['no-peer-dns'])
+            config.delete(pppoe_base + ['name-server'])
+
+        # Migrate user-id and password nodes under an 'authentication'
+        # node
+        if config.exists(pppoe_base + ['user-id']):
+            user = config.return_value(pppoe_base + ['user-id'])
+            config.set(pppoe_base + ['authentication', 'user'], value=user)
+            config.delete(pppoe_base + ['user-id'])
+
+        if config.exists(pppoe_base + ['password']):
+            pwd = config.return_value(pppoe_base + ['password'])
+            config.set(pppoe_base + ['authentication', 'password'], value=pwd)
+            config.delete(pppoe_base + ['password'])
+
+        # remove enable-ipv6 node and rather place it under ipv6 node
+        if config.exists(pppoe_base + ['enable-ipv6']):
+            config.set(pppoe_base + ['ipv6', 'enable'])
+            config.delete(pppoe_base + ['enable-ipv6'])
+
+        # Source interface migration
+        config.set(pppoe_base + ['source-interface'], value=intf)
+
+        # Remove IPv6 router-advert nodes as this makes no sense on a
+        # client diale rinterface to send RAs back into the network
+        # https://phabricator.vyos.net/T2055
+        ipv6_ra = pppoe_base + ['ipv6', 'router-advert']
+        if config.exists(ipv6_ra):
+            config.delete(ipv6_ra)
+
+
+if __name__ == '__main__':
+    if (len(sys.argv) < 1):
+        print("Must specify file name!")
+        exit(1)
+
+    file_name = sys.argv[1]
+
+    with open(file_name, 'r') as f:
+        config_file = f.read()
+
+    config = ConfigTree(config_file)
+    pppoe_links = ['bonding', 'ethernet']
+
+    for link_type in pppoe_links:
+        if not config.exists(['interfaces', link_type]):
+            continue
+
+        for interface in config.list_nodes(['interfaces', link_type]):
+            # check if PPPoE exists
+            base_if = ['interfaces', link_type, interface]
+            pppoe_if = base_if + ['pppoe']
+            if config.exists(pppoe_if):
+                for dialer in  config.list_nodes(pppoe_if):
+                    migrate_dialer(config, pppoe_if, interface)
+
+                # Delete old PPPoE interface
+                config.delete(pppoe_if)
+
+            # bail out early if there are no VLAN interfaces to migrate
+            if not config.exists(base_if + ['vif']):
+                continue
+
+            # Migrate PPPoE interfaces attached to a VLAN
+            for vlan in config.list_nodes(base_if + ['vif']):
+                vlan_if = base_if + ['vif', vlan]
+                pppoe_if = vlan_if + ['pppoe']
+                if config.exists(pppoe_if):
+                    for dialer in  config.list_nodes(pppoe_if):
+                        intf = "{}.{}".format(interface, vlan)
+                        migrate_dialer(config, pppoe_if, intf)
+
+                    # Delete old PPPoE interface
+                    config.delete(pppoe_if)
+
+                    # Add interface description that this is required for PPPoE
+                    if not config.exists(vlan_if + ['description']):
+                        config.set(vlan_if + ['description'], value='PPPoE link interface')
+
+    try:
+        with open(file_name, 'w') as f:
+            f.write(config.to_string())
+    except OSError as e:
+        print("Failed to save the modified config: {}".format(e))
+        sys.exit(1)