net: add rfc3442 (classless static routes) to EphemeralDHCP

The EphemeralDHCP context manager did not parse or handle
rfc3442 classless static routes which prevented reading
datasource metadata in some clouds.  This branch adds support
for extracting the field from the leases output, parsing the
format and then adding the required iproute2 ip commands to
apply (and teardown) the static routes.

LP: #1821102

6 files changed, 286 insertions, 6 deletions

diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index e758006f..624c9b42 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -679,7 +679,7 @@ class EphemeralIPv4Network(object):
     """
 
     def __init__(self, interface, ip, prefix_or_mask, broadcast, router=None,
-                 connectivity_url=None):
+                 connectivity_url=None, static_routes=None):
         """Setup context manager and validate call signature.
 
         @param interface: Name of the network interface to bring up.
@@ -690,6 +690,7 @@ class EphemeralIPv4Network(object):
         @param router: Optionally the default gateway IP.
         @param connectivity_url: Optionally, a URL to verify if a usable
            connection already exists.
+        @param static_routes: Optionally a list of static routes from DHCP
         """
         if not all([interface, ip, prefix_or_mask, broadcast]):
             raise ValueError(
@@ -706,6 +707,7 @@ class EphemeralIPv4Network(object):
         self.ip = ip
         self.broadcast = broadcast
         self.router = router
+        self.static_routes = static_routes
         self.cleanup_cmds = []  # List of commands to run to cleanup state.
 
     def __enter__(self):
@@ -718,7 +720,21 @@ class EphemeralIPv4Network(object):
                 return
 
         self._bringup_device()
-        if self.router:
+
+        # rfc3442 requires us to ignore the router config *if* classless static
+        # routes are provided.
+        #
+        # https://tools.ietf.org/html/rfc3442
+        #
+        # If the DHCP server returns both a Classless Static Routes option and
+        # a Router option, the DHCP client MUST ignore the Router option.
+        #
+        # Similarly, if the DHCP server returns both a Classless Static Routes
+        # option and a Static Routes option, the DHCP client MUST ignore the
+        # Static Routes option.
+        if self.static_routes:
+            self._bringup_static_routes()
+        elif self.router:
             self._bringup_router()
 
     def __exit__(self, excp_type, excp_value, excp_traceback):
@@ -762,6 +778,20 @@ class EphemeralIPv4Network(object):
                 ['ip', '-family', 'inet', 'addr', 'del', cidr, 'dev',
                  self.interface])
 
+    def _bringup_static_routes(self):
+        # static_routes = [("169.254.169.254/32", "130.56.248.255"),
+        #                  ("0.0.0.0/0", "130.56.240.1")]
+        for net_address, gateway in self.static_routes:
+            via_arg = []
+            if gateway != "0.0.0.0/0":
+                via_arg = ['via', gateway]
+            util.subp(
+                ['ip', '-4', 'route', 'add', net_address] + via_arg +
+                ['dev', self.interface], capture=True)
+            self.cleanup_cmds.insert(
+                0, ['ip', '-4', 'route', 'del', net_address] + via_arg +
+                   ['dev', self.interface])
+
     def _bringup_router(self):
         """Perform the ip commands to fully setup the router if needed."""
         # Check if a default route exists and exit if it does
diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py
index c98a97cd..17379918 100644
--- a/cloudinit/net/dhcp.py
+++ b/cloudinit/net/dhcp.py
@@ -92,10 +92,14 @@ class EphemeralDHCPv4(object):
         nmap = {'interface': 'interface', 'ip': 'fixed-address',
                 'prefix_or_mask': 'subnet-mask',
                 'broadcast': 'broadcast-address',
+                'static_routes': 'rfc3442-classless-static-routes',
                 'router': 'routers'}
         kwargs = dict([(k, self.lease.get(v)) for k, v in nmap.items()])
         if not kwargs['broadcast']:
             kwargs['broadcast'] = bcip(kwargs['prefix_or_mask'], kwargs['ip'])
+        if kwargs['static_routes']:
+            kwargs['static_routes'] = (
+                parse_static_routes(kwargs['static_routes']))
         if self.connectivity_url:
             kwargs['connectivity_url'] = self.connectivity_url
         ephipv4 = EphemeralIPv4Network(**kwargs)
@@ -272,4 +276,90 @@ def networkd_get_option_from_leases(keyname, leases_d=None):
             return data[keyname]
     return None
 
+
+def parse_static_routes(rfc3442):
+    """ parse rfc3442 format and return a list containing tuple of strings.
+
+    The tuple is composed of the network_address (including net length) and
+    gateway for a parsed static route.
+
+    @param rfc3442: string in rfc3442 format
+    @returns: list of tuple(str, str) for all valid parsed routes until the
+              first parsing error.
+
+    E.g.
+    sr = parse_state_routes("32,169,254,169,254,130,56,248,255,0,130,56,240,1")
+    sr = [
+        ("169.254.169.254/32", "130.56.248.255"), ("0.0.0.0/0", "130.56.240.1")
+    ]
+
+    Python version of isc-dhclient's hooks:
+       /etc/dhcp/dhclient-exit-hooks.d/rfc3442-classless-routes
+    """
+    # raw strings from dhcp lease may end in semi-colon
+    rfc3442 = rfc3442.rstrip(";")
+    tokens = rfc3442.split(',')
+    static_routes = []
+
+    def _trunc_error(cidr, required, remain):
+        msg = ("RFC3442 string malformed.  Current route has CIDR of %s "
+               "and requires %s significant octets, but only %s remain. "
+               "Verify DHCP rfc3442-classless-static-routes value: %s"
+               % (cidr, required, remain, rfc3442))
+        LOG.error(msg)
+
+    current_idx = 0
+    for idx, tok in enumerate(tokens):
+        if idx < current_idx:
+            continue
+        net_length = int(tok)
+        if net_length in range(25, 33):
+            req_toks = 9
+            if len(tokens[idx:]) < req_toks:
+                _trunc_error(net_length, req_toks, len(tokens[idx:]))
+                return static_routes
+            net_address = ".".join(tokens[idx+1:idx+5])
+            gateway = ".".join(tokens[idx+5:idx+req_toks])
+            current_idx = idx + req_toks
+        elif net_length in range(17, 25):
+            req_toks = 8
+            if len(tokens[idx:]) < req_toks:
+                _trunc_error(net_length, req_toks, len(tokens[idx:]))
+                return static_routes
+            net_address = ".".join(tokens[idx+1:idx+4] + ["0"])
+            gateway = ".".join(tokens[idx+4:idx+req_toks])
+            current_idx = idx + req_toks
+        elif net_length in range(9, 17):
+            req_toks = 7
+            if len(tokens[idx:]) < req_toks:
+                _trunc_error(net_length, req_toks, len(tokens[idx:]))
+                return static_routes
+            net_address = ".".join(tokens[idx+1:idx+3] + ["0", "0"])
+            gateway = ".".join(tokens[idx+3:idx+req_toks])
+            current_idx = idx + req_toks
+        elif net_length in range(1, 9):
+            req_toks = 6
+            if len(tokens[idx:]) < req_toks:
+                _trunc_error(net_length, req_toks, len(tokens[idx:]))
+                return static_routes
+            net_address = ".".join(tokens[idx+1:idx+2] + ["0", "0", "0"])
+            gateway = ".".join(tokens[idx+2:idx+req_toks])
+            current_idx = idx + req_toks
+        elif net_length == 0:
+            req_toks = 5
+            if len(tokens[idx:]) < req_toks:
+                _trunc_error(net_length, req_toks, len(tokens[idx:]))
+                return static_routes
+            net_address = "0.0.0.0"
+            gateway = ".".join(tokens[idx+1:idx+req_toks])
+            current_idx = idx + req_toks
+        else:
+            LOG.error('Parsed invalid net length "%s".  Verify DHCP '
+                      'rfc3442-classless-static-routes value.', net_length)
+            return static_routes
+
+        static_routes.append(("%s/%s" % (net_address, net_length), gateway))
+
+    return static_routes
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py
index 51390249..91f503c9 100644
--- a/cloudinit/net/tests/test_dhcp.py
+++ b/cloudinit/net/tests/test_dhcp.py
@@ -8,7 +8,8 @@ from textwrap import dedent
 import cloudinit.net as net
 from cloudinit.net.dhcp import (
     InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery,
-    parse_dhcp_lease_file, dhcp_discovery, networkd_load_leases)
+    parse_dhcp_lease_file, dhcp_discovery, networkd_load_leases,
+    parse_static_routes)
 from cloudinit.util import ensure_file, write_file
 from cloudinit.tests.helpers import (
     CiTestCase, HttprettyTestCase, mock, populate_dir, wrap_and_call)
@@ -64,6 +65,123 @@ class TestParseDHCPLeasesFile(CiTestCase):
         self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file))
 
 
+class TestDHCPRFC3442(CiTestCase):
+
+    def test_parse_lease_finds_rfc3442_classless_static_routes(self):
+        """parse_dhcp_lease_file returns rfc3442-classless-static-routes."""
+        lease_file = self.tmp_path('leases')
+        content = dedent("""
+            lease {
+              interface "wlp3s0";
+              fixed-address 192.168.2.74;
+              option subnet-mask 255.255.255.0;
+              option routers 192.168.2.1;
+              option rfc3442-classless-static-routes 0,130,56,240,1;
+              renew 4 2017/07/27 18:02:30;
+              expire 5 2017/07/28 07:08:15;
+            }
+        """)
+        expected = [
+            {'interface': 'wlp3s0', 'fixed-address': '192.168.2.74',
+             'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1',
+             'rfc3442-classless-static-routes': '0,130,56,240,1',
+             'renew': '4 2017/07/27 18:02:30',
+             'expire': '5 2017/07/28 07:08:15'}]
+        write_file(lease_file, content)
+        self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file))
+
+    @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network')
+    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    def test_obtain_lease_parses_static_routes(self, m_maybe, m_ipv4):
+        """EphemeralDHPCv4 parses rfc3442 routes for EphemeralIPv4Network"""
+        lease = [
+            {'interface': 'wlp3s0', 'fixed-address': '192.168.2.74',
+             'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1',
+             'rfc3442-classless-static-routes': '0,130,56,240,1',
+             'renew': '4 2017/07/27 18:02:30',
+             'expire': '5 2017/07/28 07:08:15'}]
+        m_maybe.return_value = lease
+        eph = net.dhcp.EphemeralDHCPv4()
+        eph.obtain_lease()
+        expected_kwargs = {
+            'interface': 'wlp3s0',
+            'ip': '192.168.2.74',
+            'prefix_or_mask': '255.255.255.0',
+            'broadcast': '192.168.2.255',
+            'static_routes': [('0.0.0.0/0', '130.56.240.1')],
+            'router': '192.168.2.1'}
+        m_ipv4.assert_called_with(**expected_kwargs)
+
+
+class TestDHCPParseStaticRoutes(CiTestCase):
+
+    with_logs = True
+
+    def parse_static_routes_empty_string(self):
+        self.assertEqual([], parse_static_routes(""))
+
+    def test_parse_static_routes_invalid_input_returns_empty_list(self):
+        rfc3442 = "32,169,254,169,254,130,56,248"
+        self.assertEqual([], parse_static_routes(rfc3442))
+
+    def test_parse_static_routes_bogus_width_returns_empty_list(self):
+        rfc3442 = "33,169,254,169,254,130,56,248"
+        self.assertEqual([], parse_static_routes(rfc3442))
+
+    def test_parse_static_routes_single_ip(self):
+        rfc3442 = "32,169,254,169,254,130,56,248,255"
+        self.assertEqual([('169.254.169.254/32', '130.56.248.255')],
+                         parse_static_routes(rfc3442))
+
+    def test_parse_static_routes_single_ip_handles_trailing_semicolon(self):
+        rfc3442 = "32,169,254,169,254,130,56,248,255;"
+        self.assertEqual([('169.254.169.254/32', '130.56.248.255')],
+                         parse_static_routes(rfc3442))
+
+    def test_parse_static_routes_default_route(self):
+        rfc3442 = "0,130,56,240,1"
+        self.assertEqual([('0.0.0.0/0', '130.56.240.1')],
+                         parse_static_routes(rfc3442))
+
+    def test_parse_static_routes_class_c_b_a(self):
+        class_c = "24,192,168,74,192,168,0,4"
+        class_b = "16,172,16,172,16,0,4"
+        class_a = "8,10,10,0,0,4"
+        rfc3442 = ",".join([class_c, class_b, class_a])
+        self.assertEqual(sorted([
+            ("192.168.74.0/24", "192.168.0.4"),
+            ("172.16.0.0/16", "172.16.0.4"),
+            ("10.0.0.0/8", "10.0.0.4")
+        ]), sorted(parse_static_routes(rfc3442)))
+
+    def test_parse_static_routes_logs_error_truncated(self):
+        bad_rfc3442 = {
+            "class_c": "24,169,254,169,10",
+            "class_b": "16,172,16,10",
+            "class_a": "8,10,10",
+            "gateway": "0,0",
+            "netlen":  "33,0",
+        }
+        for rfc3442 in bad_rfc3442.values():
+            self.assertEqual([], parse_static_routes(rfc3442))
+
+        logs = self.logs.getvalue()
+        self.assertEqual(len(bad_rfc3442.keys()), len(logs.splitlines()))
+
+    def test_parse_static_routes_returns_valid_routes_until_parse_err(self):
+        class_c = "24,192,168,74,192,168,0,4"
+        class_b = "16,172,16,172,16,0,4"
+        class_a_error = "8,10,10,0,0"
+        rfc3442 = ",".join([class_c, class_b, class_a_error])
+        self.assertEqual(sorted([
+            ("192.168.74.0/24", "192.168.0.4"),
+            ("172.16.0.0/16", "172.16.0.4"),
+        ]), sorted(parse_static_routes(rfc3442)))
+
+        logs = self.logs.getvalue()
+        self.assertIn(rfc3442, logs.splitlines()[0])
+
+
 class TestDHCPDiscoveryClean(CiTestCase):
     with_logs = True
 
diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py
index 6d2affe7..d393e6ad 100644
--- a/cloudinit/net/tests/test_init.py
+++ b/cloudinit/net/tests/test_init.py
@@ -549,6 +549,45 @@ class TestEphemeralIPV4Network(CiTestCase):
             self.assertEqual(expected_setup_calls, m_subp.call_args_list)
         m_subp.assert_has_calls(expected_teardown_calls)
 
+    def test_ephemeral_ipv4_network_with_rfc3442_static_routes(self, m_subp):
+        params = {
+            'interface': 'eth0', 'ip': '192.168.2.2',
+            'prefix_or_mask': '255.255.255.0', 'broadcast': '192.168.2.255',
+            'static_routes': [('169.254.169.254/32', '192.168.2.1'),
+                              ('0.0.0.0/0', '192.168.2.1')],
+            'router': '192.168.2.1'}
+        expected_setup_calls = [
+            mock.call(
+                ['ip', '-family', 'inet', 'addr', 'add', '192.168.2.2/24',
+                 'broadcast', '192.168.2.255', 'dev', 'eth0'],
+                capture=True, update_env={'LANG': 'C'}),
+            mock.call(
+                ['ip', '-family', 'inet', 'link', 'set', 'dev', 'eth0', 'up'],
+                capture=True),
+            mock.call(
+                ['ip', '-4', 'route', 'add', '169.254.169.254/32',
+                 'via', '192.168.2.1', 'dev', 'eth0'], capture=True),
+            mock.call(
+                ['ip', '-4', 'route', 'add', '0.0.0.0/0',
+                 'via', '192.168.2.1', 'dev', 'eth0'], capture=True)]
+        expected_teardown_calls = [
+            mock.call(
+                ['ip', '-4', 'route', 'del', '0.0.0.0/0',
+                 'via', '192.168.2.1', 'dev', 'eth0'], capture=True),
+            mock.call(
+                ['ip', '-4', 'route', 'del', '169.254.169.254/32',
+                 'via', '192.168.2.1', 'dev', 'eth0'], capture=True),
+            mock.call(
+                ['ip', '-family', 'inet', 'link', 'set', 'dev',
+                 'eth0', 'down'], capture=True),
+            mock.call(
+                ['ip', '-family', 'inet', 'addr', 'del',
+                 '192.168.2.2/24', 'dev', 'eth0'], capture=True)
+        ]
+        with net.EphemeralIPv4Network(**params):
+            self.assertEqual(expected_setup_calls, m_subp.call_args_list)
+        m_subp.assert_has_calls(expected_setup_calls + expected_teardown_calls)
+
 
 class TestApplyNetworkCfgNames(CiTestCase):
     V1_CONFIG = textwrap.dedent("""\
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index f27ef21b..2de2aea2 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -1807,7 +1807,8 @@ class TestAzureDataSourcePreprovisioning(CiTestCase):
         self.assertEqual(m_dhcp.call_count, 2)
         m_net.assert_any_call(
             broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9',
-            prefix_or_mask='255.255.255.0', router='192.168.2.1')
+            prefix_or_mask='255.255.255.0', router='192.168.2.1',
+            static_routes=None)
         self.assertEqual(m_net.call_count, 2)
 
     def test__reprovision_calls__poll_imds(self, fake_resp,
@@ -1845,7 +1846,8 @@ class TestAzureDataSourcePreprovisioning(CiTestCase):
         self.assertEqual(m_dhcp.call_count, 2)
         m_net.assert_any_call(
             broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9',
-            prefix_or_mask='255.255.255.0', router='192.168.2.1')
+            prefix_or_mask='255.255.255.0', router='192.168.2.1',
+            static_routes=None)
         self.assertEqual(m_net.call_count, 2)
 
 
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index 20d59bfd..1ec8e009 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -538,7 +538,8 @@ class TestEc2(test_helpers.HttprettyTestCase):
         m_dhcp.assert_called_once_with('eth9')
         m_net.assert_called_once_with(
             broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9',
-            prefix_or_mask='255.255.255.0', router='192.168.2.1')
+            prefix_or_mask='255.255.255.0', router='192.168.2.1',
+            static_routes=None)
         self.assertIn('Crawl of metadata service took', self.logs.getvalue())