Disable ec2 mirror for non aws instances (#390)

For versions before 20.2, we allowed the use of ec2 mirrors if the datasource availability_zone matches one of the ec2 regions. We are now updating that behavior to allow allow the use of ec2 mirrors on ec2 instances or if the user directly passes an an ec2 mirror url through #cloud-config apt directives.

LP: #1456277

4 files changed, 145 insertions, 81 deletions

diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py
index 6b6c1566..44607489 100644
--- a/tests/unittests/test_distros/test_generic.py
+++ b/tests/unittests/test_distros/test_generic.py
@@ -6,6 +6,7 @@ from cloudinit import util
 from cloudinit.tests import helpers
 
 import os
+import pytest
 import shutil
 import tempfile
 from unittest import mock
@@ -37,24 +38,6 @@ gapmi = distros._get_arch_package_mirror_info
 
 class TestGenericDistro(helpers.FilesystemMockingTestCase):
 
-    def return_first(self, mlist):
-        if not mlist:
-            return None
-        return mlist[0]
-
-    def return_second(self, mlist):
-        if not mlist:
-            return None
-        return mlist[1]
-
-    def return_none(self, _mlist):
-        return None
-
-    def return_last(self, mlist):
-        if not mlist:
-            return None
-        return(mlist[-1])
-
     def setUp(self):
         super(TestGenericDistro, self).setUp()
         # Make a temp directoy for tests to use.
@@ -145,61 +128,6 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase):
         arch_mirrors = gapmi(package_mirrors, arch="amd64")
         self.assertEqual(package_mirrors[0], arch_mirrors)
 
-    def test_get_package_mirror_info_az_ec2(self):
-        arch_mirrors = gapmi(package_mirrors, arch="amd64")
-        data_source_mock = mock.Mock(availability_zone="us-east-1a")
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_first)
-        self.assertEqual(results,
-                         {'primary': 'http://us-east-1.ec2/',
-                          'security': 'http://security-mirror1-intel'})
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_second)
-        self.assertEqual(results,
-                         {'primary': 'http://us-east-1a.clouds/',
-                          'security': 'http://security-mirror2-intel'})
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_none)
-        self.assertEqual(results, package_mirrors[0]['failsafe'])
-
-    def test_get_package_mirror_info_az_non_ec2(self):
-        arch_mirrors = gapmi(package_mirrors, arch="amd64")
-        data_source_mock = mock.Mock(availability_zone="nova.cloudvendor")
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_first)
-        self.assertEqual(results,
-                         {'primary': 'http://nova.cloudvendor.clouds/',
-                          'security': 'http://security-mirror1-intel'})
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_last)
-        self.assertEqual(results,
-                         {'primary': 'http://nova.cloudvendor.clouds/',
-                          'security': 'http://security-mirror2-intel'})
-
-    def test_get_package_mirror_info_none(self):
-        arch_mirrors = gapmi(package_mirrors, arch="amd64")
-        data_source_mock = mock.Mock(availability_zone=None)
-
-        # because both search entries here replacement based on
-        # availability-zone, the filter will be called with an empty list and
-        # failsafe should be taken.
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_first)
-        self.assertEqual(results,
-                         {'primary': 'http://fs-primary-intel',
-                          'security': 'http://security-mirror1-intel'})
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_last)
-        self.assertEqual(results,
-                         {'primary': 'http://fs-primary-intel',
-                          'security': 'http://security-mirror2-intel'})
-
     def test_systemd_in_use(self):
         cls = distros.fetch("ubuntu")
         d = cls("ubuntu", {}, None)
@@ -259,4 +187,116 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase):
             ["pw", "usermod", "myuser", "-p", "01-Jan-1970"])
 
 
+class TestGetPackageMirrors:
+
+    def return_first(self, mlist):
+        if not mlist:
+            return None
+        return mlist[0]
+
+    def return_second(self, mlist):
+        if not mlist:
+            return None
+
+        return mlist[1] if len(mlist) > 1 else None
+
+    def return_none(self, _mlist):
+        return None
+
+    def return_last(self, mlist):
+        if not mlist:
+            return None
+        return(mlist[-1])
+
+    @pytest.mark.parametrize(
+        "allow_ec2_mirror, platform_type, mirrors",
+        [
+            (True, "ec2", [
+                {'primary': 'http://us-east-1.ec2/',
+                 'security': 'http://security-mirror1-intel'},
+                {'primary': 'http://us-east-1a.clouds/',
+                 'security': 'http://security-mirror2-intel'}
+            ]),
+            (True, "other", [
+                {'primary': 'http://us-east-1.ec2/',
+                 'security': 'http://security-mirror1-intel'},
+                {'primary': 'http://us-east-1a.clouds/',
+                 'security': 'http://security-mirror2-intel'}
+            ]),
+            (False, "ec2", [
+                {'primary': 'http://us-east-1.ec2/',
+                 'security': 'http://security-mirror1-intel'},
+                {'primary': 'http://us-east-1a.clouds/',
+                 'security': 'http://security-mirror2-intel'}
+            ]),
+            (False, "other", [
+                {'primary': 'http://us-east-1a.clouds/',
+                 'security': 'http://security-mirror1-intel'},
+                {'primary': 'http://fs-primary-intel',
+                 'security': 'http://security-mirror2-intel'}
+            ])
+        ])
+    def test_get_package_mirror_info_az_ec2(self,
+                                            allow_ec2_mirror,
+                                            platform_type,
+                                            mirrors):
+        flag_path = "cloudinit.distros." \
+                    "ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES"
+        with mock.patch(flag_path, allow_ec2_mirror):
+            arch_mirrors = gapmi(package_mirrors, arch="amd64")
+            data_source_mock = mock.Mock(
+                availability_zone="us-east-1a",
+                platform_type=platform_type)
+
+            results = gpmi(arch_mirrors, data_source=data_source_mock,
+                           mirror_filter=self.return_first)
+            assert(results == mirrors[0])
+
+            results = gpmi(arch_mirrors, data_source=data_source_mock,
+                           mirror_filter=self.return_second)
+            assert(results == mirrors[1])
+
+            results = gpmi(arch_mirrors, data_source=data_source_mock,
+                           mirror_filter=self.return_none)
+            assert(results == package_mirrors[0]['failsafe'])
+
+    def test_get_package_mirror_info_az_non_ec2(self):
+        arch_mirrors = gapmi(package_mirrors, arch="amd64")
+        data_source_mock = mock.Mock(availability_zone="nova.cloudvendor")
+
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
+                       mirror_filter=self.return_first)
+        assert(results == {
+            'primary': 'http://nova.cloudvendor.clouds/',
+            'security': 'http://security-mirror1-intel'}
+        )
+
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
+                       mirror_filter=self.return_last)
+        assert(results == {
+            'primary': 'http://nova.cloudvendor.clouds/',
+            'security': 'http://security-mirror2-intel'}
+        )
+
+    def test_get_package_mirror_info_none(self):
+        arch_mirrors = gapmi(package_mirrors, arch="amd64")
+        data_source_mock = mock.Mock(availability_zone=None)
+
+        # because both search entries here replacement based on
+        # availability-zone, the filter will be called with an empty list and
+        # failsafe should be taken.
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
+                       mirror_filter=self.return_first)
+        assert(results == {
+            'primary': 'http://fs-primary-intel',
+            'security': 'http://security-mirror1-intel'}
+        )
+
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
+                       mirror_filter=self.return_last)
+        assert(results == {
+            'primary': 'http://fs-primary-intel',
+            'security': 'http://security-mirror2-intel'}
+        )
+
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
index e5382544..369480be 100644
--- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
@@ -101,6 +101,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
             cfg = {'apt_mirror_search': mirror}
         else:
             cfg = {'apt_mirror': mirror}
+
         mycloud = self._get_cloud(distro)
 
         with mock.patch.object(util, 'write_file') as mockwf:
@@ -108,8 +109,9 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
                                    return_value="faketmpl") as mocklf:
                 with mock.patch.object(os.path, 'isfile',
                                        return_value=True) as mockisfile:
-                    with mock.patch.object(templater, 'render_string',
-                                           return_value="fake") as mockrnd:
+                    with mock.patch.object(
+                        templater, 'render_string',
+                            return_value='fake') as mockrnd:
                         with mock.patch.object(util, 'rename'):
                             cc_apt_configure.handle("test", cfg, mycloud,
                                                     LOG, None)
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py
index f2349157..367971cb 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py
@@ -43,10 +43,17 @@ class FakeDistro(object):
         return
 
 
+class FakeDatasource:
+    """Fake Datasource helper object"""
+    def __init__(self):
+        self.region = 'region'
+
+
 class FakeCloud(object):
     """Fake Cloud helper object"""
     def __init__(self):
         self.distro = FakeDistro()
+        self.datasource = FakeDatasource()
 
 
 class TestAptSourceConfig(TestCase):
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py
index 220100e2..ac847238 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v3.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py
@@ -49,6 +49,18 @@ MOCK_LSB_RELEASE_DATA = {
     'release': '18.04', 'codename': 'bionic'}
 
 
+class FakeDatasource:
+    """Fake Datasource helper object"""
+    def __init__(self):
+        self.region = 'region'
+
+
+class FakeCloud:
+    """Fake Cloud helper object"""
+    def __init__(self):
+        self.datasource = FakeDatasource()
+
+
 class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
     """TestAptSourceConfig
     Main Class to test apt configs
@@ -471,7 +483,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
         fromfn = ("%s/%s_%s" % (pre, archive, post))
         tofn = ("%s/test.ubuntu.com_%s" % (pre, post))
 
-        mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, arch)
+        mirrors = cc_apt_configure.find_apt_mirror_info(cfg, FakeCloud(), arch)
 
         self.assertEqual(mirrors['MIRROR'],
                          "http://test.ubuntu.com/%s/" % component)
@@ -559,7 +571,8 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
                "security": [{'arches': ["default"],
                              "uri": smir}]}
 
-        mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, 'amd64')
+        mirrors = cc_apt_configure.find_apt_mirror_info(
+            cfg, FakeCloud(), 'amd64')
 
         self.assertEqual(mirrors['MIRROR'],
                          pmir)
@@ -594,7 +607,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
                "security": [{'arches': ["default"], "uri": "nothis-security"},
                             {'arches': [arch], "uri": smir}]}
 
-        mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, arch)
+        mirrors = cc_apt_configure.find_apt_mirror_info(cfg, FakeCloud(), arch)
 
         self.assertEqual(mirrors['PRIMARY'], pmir)
         self.assertEqual(mirrors['MIRROR'], pmir)
@@ -613,7 +626,8 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
                             {'arches': ["default"],
                              "uri": smir}]}
 
-        mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, 'amd64')
+        mirrors = cc_apt_configure.find_apt_mirror_info(
+            cfg, FakeCloud(), 'amd64')
 
         self.assertEqual(mirrors['MIRROR'],
                          pmir)
@@ -673,7 +687,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
 
         with mock.patch.object(cc_apt_configure.util, 'search_for_mirror',
                                side_effect=[pmir, smir]) as mocksearch:
-            mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None,
+            mirrors = cc_apt_configure.find_apt_mirror_info(cfg, FakeCloud(),
                                                             'amd64')
 
         calls = [call(["pfailme", pmir]),
@@ -712,7 +726,8 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
         # should not be called, since primary is specified
         with mock.patch.object(cc_apt_configure.util,
                                'search_for_mirror') as mockse:
-            mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, arch)
+            mirrors = cc_apt_configure.find_apt_mirror_info(
+                cfg, FakeCloud(), arch)
         mockse.assert_not_called()
 
         self.assertEqual(mirrors['MIRROR'],