From 1081962eacf2814fea6f4fa3255c530de14e4a24 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Thu, 19 Apr 2018 21:30:08 -0600
Subject: pylint: pay attention to unused variable warnings.

This enables warnings produced by pylint for unused variables (W0612),
and fixes the existing errors.
---
 cloudinit/sources/helpers/digitalocean.py              | 7 +++----
 cloudinit/sources/helpers/openstack.py                 | 2 +-
 cloudinit/sources/helpers/vmware/imc/config_nic.py     | 2 +-
 cloudinit/sources/helpers/vmware/imc/config_passwd.py  | 4 ++--
 cloudinit/sources/helpers/vmware/imc/guestcust_util.py | 4 ++--
 5 files changed, 9 insertions(+), 10 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/helpers/digitalocean.py b/cloudinit/sources/helpers/digitalocean.py
index 693f8d5c..0e7cccac 100644
--- a/cloudinit/sources/helpers/digitalocean.py
+++ b/cloudinit/sources/helpers/digitalocean.py
@@ -41,10 +41,9 @@ def assign_ipv4_link_local(nic=None):
                            "address")
 
     try:
-        (result, _err) = util.subp(ip_addr_cmd)
+        util.subp(ip_addr_cmd)
         LOG.debug("assigned ip4LL address '%s' to '%s'", addr, nic)
-
-        (result, _err) = util.subp(ip_link_cmd)
+        util.subp(ip_link_cmd)
         LOG.debug("brought device '%s' up", nic)
     except Exception:
         util.logexc(LOG, "ip4LL address assignment of '%s' to '%s' failed."
@@ -75,7 +74,7 @@ def del_ipv4_link_local(nic=None):
     ip_addr_cmd = ['ip', 'addr', 'flush', 'dev', nic]
 
     try:
-        (result, _err) = util.subp(ip_addr_cmd)
+        util.subp(ip_addr_cmd)
         LOG.debug("removed ip4LL addresses from %s", nic)
 
     except Exception as e:
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 26f3168d..a4cf0667 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -638,7 +638,7 @@ def convert_net_json(network_json=None, known_macs=None):
             known_macs = net.get_interfaces_by_mac()
 
         # go through and fill out the link_id_info with names
-        for link_id, info in link_id_info.items():
+        for _link_id, info in link_id_info.items():
             if info.get('name'):
                 continue
             if info.get('mac') in known_macs:
diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py
index 2d8900e2..3ef8c624 100644
--- a/cloudinit/sources/helpers/vmware/imc/config_nic.py
+++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py
@@ -73,7 +73,7 @@ class NicConfigurator(object):
         The mac address(es) are in the lower case
         """
         cmd = ['ip', 'addr', 'show']
-        (output, err) = util.subp(cmd)
+        output, _err = util.subp(cmd)
         sections = re.split(r'\n\d+: ', '\n' + output)[1:]
 
         macPat = r'link/ether (([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2}))'
diff --git a/cloudinit/sources/helpers/vmware/imc/config_passwd.py b/cloudinit/sources/helpers/vmware/imc/config_passwd.py
index 75cfbaaf..8c91fa41 100644
--- a/cloudinit/sources/helpers/vmware/imc/config_passwd.py
+++ b/cloudinit/sources/helpers/vmware/imc/config_passwd.py
@@ -56,10 +56,10 @@ class PasswordConfigurator(object):
         LOG.info('Expiring password.')
         for user in uidUserList:
             try:
-                out, err = util.subp(['passwd', '--expire', user])
+                util.subp(['passwd', '--expire', user])
             except util.ProcessExecutionError as e:
                 if os.path.exists('/usr/bin/chage'):
-                    out, e = util.subp(['chage', '-d', '0', user])
+                    util.subp(['chage', '-d', '0', user])
                 else:
                     LOG.warning('Failed to expire password for %s with error: '
                                 '%s', user, e)
diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py
index 44075255..a590f323 100644
--- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py
+++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py
@@ -91,7 +91,7 @@ def enable_nics(nics):
 
     for attempt in range(0, enableNicsWaitRetries):
         logger.debug("Trying to connect interfaces, attempt %d", attempt)
-        (out, err) = set_customization_status(
+        (out, _err) = set_customization_status(
             GuestCustStateEnum.GUESTCUST_STATE_RUNNING,
             GuestCustEventEnum.GUESTCUST_EVENT_ENABLE_NICS,
             nics)
@@ -104,7 +104,7 @@ def enable_nics(nics):
             return
 
         for count in range(0, enableNicsWaitCount):
-            (out, err) = set_customization_status(
+            (out, _err) = set_customization_status(
                 GuestCustStateEnum.GUESTCUST_STATE_RUNNING,
                 GuestCustEventEnum.GUESTCUST_EVENT_QUERY_NICS,
                 nics)
-- 
cgit v1.2.3


From 30e730f7ca111487d243ba9f40c66df6d7a49953 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Thu, 17 May 2018 14:59:54 -0600
Subject: read_file_or_url: move to url_helper, fix bug in its FileResponse.

The result of a read_file_or_url on a file and on a url would differ
in behavior.
  str(UrlResponse) would return UrlResponse.contents.decode('utf-8')
while
  str(FileResponse) would return str(FileResponse.contents)

The difference being "b'foo'" versus "foo".

As part of the general goal of cleaning util, move read_file_or_url
into url_helper.
---
 cloudinit/cmd/main.py                              |  2 +-
 cloudinit/config/cc_phone_home.py                  |  7 +--
 cloudinit/config/schema.py                         |  4 +-
 cloudinit/ec2_utils.py                             | 14 +++---
 cloudinit/sources/DataSourceMAAS.py                |  2 +-
 cloudinit/sources/helpers/azure.py                 |  5 ++-
 cloudinit/tests/test_url_helper.py                 | 28 +++++++++++-
 cloudinit/url_helper.py                            | 29 +++++++++++-
 cloudinit/user_data.py                             |  6 +--
 cloudinit/util.py                                  | 37 ++--------------
 tests/unittests/test__init__.py                    |  8 ++--
 .../unittests/test_datasource/test_azure_helper.py |  2 +-
 .../test_handler/test_handler_apt_conf_v1.py       | 16 +++----
 .../test_handler_apt_configure_sources_list_v1.py  |  7 ---
 .../test_handler/test_handler_apt_source_v1.py     | 27 +++++-------
 .../test_handler/test_handler_apt_source_v3.py     | 27 +++++-------
 tests/unittests/test_handler/test_handler_ntp.py   | 51 +++++++++-------------
 17 files changed, 131 insertions(+), 141 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index 3f2dbb93..d6ba90f4 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -187,7 +187,7 @@ def attempt_cmdline_url(path, network=True, cmdline=None):
     data = None
     header = b'#cloud-config'
     try:
-        resp = util.read_file_or_url(**kwargs)
+        resp = url_helper.read_file_or_url(**kwargs)
         if resp.ok():
             data = resp.contents
             if not resp.contents.startswith(header):
diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py
index 878069b7..3be0d1c1 100644
--- a/cloudinit/config/cc_phone_home.py
+++ b/cloudinit/config/cc_phone_home.py
@@ -41,6 +41,7 @@ keys to post. Available keys are:
 """
 
 from cloudinit import templater
+from cloudinit import url_helper
 from cloudinit import util
 
 from cloudinit.settings import PER_INSTANCE
@@ -136,9 +137,9 @@ def handle(name, cfg, cloud, log, args):
     }
     url = templater.render_string(url, url_params)
     try:
-        util.read_file_or_url(url, data=real_submit_keys,
-                              retries=tries, sec_between=3,
-                              ssl_details=util.fetch_ssl_details(cloud.paths))
+        url_helper.read_file_or_url(
+            url, data=real_submit_keys, retries=tries, sec_between=3,
+            ssl_details=util.fetch_ssl_details(cloud.paths))
     except Exception:
         util.logexc(log, "Failed to post phone home data to %s in %s tries",
                     url, tries)
diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py
index 76826e05..3bad8e22 100644
--- a/cloudinit/config/schema.py
+++ b/cloudinit/config/schema.py
@@ -4,7 +4,7 @@
 from __future__ import print_function
 
 from cloudinit import importer
-from cloudinit.util import find_modules, read_file_or_url
+from cloudinit.util import find_modules, load_file
 
 import argparse
 from collections import defaultdict
@@ -139,7 +139,7 @@ def validate_cloudconfig_file(config_path, schema, annotate=False):
     """
     if not os.path.exists(config_path):
         raise RuntimeError('Configfile {0} does not exist'.format(config_path))
-    content = read_file_or_url('file://{0}'.format(config_path)).contents
+    content = load_file(config_path, decode=False)
     if not content.startswith(CLOUD_CONFIG_HEADER):
         errors = (
             ('header', 'File {0} needs to begin with "{1}"'.format(
diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py
index dc3f0fc3..3b7b17f1 100644
--- a/cloudinit/ec2_utils.py
+++ b/cloudinit/ec2_utils.py
@@ -150,11 +150,9 @@ def get_instance_userdata(api_version='latest',
         # NOT_FOUND occurs) and just in that case returning an empty string.
         exception_cb = functools.partial(_skip_retry_on_codes,
                                          SKIP_USERDATA_CODES)
-        response = util.read_file_or_url(ud_url,
-                                         ssl_details=ssl_details,
-                                         timeout=timeout,
-                                         retries=retries,
-                                         exception_cb=exception_cb)
+        response = url_helper.read_file_or_url(
+            ud_url, ssl_details=ssl_details, timeout=timeout,
+            retries=retries, exception_cb=exception_cb)
         user_data = response.contents
     except url_helper.UrlError as e:
         if e.code not in SKIP_USERDATA_CODES:
@@ -169,9 +167,9 @@ def _get_instance_metadata(tree, api_version='latest',
                            ssl_details=None, timeout=5, retries=5,
                            leaf_decoder=None):
     md_url = url_helper.combine_url(metadata_address, api_version, tree)
-    caller = functools.partial(util.read_file_or_url,
-                               ssl_details=ssl_details, timeout=timeout,
-                               retries=retries)
+    caller = functools.partial(
+        url_helper.read_file_or_url, ssl_details=ssl_details,
+        timeout=timeout, retries=retries)
 
     def mcaller(url):
         return caller(url).contents
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index aa56addb..bcb38544 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -198,7 +198,7 @@ def read_maas_seed_url(seed_url, read_file_or_url=None, timeout=None,
     If version is None, then <version>/ will not be used.
     """
     if read_file_or_url is None:
-        read_file_or_url = util.read_file_or_url
+        read_file_or_url = url_helper.read_file_or_url
 
     if seed_url.endswith("/"):
         seed_url = seed_url[:-1]
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index 90c12df1..e5696b1f 100644
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
@@ -14,6 +14,7 @@ from cloudinit import temp_utils
 from contextlib import contextmanager
 from xml.etree import ElementTree
 
+from cloudinit import url_helper
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -55,14 +56,14 @@ class AzureEndpointHttpClient(object):
         if secure:
             headers = self.headers.copy()
             headers.update(self.extra_secure_headers)
-        return util.read_file_or_url(url, headers=headers)
+        return url_helper.read_file_or_url(url, headers=headers)
 
     def post(self, url, data=None, extra_headers=None):
         headers = self.headers
         if extra_headers is not None:
             headers = self.headers.copy()
             headers.update(extra_headers)
-        return util.read_file_or_url(url, data=data, headers=headers)
+        return url_helper.read_file_or_url(url, data=data, headers=headers)
 
 
 class GoalState(object):
diff --git a/cloudinit/tests/test_url_helper.py b/cloudinit/tests/test_url_helper.py
index b778a3a7..113249d9 100644
--- a/cloudinit/tests/test_url_helper.py
+++ b/cloudinit/tests/test_url_helper.py
@@ -1,7 +1,10 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-from cloudinit.url_helper import oauth_headers
+from cloudinit.url_helper import oauth_headers, read_file_or_url
 from cloudinit.tests.helpers import CiTestCase, mock, skipIf
+from cloudinit import util
+
+import httpretty
 
 
 try:
@@ -38,3 +41,26 @@ class TestOAuthHeaders(CiTestCase):
             'url', 'consumer_key', 'token_key', 'token_secret',
             'consumer_secret')
         self.assertEqual('url', return_value)
+
+
+class TestReadFileOrUrl(CiTestCase):
+    def test_read_file_or_url_str_from_file(self):
+        """Test that str(result.contents) on file is text version of contents.
+        It should not be "b'data'", but just "'data'" """
+        tmpf = self.tmp_path("myfile1")
+        data = b'This is my file content\n'
+        util.write_file(tmpf, data, omode="wb")
+        result = read_file_or_url("file://%s" % tmpf)
+        self.assertEqual(result.contents, data)
+        self.assertEqual(str(result), data.decode('utf-8'))
+
+    @httpretty.activate
+    def test_read_file_or_url_str_from_url(self):
+        """Test that str(result.contents) on url is text version of contents.
+        It should not be "b'data'", but just "'data'" """
+        url = 'http://hostname/path'
+        data = b'This is my url content\n'
+        httpretty.register_uri(httpretty.GET, url, data)
+        result = read_file_or_url(url)
+        self.assertEqual(result.contents, data)
+        self.assertEqual(str(result), data.decode('utf-8'))
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 1de07b1c..8067979e 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -15,6 +15,7 @@ import six
 import time
 
 from email.utils import parsedate
+from errno import ENOENT
 from functools import partial
 from itertools import count
 from requests import exceptions
@@ -80,6 +81,32 @@ def combine_url(base, *add_ons):
     return url
 
 
+def read_file_or_url(url, timeout=5, retries=10,
+                     headers=None, data=None, sec_between=1, ssl_details=None,
+                     headers_cb=None, exception_cb=None):
+    url = url.lstrip()
+    if url.startswith("/"):
+        url = "file://%s" % url
+    if url.lower().startswith("file://"):
+        if data:
+            LOG.warning("Unable to post data to file resource %s", url)
+        file_path = url[len("file://"):]
+        try:
+            with open(file_path, "rb") as fp:
+                contents = fp.read()
+        except IOError as e:
+            code = e.errno
+            if e.errno == ENOENT:
+                code = NOT_FOUND
+            raise UrlError(cause=e, code=code, headers=None, url=url)
+        return FileResponse(file_path, contents=contents)
+    else:
+        return readurl(url, timeout=timeout, retries=retries, headers=headers,
+                       headers_cb=headers_cb, data=data,
+                       sec_between=sec_between, ssl_details=ssl_details,
+                       exception_cb=exception_cb)
+
+
 # Made to have same accessors as UrlResponse so that the
 # read_file_or_url can return this or that object and the
 # 'user' of those objects will not need to know the difference.
@@ -96,7 +123,7 @@ class StringResponse(object):
         return True
 
     def __str__(self):
-        return self.contents
+        return self.contents.decode('utf-8')
 
 
 class FileResponse(StringResponse):
diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py
index cc55daf8..8f6aba1e 100644
--- a/cloudinit/user_data.py
+++ b/cloudinit/user_data.py
@@ -19,7 +19,7 @@ import six
 
 from cloudinit import handlers
 from cloudinit import log as logging
-from cloudinit.url_helper import UrlError
+from cloudinit.url_helper import read_file_or_url, UrlError
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -224,8 +224,8 @@ class UserDataProcessor(object):
                 content = util.load_file(include_once_fn)
             else:
                 try:
-                    resp = util.read_file_or_url(include_url,
-                                                 ssl_details=self.ssl_details)
+                    resp = read_file_or_url(include_url,
+                                            ssl_details=self.ssl_details)
                     if include_once_on and resp.ok():
                         util.write_file(include_once_fn, resp.contents,
                                         mode=0o600)
diff --git a/cloudinit/util.py b/cloudinit/util.py
index fc30018c..edfedc7d 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -857,37 +857,6 @@ def fetch_ssl_details(paths=None):
     return ssl_details
 
 
-def read_file_or_url(url, timeout=5, retries=10,
-                     headers=None, data=None, sec_between=1, ssl_details=None,
-                     headers_cb=None, exception_cb=None):
-    url = url.lstrip()
-    if url.startswith("/"):
-        url = "file://%s" % url
-    if url.lower().startswith("file://"):
-        if data:
-            LOG.warning("Unable to post data to file resource %s", url)
-        file_path = url[len("file://"):]
-        try:
-            contents = load_file(file_path, decode=False)
-        except IOError as e:
-            code = e.errno
-            if e.errno == ENOENT:
-                code = url_helper.NOT_FOUND
-            raise url_helper.UrlError(cause=e, code=code, headers=None,
-                                      url=url)
-        return url_helper.FileResponse(file_path, contents=contents)
-    else:
-        return url_helper.readurl(url,
-                                  timeout=timeout,
-                                  retries=retries,
-                                  headers=headers,
-                                  headers_cb=headers_cb,
-                                  data=data,
-                                  sec_between=sec_between,
-                                  ssl_details=ssl_details,
-                                  exception_cb=exception_cb)
-
-
 def load_yaml(blob, default=None, allowed=(dict,)):
     loaded = default
     blob = decode_binary(blob)
@@ -925,12 +894,14 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0):
         ud_url = "%s%s%s" % (base, "user-data", ext)
         md_url = "%s%s%s" % (base, "meta-data", ext)
 
-    md_resp = read_file_or_url(md_url, timeout, retries, file_retries)
+    md_resp = url_helper.read_file_or_url(md_url, timeout, retries,
+                                          file_retries)
     md = None
     if md_resp.ok():
         md = load_yaml(decode_binary(md_resp.contents), default={})
 
-    ud_resp = read_file_or_url(ud_url, timeout, retries, file_retries)
+    ud_resp = url_helper.read_file_or_url(ud_url, timeout, retries,
+                                          file_retries)
     ud = None
     if ud_resp.ok():
         ud = ud_resp.contents
diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py
index f1ab02e9..739bbebf 100644
--- a/tests/unittests/test__init__.py
+++ b/tests/unittests/test__init__.py
@@ -182,7 +182,7 @@ class TestCmdlineUrl(CiTestCase):
         self.assertEqual(
             ('url', 'http://example.com'), main.parse_cmdline_url(cmdline))
 
-    @mock.patch('cloudinit.cmd.main.util.read_file_or_url')
+    @mock.patch('cloudinit.cmd.main.url_helper.read_file_or_url')
     def test_invalid_content(self, m_read):
         key = "cloud-config-url"
         url = 'http://example.com/foo'
@@ -196,7 +196,7 @@ class TestCmdlineUrl(CiTestCase):
         self.assertIn(url, msg)
         self.assertFalse(os.path.exists(fpath))
 
-    @mock.patch('cloudinit.cmd.main.util.read_file_or_url')
+    @mock.patch('cloudinit.cmd.main.url_helper.read_file_or_url')
     def test_valid_content(self, m_read):
         url = "http://example.com/foo"
         payload = b"#cloud-config\nmydata: foo\nbar: wark\n"
@@ -210,7 +210,7 @@ class TestCmdlineUrl(CiTestCase):
         self.assertEqual(logging.INFO, lvl)
         self.assertIn(url, msg)
 
-    @mock.patch('cloudinit.cmd.main.util.read_file_or_url')
+    @mock.patch('cloudinit.cmd.main.url_helper.read_file_or_url')
     def test_no_key_found(self, m_read):
         cmdline = "ro mykey=http://example.com/foo root=foo"
         fpath = self.tmp_path("ccpath")
@@ -221,7 +221,7 @@ class TestCmdlineUrl(CiTestCase):
         self.assertFalse(os.path.exists(fpath))
         self.assertEqual(logging.DEBUG, lvl)
 
-    @mock.patch('cloudinit.cmd.main.util.read_file_or_url')
+    @mock.patch('cloudinit.cmd.main.url_helper.read_file_or_url')
     def test_exception_warns(self, m_read):
         url = "http://example.com/foo"
         cmdline = "ro cloud-config-url=%s root=LABEL=bar" % url
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
index b42b073f..af9d3e1a 100644
--- a/tests/unittests/test_datasource/test_azure_helper.py
+++ b/tests/unittests/test_datasource/test_azure_helper.py
@@ -195,7 +195,7 @@ class TestAzureEndpointHttpClient(CiTestCase):
         self.addCleanup(patches.close)
 
         self.read_file_or_url = patches.enter_context(
-            mock.patch.object(azure_helper.util, 'read_file_or_url'))
+            mock.patch.object(azure_helper.url_helper, 'read_file_or_url'))
 
     def test_non_secure_get(self):
         client = azure_helper.AzureEndpointHttpClient(mock.MagicMock())
diff --git a/tests/unittests/test_handler/test_handler_apt_conf_v1.py b/tests/unittests/test_handler/test_handler_apt_conf_v1.py
index 83f962a9..6a4b03ee 100644
--- a/tests/unittests/test_handler/test_handler_apt_conf_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_conf_v1.py
@@ -12,10 +12,6 @@ import shutil
 import tempfile
 
 
-def load_tfile_or_url(*args, **kwargs):
-    return(util.decode_binary(util.read_file_or_url(*args, **kwargs).contents))
-
-
 class TestAptProxyConfig(TestCase):
     def setUp(self):
         super(TestAptProxyConfig, self).setUp()
@@ -36,7 +32,7 @@ class TestAptProxyConfig(TestCase):
         self.assertTrue(os.path.isfile(self.pfile))
         self.assertFalse(os.path.isfile(self.cfile))
 
-        contents = load_tfile_or_url(self.pfile)
+        contents = util.load_file(self.pfile)
         self.assertTrue(self._search_apt_config(contents, "http", "myproxy"))
 
     def test_apt_http_proxy_written(self):
@@ -46,7 +42,7 @@ class TestAptProxyConfig(TestCase):
         self.assertTrue(os.path.isfile(self.pfile))
         self.assertFalse(os.path.isfile(self.cfile))
 
-        contents = load_tfile_or_url(self.pfile)
+        contents = util.load_file(self.pfile)
         self.assertTrue(self._search_apt_config(contents, "http", "myproxy"))
 
     def test_apt_all_proxy_written(self):
@@ -64,7 +60,7 @@ class TestAptProxyConfig(TestCase):
         self.assertTrue(os.path.isfile(self.pfile))
         self.assertFalse(os.path.isfile(self.cfile))
 
-        contents = load_tfile_or_url(self.pfile)
+        contents = util.load_file(self.pfile)
 
         for ptype, pval in values.items():
             self.assertTrue(self._search_apt_config(contents, ptype, pval))
@@ -80,7 +76,7 @@ class TestAptProxyConfig(TestCase):
         cc_apt_configure.apply_apt_config({'proxy': "foo"},
                                           self.pfile, self.cfile)
         self.assertTrue(os.path.isfile(self.pfile))
-        contents = load_tfile_or_url(self.pfile)
+        contents = util.load_file(self.pfile)
         self.assertTrue(self._search_apt_config(contents, "http", "foo"))
 
     def test_config_written(self):
@@ -92,14 +88,14 @@ class TestAptProxyConfig(TestCase):
         self.assertTrue(os.path.isfile(self.cfile))
         self.assertFalse(os.path.isfile(self.pfile))
 
-        self.assertEqual(load_tfile_or_url(self.cfile), payload)
+        self.assertEqual(util.load_file(self.cfile), payload)
 
     def test_config_replaced(self):
         util.write_file(self.pfile, "content doesnt matter")
         cc_apt_configure.apply_apt_config({'conf': "foo"},
                                           self.pfile, self.cfile)
         self.assertTrue(os.path.isfile(self.cfile))
-        self.assertEqual(load_tfile_or_url(self.cfile), "foo")
+        self.assertEqual(util.load_file(self.cfile), "foo")
 
     def test_config_deleted(self):
         # if no 'conf' is provided, delete any previously written file
diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
index d2b96f0b..23bd6e10 100644
--- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
@@ -64,13 +64,6 @@ deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted
 """)
 
 
-def load_tfile_or_url(*args, **kwargs):
-    """load_tfile_or_url
-    load file and return content after decoding
-    """
-    return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)
-
-
 class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
     """TestAptSourceConfigSourceList
     Main Class to test sources list rendering
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py
index 46ca4ce4..a3132fbd 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py
@@ -39,13 +39,6 @@ S0ORP6HXET3+jC8BMG4tBWCTK/XEZw==
 ADD_APT_REPO_MATCH = r"^[\w-]+:\w"
 
 
-def load_tfile_or_url(*args, **kwargs):
-    """load_tfile_or_url
-    load file and return content after decoding
-    """
-    return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)
-
-
 class FakeDistro(object):
     """Fake Distro helper object"""
     def update_package_sources(self):
@@ -125,7 +118,7 @@ class TestAptSourceConfig(TestCase):
 
         self.assertTrue(os.path.isfile(filename))
 
-        contents = load_tfile_or_url(filename)
+        contents = util.load_file(filename)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", "http://archive.ubuntu.com/ubuntu",
                                    "karmic-backports",
@@ -157,13 +150,13 @@ class TestAptSourceConfig(TestCase):
         self.apt_src_basic(self.aptlistfile, cfg)
 
         # extra verify on two extra files of this test
-        contents = load_tfile_or_url(self.aptlistfile2)
+        contents = util.load_file(self.aptlistfile2)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", "http://archive.ubuntu.com/ubuntu",
                                    "precise-backports",
                                    "main universe multiverse restricted"),
                                   contents, flags=re.IGNORECASE))
-        contents = load_tfile_or_url(self.aptlistfile3)
+        contents = util.load_file(self.aptlistfile3)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", "http://archive.ubuntu.com/ubuntu",
                                    "lucid-backports",
@@ -220,7 +213,7 @@ class TestAptSourceConfig(TestCase):
 
         self.assertTrue(os.path.isfile(filename))
 
-        contents = load_tfile_or_url(filename)
+        contents = util.load_file(filename)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", params['MIRROR'], params['RELEASE'],
                                    "multiverse"),
@@ -241,12 +234,12 @@ class TestAptSourceConfig(TestCase):
 
         # extra verify on two extra files of this test
         params = self._get_default_params()
-        contents = load_tfile_or_url(self.aptlistfile2)
+        contents = util.load_file(self.aptlistfile2)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", params['MIRROR'], params['RELEASE'],
                                    "main"),
                                   contents, flags=re.IGNORECASE))
-        contents = load_tfile_or_url(self.aptlistfile3)
+        contents = util.load_file(self.aptlistfile3)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", params['MIRROR'], params['RELEASE'],
                                    "universe"),
@@ -296,7 +289,7 @@ class TestAptSourceConfig(TestCase):
 
         self.assertTrue(os.path.isfile(filename))
 
-        contents = load_tfile_or_url(filename)
+        contents = util.load_file(filename)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb",
                                    ('http://ppa.launchpad.net/smoser/'
@@ -336,14 +329,14 @@ class TestAptSourceConfig(TestCase):
                 'filename': self.aptlistfile3}
 
         self.apt_src_keyid(self.aptlistfile, [cfg1, cfg2, cfg3], 3)
-        contents = load_tfile_or_url(self.aptlistfile2)
+        contents = util.load_file(self.aptlistfile2)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb",
                                    ('http://ppa.launchpad.net/smoser/'
                                     'cloud-init-test/ubuntu'),
                                    "xenial", "universe"),
                                   contents, flags=re.IGNORECASE))
-        contents = load_tfile_or_url(self.aptlistfile3)
+        contents = util.load_file(self.aptlistfile3)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb",
                                    ('http://ppa.launchpad.net/smoser/'
@@ -375,7 +368,7 @@ class TestAptSourceConfig(TestCase):
 
         self.assertTrue(os.path.isfile(filename))
 
-        contents = load_tfile_or_url(filename)
+        contents = util.load_file(filename)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb",
                                    ('http://ppa.launchpad.net/smoser/'
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py
index e486862d..7a64c230 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v3.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py
@@ -49,13 +49,6 @@ ADD_APT_REPO_MATCH = r"^[\w-]+:\w"
 TARGET = None
 
 
-def load_tfile(*args, **kwargs):
-    """load_tfile_or_url
-    load file and return content after decoding
-    """
-    return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)
-
-
 class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
     """TestAptSourceConfig
     Main Class to test apt configs
@@ -119,7 +112,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
 
         self.assertTrue(os.path.isfile(filename))
 
-        contents = load_tfile(filename)
+        contents = util.load_file(filename)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", "http://test.ubuntu.com/ubuntu",
                                    "karmic-backports",
@@ -151,13 +144,13 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
         self._apt_src_basic(self.aptlistfile, cfg)
 
         # extra verify on two extra files of this test
-        contents = load_tfile(self.aptlistfile2)
+        contents = util.load_file(self.aptlistfile2)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", "http://test.ubuntu.com/ubuntu",
                                    "precise-backports",
                                    "main universe multiverse restricted"),
                                   contents, flags=re.IGNORECASE))
-        contents = load_tfile(self.aptlistfile3)
+        contents = util.load_file(self.aptlistfile3)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", "http://test.ubuntu.com/ubuntu",
                                    "lucid-backports",
@@ -174,7 +167,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
 
         self.assertTrue(os.path.isfile(filename))
 
-        contents = load_tfile(filename)
+        contents = util.load_file(filename)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", params['MIRROR'], params['RELEASE'],
                                    "multiverse"),
@@ -201,12 +194,12 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
 
         # extra verify on two extra files of this test
         params = self._get_default_params()
-        contents = load_tfile(self.aptlistfile2)
+        contents = util.load_file(self.aptlistfile2)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", params['MIRROR'], params['RELEASE'],
                                    "main"),
                                   contents, flags=re.IGNORECASE))
-        contents = load_tfile(self.aptlistfile3)
+        contents = util.load_file(self.aptlistfile3)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", params['MIRROR'], params['RELEASE'],
                                    "universe"),
@@ -240,7 +233,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
 
         self.assertTrue(os.path.isfile(filename))
 
-        contents = load_tfile(filename)
+        contents = util.load_file(filename)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb",
                                    ('http://ppa.launchpad.net/smoser/'
@@ -277,14 +270,14 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
                                    'keyid': "03683F77"}}
 
         self._apt_src_keyid(self.aptlistfile, cfg, 3)
-        contents = load_tfile(self.aptlistfile2)
+        contents = util.load_file(self.aptlistfile2)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb",
                                    ('http://ppa.launchpad.net/smoser/'
                                     'cloud-init-test/ubuntu'),
                                    "xenial", "universe"),
                                   contents, flags=re.IGNORECASE))
-        contents = load_tfile(self.aptlistfile3)
+        contents = util.load_file(self.aptlistfile3)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb",
                                    ('http://ppa.launchpad.net/smoser/'
@@ -310,7 +303,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
 
         self.assertTrue(os.path.isfile(self.aptlistfile))
 
-        contents = load_tfile(self.aptlistfile)
+        contents = util.load_file(self.aptlistfile)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb",
                                    ('http://ppa.launchpad.net/smoser/'
diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py
index 6da4564e..6fe3659d 100644
--- a/tests/unittests/test_handler/test_handler_ntp.py
+++ b/tests/unittests/test_handler/test_handler_ntp.py
@@ -155,9 +155,9 @@ class TestNtp(FilesystemMockingTestCase):
                                              path=confpath,
                                              template_fn=template_fn,
                                              template=None)
-        content = util.read_file_or_url('file://' + confpath).contents
         self.assertEqual(
-            "servers []\npools ['10.0.0.1', '10.0.0.2']\n", content.decode())
+            "servers []\npools ['10.0.0.1', '10.0.0.2']\n",
+            util.load_file(confpath))
 
     def test_write_ntp_config_template_defaults_pools_w_empty_lists(self):
         """write_ntp_config_template defaults pools servers upon empty config.
@@ -176,10 +176,9 @@ class TestNtp(FilesystemMockingTestCase):
                                              path=confpath,
                                              template_fn=template_fn,
                                              template=None)
-        content = util.read_file_or_url('file://' + confpath).contents
         self.assertEqual(
             "servers []\npools {0}\n".format(pools),
-            content.decode())
+            util.load_file(confpath))
 
     def test_defaults_pools_empty_lists_sles(self):
         """write_ntp_config_template defaults opensuse pools upon empty config.
@@ -196,11 +195,11 @@ class TestNtp(FilesystemMockingTestCase):
                                          path=confpath,
                                          template_fn=template_fn,
                                          template=None)
-        content = util.read_file_or_url('file://' + confpath).contents
         for pool in default_pools:
             self.assertIn('opensuse', pool)
         self.assertEqual(
-            "servers []\npools {0}\n".format(default_pools), content.decode())
+            "servers []\npools {0}\n".format(default_pools),
+            util.load_file(confpath))
         self.assertIn(
             "Adding distro default ntp pool servers: {0}".format(
                 ",".join(default_pools)),
@@ -217,10 +216,9 @@ class TestNtp(FilesystemMockingTestCase):
                                          path=confpath,
                                          template_fn=template_fn,
                                          template=None)
-        content = util.read_file_or_url('file://' + confpath).contents
         self.assertEqual(
             "[Time]\nNTP=%s %s \n" % (" ".join(servers), " ".join(pools)),
-            content.decode())
+            util.load_file(confpath))
 
     def test_distro_ntp_client_configs(self):
         """Test we have updated ntp client configs on different distros"""
@@ -267,17 +265,17 @@ class TestNtp(FilesystemMockingTestCase):
                 cc_ntp.write_ntp_config_template(distro, servers=servers,
                                                  pools=pools, path=confpath,
                                                  template_fn=template_fn)
-                content = util.read_file_or_url('file://' + confpath).contents
+                content = util.load_file(confpath)
                 if client in ['ntp', 'chrony']:
                     expected_servers = '\n'.join([
                         'server {0} iburst'.format(srv) for srv in servers])
                     print('distro=%s client=%s' % (distro, client))
-                    self.assertIn(expected_servers, content.decode('utf-8'),
+                    self.assertIn(expected_servers, content,
                                   ('failed to render {0} conf'
                                    ' for distro:{1}'.format(client, distro)))
                     expected_pools = '\n'.join([
                         'pool {0} iburst'.format(pool) for pool in pools])
-                    self.assertIn(expected_pools, content.decode('utf-8'),
+                    self.assertIn(expected_pools, content,
                                   ('failed to render {0} conf'
                                    ' for distro:{1}'.format(client, distro)))
                 elif client == 'systemd-timesyncd':
@@ -286,7 +284,7 @@ class TestNtp(FilesystemMockingTestCase):
                         "# See timesyncd.conf(5) for details.\n\n" +
                         "[Time]\nNTP=%s %s \n" % (" ".join(servers),
                                                   " ".join(pools)))
-                    self.assertEqual(expected_content, content.decode())
+                    self.assertEqual(expected_content, content)
 
     def test_no_ntpcfg_does_nothing(self):
         """When no ntp section is defined handler logs a warning and noops."""
@@ -308,10 +306,10 @@ class TestNtp(FilesystemMockingTestCase):
                 confpath = ntpconfig['confpath']
                 m_select.return_value = ntpconfig
                 cc_ntp.handle('cc_ntp', valid_empty_config, mycloud, None, [])
-                content = util.read_file_or_url('file://' + confpath).contents
                 pools = cc_ntp.generate_server_names(mycloud.distro.name)
                 self.assertEqual(
-                    "servers []\npools {0}\n".format(pools), content.decode())
+                    "servers []\npools {0}\n".format(pools),
+                    util.load_file(confpath))
             self.assertNotIn('Invalid config:', self.logs.getvalue())
 
     @skipUnlessJsonSchema()
@@ -333,9 +331,8 @@ class TestNtp(FilesystemMockingTestCase):
                 "Invalid config:\nntp.pools.0: 123 is not of type 'string'\n"
                 "ntp.servers.1: None is not of type 'string'",
                 self.logs.getvalue())
-            content = util.read_file_or_url('file://' + confpath).contents
             self.assertEqual("servers ['valid', None]\npools [123]\n",
-                             content.decode())
+                             util.load_file(confpath))
 
     @skipUnlessJsonSchema()
     @mock.patch('cloudinit.config.cc_ntp.select_ntp_client')
@@ -357,9 +354,8 @@ class TestNtp(FilesystemMockingTestCase):
                 "Invalid config:\nntp.pools: 123 is not of type 'array'\n"
                 "ntp.servers: 'non-array' is not of type 'array'",
                 self.logs.getvalue())
-            content = util.read_file_or_url('file://' + confpath).contents
             self.assertEqual("servers non-array\npools 123\n",
-                             content.decode())
+                             util.load_file(confpath))
 
     @skipUnlessJsonSchema()
     @mock.patch('cloudinit.config.cc_ntp.select_ntp_client')
@@ -381,10 +377,9 @@ class TestNtp(FilesystemMockingTestCase):
                 "Invalid config:\nntp: Additional properties are not allowed "
                 "('invalidkey' was unexpected)",
                 self.logs.getvalue())
-            content = util.read_file_or_url('file://' + confpath).contents
             self.assertEqual(
                 "servers []\npools ['0.mycompany.pool.ntp.org']\n",
-                content.decode())
+                util.load_file(confpath))
 
     @skipUnlessJsonSchema()
     @mock.patch('cloudinit.config.cc_ntp.select_ntp_client')
@@ -407,10 +402,10 @@ class TestNtp(FilesystemMockingTestCase):
                 " has non-unique elements\nntp.servers: "
                 "['10.0.0.1', '10.0.0.1'] has non-unique elements",
                 self.logs.getvalue())
-            content = util.read_file_or_url('file://' + confpath).contents
             self.assertEqual(
                 "servers ['10.0.0.1', '10.0.0.1']\n"
-                "pools ['0.mypool.org', '0.mypool.org']\n", content.decode())
+                "pools ['0.mypool.org', '0.mypool.org']\n",
+                util.load_file(confpath))
 
     @mock.patch('cloudinit.config.cc_ntp.select_ntp_client')
     def test_ntp_handler_timesyncd(self, m_select):
@@ -426,10 +421,9 @@ class TestNtp(FilesystemMockingTestCase):
             confpath = ntpconfig['confpath']
             m_select.return_value = ntpconfig
             cc_ntp.handle('cc_ntp', cfg, mycloud, None, [])
-            content = util.read_file_or_url('file://' + confpath).contents
             self.assertEqual(
                 "[Time]\nNTP=192.168.2.1 192.168.2.2 0.mypool.org \n",
-                content.decode())
+                util.load_file(confpath))
 
     @mock.patch('cloudinit.config.cc_ntp.select_ntp_client')
     def test_ntp_handler_enabled_false(self, m_select):
@@ -466,10 +460,9 @@ class TestNtp(FilesystemMockingTestCase):
                 m_util.subp.assert_called_with(
                     ['systemctl', 'reload-or-restart',
                      service_name], capture=True)
-            content = util.read_file_or_url('file://' + confpath).contents
             self.assertEqual(
                 "servers []\npools {0}\n".format(pools),
-                content.decode())
+                util.load_file(confpath))
 
     def test_opensuse_picks_chrony(self):
         """Test opensuse picks chrony or ntp on certain distro versions"""
@@ -638,10 +631,9 @@ class TestNtp(FilesystemMockingTestCase):
             mock_path = 'cloudinit.config.cc_ntp.temp_utils._TMPDIR'
             with mock.patch(mock_path, self.new_root):
                 cc_ntp.handle('notimportant', cfg, mycloud, None, None)
-            content = util.read_file_or_url('file://' + confpath).contents
             self.assertEqual(
                 "servers []\npools ['mypool.org']\n%s" % custom,
-                content.decode())
+                util.load_file(confpath))
 
     @mock.patch('cloudinit.config.cc_ntp.supplemental_schema_validation')
     @mock.patch('cloudinit.config.cc_ntp.reload_ntp')
@@ -675,10 +667,9 @@ class TestNtp(FilesystemMockingTestCase):
             with mock.patch(mock_path, self.new_root):
                 cc_ntp.handle('notimportant',
                               {'ntp': cfg}, mycloud, None, None)
-            content = util.read_file_or_url('file://' + confpath).contents
             self.assertEqual(
                 "servers []\npools ['mypool.org']\n%s" % custom,
-                content.decode())
+                util.load_file(confpath))
         m_schema.assert_called_with(expected_merged_cfg)
 
 
-- 
cgit v1.2.3


From aaffd59431fe05932a66016db941fe197c4e7620 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Fri, 17 Aug 2018 20:25:31 +0000
Subject: Add datasource Oracle Compute Infrastructure (OCI).

This adds a Oracle specific datasource that functions with OCI.
It is a simplified version of the OpenStack metadata server
with support for vendor-data.

It does not support the OCI-C (classic) platform.

Also here is a move of BrokenMetadata to common 'sources'
as this was the third occurrence of that class.
---
 .pylintrc                                         |   3 +-
 cloudinit/apport.py                               |   1 +
 cloudinit/settings.py                             |   1 +
 cloudinit/sources/DataSourceIBMCloud.py           |  13 +-
 cloudinit/sources/DataSourceOpenStack.py          |  12 +-
 cloudinit/sources/DataSourceOracle.py             | 233 +++++++++++++++
 cloudinit/sources/__init__.py                     |   4 +
 cloudinit/sources/helpers/openstack.py            |   6 +-
 cloudinit/sources/tests/test_oracle.py            | 331 ++++++++++++++++++++++
 doc/rtd/topics/datasources.rst                    |   1 +
 doc/rtd/topics/datasources/oracle.rst             |  26 ++
 tests/unittests/test_datasource/test_common.py    |   2 +
 tests/unittests/test_datasource/test_openstack.py |  13 +-
 tests/unittests/test_ds_identify.py               |  19 ++
 tools/ds-identify                                 |   8 +-
 15 files changed, 650 insertions(+), 23 deletions(-)
 create mode 100644 cloudinit/sources/DataSourceOracle.py
 create mode 100644 cloudinit/sources/tests/test_oracle.py
 create mode 100644 doc/rtd/topics/datasources/oracle.rst

(limited to 'cloudinit/sources/helpers')

diff --git a/.pylintrc b/.pylintrc
index 3bfa0c81..e376b48b 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -61,7 +61,8 @@ ignored-modules=
 # List of class names for which member attributes should not be checked (useful
 # for classes with dynamically set attributes). This supports the use of
 # qualified names.
-ignored-classes=optparse.Values,thread._local
+# argparse.Namespace from https://github.com/PyCQA/pylint/issues/2413
+ignored-classes=argparse.Namespace,optparse.Values,thread._local
 
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
diff --git a/cloudinit/apport.py b/cloudinit/apport.py
index 130ff269..22cb7fde 100644
--- a/cloudinit/apport.py
+++ b/cloudinit/apport.py
@@ -30,6 +30,7 @@ KNOWN_CLOUD_NAMES = [
     'NoCloud',
     'OpenNebula',
     'OpenStack',
+    'Oracle',
     'OVF',
     'OpenTelekomCloud',
     'Scaleway',
diff --git a/cloudinit/settings.py b/cloudinit/settings.py
index dde5749d..ea367cb7 100644
--- a/cloudinit/settings.py
+++ b/cloudinit/settings.py
@@ -38,6 +38,7 @@ CFG_BUILTIN = {
         'Scaleway',
         'Hetzner',
         'IBMCloud',
+        'Oracle',
         # At the end to act as a 'catch' when none of the above work...
         'None',
     ],
diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py
index 01106ec0..a5358148 100644
--- a/cloudinit/sources/DataSourceIBMCloud.py
+++ b/cloudinit/sources/DataSourceIBMCloud.py
@@ -295,7 +295,7 @@ def read_md():
             results = metadata_from_dir(path)
         else:
             results = util.mount_cb(path, metadata_from_dir)
-    except BrokenMetadata as e:
+    except sources.BrokenMetadata as e:
         raise RuntimeError(
             "Failed reading IBM config disk (platform=%s path=%s): %s" %
             (platform, path, e))
@@ -304,10 +304,6 @@ def read_md():
     return ret
 
 
-class BrokenMetadata(IOError):
-    pass
-
-
 def metadata_from_dir(source_dir):
     """Walk source_dir extracting standardized metadata.
 
@@ -352,12 +348,13 @@ def metadata_from_dir(source_dir):
             try:
                 data = transl(raw)
             except Exception as e:
-                raise BrokenMetadata("Failed decoding %s: %s" % (path, e))
+                raise sources.BrokenMetadata(
+                    "Failed decoding %s: %s" % (path, e))
 
         results[name] = data
 
     if results.get('metadata_raw') is None:
-        raise BrokenMetadata(
+        raise sources.BrokenMetadata(
             "%s missing required file 'meta_data.json'" % source_dir)
 
     results['metadata'] = {}
@@ -368,7 +365,7 @@ def metadata_from_dir(source_dir):
         try:
             md['random_seed'] = base64.b64decode(md_raw['random_seed'])
         except (ValueError, TypeError) as e:
-            raise BrokenMetadata(
+            raise sources.BrokenMetadata(
                 "Badly formatted metadata random_seed entry: %s" % e)
 
     renames = (
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index b9ade90d..4a015240 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -13,6 +13,7 @@ from cloudinit import url_helper
 from cloudinit import util
 
 from cloudinit.sources.helpers import openstack
+from cloudinit.sources import DataSourceOracle as oracle
 
 LOG = logging.getLogger(__name__)
 
@@ -28,8 +29,7 @@ DMI_PRODUCT_NOVA = 'OpenStack Nova'
 DMI_PRODUCT_COMPUTE = 'OpenStack Compute'
 VALID_DMI_PRODUCT_NAMES = [DMI_PRODUCT_NOVA, DMI_PRODUCT_COMPUTE]
 DMI_ASSET_TAG_OPENTELEKOM = 'OpenTelekomCloud'
-DMI_ASSET_TAG_ORACLE_CLOUD = 'OracleCloud.com'
-VALID_DMI_ASSET_TAGS = [DMI_ASSET_TAG_OPENTELEKOM, DMI_ASSET_TAG_ORACLE_CLOUD]
+VALID_DMI_ASSET_TAGS = [DMI_ASSET_TAG_OPENTELEKOM]
 
 
 class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
@@ -122,8 +122,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
             False when unable to contact metadata service or when metadata
             format is invalid or disabled.
         """
-        if not detect_openstack():
+        oracle_considered = 'Oracle' in self.sys_cfg.get('datasource_list')
+        if not detect_openstack(accept_oracle=not oracle_considered):
             return False
+
         if self.perform_dhcp_setup:  # Setup networking in init-local stage.
             try:
                 with EphemeralDHCPv4(self.fallback_interface):
@@ -215,7 +217,7 @@ def read_metadata_service(base_url, ssl_details=None,
     return reader.read_v2()
 
 
-def detect_openstack():
+def detect_openstack(accept_oracle=False):
     """Return True when a potential OpenStack platform is detected."""
     if not util.is_x86():
         return True  # Non-Intel cpus don't properly report dmi product names
@@ -224,6 +226,8 @@ def detect_openstack():
         return True
     elif util.read_dmi_data('chassis-asset-tag') in VALID_DMI_ASSET_TAGS:
         return True
+    elif accept_oracle and oracle._is_platform_viable():
+        return True
     elif util.get_proc_env(1).get('product_name') == DMI_PRODUCT_NOVA:
         return True
     return False
diff --git a/cloudinit/sources/DataSourceOracle.py b/cloudinit/sources/DataSourceOracle.py
new file mode 100644
index 00000000..fab39af3
--- /dev/null
+++ b/cloudinit/sources/DataSourceOracle.py
@@ -0,0 +1,233 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+"""Datasource for Oracle (OCI/Oracle Cloud Infrastructure)
+
+OCI provides a OpenStack like metadata service which provides only
+'2013-10-17' and 'latest' versions..
+
+Notes:
+ * This datasource does not support the OCI-Classic. OCI-Classic
+   provides an EC2 lookalike metadata service.
+ * The uuid provided in DMI data is not the same as the meta-data provided
+   instance-id, but has an equivalent lifespan.
+ * We do need to support upgrade from an instance that cloud-init
+   identified as OpenStack.
+ * Both bare-metal and vms use iscsi root
+ * Both bare-metal and vms provide chassis-asset-tag of OracleCloud.com
+"""
+
+from cloudinit.url_helper import combine_url, readurl, UrlError
+from cloudinit.net import dhcp
+from cloudinit import net
+from cloudinit import sources
+from cloudinit import util
+from cloudinit.net import cmdline
+from cloudinit import log as logging
+
+import json
+import re
+
+LOG = logging.getLogger(__name__)
+
+CHASSIS_ASSET_TAG = "OracleCloud.com"
+METADATA_ENDPOINT = "http://169.254.169.254/openstack/"
+
+
+class DataSourceOracle(sources.DataSource):
+
+    dsname = 'Oracle'
+    system_uuid = None
+    vendordata_pure = None
+    _network_config = sources.UNSET
+
+    def _is_platform_viable(self):
+        """Check platform environment to report if this datasource may run."""
+        return _is_platform_viable()
+
+    def _get_data(self):
+        if not self._is_platform_viable():
+            return False
+
+        # network may be configured if iscsi root.  If that is the case
+        # then read_kernel_cmdline_config will return non-None.
+        if _is_iscsi_root():
+            data = self.crawl_metadata()
+        else:
+            with dhcp.EphemeralDHCPv4(net.find_fallback_nic()):
+                data = self.crawl_metadata()
+
+        self._crawled_metadata = data
+        vdata = data['2013-10-17']
+
+        self.userdata_raw = vdata.get('user_data')
+        self.system_uuid = vdata['system_uuid']
+
+        vd = vdata.get('vendor_data')
+        if vd:
+            self.vendordata_pure = vd
+            try:
+                self.vendordata_raw = sources.convert_vendordata(vd)
+            except ValueError as e:
+                LOG.warning("Invalid content in vendor-data: %s", e)
+                self.vendordata_raw = None
+
+        mdcopies = ('public_keys',)
+        md = dict([(k, vdata['meta_data'].get(k))
+                   for k in mdcopies if k in vdata['meta_data']])
+
+        mdtrans = (
+            # oracle meta_data.json name, cloudinit.datasource.metadata name
+            ('availability_zone', 'availability-zone'),
+            ('hostname', 'local-hostname'),
+            ('launch_index', 'launch-index'),
+            ('uuid', 'instance-id'),
+        )
+        for dsname, ciname in mdtrans:
+            if dsname in vdata['meta_data']:
+                md[ciname] = vdata['meta_data'][dsname]
+
+        self.metadata = md
+        return True
+
+    def crawl_metadata(self):
+        return read_metadata()
+
+    def check_instance_id(self, sys_cfg):
+        """quickly check (local only) if self.instance_id is still valid
+
+        On Oracle, the dmi-provided system uuid differs from the instance-id
+        but has the same life-span."""
+        return sources.instance_id_matches_system_uuid(self.system_uuid)
+
+    def get_public_ssh_keys(self):
+        return sources.normalize_pubkey_data(self.metadata.get('public_keys'))
+
+    @property
+    def network_config(self):
+        """Network config is read from initramfs provided files
+        If none is present, then we fall back to fallback configuration.
+
+        One thing to note here is that this method is not currently
+        considered at all if there is is kernel/initramfs provided
+        data.  In that case, stages considers that the cmdline data
+        overrides datasource provided data and does not consult here.
+
+        We nonetheless return cmdline provided config if present
+        and fallback to generate fallback."""
+        if self._network_config == sources.UNSET:
+            cmdline_cfg = cmdline.read_kernel_cmdline_config()
+            if cmdline_cfg:
+                self._network_config = cmdline_cfg
+            else:
+                self._network_config = self.distro.generate_fallback_config()
+        return self._network_config
+
+
+def _read_system_uuid():
+    sys_uuid = util.read_dmi_data('system-uuid')
+    return None if sys_uuid is None else sys_uuid.lower()
+
+
+def _is_platform_viable():
+    asset_tag = util.read_dmi_data('chassis-asset-tag')
+    return asset_tag == CHASSIS_ASSET_TAG
+
+
+def _is_iscsi_root():
+    return bool(cmdline.read_kernel_cmdline_config())
+
+
+def _load_index(content):
+    """Return a list entries parsed from content.
+
+    OpenStack's metadata service returns a newline delimited list
+    of items.  Oracle's implementation has html formatted list of links.
+    The parser here just grabs targets from <a href="target">
+    and throws away "../".
+
+    Oracle has accepted that to be buggy and may fix in the future
+    to instead return a '\n' delimited plain text list.  This function
+    will continue to work if that change is made."""
+    if not content.lower().startswith("<html>"):
+        return content.splitlines()
+    items = re.findall(
+        r'href="(?P<target>[^"]*)"', content, re.MULTILINE | re.IGNORECASE)
+    return [i for i in items if not i.startswith(".")]
+
+
+def read_metadata(endpoint_base=METADATA_ENDPOINT, sys_uuid=None,
+                  version='2013-10-17'):
+    """Read metadata, return a dictionary.
+
+    Each path listed in the index will be represented in the dictionary.
+    If the path ends in .json, then the content will be decoded and
+    populated into the dictionary.
+
+    The system uuid (/sys/class/dmi/id/product_uuid) is also populated.
+    Example: given paths = ('user_data', 'meta_data.json')
+    This would return:
+      {version: {'user_data': b'blob', 'meta_data': json.loads(blob.decode())
+                 'system_uuid': '3b54f2e0-3ab2-458d-b770-af9926eee3b2'}}
+    """
+    endpoint = combine_url(endpoint_base, version) + "/"
+    if sys_uuid is None:
+        sys_uuid = _read_system_uuid()
+    if not sys_uuid:
+        raise sources.BrokenMetadata("Failed to read system uuid.")
+
+    try:
+        resp = readurl(endpoint)
+        if not resp.ok():
+            raise sources.BrokenMetadata(
+                "Bad response from %s: %s" % (endpoint, resp.code))
+    except UrlError as e:
+        raise sources.BrokenMetadata(
+            "Failed to read index at %s: %s" % (endpoint, e))
+
+    entries = _load_index(resp.contents.decode('utf-8'))
+    LOG.debug("index url %s contained: %s", endpoint, entries)
+
+    # meta_data.json is required.
+    mdj = 'meta_data.json'
+    if mdj not in entries:
+        raise sources.BrokenMetadata(
+            "Required field '%s' missing in index at %s" % (mdj, endpoint))
+
+    ret = {'system_uuid': sys_uuid}
+    for path in entries:
+        response = readurl(combine_url(endpoint, path))
+        if path.endswith(".json"):
+            ret[path.rpartition(".")[0]] = (
+                json.loads(response.contents.decode('utf-8')))
+        else:
+            ret[path] = response.contents
+
+    return {version: ret}
+
+
+# Used to match classes to dependencies
+datasources = [
+    (DataSourceOracle, (sources.DEP_FILESYSTEM,)),
+]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
+
+
+if __name__ == "__main__":
+    import argparse
+    import os
+
+    parser = argparse.ArgumentParser(description='Query Oracle Cloud Metadata')
+    parser.add_argument("--endpoint", metavar="URL",
+                        help="The url of the metadata service.",
+                        default=METADATA_ENDPOINT)
+    args = parser.parse_args()
+    sys_uuid = "uuid-not-available-not-root" if os.geteuid() != 0 else None
+
+    data = read_metadata(endpoint_base=args.endpoint, sys_uuid=sys_uuid)
+    data['is_platform_viable'] = _is_platform_viable()
+    print(util.json_dumps(data))
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 06e613f8..41fde9ba 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -671,6 +671,10 @@ def convert_vendordata(data, recurse=True):
     raise ValueError("Unknown data type for vendordata: %s" % type(data))
 
 
+class BrokenMetadata(IOError):
+    pass
+
+
 # 'depends' is a list of dependencies (DEP_FILESYSTEM)
 # ds_list is a list of 2 item lists
 # ds_list = [
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index a4cf0667..8f9c1441 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -21,6 +21,8 @@ from cloudinit import sources
 from cloudinit import url_helper
 from cloudinit import util
 
+from cloudinit.sources import BrokenMetadata
+
 # See https://docs.openstack.org/user-guide/cli-config-drive.html
 
 LOG = logging.getLogger(__name__)
@@ -68,10 +70,6 @@ class NonReadable(IOError):
     pass
 
 
-class BrokenMetadata(IOError):
-    pass
-
-
 class SourceMixin(object):
     def _ec2_name_to_device(self, name):
         if not self.ec2_metadata:
diff --git a/cloudinit/sources/tests/test_oracle.py b/cloudinit/sources/tests/test_oracle.py
new file mode 100644
index 00000000..7599126c
--- /dev/null
+++ b/cloudinit/sources/tests/test_oracle.py
@@ -0,0 +1,331 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit.sources import DataSourceOracle as oracle
+from cloudinit.sources import BrokenMetadata
+from cloudinit import helpers
+
+from cloudinit.tests import helpers as test_helpers
+
+from textwrap import dedent
+import argparse
+import httpretty
+import json
+import mock
+import os
+import six
+import uuid
+
+DS_PATH = "cloudinit.sources.DataSourceOracle"
+MD_VER = "2013-10-17"
+
+
+class TestDataSourceOracle(test_helpers.CiTestCase):
+    """Test datasource DataSourceOracle."""
+
+    ds_class = oracle.DataSourceOracle
+
+    my_uuid = str(uuid.uuid4())
+    my_md = {"uuid": "ocid1.instance.oc1.phx.abyhqlj",
+             "name": "ci-vm1", "availability_zone": "phx-ad-3",
+             "hostname": "ci-vm1hostname",
+             "launch_index": 0, "files": [],
+             "public_keys": {"0": "ssh-rsa AAAAB3N...== user@host"},
+             "meta": {}}
+
+    def _patch_instance(self, inst, patches):
+        """Patch an instance of a class 'inst'.
+        for each name, kwargs in patches:
+             inst.name = mock.Mock(**kwargs)
+        returns a namespace object that has
+             namespace.name = mock.Mock(**kwargs)
+        Do not bother with cleanup as instance is assumed transient."""
+        mocks = argparse.Namespace()
+        for name, kwargs in patches.items():
+            imock = mock.Mock(name=name, spec=getattr(inst, name), **kwargs)
+            setattr(mocks, name, imock)
+            setattr(inst, name, imock)
+        return mocks
+
+    def _get_ds(self, sys_cfg=None, distro=None, paths=None, ud_proc=None,
+                patches=None):
+        if sys_cfg is None:
+            sys_cfg = {}
+        if patches is None:
+            patches = {}
+        if paths is None:
+            tmpd = self.tmp_dir()
+            dirs = {'cloud_dir': self.tmp_path('cloud_dir', tmpd),
+                    'run_dir': self.tmp_path('run_dir')}
+            for d in dirs.values():
+                os.mkdir(d)
+            paths = helpers.Paths(dirs)
+
+        ds = self.ds_class(sys_cfg=sys_cfg, distro=distro,
+                           paths=paths, ud_proc=ud_proc)
+
+        return ds, self._patch_instance(ds, patches)
+
+    def test_platform_not_viable_returns_false(self):
+        ds, mocks = self._get_ds(
+            patches={'_is_platform_viable': {'return_value': False}})
+        self.assertFalse(ds._get_data())
+        mocks._is_platform_viable.assert_called_once_with()
+
+    @mock.patch(DS_PATH + "._is_iscsi_root", return_value=True)
+    def test_without_userdata(self, m_is_iscsi_root):
+        """If no user-data is provided, it should not be in return dict."""
+        ds, mocks = self._get_ds(patches={
+            '_is_platform_viable': {'return_value': True},
+            'crawl_metadata': {
+                'return_value': {
+                    MD_VER: {'system_uuid': self.my_uuid,
+                             'meta_data': self.my_md}}}})
+        self.assertTrue(ds._get_data())
+        mocks._is_platform_viable.assert_called_once_with()
+        mocks.crawl_metadata.assert_called_once_with()
+        self.assertEqual(self.my_uuid, ds.system_uuid)
+        self.assertEqual(self.my_md['availability_zone'], ds.availability_zone)
+        self.assertIn(self.my_md["public_keys"]["0"], ds.get_public_ssh_keys())
+        self.assertEqual(self.my_md['uuid'], ds.get_instance_id())
+        self.assertIsNone(ds.userdata_raw)
+
+    @mock.patch(DS_PATH + "._is_iscsi_root", return_value=True)
+    def test_with_vendordata(self, m_is_iscsi_root):
+        """Test with vendor data."""
+        vd = {'cloud-init': '#cloud-config\nkey: value'}
+        ds, mocks = self._get_ds(patches={
+            '_is_platform_viable': {'return_value': True},
+            'crawl_metadata': {
+                'return_value': {
+                    MD_VER: {'system_uuid': self.my_uuid,
+                             'meta_data': self.my_md,
+                             'vendor_data': vd}}}})
+        self.assertTrue(ds._get_data())
+        mocks._is_platform_viable.assert_called_once_with()
+        mocks.crawl_metadata.assert_called_once_with()
+        self.assertEqual(vd, ds.vendordata_pure)
+        self.assertEqual(vd['cloud-init'], ds.vendordata_raw)
+
+    @mock.patch(DS_PATH + "._is_iscsi_root", return_value=True)
+    def test_with_userdata(self, m_is_iscsi_root):
+        """Ensure user-data is populated if present and is binary."""
+        my_userdata = b'abcdefg'
+        ds, mocks = self._get_ds(patches={
+            '_is_platform_viable': {'return_value': True},
+            'crawl_metadata': {
+                'return_value': {
+                    MD_VER: {'system_uuid': self.my_uuid,
+                             'meta_data': self.my_md,
+                             'user_data': my_userdata}}}})
+        self.assertTrue(ds._get_data())
+        mocks._is_platform_viable.assert_called_once_with()
+        mocks.crawl_metadata.assert_called_once_with()
+        self.assertEqual(self.my_uuid, ds.system_uuid)
+        self.assertIn(self.my_md["public_keys"]["0"], ds.get_public_ssh_keys())
+        self.assertEqual(self.my_md['uuid'], ds.get_instance_id())
+        self.assertEqual(my_userdata, ds.userdata_raw)
+
+    @mock.patch(DS_PATH + ".cmdline.read_kernel_cmdline_config")
+    @mock.patch(DS_PATH + "._is_iscsi_root", return_value=True)
+    def test_network_cmdline(self, m_is_iscsi_root, m_cmdline_config):
+        """network_config should read kernel cmdline."""
+        distro = mock.MagicMock()
+        ds, _ = self._get_ds(distro=distro, patches={
+            '_is_platform_viable': {'return_value': True},
+            'crawl_metadata': {
+                'return_value': {
+                    MD_VER: {'system_uuid': self.my_uuid,
+                             'meta_data': self.my_md}}}})
+        ncfg = {'version': 1, 'config': [{'a': 'b'}]}
+        m_cmdline_config.return_value = ncfg
+        self.assertTrue(ds._get_data())
+        self.assertEqual(ncfg, ds.network_config)
+        m_cmdline_config.assert_called_once_with()
+        self.assertFalse(distro.generate_fallback_config.called)
+
+    @mock.patch(DS_PATH + ".cmdline.read_kernel_cmdline_config")
+    @mock.patch(DS_PATH + "._is_iscsi_root", return_value=True)
+    def test_network_fallback(self, m_is_iscsi_root, m_cmdline_config):
+        """test that fallback network is generated if no kernel cmdline."""
+        distro = mock.MagicMock()
+        ds, _ = self._get_ds(distro=distro, patches={
+            '_is_platform_viable': {'return_value': True},
+            'crawl_metadata': {
+                'return_value': {
+                    MD_VER: {'system_uuid': self.my_uuid,
+                             'meta_data': self.my_md}}}})
+        ncfg = {'version': 1, 'config': [{'a': 'b'}]}
+        m_cmdline_config.return_value = None
+        self.assertTrue(ds._get_data())
+        ncfg = {'version': 1, 'config': [{'distro1': 'value'}]}
+        distro.generate_fallback_config.return_value = ncfg
+        self.assertEqual(ncfg, ds.network_config)
+        m_cmdline_config.assert_called_once_with()
+        distro.generate_fallback_config.assert_called_once_with()
+        self.assertEqual(1, m_cmdline_config.call_count)
+
+        # test that the result got cached, and the methods not re-called.
+        self.assertEqual(ncfg, ds.network_config)
+        self.assertEqual(1, m_cmdline_config.call_count)
+
+
+@mock.patch(DS_PATH + "._read_system_uuid", return_value=str(uuid.uuid4()))
+class TestReadMetaData(test_helpers.HttprettyTestCase):
+    """Test the read_metadata which interacts with http metadata service."""
+
+    mdurl = oracle.METADATA_ENDPOINT
+    my_md = {"uuid": "ocid1.instance.oc1.phx.abyhqlj",
+             "name": "ci-vm1", "availability_zone": "phx-ad-3",
+             "hostname": "ci-vm1hostname",
+             "launch_index": 0, "files": [],
+             "public_keys": {"0": "ssh-rsa AAAAB3N...== user@host"},
+             "meta": {}}
+
+    def populate_md(self, data):
+        """call httppretty.register_url for each item dict 'data',
+           including valid indexes. Text values converted to bytes."""
+        httpretty.register_uri(
+            httpretty.GET, self.mdurl + MD_VER + "/",
+            '\n'.join(data.keys()).encode('utf-8'))
+        for k, v in data.items():
+            httpretty.register_uri(
+                httpretty.GET, self.mdurl + MD_VER + "/" + k,
+                v if not isinstance(v, six.text_type) else v.encode('utf-8'))
+
+    def test_broken_no_sys_uuid(self, m_read_system_uuid):
+        """Datasource requires ability to read system_uuid and true return."""
+        m_read_system_uuid.return_value = None
+        self.assertRaises(BrokenMetadata, oracle.read_metadata)
+
+    def test_broken_no_metadata_json(self, m_read_system_uuid):
+        """Datasource requires meta_data.json."""
+        httpretty.register_uri(
+            httpretty.GET, self.mdurl + MD_VER + "/",
+            '\n'.join(['user_data']).encode('utf-8'))
+        with self.assertRaises(BrokenMetadata) as cm:
+            oracle.read_metadata()
+        self.assertIn("Required field 'meta_data.json' missing",
+                      str(cm.exception))
+
+    def test_with_userdata(self, m_read_system_uuid):
+        data = {'user_data': b'#!/bin/sh\necho hi world\n',
+                'meta_data.json': json.dumps(self.my_md)}
+        self.populate_md(data)
+        result = oracle.read_metadata()[MD_VER]
+        self.assertEqual(data['user_data'], result['user_data'])
+        self.assertEqual(self.my_md, result['meta_data'])
+
+    def test_without_userdata(self, m_read_system_uuid):
+        data = {'meta_data.json': json.dumps(self.my_md)}
+        self.populate_md(data)
+        result = oracle.read_metadata()[MD_VER]
+        self.assertNotIn('user_data', result)
+        self.assertEqual(self.my_md, result['meta_data'])
+
+    def test_unknown_fields_included(self, m_read_system_uuid):
+        """Unknown fields listed in index should be included.
+        And those ending in .json should be decoded."""
+        some_data = {'key1': 'data1', 'subk1': {'subd1': 'subv'}}
+        some_vendor_data = {'cloud-init': 'foo'}
+        data = {'meta_data.json': json.dumps(self.my_md),
+                'some_data.json': json.dumps(some_data),
+                'vendor_data.json': json.dumps(some_vendor_data),
+                'other_blob': b'this is blob'}
+        self.populate_md(data)
+        result = oracle.read_metadata()[MD_VER]
+        self.assertNotIn('user_data', result)
+        self.assertEqual(self.my_md, result['meta_data'])
+        self.assertEqual(some_data, result['some_data'])
+        self.assertEqual(some_vendor_data, result['vendor_data'])
+        self.assertEqual(data['other_blob'], result['other_blob'])
+
+
+class TestIsPlatformViable(test_helpers.CiTestCase):
+    @mock.patch(DS_PATH + ".util.read_dmi_data",
+                return_value=oracle.CHASSIS_ASSET_TAG)
+    def test_expected_viable(self, m_read_dmi_data):
+        """System with known chassis tag is viable."""
+        self.assertTrue(oracle._is_platform_viable())
+        m_read_dmi_data.assert_has_calls([mock.call('chassis-asset-tag')])
+
+    @mock.patch(DS_PATH + ".util.read_dmi_data", return_value=None)
+    def test_expected_not_viable_dmi_data_none(self, m_read_dmi_data):
+        """System without known chassis tag is not viable."""
+        self.assertFalse(oracle._is_platform_viable())
+        m_read_dmi_data.assert_has_calls([mock.call('chassis-asset-tag')])
+
+    @mock.patch(DS_PATH + ".util.read_dmi_data", return_value="LetsGoCubs")
+    def test_expected_not_viable_other(self, m_read_dmi_data):
+        """System with unnown chassis tag is not viable."""
+        self.assertFalse(oracle._is_platform_viable())
+        m_read_dmi_data.assert_has_calls([mock.call('chassis-asset-tag')])
+
+
+class TestLoadIndex(test_helpers.CiTestCase):
+    """_load_index handles parsing of an index into a proper list.
+    The tests here guarantee correct parsing of html version or
+    a fixed version.  See the function docstring for more doc."""
+
+    _known_html_api_versions = dedent("""\
+        <html>
+        <head><title>Index of /openstack/</title></head>
+        <body bgcolor="white">
+        <h1>Index of /openstack/</h1><hr><pre><a href="../">../</a>
+        <a href="2013-10-17/">2013-10-17/</a>   27-Jun-2018 12:22  -
+        <a href="latest/">latest/</a>           27-Jun-2018 12:22  -
+        </pre><hr></body>
+        </html>""")
+
+    _known_html_contents = dedent("""\
+        <html>
+        <head><title>Index of /openstack/2013-10-17/</title></head>
+        <body bgcolor="white">
+        <h1>Index of /openstack/2013-10-17/</h1><hr><pre><a href="../">../</a>
+        <a href="meta_data.json">meta_data.json</a>  27-Jun-2018 12:22  679
+        <a href="user_data">user_data</a>            27-Jun-2018 12:22  146
+        </pre><hr></body>
+        </html>""")
+
+    def test_parse_html(self):
+        """Test parsing of lower case html."""
+        self.assertEqual(
+            ['2013-10-17/', 'latest/'],
+            oracle._load_index(self._known_html_api_versions))
+        self.assertEqual(
+            ['meta_data.json', 'user_data'],
+            oracle._load_index(self._known_html_contents))
+
+    def test_parse_html_upper(self):
+        """Test parsing of upper case html, although known content is lower."""
+        def _toupper(data):
+            return data.replace("<a", "<A").replace("html>", "HTML>")
+
+        self.assertEqual(
+            ['2013-10-17/', 'latest/'],
+            oracle._load_index(_toupper(self._known_html_api_versions)))
+        self.assertEqual(
+            ['meta_data.json', 'user_data'],
+            oracle._load_index(_toupper(self._known_html_contents)))
+
+    def test_parse_newline_list_with_endl(self):
+        """Test parsing of newline separated list with ending newline."""
+        self.assertEqual(
+            ['2013-10-17/', 'latest/'],
+            oracle._load_index("\n".join(["2013-10-17/", "latest/", ""])))
+        self.assertEqual(
+            ['meta_data.json', 'user_data'],
+            oracle._load_index("\n".join(["meta_data.json", "user_data", ""])))
+
+    def test_parse_newline_list_without_endl(self):
+        """Test parsing of newline separated list with no ending newline.
+
+        Actual openstack implementation does not include trailing newline."""
+        self.assertEqual(
+            ['2013-10-17/', 'latest/'],
+            oracle._load_index("\n".join(["2013-10-17/", "latest/"])))
+        self.assertEqual(
+            ['meta_data.json', 'user_data'],
+            oracle._load_index("\n".join(["meta_data.json", "user_data"])))
+
+
+# vi: ts=4 expandtab
diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst
index 30e57d85..83034589 100644
--- a/doc/rtd/topics/datasources.rst
+++ b/doc/rtd/topics/datasources.rst
@@ -189,6 +189,7 @@ Follow for more information.
    datasources/nocloud.rst
    datasources/opennebula.rst
    datasources/openstack.rst
+   datasources/oracle.rst
    datasources/ovf.rst
    datasources/smartos.rst
    datasources/fallback.rst
diff --git a/doc/rtd/topics/datasources/oracle.rst b/doc/rtd/topics/datasources/oracle.rst
new file mode 100644
index 00000000..f2383cee
--- /dev/null
+++ b/doc/rtd/topics/datasources/oracle.rst
@@ -0,0 +1,26 @@
+.. _datasource_oracle:
+
+Oracle
+======
+
+This datasource reads metadata, vendor-data and user-data from
+`Oracle Compute Infrastructure`_ (OCI).
+
+Oracle Platform
+---------------
+OCI provides bare metal and virtual machines.  In both cases, 
+the platform identifies itself via DMI data in the chassis asset tag
+with the string 'OracleCloud.com'.
+
+Oracle's platform provides a metadata service that mimics the 2013-10-17
+version of OpenStack metadata service.  Initially support for Oracle
+was done via the OpenStack datasource.
+
+Cloud-init has a specific datasource for Oracle in order to:
+ a. allow and support future growth of the OCI platform.
+ b. address small differences between OpenStack and Oracle metadata
+    implementation.
+
+
+.. _Oracle Compute Infrastructure: https://cloud.oracle.com/
+.. vi: textwidth=78
diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py
index 1a5a3db2..6b01a4ea 100644
--- a/tests/unittests/test_datasource/test_common.py
+++ b/tests/unittests/test_datasource/test_common.py
@@ -20,6 +20,7 @@ from cloudinit.sources import (
     DataSourceNoCloud as NoCloud,
     DataSourceOpenNebula as OpenNebula,
     DataSourceOpenStack as OpenStack,
+    DataSourceOracle as Oracle,
     DataSourceOVF as OVF,
     DataSourceScaleway as Scaleway,
     DataSourceSmartOS as SmartOS,
@@ -37,6 +38,7 @@ DEFAULT_LOCAL = [
     IBMCloud.DataSourceIBMCloud,
     NoCloud.DataSourceNoCloud,
     OpenNebula.DataSourceOpenNebula,
+    Oracle.DataSourceOracle,
     OVF.DataSourceOVF,
     SmartOS.DataSourceSmartOS,
     Ec2.DataSourceEc2Local,
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index d862f4bc..6e1e971b 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -16,7 +16,7 @@ from six import StringIO
 
 from cloudinit import helpers
 from cloudinit import settings
-from cloudinit.sources import convert_vendordata, UNSET
+from cloudinit.sources import BrokenMetadata, convert_vendordata, UNSET
 from cloudinit.sources import DataSourceOpenStack as ds
 from cloudinit.sources.helpers import openstack
 from cloudinit import util
@@ -186,7 +186,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             if k.endswith('meta_data.json'):
                 os_files[k] = json.dumps(os_meta)
         _register_uris(self.VERSION, {}, {}, os_files)
-        self.assertRaises(openstack.BrokenMetadata, _read_metadata_service)
+        self.assertRaises(BrokenMetadata, _read_metadata_service)
 
     def test_userdata_empty(self):
         os_files = copy.deepcopy(OS_FILES)
@@ -217,7 +217,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             if k.endswith('vendor_data.json'):
                 os_files[k] = '{'  # some invalid json
         _register_uris(self.VERSION, {}, {}, os_files)
-        self.assertRaises(openstack.BrokenMetadata, _read_metadata_service)
+        self.assertRaises(BrokenMetadata, _read_metadata_service)
 
     def test_metadata_invalid(self):
         os_files = copy.deepcopy(OS_FILES)
@@ -225,7 +225,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             if k.endswith('meta_data.json'):
                 os_files[k] = '{'  # some invalid json
         _register_uris(self.VERSION, {}, {}, os_files)
-        self.assertRaises(openstack.BrokenMetadata, _read_metadata_service)
+        self.assertRaises(BrokenMetadata, _read_metadata_service)
 
     @test_helpers.mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
     def test_datasource(self, m_dhcp):
@@ -525,8 +525,11 @@ class TestDetectOpenStack(test_helpers.CiTestCase):
 
         m_dmi.side_effect = fake_dmi_read
         self.assertTrue(
-            ds.detect_openstack(),
+            ds.detect_openstack(accept_oracle=True),
             'Expected detect_openstack == True on OracleCloud.com')
+        self.assertFalse(
+            ds.detect_openstack(accept_oracle=False),
+            'Expected detect_openstack == False.')
 
     @test_helpers.mock.patch(MOCK_PATH + 'util.get_proc_env')
     @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data')
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 64d9f9f8..e08e7908 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -12,6 +12,7 @@ from cloudinit.tests.helpers import (
 
 from cloudinit.sources import DataSourceIBMCloud as ds_ibm
 from cloudinit.sources import DataSourceSmartOS as ds_smartos
+from cloudinit.sources import DataSourceOracle as ds_oracle
 
 UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu "
                "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux")
@@ -598,6 +599,18 @@ class TestIsIBMProvisioning(DsIdentifyBase):
         self.assertIn("from current boot", ret.stderr)
 
 
+class TestOracle(DsIdentifyBase):
+    def test_found_by_chassis(self):
+        """Simple positive test of Oracle by chassis id."""
+        self._test_ds_found('Oracle')
+
+    def test_not_found(self):
+        """Simple negative test of Oracle."""
+        mycfg = copy.deepcopy(VALID_CFG['Oracle'])
+        mycfg['files'][P_CHASSIS_ASSET_TAG] = "Not Oracle"
+        self._check_via_dict(mycfg, rc=RC_NOT_FOUND)
+
+
 def blkid_out(disks=None):
     """Convert a list of disk dictionaries into blkid content."""
     if disks is None:
@@ -838,6 +851,12 @@ VALID_CFG = {
              },
         ],
     },
+    'Oracle': {
+        'ds': 'Oracle',
+        'files': {
+            P_CHASSIS_ASSET_TAG: ds_oracle.CHASSIS_ASSET_TAG + '\n',
+        }
+    },
     'SmartOS-bhyve': {
         'ds': 'SmartOS',
         'mocks': [
diff --git a/tools/ds-identify b/tools/ds-identify
index ce0477a5..fcc60149 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -116,7 +116,7 @@ DI_DSNAME=""
 # be searched if there is no setting found in config.
 DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \
 CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \
-OVF SmartOS Scaleway Hetzner IBMCloud"
+OVF SmartOS Scaleway Hetzner IBMCloud Oracle"
 DI_DSLIST=""
 DI_MODE=""
 DI_ON_FOUND=""
@@ -1036,6 +1036,12 @@ dscheck_Hetzner() {
     return ${DS_NOT_FOUND}
 }
 
+dscheck_Oracle() {
+    local asset_tag="OracleCloud.com"
+    dmi_chassis_asset_tag_matches "${asset_tag}" && return ${DS_FOUND}
+    return ${DS_NOT_FOUND}
+}
+
 is_ibm_provisioning() {
     local pcfg="${PATH_ROOT}/root/provisioningConfiguration.cfg"
     local logf="${PATH_ROOT}/root/swinstall.log"
-- 
cgit v1.2.3


From 9c35f9762028b8bf15cdcd6b42c0fafc233ddda3 Mon Sep 17 00:00:00 2001
From: Pengpeng Sun <pengpengs@vmware.com>
Date: Tue, 28 Aug 2018 22:52:28 +0000
Subject: VMWare: Fix a network config bug in vm with static IPv4 and no
 gateway.

The issue is when customize a VM with static IPv4 and without gateway, it
will still extend route list and will loop a gateways list which is None.
This fix is to make sure when no gateway is here, it will not extend route
list.

LP: #1766538
---
 cloudinit/sources/helpers/vmware/imc/config_nic.py |   2 +-
 tests/unittests/test_vmware_config_file.py         | 115 +++++++++++++++++++++
 2 files changed, 116 insertions(+), 1 deletion(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py
index 3ef8c624..e1890e23 100644
--- a/cloudinit/sources/helpers/vmware/imc/config_nic.py
+++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py
@@ -164,7 +164,7 @@ class NicConfigurator(object):
             return ([subnet], route_list)
 
         # Add routes if there is no primary nic
-        if not self._primaryNic:
+        if not self._primaryNic and v4.gateways:
             route_list.extend(self.gen_ipv4_route(nic,
                                                   v4.gateways,
                                                   v4.netmask))
diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py
index 036f6879..602dedb0 100644
--- a/tests/unittests/test_vmware_config_file.py
+++ b/tests/unittests/test_vmware_config_file.py
@@ -2,11 +2,15 @@
 # Copyright (C) 2016 VMware INC.
 #
 # Author: Sankar Tanguturi <stanguturi@vmware.com>
+#         Pengpeng Sun <pengpengs@vmware.com>
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
 import logging
+import os
 import sys
+import tempfile
+import textwrap
 
 from cloudinit.sources.DataSourceOVF import get_network_config_from_conf
 from cloudinit.sources.DataSourceOVF import read_vmware_imc
@@ -343,4 +347,115 @@ class TestVmwareConfigFile(CiTestCase):
         conf = Config(cf)
         self.assertEqual("test-script", conf.custom_script_name)
 
+
+class TestVmwareNetConfig(CiTestCase):
+    """Test conversion of vmware config to cloud-init config."""
+
+    def _get_NicConfigurator(self, text):
+        fp = None
+        try:
+            with tempfile.NamedTemporaryFile(mode="w", dir=self.tmp_dir(),
+                                             delete=False) as fp:
+                fp.write(text)
+                fp.close()
+            cfg = Config(ConfigFile(fp.name))
+            return NicConfigurator(cfg.nics, use_system_devices=False)
+        finally:
+            if fp:
+                os.unlink(fp.name)
+
+    def test_non_primary_nic_without_gateway(self):
+        """A non primary nic set is not required to have a gateway."""
+        config = textwrap.dedent("""\
+            [NETWORK]
+            NETWORKING = yes
+            BOOTPROTO = dhcp
+            HOSTNAME = myhost1
+            DOMAINNAME = eng.vmware.com
+
+            [NIC-CONFIG]
+            NICS = NIC1
+
+            [NIC1]
+            MACADDR = 00:50:56:a6:8c:08
+            ONBOOT = yes
+            IPv4_MODE = BACKWARDS_COMPATIBLE
+            BOOTPROTO = static
+            IPADDR = 10.20.87.154
+            NETMASK = 255.255.252.0
+            """)
+        nc = self._get_NicConfigurator(config)
+        self.assertEqual(
+            [{'type': 'physical', 'name': 'NIC1',
+              'mac_address': '00:50:56:a6:8c:08',
+              'subnets': [
+                  {'control': 'auto', 'type': 'static',
+                   'address': '10.20.87.154', 'netmask': '255.255.252.0'}]}],
+            nc.generate())
+
+    def test_non_primary_nic_with_gateway(self):
+        """A non primary nic set can have a gateway."""
+        config = textwrap.dedent("""\
+            [NETWORK]
+            NETWORKING = yes
+            BOOTPROTO = dhcp
+            HOSTNAME = myhost1
+            DOMAINNAME = eng.vmware.com
+
+            [NIC-CONFIG]
+            NICS = NIC1
+
+            [NIC1]
+            MACADDR = 00:50:56:a6:8c:08
+            ONBOOT = yes
+            IPv4_MODE = BACKWARDS_COMPATIBLE
+            BOOTPROTO = static
+            IPADDR = 10.20.87.154
+            NETMASK = 255.255.252.0
+            GATEWAY = 10.20.87.253
+            """)
+        nc = self._get_NicConfigurator(config)
+        self.assertEqual(
+            [{'type': 'physical', 'name': 'NIC1',
+              'mac_address': '00:50:56:a6:8c:08',
+              'subnets': [
+                  {'control': 'auto', 'type': 'static',
+                   'address': '10.20.87.154', 'netmask': '255.255.252.0'}]},
+             {'type': 'route', 'destination': '10.20.84.0/22',
+              'gateway': '10.20.87.253', 'metric': 10000}],
+            nc.generate())
+
+    def test_a_primary_nic_with_gateway(self):
+        """A primary nic set can have a gateway."""
+        config = textwrap.dedent("""\
+            [NETWORK]
+            NETWORKING = yes
+            BOOTPROTO = dhcp
+            HOSTNAME = myhost1
+            DOMAINNAME = eng.vmware.com
+
+            [NIC-CONFIG]
+            NICS = NIC1
+
+            [NIC1]
+            MACADDR = 00:50:56:a6:8c:08
+            ONBOOT = yes
+            IPv4_MODE = BACKWARDS_COMPATIBLE
+            BOOTPROTO = static
+            IPADDR = 10.20.87.154
+            NETMASK = 255.255.252.0
+            PRIMARY = true
+            GATEWAY = 10.20.87.253
+            """)
+        nc = self._get_NicConfigurator(config)
+        self.assertEqual(
+            [{'type': 'physical', 'name': 'NIC1',
+              'mac_address': '00:50:56:a6:8c:08',
+              'subnets': [
+                  {'control': 'auto', 'type': 'static',
+                   'address': '10.20.87.154', 'netmask': '255.255.252.0',
+                   'gateway': '10.20.87.253'}]}],
+            nc.generate())
+
+
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From c75c582ed1824dc3ff39cefdbddccfc2924b868c Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 12 Sep 2018 18:44:19 +0000
Subject: OpenStack: fix bug causing 'latest' version to be used from network.

Cloud-init was reading a list of versions from the OpenStack metadata
service (http://169.254.169.254/openstack/) and attempt to select the
newest known supported version.  The problem was that the list
of versions was not being decoded, so we were comparing a list of
bytes (found versions) to a list of strings (known versions).

LP: #1792157
---
 cloudinit/sources/helpers/openstack.py            |  2 +-
 tests/unittests/test_datasource/test_openstack.py | 30 +++++++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 8f9c1441..e3d372b6 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -439,7 +439,7 @@ class MetadataReader(BaseReader):
             return self._versions
         found = []
         version_path = self._path_join(self.base_path, "openstack")
-        content = self._path_read(version_path)
+        content = self._path_read(version_path, decode=True)
         for line in content.splitlines():
             line = line.strip()
             if not line:
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index 6e1e971b..ab5d3adc 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -555,4 +555,34 @@ class TestDetectOpenStack(test_helpers.CiTestCase):
         m_proc_env.assert_called_with(1)
 
 
+class TestMetadataReader(test_helpers.HttprettyTestCase):
+    """Test the MetadataReader."""
+    burl = 'http://169.254.169.254/'
+
+    def register(self, path, body=None, status=200):
+        hp.register_uri(
+            hp.GET, self.burl + "openstack" + path, status=status, body=body)
+
+    def register_versions(self, versions):
+        self.register("", '\n'.join(versions).encode('utf-8'))
+
+    def test__find_working_version(self):
+        """Test a working version ignores unsupported."""
+        unsup = "2016-11-09"
+        self.register_versions(
+            [openstack.OS_FOLSOM, openstack.OS_LIBERTY, unsup,
+             openstack.OS_LATEST])
+        self.assertEqual(
+            openstack.OS_LIBERTY,
+            openstack.MetadataReader(self.burl)._find_working_version())
+
+    def test__find_working_version_uses_latest(self):
+        """'latest' should be used if no supported versions."""
+        unsup1, unsup2 = ("2016-11-09", '2017-06-06')
+        self.register_versions([unsup1, unsup2, openstack.OS_LATEST])
+        self.assertEqual(
+            openstack.OS_LATEST,
+            openstack.MetadataReader(self.burl)._find_working_version())
+
+
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From 4361e0e42a34ebf9f49dde2356b7261715cf0cd5 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Thu, 13 Sep 2018 18:44:20 +0000
Subject: OpenStack: support reading of newer versions of metdata.

Mark as supported for reading some newer versions of openstack metadata:
 2016-06-30 : Newton one
 2016-10-06 : Newton two
 2017-02-22 : Ocata
 2018-08-27 : Rocky
---
 cloudinit/sources/helpers/openstack.py            | 16 ++++++
 tests/unittests/test_datasource/test_openstack.py | 66 +++++++++++++++++++++--
 2 files changed, 79 insertions(+), 3 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index e3d372b6..1e651440 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -38,17 +38,33 @@ KEY_COPIES = (
     ('local-hostname', 'hostname', False),
     ('instance-id', 'uuid', True),
 )
+
+# Versions and names taken from nova source nova/api/metadata/base.py
 OS_LATEST = 'latest'
 OS_FOLSOM = '2012-08-10'
 OS_GRIZZLY = '2013-04-04'
 OS_HAVANA = '2013-10-17'
 OS_LIBERTY = '2015-10-15'
+# NEWTON_ONE adds 'devices' to md (sriov-pf-passthrough-neutron-port-vlan)
+OS_NEWTON_ONE = '2016-06-30'
+# NEWTON_TWO adds vendor_data2.json (vendordata-reboot)
+OS_NEWTON_TWO = '2016-10-06'
+# OS_OCATA adds 'vif' field to devices (sriov-pf-passthrough-neutron-port-vlan)
+OS_OCATA = '2017-02-22'
+# OS_ROCKY adds a vf_trusted field to devices (sriov-trusted-vfs)
+OS_ROCKY = '2018-08-27'
+
+
 # keep this in chronological order. new supported versions go at the end.
 OS_VERSIONS = (
     OS_FOLSOM,
     OS_GRIZZLY,
     OS_HAVANA,
     OS_LIBERTY,
+    OS_NEWTON_ONE,
+    OS_NEWTON_TWO,
+    OS_OCATA,
+    OS_ROCKY,
 )
 
 PHYSICAL_TYPES = (
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index ab5d3adc..a731f1ed 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -12,7 +12,7 @@ import re
 from cloudinit.tests import helpers as test_helpers
 
 from six.moves.urllib.parse import urlparse
-from six import StringIO
+from six import StringIO, text_type
 
 from cloudinit import helpers
 from cloudinit import settings
@@ -558,13 +558,36 @@ class TestDetectOpenStack(test_helpers.CiTestCase):
 class TestMetadataReader(test_helpers.HttprettyTestCase):
     """Test the MetadataReader."""
     burl = 'http://169.254.169.254/'
+    md_base = {
+        'availability_zone': 'myaz1',
+        'hostname': 'sm-foo-test.novalocal',
+        "keys": [{"data": PUBKEY, "name": "brickies", "type": "ssh"}],
+        'launch_index': 0,
+        'name': 'sm-foo-test',
+        'public_keys': {'mykey': PUBKEY},
+        'project_id': '6a103f813b774b9fb15a4fcd36e1c056',
+        'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c'}
 
     def register(self, path, body=None, status=200):
+        content = (body if not isinstance(body, text_type)
+                   else body.encode('utf-8'))
         hp.register_uri(
-            hp.GET, self.burl + "openstack" + path, status=status, body=body)
+            hp.GET, self.burl + "openstack" + path, status=status,
+            body=content)
 
     def register_versions(self, versions):
-        self.register("", '\n'.join(versions).encode('utf-8'))
+        self.register("", '\n'.join(versions))
+        self.register("/", '\n'.join(versions))
+
+    def register_version(self, version, data):
+        content = '\n'.join(sorted(data.keys()))
+        self.register(version, content)
+        self.register(version + "/", content)
+        for path, content in data.items():
+            self.register("/%s/%s" % (version, path), content)
+            self.register("/%s/%s" % (version, path), content)
+        if 'user_data' not in data:
+            self.register("/%s/user_data" % version, "nodata", status=404)
 
     def test__find_working_version(self):
         """Test a working version ignores unsupported."""
@@ -584,5 +607,42 @@ class TestMetadataReader(test_helpers.HttprettyTestCase):
             openstack.OS_LATEST,
             openstack.MetadataReader(self.burl)._find_working_version())
 
+    def test_read_v2_os_ocata(self):
+        """Validate return value of read_v2 for os_ocata data."""
+        md = copy.deepcopy(self.md_base)
+        md['devices'] = []
+        network_data = {'links': [], 'networks': [], 'services': []}
+        vendor_data = {}
+        vendor_data2 = {"static": {}}
+
+        data = {
+            'meta_data.json': json.dumps(md),
+            'network_data.json': json.dumps(network_data),
+            'vendor_data.json': json.dumps(vendor_data),
+            'vendor_data2.json': json.dumps(vendor_data2),
+        }
+
+        self.register_versions([openstack.OS_OCATA, openstack.OS_LATEST])
+        self.register_version(openstack.OS_OCATA, data)
+
+        mock_read_ec2 = test_helpers.mock.MagicMock(
+            return_value={'instance-id': 'unused-ec2'})
+        expected_md = copy.deepcopy(md)
+        expected_md.update(
+            {'instance-id': md['uuid'], 'local-hostname': md['hostname']})
+        expected = {
+            'userdata': '',  # Annoying, no user-data results in empty string.
+            'version': 2,
+            'metadata': expected_md,
+            'vendordata': vendor_data,
+            'networkdata': network_data,
+            'ec2-metadata': mock_read_ec2.return_value,
+            'files': {},
+        }
+        reader = openstack.MetadataReader(self.burl)
+        reader._read_ec2_metadata = mock_read_ec2
+        self.assertEqual(expected, reader.read_v2())
+        self.assertEqual(1, mock_read_ec2.call_count)
+
 
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From 12066fc8f6518d294501aa126b5332ea90c4896c Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Thu, 13 Sep 2018 22:15:39 +0000
Subject: EphemeralIPv4Network: Be more explicit when adding default route.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

On OpenStack based OVH public cloud, we got DHCP response with
  fixed-address 54.36.113.86;
  option subnet-mask 255.255.255.255;
  option routers 54.36.112.1;

The router clearly is not on the subnet.  So 'ip' would fail when
we tried to add the default route.
The solution here is to add an explicit route on that interface
to the router and then add the default route.

Also add 'bgpovs' to the list of 'physical' types for OpenStack
network configuration.  That type is used on OVH public cloud.

LP: #1792415
---
 cloudinit/net/__init__.py              |  7 +++++++
 cloudinit/net/tests/test_init.py       | 11 ++++++++---
 cloudinit/sources/helpers/openstack.py |  1 +
 3 files changed, 16 insertions(+), 3 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 3ffde52c..5e87bcad 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -697,6 +697,13 @@ class EphemeralIPv4Network(object):
                 'Skip ephemeral route setup. %s already has default route: %s',
                 self.interface, out.strip())
             return
+        util.subp(
+            ['ip', '-4', 'route', 'add', self.router, 'dev', self.interface,
+             'src', self.ip], capture=True)
+        self.cleanup_cmds.insert(
+            0,
+            ['ip', '-4', 'route', 'del', self.router, 'dev', self.interface,
+             'src', self.ip])
         util.subp(
             ['ip', '-4', 'route', 'add', 'default', 'via', self.router,
              'dev', self.interface], capture=True)
diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py
index 8b444f14..58e0a591 100644
--- a/cloudinit/net/tests/test_init.py
+++ b/cloudinit/net/tests/test_init.py
@@ -515,12 +515,17 @@ class TestEphemeralIPV4Network(CiTestCase):
                 capture=True),
             mock.call(
                 ['ip', 'route', 'show', '0.0.0.0/0'], capture=True),
+            mock.call(['ip', '-4', 'route', 'add', '192.168.2.1',
+                       'dev', 'eth0', 'src', '192.168.2.2'], capture=True),
             mock.call(
                 ['ip', '-4', 'route', 'add', 'default', 'via',
                  '192.168.2.1', 'dev', 'eth0'], capture=True)]
-        expected_teardown_calls = [mock.call(
-            ['ip', '-4', 'route', 'del', 'default', 'dev', 'eth0'],
-            capture=True)]
+        expected_teardown_calls = [
+            mock.call(['ip', '-4', 'route', 'del', 'default', 'dev', 'eth0'],
+                      capture=True),
+            mock.call(['ip', '-4', 'route', 'del', '192.168.2.1',
+                       'dev', 'eth0', 'src', '192.168.2.2'], capture=True),
+        ]
 
         with net.EphemeralIPv4Network(**params):
             self.assertEqual(expected_setup_calls, m_subp.call_args_list)
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 1e651440..d6f39a1e 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -69,6 +69,7 @@ OS_VERSIONS = (
 
 PHYSICAL_TYPES = (
     None,
+    'bgpovs',  # not present in OpenStack upstream but used on OVH cloud.
     'bridge',
     'dvs',
     'ethernet',
-- 
cgit v1.2.3


From c6cfed7fb58be76a3f95963a5de469aa03542cd3 Mon Sep 17 00:00:00 2001
From: Fabian Wiesel <fabian.wiesel@sap.com>
Date: Fri, 14 Sep 2018 19:33:01 +0000
Subject: OpenStack: Support setting mac address on bond.

Fix a bug where setting of mac address on a bond device was
ignored when provided in OpenStack network_config.json.

LP: #1682064
---
 cloudinit/sources/helpers/openstack.py              | 2 ++
 tests/unittests/test_datasource/test_configdrive.py | 6 +++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index d6f39a1e..76a6e310 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -604,6 +604,8 @@ def convert_net_json(network_json=None, known_macs=None):
             cfg.update({'type': 'physical', 'mac_address': link_mac_addr})
         elif link['type'] in ['bond']:
             params = {}
+            if link_mac_addr:
+                params['mac_address'] = link_mac_addr
             for k, v in link.items():
                 if k == 'bond_links':
                     continue
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index b0abfc5f..231619c9 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -136,6 +136,7 @@ NETWORK_DATA_3 = {
     ]
 }
 
+BOND_MAC = "fa:16:3e:b3:72:36"
 NETWORK_DATA_BOND = {
     "services": [
         {"type": "dns", "address": "1.1.1.191"},
@@ -163,7 +164,7 @@ NETWORK_DATA_BOND = {
         {"bond_links": ["eth0", "eth1"],
          "bond_miimon": 100, "bond_mode": "4",
          "bond_xmit_hash_policy": "layer3+4",
-         "ethernet_mac_address": "0c:c4:7a:34:6e:3c",
+         "ethernet_mac_address": BOND_MAC,
          "id": "bond0", "type": "bond"},
         {"ethernet_mac_address": "fa:16:3e:b3:72:30",
          "id": "vlan2", "type": "vlan", "vlan_id": 602,
@@ -691,6 +692,9 @@ class TestConvertNetworkData(CiTestCase):
         self.assertIn("auto oeth0", eni_rendering)
         self.assertIn("auto oeth1", eni_rendering)
         self.assertIn("auto bond0", eni_rendering)
+        # The bond should have the given mac address
+        pos = eni_rendering.find("auto bond0")
+        self.assertIn(BOND_MAC, eni_rendering[pos:])
 
     def test_vlan(self):
         # light testing of vlan config conversion and eni rendering
-- 
cgit v1.2.3


From e7b0e5f72e134779cfe22cd07b09a42c22d2bfe1 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Wed, 26 Sep 2018 17:59:07 +0000
Subject: Add support for Infiniband network interfaces (IPoIB).

OpenStack ironic references Infiniband interfaces via a 6 byte 'MAC
address' formed from bytes 13-15 and 18-20 of interface's hardware
address. This address is used as the ethernet_mac_address of Infiniband
links in network_data.json in configdrives generated by OpenStack nova.
We can use this address to map links in network_data.json to their
corresponding interface names.

When generating interface configuration files, we need to use the
interface's full hardware address as the HWADDR, rather than the 6 byte
MAC address provided by network_data.json.

This change allows IB interfaces to be referenced in this dual mode - by
MAC address and hardware address, depending on the context.

Support TYPE=InfiniBand for sysconfig configuration of IB interfaces.
---
 cloudinit/net/__init__.py              | 38 ++++++++++++++++
 cloudinit/net/network_state.py         |  4 ++
 cloudinit/net/sysconfig.py             | 14 ++++++
 cloudinit/sources/helpers/openstack.py | 11 +++++
 tests/unittests/test_net.py            | 81 +++++++++++++++++++++++++++++++++-
 5 files changed, 147 insertions(+), 1 deletion(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 5e87bcad..f83d3681 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -569,6 +569,20 @@ def get_interface_mac(ifname):
     return read_sys_net_safe(ifname, path)
 
 
+def get_ib_interface_hwaddr(ifname, ethernet_format):
+    """Returns the string value of an Infiniband interface's hardware
+    address. If ethernet_format is True, an Ethernet MAC-style 6 byte
+    representation of the address will be returned.
+    """
+    # Type 32 is Infiniband.
+    if read_sys_net_safe(ifname, 'type') == '32':
+        mac = get_interface_mac(ifname)
+        if mac and ethernet_format:
+            # Use bytes 13-15 and 18-20 of the hardware address.
+            mac = mac[36:-14] + mac[51:]
+        return mac
+
+
 def get_interfaces_by_mac():
     """Build a dictionary of tuples {mac: name}.
 
@@ -580,6 +594,15 @@ def get_interfaces_by_mac():
                 "duplicate mac found! both '%s' and '%s' have mac '%s'" %
                 (name, ret[mac], mac))
         ret[mac] = name
+        # Try to get an Infiniband hardware address (in 6 byte Ethernet format)
+        # for the interface.
+        ib_mac = get_ib_interface_hwaddr(name, True)
+        if ib_mac:
+            if ib_mac in ret:
+                raise RuntimeError(
+                    "duplicate mac found! both '%s' and '%s' have mac '%s'" %
+                    (name, ret[ib_mac], ib_mac))
+            ret[ib_mac] = name
     return ret
 
 
@@ -607,6 +630,21 @@ def get_interfaces():
     return ret
 
 
+def get_ib_hwaddrs_by_interface():
+    """Build a dictionary mapping Infiniband interface names to their hardware
+    address."""
+    ret = {}
+    for name, _, _, _ in get_interfaces():
+        ib_mac = get_ib_interface_hwaddr(name, False)
+        if ib_mac:
+            if ib_mac in ret:
+                raise RuntimeError(
+                    "duplicate mac found! both '%s' and '%s' have mac '%s'" %
+                    (name, ret[ib_mac], ib_mac))
+            ret[name] = ib_mac
+    return ret
+
+
 class EphemeralIPv4Network(object):
     """Context manager which sets up temporary static network configuration.
 
diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py
index 72c803eb..f76e508a 100644
--- a/cloudinit/net/network_state.py
+++ b/cloudinit/net/network_state.py
@@ -483,6 +483,10 @@ class NetworkStateInterpreter(object):
 
         interfaces.update({iface['name']: iface})
 
+    @ensure_command_keys(['name'])
+    def handle_infiniband(self, command):
+        self.handle_physical(command)
+
     @ensure_command_keys(['address'])
     def handle_nameserver(self, command):
         dns = self._network_state.get('dns')
diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py
index 66e970e0..9c16d3a7 100644
--- a/cloudinit/net/sysconfig.py
+++ b/cloudinit/net/sysconfig.py
@@ -174,6 +174,7 @@ class NetInterface(ConfigMap):
         'ethernet': 'Ethernet',
         'bond': 'Bond',
         'bridge': 'Bridge',
+        'infiniband': 'InfiniBand',
     }
 
     def __init__(self, iface_name, base_sysconf_dir, templates,
@@ -568,6 +569,18 @@ class Renderer(renderer.Renderer):
             cls._render_subnets(iface_cfg, iface_subnets)
             cls._render_subnet_routes(iface_cfg, route_cfg, iface_subnets)
 
+    @classmethod
+    def _render_ib_interfaces(cls, network_state, iface_contents):
+        ib_filter = renderer.filter_by_type('infiniband')
+        for iface in network_state.iter_interfaces(ib_filter):
+            iface_name = iface['name']
+            iface_cfg = iface_contents[iface_name]
+            iface_cfg.kind = 'infiniband'
+            iface_subnets = iface.get("subnets", [])
+            route_cfg = iface_cfg.routes
+            cls._render_subnets(iface_cfg, iface_subnets)
+            cls._render_subnet_routes(iface_cfg, route_cfg, iface_subnets)
+
     @classmethod
     def _render_sysconfig(cls, base_sysconf_dir, network_state,
                           templates=None):
@@ -586,6 +599,7 @@ class Renderer(renderer.Renderer):
         cls._render_bond_interfaces(network_state, iface_contents)
         cls._render_vlan_interfaces(network_state, iface_contents)
         cls._render_bridge_interfaces(network_state, iface_contents)
+        cls._render_ib_interfaces(network_state, iface_contents)
         contents = {}
         for iface_name, iface_cfg in iface_contents.items():
             if iface_cfg or iface_cfg.children:
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 76a6e310..9c29ceac 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -675,6 +675,17 @@ def convert_net_json(network_json=None, known_macs=None):
             else:
                 cfg[key] = fmt % link_id_info[target]['name']
 
+    # Infiniband interfaces may be referenced in network_data.json by a 6 byte
+    # Ethernet MAC-style address, and we use that address to look up the
+    # interface name above. Now ensure that the hardware address is set to the
+    # full 20 byte address.
+    ib_known_hwaddrs = net.get_ib_hwaddrs_by_interface()
+    if ib_known_hwaddrs:
+        for cfg in config:
+            if cfg['name'] in ib_known_hwaddrs:
+                cfg['mac_address'] = ib_known_hwaddrs[cfg['name']]
+                cfg['type'] = 'infiniband'
+
     for service in services:
         cfg = service
         cfg.update({'type': 'nameserver'})
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index f3165da9..5d9c7d92 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -3260,11 +3260,15 @@ class TestGetInterfacesByMac(CiTestCase):
     def _se_interface_has_own_mac(self, name):
         return name in self.data['own_macs']
 
+    def _se_get_ib_interface_hwaddr(self, name, ethernet_format):
+        ib_hwaddr = self.data.get('ib_hwaddr', {})
+        return ib_hwaddr.get(name, {}).get(ethernet_format)
+
     def _mock_setup(self):
         self.data = copy.deepcopy(self._data)
         self.data['devices'] = set(list(self.data['macs'].keys()))
         mocks = ('get_devicelist', 'get_interface_mac', 'is_bridge',
-                 'interface_has_own_mac', 'is_vlan')
+                 'interface_has_own_mac', 'is_vlan', 'get_ib_interface_hwaddr')
         self.mocks = {}
         for n in mocks:
             m = mock.patch('cloudinit.net.' + n,
@@ -3338,6 +3342,20 @@ class TestGetInterfacesByMac(CiTestCase):
         ret = net.get_interfaces_by_mac()
         self.assertEqual('lo', ret[empty_mac])
 
+    def test_ib(self):
+        ib_addr = '80:00:00:28:fe:80:00:00:00:00:00:00:00:11:22:03:00:33:44:56'
+        ib_addr_eth_format = '00:11:22:33:44:56'
+        self._mock_setup()
+        self.data['devices'] = ['enp0s1', 'ib0']
+        self.data['own_macs'].append('ib0')
+        self.data['macs']['ib0'] = ib_addr
+        self.data['ib_hwaddr'] = {'ib0': {True: ib_addr_eth_format,
+                                          False: ib_addr}}
+        result = net.get_interfaces_by_mac()
+        expected = {'aa:aa:aa:aa:aa:01': 'enp0s1',
+                    ib_addr_eth_format: 'ib0', ib_addr: 'ib0'}
+        self.assertEqual(expected, result)
+
 
 class TestInterfacesSorting(CiTestCase):
 
@@ -3352,6 +3370,67 @@ class TestInterfacesSorting(CiTestCase):
             ['enp0s3', 'enp0s8', 'enp0s13', 'enp1s2', 'enp2s0', 'enp2s3'])
 
 
+class TestGetIBHwaddrsByInterface(CiTestCase):
+
+    _ib_addr = '80:00:00:28:fe:80:00:00:00:00:00:00:00:11:22:03:00:33:44:56'
+    _ib_addr_eth_format = '00:11:22:33:44:56'
+    _data = {'devices': ['enp0s1', 'enp0s2', 'bond1', 'bridge1',
+                         'bridge1-nic', 'tun0', 'ib0'],
+             'bonds': ['bond1'],
+             'bridges': ['bridge1'],
+             'own_macs': ['enp0s1', 'enp0s2', 'bridge1-nic', 'bridge1', 'ib0'],
+             'macs': {'enp0s1': 'aa:aa:aa:aa:aa:01',
+                      'enp0s2': 'aa:aa:aa:aa:aa:02',
+                      'bond1': 'aa:aa:aa:aa:aa:01',
+                      'bridge1': 'aa:aa:aa:aa:aa:03',
+                      'bridge1-nic': 'aa:aa:aa:aa:aa:03',
+                      'tun0': None,
+                      'ib0': _ib_addr},
+             'ib_hwaddr': {'ib0': {True: _ib_addr_eth_format,
+                                   False: _ib_addr}}}
+    data = {}
+
+    def _mock_setup(self):
+        self.data = copy.deepcopy(self._data)
+        mocks = ('get_devicelist', 'get_interface_mac', 'is_bridge',
+                 'interface_has_own_mac', 'get_ib_interface_hwaddr')
+        self.mocks = {}
+        for n in mocks:
+            m = mock.patch('cloudinit.net.' + n,
+                           side_effect=getattr(self, '_se_' + n))
+            self.addCleanup(m.stop)
+            self.mocks[n] = m.start()
+
+    def _se_get_devicelist(self):
+        return self.data['devices']
+
+    def _se_get_interface_mac(self, name):
+        return self.data['macs'][name]
+
+    def _se_is_bridge(self, name):
+        return name in self.data['bridges']
+
+    def _se_interface_has_own_mac(self, name):
+        return name in self.data['own_macs']
+
+    def _se_get_ib_interface_hwaddr(self, name, ethernet_format):
+        ib_hwaddr = self.data.get('ib_hwaddr', {})
+        return ib_hwaddr.get(name, {}).get(ethernet_format)
+
+    def test_ethernet(self):
+        self._mock_setup()
+        self.data['devices'].remove('ib0')
+        result = net.get_ib_hwaddrs_by_interface()
+        expected = {}
+        self.assertEqual(expected, result)
+
+    def test_ib(self):
+        self._mock_setup()
+        result = net.get_ib_hwaddrs_by_interface()
+        expected = {'ib0': self._ib_addr}
+        self.assertEqual(expected, result)
+
+
 def _gzip_data(data):
     with io.BytesIO() as iobuf:
         gzfp = gzip.GzipFile(mode="wb", fileobj=iobuf)
-- 
cgit v1.2.3