Add RbxCloud datasource

5 files changed, 505 insertions, 2 deletions

diff --git a/cloudinit/sources/DataSourceRbxCloud.py b/cloudinit/sources/DataSourceRbxCloud.py
new file mode 100644
index 00000000..9a8c3d5c
--- /dev/null
+++ b/cloudinit/sources/DataSourceRbxCloud.py
@@ -0,0 +1,250 @@
+# Copyright (C) 2018 Warsaw Data Center
+#
+# Author: Malwina Leis <m.leis@rootbox.com>
+# Author: Grzegorz Brzeski <gregory@rootbox.io>
+# Author: Adam Dobrawy <a.dobrawy@hyperone.com>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+"""
+This file contains code used to gather the user data passed to an
+instance on rootbox / hyperone cloud platforms
+"""
+import errno
+import os
+import os.path
+
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import util
+from cloudinit.event import EventType
+
+LOG = logging.getLogger(__name__)
+ETC_HOSTS = '/etc/hosts'
+
+
+def get_manage_etc_hosts():
+    hosts = util.load_file(ETC_HOSTS, quiet=True)
+    if hosts:
+        LOG.debug('/etc/hosts exists - setting manage_etc_hosts to False')
+        return False
+    LOG.debug('/etc/hosts does not exists - setting manage_etc_hosts to True')
+    return True
+
+
+def ip2int(addr):
+    parts = addr.split('.')
+    return (int(parts[0]) << 24) + (int(parts[1]) << 16) + \
+           (int(parts[2]) << 8) + int(parts[3])
+
+
+def int2ip(addr):
+    return '.'.join([str(addr >> (i << 3) & 0xFF) for i in range(4)[::-1]])
+
+
+def _sub_arp(cmd):
+    """
+    Uses the prefered cloud-init subprocess def of util.subp
+    and runs arping.  Breaking this to a separate function
+    for later use in mocking and unittests
+    """
+    return util.subp(['arping'] + cmd)
+
+
+def gratuitous_arp(items, distro):
+    source_param = '-S'
+    if distro.name in ['fedora', 'centos', 'rhel']:
+        source_param = '-s'
+    for item in items:
+        _sub_arp([
+            '-c', '2',
+            source_param, item['source'],
+            item['destination']
+        ])
+
+
+def get_md():
+    rbx_data = None
+    devices = [
+        dev
+        for dev, bdata in util.blkid().items()
+        if bdata.get('LABEL', '').upper() == 'CLOUDMD'
+    ]
+    for device in devices:
+        try:
+            rbx_data = util.mount_cb(
+                device=device,
+                callback=read_user_data_callback,
+                mtype=['vfat', 'fat']
+            )
+            if rbx_data:
+                break
+        except OSError as err:
+            if err.errno != errno.ENOENT:
+                raise
+        except util.MountFailedError:
+            util.logexc(LOG, "Failed to mount %s when looking for user "
+                             "data", device)
+    if not rbx_data:
+        util.logexc(LOG, "Failed to load metadata and userdata")
+        return False
+    return rbx_data
+
+
+def generate_network_config(netadps):
+    """Generate network configuration
+
+    @param netadps: A list of network adapter settings
+
+    @returns: A dict containing network config
+    """
+    return {
+        'version': 1,
+        'config': [
+            {
+                'type': 'physical',
+                'name': 'eth{}'.format(str(i)),
+                'mac_address': netadp['macaddress'].lower(),
+                'subnets': [
+                    {
+                        'type': 'static',
+                        'address': ip['address'],
+                        'netmask': netadp['network']['netmask'],
+                        'control': 'auto',
+                        'gateway': netadp['network']['gateway'],
+                        'dns_nameservers': netadp['network']['dns'][
+                            'nameservers']
+                    } for ip in netadp['ip']
+                ],
+            } for i, netadp in enumerate(netadps)
+        ]
+    }
+
+
+def read_user_data_callback(mount_dir):
+    """This callback will be applied by util.mount_cb() on the mounted
+    drive.
+
+    @param mount_dir: String representing path of directory where mounted drive
+    is available
+
+    @returns: A dict containing userdata, metadata and cfg based on metadata.
+    """
+    meta_data = util.load_json(
+        text=util.load_file(
+            fname=os.path.join(mount_dir, 'cloud.json'),
+            decode=False
+        )
+    )
+    user_data = util.load_file(
+        fname=os.path.join(mount_dir, 'user.data'),
+        quiet=True
+    )
+    if 'vm' not in meta_data or 'netadp' not in meta_data:
+        util.logexc(LOG, "Failed to load metadata. Invalid format.")
+        return None
+    username = meta_data.get('additionalMetadata', {}).get('username')
+    ssh_keys = meta_data.get('additionalMetadata', {}).get('sshKeys', [])
+
+    hash = None
+    if meta_data.get('additionalMetadata', {}).get('password'):
+        hash = meta_data['additionalMetadata']['password']['sha512']
+
+    network = generate_network_config(meta_data['netadp'])
+
+    data = {
+        'userdata': user_data,
+        'metadata': {
+            'instance-id': meta_data['vm']['_id'],
+            'local-hostname': meta_data['vm']['name'],
+            'public-keys': []
+        },
+        'gratuitous_arp': [
+            {
+                "source": ip["address"],
+                "destination": target
+            }
+            for netadp in meta_data['netadp']
+            for ip in netadp['ip']
+            for target in [
+                netadp['network']["gateway"],
+                int2ip(ip2int(netadp['network']["gateway"]) + 2),
+                int2ip(ip2int(netadp['network']["gateway"]) + 3)
+            ]
+        ],
+        'cfg': {
+            'ssh_pwauth': True,
+            'disable_root': True,
+            'system_info': {
+                'default_user': {
+                    'name': username,
+                    'gecos': username,
+                    'sudo': ['ALL=(ALL) NOPASSWD:ALL'],
+                    'passwd': hash,
+                    'lock_passwd': False,
+                    'ssh_authorized_keys': ssh_keys,
+                    'shell': '/bin/bash'
+                }
+            },
+            'network_config': network,
+            'manage_etc_hosts': get_manage_etc_hosts(),
+        },
+    }
+
+    LOG.debug('returning DATA object:')
+    LOG.debug(data)
+
+    return data
+
+
+class DataSourceRbxCloud(sources.DataSource):
+    update_events = {'network': [
+        EventType.BOOT_NEW_INSTANCE,
+        EventType.BOOT
+    ]}
+
+    def __init__(self, sys_cfg, distro, paths):
+        sources.DataSource.__init__(self, sys_cfg, distro, paths)
+        self.seed = None
+
+    def __str__(self):
+        root = sources.DataSource.__str__(self)
+        return "%s [seed=%s]" % (root, self.seed)
+
+    def _get_data(self):
+        """
+        Metadata is passed to the launching instance which
+        is used to perform instance configuration.
+        """
+        rbx_data = get_md()
+        self.userdata_raw = rbx_data['userdata']
+        self.metadata = rbx_data['metadata']
+        self.gratuitous_arp = rbx_data['gratuitous_arp']
+        self.cfg = rbx_data['cfg']
+        return True
+
+    @property
+    def network_config(self):
+        return self.cfg['network_config']
+
+    def get_public_ssh_keys(self):
+        return self.metadata['public-keys']
+
+    def get_userdata_raw(self):
+        return self.userdata_raw
+
+    def get_config_obj(self):
+        return self.cfg
+
+    def activate(self, cfg, is_new_instance):
+        gratuitous_arp(self.gratuitous_arp, self.distro)
+
+
+# Used to match classes to dependencies
+datasources = [
+    (DataSourceRbxCloud, (sources.DEP_FILESYSTEM,)),
+]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
diff --git a/doc/rtd/topics/datasources/rbxcloud.rst b/doc/rtd/topics/datasources/rbxcloud.rst
new file mode 100644
index 00000000..3d465bed
--- /dev/null
+++ b/doc/rtd/topics/datasources/rbxcloud.rst
@@ -0,0 +1,25 @@
+.. _datasource_config_drive:
+
+Rbx Cloud
+=========
+
+The Rbx datasource consumes the metadata drive available on platform
+`HyperOne`_ and `Rootbox`_ platform.
+
+Datasource supports, in particular, network configurations, hostname,
+user accounts and user metadata.
+
+Metadata drive
+--------------
+
+Drive metadata is a `FAT`_-formatted partition with the ```CLOUDMD``` label on
+the system disk. Its contents are refreshed each time the virtual machine
+is restarted, if the partition exists. For more information see
+`HyperOne docs`_.
+
+.. _HyperOne: http://www.hyperone.com/
+.. _Rootbox: https://rootbox.com/
+.. _HyperOne Virtual Machine docs: http://www.hyperone.com/
+.. _vfat: https://en.wikipedia.org/wiki/File_Allocation_Table
+
+.. vi: textwidth=78
diff --git a/tests/unittests/test_datasource/test_rbx.py b/tests/unittests/test_datasource/test_rbx.py
new file mode 100644
index 00000000..aabf1f18
--- /dev/null
+++ b/tests/unittests/test_datasource/test_rbx.py
@@ -0,0 +1,208 @@
+import json
+
+from cloudinit import helpers
+from cloudinit import distros
+from cloudinit.sources import DataSourceRbxCloud as ds
+from cloudinit.tests.helpers import mock, CiTestCase, populate_dir
+
+DS_PATH = "cloudinit.sources.DataSourceRbxCloud"
+
+CRYPTO_PASS = "$6$uktth46t$FvpDzFD2iL9YNZIG1Epz7957hJqbH0f" \
+              "QKhnzcfBcUhEodGAWRqTy7tYG4nEW7SUOYBjxOSFIQW5" \
+              "tToyGP41.s1"
+
+CLOUD_METADATA = {
+    "vm": {
+        "memory": 4,
+        "cpu": 2,
+        "name": "vm-image-builder",
+        "_id": "5beab44f680cffd11f0e60fc"
+    },
+    "additionalMetadata": {
+        "username": "guru",
+        "sshKeys": ["ssh-rsa ..."],
+        "password": {
+            "sha512": CRYPTO_PASS
+        }
+    },
+    "disk": [
+        {"size": 10, "type": "ssd",
+         "name": "vm-image-builder-os",
+         "_id": "5beab450680cffd11f0e60fe"},
+        {"size": 2, "type": "ssd",
+         "name": "ubuntu-1804-bionic",
+         "_id": "5bef002c680cffd11f107590"}
+    ],
+    "netadp": [
+        {
+            "ip": [{"address": "62.181.8.174"}],
+            "network": {
+                "dns": {"nameservers": ["8.8.8.8", "8.8.4.4"]},
+                "routing": [],
+                "gateway": "62.181.8.1",
+                "netmask": "255.255.248.0",
+                "name": "public",
+                "type": "public",
+                "_id": "5784e97be2627505227b578c"
+            },
+            "speed": 1000,
+            "type": "hv",
+            "macaddress": "00:15:5D:FF:0F:03",
+            "_id": "5beab450680cffd11f0e6102"
+        },
+        {
+            "ip": [{"address": "10.209.78.11"}],
+            "network": {
+                "dns": {"nameservers": ["9.9.9.9", "8.8.8.8"]},
+                "routing": [],
+                "gateway": "10.209.78.1",
+                "netmask": "255.255.255.0",
+                "name": "network-determined-bardeen",
+                "type": "private",
+                "_id": "5beaec64680cffd11f0e7c31"
+            },
+            "speed": 1000,
+            "type": "hv",
+            "macaddress": "00:15:5D:FF:0F:24",
+            "_id": "5bec18c6680cffd11f0f0d8b"
+        }
+    ],
+    "dvddrive": [{"iso": {}}]
+}
+
+
+class TestRbxDataSource(CiTestCase):
+    parsed_user = None
+    allowed_subp = ['bash']
+
+    def _fetch_distro(self, kind):
+        cls = distros.fetch(kind)
+        paths = helpers.Paths({})
+        return cls(kind, {}, paths)
+
+    def setUp(self):
+        super(TestRbxDataSource, self).setUp()
+        self.tmp = self.tmp_dir()
+        self.paths = helpers.Paths(
+            {'cloud_dir': self.tmp, 'run_dir': self.tmp}
+        )
+
+        # defaults for few tests
+        self.ds = ds.DataSourceRbxCloud
+        self.seed_dir = self.paths.seed_dir
+        self.sys_cfg = {'datasource': {'RbxCloud': {'dsmode': 'local'}}}
+
+    def test_seed_read_user_data_callback_empty_file(self):
+        populate_user_metadata(self.seed_dir, '')
+        populate_cloud_metadata(self.seed_dir, {})
+        results = ds.read_user_data_callback(self.seed_dir)
+
+        self.assertIsNone(results)
+
+    def test_seed_read_user_data_callback_valid_disk(self):
+        populate_user_metadata(self.seed_dir, '')
+        populate_cloud_metadata(self.seed_dir, CLOUD_METADATA)
+        results = ds.read_user_data_callback(self.seed_dir)
+
+        self.assertNotEqual(results, None)
+        self.assertTrue('userdata' in results)
+        self.assertTrue('metadata' in results)
+        self.assertTrue('cfg' in results)
+
+    def test_seed_read_user_data_callback_userdata(self):
+        userdata = "#!/bin/sh\nexit 1"
+        populate_user_metadata(self.seed_dir, userdata)
+        populate_cloud_metadata(self.seed_dir, CLOUD_METADATA)
+
+        results = ds.read_user_data_callback(self.seed_dir)
+
+        self.assertNotEqual(results, None)
+        self.assertTrue('userdata' in results)
+        self.assertEqual(results['userdata'], userdata)
+
+    def test_generate_network_config(self):
+        expected = {
+            'version': 1,
+            'config': [
+                {
+                    'subnets': [
+                        {'control': 'auto',
+                         'dns_nameservers': ['8.8.8.8', '8.8.4.4'],
+                         'netmask': '255.255.248.0',
+                         'address': '62.181.8.174',
+                         'type': 'static', 'gateway': '62.181.8.1'}
+                    ],
+                    'type': 'physical',
+                    'name': 'eth0',
+                    'mac_address': '00:15:5d:ff:0f:03'
+                },
+                {
+                    'subnets': [
+                        {'control': 'auto',
+                         'dns_nameservers': ['9.9.9.9', '8.8.8.8'],
+                         'netmask': '255.255.255.0',
+                         'address': '10.209.78.11',
+                         'type': 'static',
+                         'gateway': '10.209.78.1'}
+                    ],
+                    'type': 'physical',
+                    'name': 'eth1',
+                    'mac_address': '00:15:5d:ff:0f:24'
+                }
+            ]
+        }
+        self.assertTrue(
+            ds.generate_network_config(CLOUD_METADATA['netadp']),
+            expected
+        )
+
+    @mock.patch(DS_PATH + '.util.subp')
+    def test_gratuitous_arp_run_standard_arping(self, m_subp):
+        """Test handle run arping & parameters."""
+        items = [
+            {
+                'destination': '172.17.0.2',
+                'source': '172.16.6.104'
+            },
+            {
+                'destination': '172.17.0.2',
+                'source': '172.16.6.104',
+            },
+        ]
+        ds.gratuitous_arp(items, self._fetch_distro('ubuntu'))
+        self.assertEqual([
+            mock.call([
+                'arping', '-c', '2', '-S',
+                '172.16.6.104', '172.17.0.2'
+            ]),
+            mock.call([
+                'arping', '-c', '2', '-S',
+                '172.16.6.104', '172.17.0.2'
+            ])
+        ], m_subp.call_args_list
+        )
+
+    @mock.patch(DS_PATH + '.util.subp')
+    def test_handle_rhel_like_arping(self, m_subp):
+        """Test handle on RHEL-like distros."""
+        items = [
+            {
+                'source': '172.16.6.104',
+                'destination': '172.17.0.2',
+            }
+        ]
+        ds.gratuitous_arp(items, self._fetch_distro('fedora'))
+        self.assertEqual([
+            mock.call(
+                ['arping', '-c', '2', '-s', '172.16.6.104', '172.17.0.2']
+            )],
+            m_subp.call_args_list
+        )
+
+
+def populate_cloud_metadata(path, data):
+    populate_dir(path, {'cloud.json': json.dumps(data)})
+
+
+def populate_user_metadata(path, data):
+    populate_dir(path, {'user.data': data})
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 7aeeb91c..c5b5c46c 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -267,10 +267,13 @@ class TestDsIdentify(DsIdentifyBase):
         """ConfigDrive datasource has a disk with LABEL=config-2."""
         self._test_ds_found('ConfigDrive')
 
+    def test_rbx_cloud(self):
+        """Rbx datasource has a disk with LABEL=CLOUDMD."""
+        self._test_ds_found('RbxCloud')
+
     def test_config_drive_upper(self):
         """ConfigDrive datasource has a disk with LABEL=CONFIG-2."""
         self._test_ds_found('ConfigDriveUpper')
-        return
 
     def test_config_drive_seed(self):
         """Config Drive seed directory."""
@@ -896,6 +899,18 @@ VALID_CFG = {
             os.path.join(P_SEED_DIR, 'config_drive', 'openstack',
                          'latest', 'meta_data.json'): 'md\n'},
     },
+    'RbxCloud': {
+        'ds': 'RbxCloud',
+        'mocks': [
+            {'name': 'blkid', 'ret': 0,
+             'out': blkid_out(
+                 [{'DEVNAME': 'vda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()},
+                  {'DEVNAME': 'vda2', 'TYPE': 'ext4',
+                   'LABEL': 'cloudimg-rootfs', 'PARTUUID': uuid4()},
+                  {'DEVNAME': 'vdb', 'TYPE': 'vfat', 'LABEL': 'CLOUDMD'}]
+             )},
+        ],
+    },
     'Hetzner': {
         'ds': 'Hetzner',
         'files': {P_SYS_VENDOR: 'Hetzner\n'},
diff --git a/tools/ds-identify b/tools/ds-identify
index f76f2a6e..40fc0604 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -124,7 +124,7 @@ DI_DSNAME=""
 # be searched if there is no setting found in config.
 DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \
 CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \
-OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale"
+OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud"
 DI_DSLIST=""
 DI_MODE=""
 DI_ON_FOUND=""
@@ -702,6 +702,11 @@ dscheck_OpenNebula() {
     return ${DS_NOT_FOUND}
 }
 
+dscheck_RbxCloud() {
+    has_fs_with_label "CLOUDMD" "cloudmd" && return ${DS_FOUND}
+    return ${DS_NOT_FOUND}
+}
+
 ovf_vmware_guest_customization() {
     # vmware guest customization