From 4afa528dcf2938e209b6dc852f6d4c4076084fa5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 8 Jul 2013 23:21:29 -0400 Subject: fix test_nocloud testcase --- tests/unittests/test_datasource/test_nocloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 62fc5358..7328b240 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -22,7 +22,7 @@ class TestNoCloudDataSource(MockerTestCase): def tearDown(self): apply_patches([i for i in reversed(self.unapply)]) - super(TestNoCloudDataSource, self).setUp() + super(TestNoCloudDataSource, self).tearDown() def apply_patches(self, patches): ret = apply_patches(patches) -- cgit v1.2.3 From 21ea6154ab2aafbe51c7b23fd56e43bd1cc26b00 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 9 Jul 2013 01:35:49 -0400 Subject: add some unit tests, fix things found by doing so --- cloudinit/sources/DataSourceAzure.py | 26 +++- tests/unittests/test_datasource/test_azure.py | 168 ++++++++++++++++++++++++++ 2 files changed, 188 insertions(+), 6 deletions(-) create mode 100644 tests/unittests/test_datasource/test_azure.py (limited to 'tests') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 83c4603c..6a04b333 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -18,6 +18,7 @@ import base64 import os +import os.path from xml.dom import minidom from cloudinit import log as logging @@ -29,7 +30,7 @@ LOG = logging.getLogger(__name__) DS_NAME = 'Azure' DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] -DEFAULT_DS_CONFIG = {'datasource': {DS_NAME: {'agent_command': AGENT_START}}} +BUILTIN_DS_CONFIG = {'datasource': {DS_NAME: {'agent_command': AGENT_START}}} class DataSourceAzureNet(sources.DataSource): @@ -56,16 +57,17 @@ class DataSourceAzureNet(sources.DataSource): ret = load_azure_ds_dir(cdev) except NonAzureDataSource: - pass + continue except BrokenAzureDataSource as exc: raise exc except util.MountFailedError: LOG.warn("%s was not mountable" % cdev) + continue (md, self.userdata_raw, cfg) = ret self.seed = cdev self.metadata = util.mergemanydict([md, DEFAULT_METADATA]) - self.cfg = util.mergemanydict([cfg, DEFAULT_DS_CONFIG]) + self.cfg = cfg found = cdev LOG.debug("found datasource in %s", cdev) @@ -76,17 +78,25 @@ class DataSourceAzureNet(sources.DataSource): path = ['datasource', DS_NAME, 'agent_command'] cmd = None - for cfg in (self.cfg, self.sys_cfg): + for cfg in (self.cfg, self.sys_cfg, BUILTIN_DS_CONFIG): cmd = util.get_cfg_by_path(cfg, keyp=path) if cmd is not None: break - invoke_agent(cmd) + + try: + invoke_agent(cmd) + except util.ProcessExecutionError: + # claim the datasource even if the command failed + util.logexc(LOG, "agent command '%s' failed.", cmd) + + return True def get_config_obj(self): return self.cfg def invoke_agent(cmd): + # this is a function itself to simplify patching it for test if cmd: LOG.debug("invoking agent: %s" % cmd) util.subp(cmd, shell=(not isinstance(cmd, list))) @@ -105,7 +115,11 @@ def find_child(node, filter_func): def read_azure_ovf(contents): - dom = minidom.parseString(contents) + try: + dom = minidom.parseString(contents) + except Exception as e: + raise NonAzureDataSource("invalid xml: %s" % e) + results = find_child(dom.documentElement, lambda n: n.localName == "ProvisioningSection") diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py new file mode 100644 index 00000000..179fb50a --- /dev/null +++ b/tests/unittests/test_datasource/test_azure.py @@ -0,0 +1,168 @@ +from cloudinit import helpers +from cloudinit.sources import DataSourceAzure +from tests.unittests.helpers import populate_dir + +import base64 +from mocker import MockerTestCase +import os +import yaml + + +def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): + if data is None: + data = {'HostName': 'FOOHOST'} + if pubkeys is None: + pubkeys = {} + + content = """ + + + 1.0 + + LinuxProvisioningConfiguration + """ + for key, val in data.items(): + content += "<%s>%s\n" % (key, val, key) + + if userdata: + content += "%s\n" % (base64.b64encode(userdata)) + + if pubkeys: + content += "\n" + for fp, path in pubkeys.items(): + content += " " + content += ("%s%s" % + (fp, path)) + content += " " + content += """ + + + 1.0 + + kms.core.windows.net + false + + + + """ + + return content + + +class TestAzureDataSource(MockerTestCase): + + def setUp(self): + # makeDir comes from MockerTestCase + self.tmp = self.makeDir() + + # patch cloud_dir, so our 'seed_dir' is guaranteed empty + self.paths = helpers.Paths({'cloud_dir': self.tmp}) + + self.unapply = [] + super(TestAzureDataSource, self).setUp() + + def tearDown(self): + apply_patches([i for i in reversed(self.unapply)]) + super(TestAzureDataSource, self).tearDown() + + def apply_patches(self, patches): + ret = apply_patches(patches) + self.unapply += ret + + def _get_ds(self, data): + + def dsdevs(): + return data.get('dsdevs', []) + + def invoker(cmd): + data['agent_invoked'] = cmd + + if data.get('ovfcontent') is not None: + populate_dir(os.path.join(self.paths.seed_dir, "azure"), + {'ovf-env.xml': data['ovfcontent']}) + + mod = DataSourceAzure + + if data.get('dsdevs'): + self.apply_patches([(mod, 'list_possible_azure_ds_devs', dsdevs)]) + + self.apply_patches([(mod, 'invoke_agent', invoker)]) + + dsrc = mod.DataSourceAzureNet( + data.get('sys_cfg', {}), distro=None, paths=self.paths) + + return dsrc + + def test_basic_seed_dir(self): + odata = {'HostName': "myhost", 'UserName': "myuser"} + data = {'ovfcontent': construct_valid_ovf_env(data=odata), + 'sys_cfg': {}} + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(dsrc.userdata_raw, "") + self.assertEqual(dsrc.metadata['local-hostname'], odata['HostName']) + + def test_user_cfg_set_agent_command(self): + cfg = {'agent_command': "my_command"} + odata = {'HostName': "myhost", 'UserName': "myuser", + 'dscfg': yaml.dump(cfg)} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(data['agent_invoked'], cfg['agent_command']) + + def test_sys_cfg_set_agent_command(self): + sys_cfg = {'datasource': {'Azure': {'agent_command': '_COMMAND'}}} + data = {'ovfcontent': construct_valid_ovf_env(data={}), + 'sys_cfg': sys_cfg} + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(data['agent_invoked'], '_COMMAND') + + def test_userdata_found(self): + mydata = "FOOBAR" + odata = {'UserData': base64.b64encode(mydata)} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(dsrc.userdata_raw, mydata) + + def test_no_datasource_expected(self): + #no source should be found if no seed_dir and no devs + data = {} + dsrc = self._get_ds({}) + ret = dsrc.get_data() + self.assertFalse(ret) + self.assertFalse('agent_invoked' in data) + + +class TestReadAzureOvf(MockerTestCase): + def test_invalid_xml_raises_non_azure_ds(self): + invalid_xml = "" + construct_valid_ovf_env(data={}) + self.assertRaises(DataSourceAzure.NonAzureDataSource, + DataSourceAzure.read_azure_ovf, invalid_xml) + + +def apply_patches(patches): + ret = [] + for (ref, name, replace) in patches: + if replace is None: + continue + orig = getattr(ref, name) + setattr(ref, name, replace) + ret.append((ref, name, orig)) + return ret -- cgit v1.2.3 From 1c76b49ccbe0c31187aeea12b0e395774aa90faa Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 9 Jul 2013 02:33:32 -0400 Subject: re-enable test case because 1124384 is fixed --- tests/unittests/test_builtin_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 9cf28215..e25a5144 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -35,7 +35,6 @@ class TestBuiltins(test_helpers.FilesystemMockingTestCase): None, None, None) self.assertEquals(0, len(os.listdir(up_root))) - @unittest.skip("until LP: #1124384 fixed") def test_upstart_frequency_single(self): # files should be written out when frequency is ! per-instance new_root = self.makeDir() @@ -47,6 +46,7 @@ class TestBuiltins(test_helpers.FilesystemMockingTestCase): 'upstart_dir': "/etc/upstart", }) + upstart_job.SUITABLE_UPSTART = True util.ensure_dir("/run") util.ensure_dir("/etc/upstart") -- cgit v1.2.3 From 6bea1cb867c13e05e3548c648d5f051d2c49f07b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 9 Jul 2013 14:41:55 -0400 Subject: better handling for user/password --- cloudinit/sources/DataSourceAzure.py | 21 ++++++++++++++++++++- tests/unittests/test_datasource/test_azure.py | 27 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 143b7e4a..5037c1a3 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -160,6 +160,8 @@ def read_azure_ovf(contents): md = {'azure_data': {}} cfg = {} ud = "" + password = None + username = None for child in lpcs.childNodes: if child.nodeType == dom.TEXT_NODE or not child.localName: @@ -176,19 +178,36 @@ def read_azure_ovf(contents): if name == "userdata": ud = base64.b64decode(''.join(value.split())) elif name == "username": - cfg['system_info'] = {'default_user': {'name': value}} + username = value + elif name == "userpassword": + password = value elif name == "hostname": md['local-hostname'] = value elif name == "dscfg": cfg['datasource'] = {DS_NAME: util.load_yaml(value, default={})} elif name == "ssh": cfg['_pubkeys'] = loadAzurePubkeys(child) + elif name == "disablesshpasswordauthentication": + cfg['ssh_pwauth'] = util.is_true(value) elif simple: if name in md_props: md[name] = value else: md['azure_data'][name] = value + defuser = {} + if username: + defuser['name'] = username + if password: + defuser['password'] = password + defuser['lock_passwd'] = False + + if defuser: + cfg['system_info'] = {'default_user': defuser} + + if 'ssh_pwauth' not in cfg and password: + cfg['ssh_pwauth'] = True + return (md, ud, cfg) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 179fb50a..a2347f1b 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -131,6 +131,33 @@ class TestAzureDataSource(MockerTestCase): self.assertTrue(ret) self.assertEqual(data['agent_invoked'], '_COMMAND') + def test_username_used(self): + odata = {'HostName': "myhost", 'UserName': "myuser"} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(dsrc.cfg['system_info']['default_user']['name'], + "myuser") + + def test_password_given(self): + odata = {'HostName': "myhost", 'UserName': "myuser", + 'UserPassword': "mypass"} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertTrue('default_user' in dsrc.cfg['system_info']) + defuser = dsrc.cfg['system_info']['default_user'] + + # default user shoudl be updated for password and username + # and should not be locked. + self.assertEqual(defuser['name'], odata['UserName']) + self.assertEqual(defuser['password'], odata['UserPassword']) + self.assertFalse(defuser['lock_passwd']) + def test_userdata_found(self): mydata = "FOOBAR" odata = {'UserData': base64.b64encode(mydata)} -- cgit v1.2.3 From cf1b10900626dfa6194c77b6720291e7edbaf9f6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 9 Jul 2013 15:07:38 -0400 Subject: populate /var/lib/waagent with ovf-env.xml this will copy the ovf-env.xml file that was found to the configured directory (default /var/lib/waagent) --- cloudinit/sources/DataSourceAzure.py | 43 +++++++++++++++++++-------- tests/unittests/test_datasource/test_azure.py | 8 +++++ 2 files changed, 39 insertions(+), 12 deletions(-) (limited to 'tests') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 5037c1a3..f1c7c771 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -30,7 +30,9 @@ LOG = logging.getLogger(__name__) DS_NAME = 'Azure' DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] -BUILTIN_DS_CONFIG = {'datasource': {DS_NAME: {'agent_command': AGENT_START}}} +BUILTIN_DS_CONFIG = {'datasource': {DS_NAME: { + 'agent_command': AGENT_START, + 'data_dir': "/var/lib/waagent"}}} class DataSourceAzureNet(sources.DataSource): @@ -64,7 +66,7 @@ class DataSourceAzureNet(sources.DataSource): LOG.warn("%s was not mountable" % cdev) continue - (md, self.userdata_raw, cfg) = ret + (md, self.userdata_raw, cfg, files) = ret self.seed = cdev self.metadata = util.mergemanydict([md, DEFAULT_METADATA]) self.cfg = cfg @@ -76,18 +78,24 @@ class DataSourceAzureNet(sources.DataSource): if not found: return False - path = ['datasource', DS_NAME, 'agent_command'] - cmd = None + fields = [('cmd', ['datasource', DS_NAME, 'agent_command']), + ('datadir', ['datasource', DS_NAME, 'data_dir'])] + mycfg = {} for cfg in (self.cfg, self.sys_cfg, BUILTIN_DS_CONFIG): - cmd = util.get_cfg_by_path(cfg, keyp=path) - if cmd is not None: - break + for name, path in fields: + if name in mycfg: + continue + value = util.get_cfg_by_path(cfg, keyp=path) + if value is not None: + mycfg[name] = value + + write_files(mycfg['datadir'], files) try: - invoke_agent(cmd) + invoke_agent(mycfg['cmd']) except util.ProcessExecutionError: # claim the datasource even if the command failed - util.logexc(LOG, "agent command '%s' failed.", cmd) + util.logexc(LOG, "agent command '%s' failed.", mycfg['cmd']) return True @@ -95,6 +103,16 @@ class DataSourceAzureNet(sources.DataSource): return self.cfg +def write_files(datadir, files): + if not datadir: + return + if not files: + files = {} + for (name, content) in files.items(): + util.write_file(filename=os.path.join(datadir, name), + content=content, mode=0600) + + def invoke_agent(cmd): # this is a function itself to simplify patching it for test if cmd: @@ -114,7 +132,7 @@ def find_child(node, filter_func): return ret -def load_azure_ovf_pubkeys(sshnode): +def load_azure_ovf_pubkeys(_sshnode): # in the future this would return a list of dicts like: # [{'fp': '6BE7A7C3C8A8F4B123CCA5D0C2F1BE4CA7B63ED7', # 'path': 'where/to/go'}] @@ -186,7 +204,7 @@ def read_azure_ovf(contents): elif name == "dscfg": cfg['datasource'] = {DS_NAME: util.load_yaml(value, default={})} elif name == "ssh": - cfg['_pubkeys'] = loadAzurePubkeys(child) + cfg['_pubkeys'] = load_azure_ovf_pubkeys(child) elif name == "disablesshpasswordauthentication": cfg['ssh_pwauth'] = util.is_true(value) elif simple: @@ -230,7 +248,8 @@ def load_azure_ds_dir(source_dir): with open(ovf_file, "r") as fp: contents = fp.read() - return read_azure_ovf(contents) + md, ud, cfg = read_azure_ovf(contents) + return (md, ud, cfg, {'ovf-env.xml': contents}) class BrokenAzureDataSource(Exception): diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index a2347f1b..68f4bcca 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -83,6 +83,12 @@ class TestAzureDataSource(MockerTestCase): def invoker(cmd): data['agent_invoked'] = cmd + def file_writer(datadir, files): + data['files'] = {} + data['datadir'] = datadir + for (fname, content) in files.items(): + data['files'][fname] = content + if data.get('ovfcontent') is not None: populate_dir(os.path.join(self.paths.seed_dir, "azure"), {'ovf-env.xml': data['ovfcontent']}) @@ -93,6 +99,7 @@ class TestAzureDataSource(MockerTestCase): self.apply_patches([(mod, 'list_possible_azure_ds_devs', dsdevs)]) self.apply_patches([(mod, 'invoke_agent', invoker)]) + self.apply_patches([(mod, 'write_files', file_writer)]) dsrc = mod.DataSourceAzureNet( data.get('sys_cfg', {}), distro=None, paths=self.paths) @@ -109,6 +116,7 @@ class TestAzureDataSource(MockerTestCase): self.assertTrue(ret) self.assertEqual(dsrc.userdata_raw, "") self.assertEqual(dsrc.metadata['local-hostname'], odata['HostName']) + self.assertTrue('ovf-env.xml' in data['files']) def test_user_cfg_set_agent_command(self): cfg = {'agent_command': "my_command"} -- cgit v1.2.3 From 950762bb008d25f529c71aae4c0b04f6b0134abb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 9 Jul 2013 20:20:55 -0400 Subject: fill out load_azure_ovf_pubkeys now if there are pubkeys, the cfg['_pubkeys'] entry will have a list of dicts where each dict has 'fingerprint' and 'path' entries. The next thing to do is to block waiting for the .crt files to appear in /var/lib/waagent. --- cloudinit/sources/DataSourceAzure.py | 40 +++++++++++++++++++++++++-- tests/unittests/test_datasource/test_azure.py | 13 +++++++-- 2 files changed, 49 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index d8e39392..43a963ad 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -146,7 +146,7 @@ def find_child(node, filter_func): return ret -def load_azure_ovf_pubkeys(_sshnode): +def load_azure_ovf_pubkeys(sshnode): # in the future this would return a list of dicts like: # [{'fp': '6BE7A7C3C8A8F4B123CCA5D0C2F1BE4CA7B63ED7', # 'path': 'where/to/go'}] @@ -155,7 +155,43 @@ def load_azure_ovf_pubkeys(_sshnode): # ABC/ABC # ... # - return [] + results = find_child(sshnode, lambda n: n.localName == "PublicKeys") + if len(results) == 0: + return [] + if len(results) > 1: + raise BrokenAzureDataSource("Multiple 'PublicKeys'(%s) in SSH node" % + len(results)) + + pubkeys_node = results[0] + pubkeys = find_child(pubkeys_node, lambda n: n.localName == "PublicKey") + + if len(pubkeys) == 0: + return [] + + found = [] + text_node = minidom.Document.TEXT_NODE + + for pk_node in pubkeys: + if not pk_node.hasChildNodes(): + continue + cur = {'fingerprint': "", 'path': ""} + for child in pk_node.childNodes: + if (child.nodeType == text_node or not child.localName): + continue + + name = child.localName.lower() + + if name not in cur.keys(): + continue + + if (len(child.childNodes) != 1 or + child.childNodes[0].nodeType != text_node): + continue + + cur[name] = child.childNodes[0].wholeText.strip() + found.append(cur) + + return found def read_azure_ovf(contents): diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 68f4bcca..be6fab70 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -34,11 +34,12 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): if pubkeys: content += "\n" - for fp, path in pubkeys.items(): + for fp, path in pubkeys: content += " " content += ("%s%s" % (fp, path)) - content += " " + content += "\n" + content += "" content += """ @@ -191,6 +192,14 @@ class TestReadAzureOvf(MockerTestCase): self.assertRaises(DataSourceAzure.NonAzureDataSource, DataSourceAzure.read_azure_ovf, invalid_xml) + def test_load_with_pubkeys(self): + mypklist = [{'fingerprint': 'fp1', 'path': 'path1'}] + pubkeys = [(x['fingerprint'], x['path']) for x in mypklist] + content = construct_valid_ovf_env(pubkeys=pubkeys) + (md, ud, cfg) = DataSourceAzure.read_azure_ovf(content) + for mypk in mypklist: + self.assertIn(mypk, cfg['_pubkeys']) + def apply_patches(patches): ret = [] -- cgit v1.2.3 From ec22feeefe309187107e0fb5471136f1c8a646c9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 9 Jul 2013 20:36:28 -0400 Subject: build up the 'wait_for' list including fingerprint.crt files --- cloudinit/sources/DataSourceAzure.py | 6 ++++++ tests/unittests/test_datasource/test_azure.py | 13 +++++++++++++ 2 files changed, 19 insertions(+) (limited to 'tests') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 43a963ad..ab570344 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -111,6 +111,12 @@ class DataSourceAzureNet(sources.DataSource): # claim the datasource even if the command failed util.logexc(LOG, "agent command '%s' failed.", mycfg['cmd']) + wait_for = [os.path.join(mycfg['datadir'], "SharedConfig.xml")] + + for pk in self.cfg.get('_pubkeys', []): + bname = pk['fingerprint'] + ".crt" + wait_for += [os.path.join(mycfg['datadir'], bname)] + return True def get_config_obj(self): diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index be6fab70..a7094ec6 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -185,6 +185,19 @@ class TestAzureDataSource(MockerTestCase): self.assertFalse(ret) self.assertFalse('agent_invoked' in data) + def test_cfg_has_pubkeys(self): + odata = {'HostName': "myhost", 'UserName': "myuser"} + mypklist = [{'fingerprint': 'fp1', 'path': 'path1'}] + pubkeys = [(x['fingerprint'], x['path']) for x in mypklist] + data = {'ovfcontent': construct_valid_ovf_env(data=odata, + pubkeys=pubkeys)} + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + for mypk in mypklist: + self.assertIn(mypk, dsrc.cfg['_pubkeys']) + class TestReadAzureOvf(MockerTestCase): def test_invalid_xml_raises_non_azure_ds(self): -- cgit v1.2.3 From ce949d5b4c94caf9c1df6393abe86de2872e05ae Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 10 Jul 2013 13:08:23 -0400 Subject: add waiting for files and reading of crt keys --- cloudinit/sources/DataSourceAzure.py | 46 ++++++++++++++++++++++++++- packages/debian/changelog.in | 2 +- tests/unittests/test_datasource/test_azure.py | 21 +++++++++--- 3 files changed, 62 insertions(+), 7 deletions(-) (limited to 'tests') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index ab570344..200bede5 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -19,6 +19,7 @@ import base64 import os import os.path +import time from xml.dom import minidom from cloudinit import log as logging @@ -113,9 +114,18 @@ class DataSourceAzureNet(sources.DataSource): wait_for = [os.path.join(mycfg['datadir'], "SharedConfig.xml")] + fp_files = [] for pk in self.cfg.get('_pubkeys', []): bname = pk['fingerprint'] + ".crt" - wait_for += [os.path.join(mycfg['datadir'], bname)] + fp_files += [os.path.join(mycfg['datadir'], bname)] + + missing = wait_for_files(wait_for + fp_files) + if len(missing): + LOG.warn("Did not find files, but going on: %s" % missing) + + pubkeys = pubkeys_from_crt_files(fp_files) + + self.metadata['public-keys'] = pubkeys return True @@ -123,6 +133,40 @@ class DataSourceAzureNet(sources.DataSource): return self.cfg +def crtfile_to_pubkey(fname): + pipeline = ('openssl x509 -noout -pubkey < "$0" |' + 'ssh-keygen -i -m PKCS8 -f /dev/stdin') + (out, _err) = util.subp(['sh', '-c', pipeline, fname], capture=True) + return out.rstrip() + + +def pubkeys_from_crt_files(flist): + pubkeys = [] + errors = [] + for fname in flist: + try: + pubkeys.append(crtfile_to_pubkey(fname)) + except util.ProcessExecutionError: + errors.extend(fname) + + if errors: + LOG.warn("failed to convert the crt files to pubkey: %s" % errors) + + return pubkeys + + +def wait_for_files(flist, maxwait=60, naplen=.5): + need = set(flist) + waited = 0 + while waited < maxwait: + need -= set([f for f in need if os.path.exists(f)]) + if len(need) == 0: + return [] + time.sleep(naplen) + waited += naplen + return need + + def write_files(datadir, files): if not datadir: return diff --git a/packages/debian/changelog.in b/packages/debian/changelog.in index e3e94f54..4944230b 100644 --- a/packages/debian/changelog.in +++ b/packages/debian/changelog.in @@ -1,5 +1,5 @@ ## This is a cheetah template -cloud-init (${version}~bzr${revision}-1) UNRELEASED; urgency=low +cloud-init (${version}~bzr${revision}-1) raring; urgency=low * build diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index a7094ec6..74ed7197 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -81,15 +81,23 @@ class TestAzureDataSource(MockerTestCase): def dsdevs(): return data.get('dsdevs', []) - def invoker(cmd): + def _invoke_agent(cmd): data['agent_invoked'] = cmd - def file_writer(datadir, files): + def _write_files(datadir, files): data['files'] = {} data['datadir'] = datadir for (fname, content) in files.items(): data['files'][fname] = content + def _wait_for_files(flist, _maxwait=None, _naplen=None): + data['waited'] = flist + return [] + + def _pubkeys_from_crt_files(flist): + data['pubkey_files'] = flist + return ["pubkey_from: %s" % f for f in flist] + if data.get('ovfcontent') is not None: populate_dir(os.path.join(self.paths.seed_dir, "azure"), {'ovf-env.xml': data['ovfcontent']}) @@ -99,8 +107,11 @@ class TestAzureDataSource(MockerTestCase): if data.get('dsdevs'): self.apply_patches([(mod, 'list_possible_azure_ds_devs', dsdevs)]) - self.apply_patches([(mod, 'invoke_agent', invoker)]) - self.apply_patches([(mod, 'write_files', file_writer)]) + self.apply_patches([(mod, 'invoke_agent', _invoke_agent), + (mod, 'write_files', _write_files), + (mod, 'wait_for_files', _wait_for_files), + (mod, 'pubkeys_from_crt_files', + _pubkeys_from_crt_files)]) dsrc = mod.DataSourceAzureNet( data.get('sys_cfg', {}), distro=None, paths=self.paths) @@ -209,7 +220,7 @@ class TestReadAzureOvf(MockerTestCase): mypklist = [{'fingerprint': 'fp1', 'path': 'path1'}] pubkeys = [(x['fingerprint'], x['path']) for x in mypklist] content = construct_valid_ovf_env(pubkeys=pubkeys) - (md, ud, cfg) = DataSourceAzure.read_azure_ovf(content) + (_md, _ud, cfg) = DataSourceAzure.read_azure_ovf(content) for mypk in mypklist: self.assertIn(mypk, cfg['_pubkeys']) -- cgit v1.2.3 From 299af1d4a3e09ad9c961cb641e62a20ab5998640 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 11 Jul 2013 10:15:58 -0400 Subject: test_builtin_handlers.py: fix pylint --- tests/unittests/test_builtin_handlers.py | 1 - 1 file changed, 1 deletion(-) (limited to 'tests') diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index e25a5144..b387f13b 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -1,7 +1,6 @@ """Tests of the built-in user data handlers.""" import os -import unittest from tests.unittests import helpers as test_helpers -- cgit v1.2.3 From 8f70bb7e7144f2225b4e9a589d16ae6d15992a3d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 17 Jul 2013 13:36:32 -0400 Subject: Azure: make /var/lib/waagent with 0700 perms The walinux agent expects that the files it writes with 0644 (default umask) permissions are not globally readable. Since we were creating the directory for it, and using default umaks (0755), the files inside were readable to non-priviledged users. --- cloudinit/sources/DataSourceAzure.py | 7 +++++-- tests/unittests/test_datasource/test_azure.py | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index f1419296..c90d7b07 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -104,7 +104,9 @@ class DataSourceAzureNet(sources.DataSource): if value is not None: mycfg[name] = value - write_files(mycfg['datadir'], files) + # walinux agent writes files world readable, but expects + # the directory to be protected. + write_files(mycfg['datadir'], files, dirmode=0700) try: invoke_agent(mycfg['cmd']) @@ -171,11 +173,12 @@ def wait_for_files(flist, maxwait=60, naplen=.5): return need -def write_files(datadir, files): +def write_files(datadir, files, dirmode=None): if not datadir: return if not files: files = {} + util.ensure_dir(datadir, dirmode) for (name, content) in files.items(): util.write_file(filename=os.path.join(datadir, name), content=content, mode=0600) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 74ed7197..c79c25d8 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -84,9 +84,10 @@ class TestAzureDataSource(MockerTestCase): def _invoke_agent(cmd): data['agent_invoked'] = cmd - def _write_files(datadir, files): + def _write_files(datadir, files, dirmode): data['files'] = {} data['datadir'] = datadir + data['datadir_mode'] = dirmode for (fname, content) in files.items(): data['files'][fname] = content @@ -129,6 +130,7 @@ class TestAzureDataSource(MockerTestCase): self.assertEqual(dsrc.userdata_raw, "") self.assertEqual(dsrc.metadata['local-hostname'], odata['HostName']) self.assertTrue('ovf-env.xml' in data['files']) + self.assertEqual(0700, data['datadir_mode']) def test_user_cfg_set_agent_command(self): cfg = {'agent_command': "my_command"} -- cgit v1.2.3