From dd9499d0daa862bcf2ce11e2949134df22f07c84 Mon Sep 17 00:00:00 2001 From: Jordan Evans Date: Tue, 24 Jun 2014 12:33:12 -0700 Subject: Add wants for sshd to make sure there isn't a race condition --- systemd/cloud-init.service | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/systemd/cloud-init.service b/systemd/cloud-init.service index 018a1fa8..d12852d8 100644 --- a/systemd/cloud-init.service +++ b/systemd/cloud-init.service @@ -1,8 +1,9 @@ [Unit] Description=Initial cloud-init job (metadata service crawler) After=local-fs.target network.target cloud-init-local.service +Before=sshd.service Requires=network.target -Wants=local-fs.target cloud-init-local.service +Wants=local-fs.target cloud-init-local.service sshd.service [Service] Type=oneshot -- cgit v1.2.3 From a9f7e5cfa5bd2ce692178c3784cc08271276dcac Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Wed, 2 Jul 2014 11:20:54 -0600 Subject: Fix for cc_grub_dpkg ignoring /dev/xvda devices as candidates (LP: 1336855). --- cloudinit/config/cc_grub_dpkg.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index b3ce6fb6..d96e85cf 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -46,7 +46,8 @@ def handle(_name, cfg, _cloud, log, _args): idevs_empty = "false" if idevs is None: idevs = "/dev/sda" - for dev in ("/dev/sda", "/dev/vda", "/dev/sda1", "/dev/vda1"): + for dev in ("/dev/sda", "/dev/vda", "/dev/xvda", + "/dev/sda1", "/dev/vda1", "/dev/xvda1"): if os.path.exists(dev): idevs = dev break -- cgit v1.2.3 From 31621d585d183e09b3d857ef2a16bfb699b3a591 Mon Sep 17 00:00:00 2001 From: Jordan Evans Date: Tue, 15 Jul 2014 09:27:21 -0700 Subject: run before sshd-keygen.service --- systemd/cloud-init.service | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/systemd/cloud-init.service b/systemd/cloud-init.service index d12852d8..6b0c7229 100644 --- a/systemd/cloud-init.service +++ b/systemd/cloud-init.service @@ -1,9 +1,9 @@ [Unit] Description=Initial cloud-init job (metadata service crawler) After=local-fs.target network.target cloud-init-local.service -Before=sshd.service +Before=sshd.service sshd-keygen.service Requires=network.target -Wants=local-fs.target cloud-init-local.service sshd.service +Wants=local-fs.target cloud-init-local.service sshd.service sshd-keygen.service [Service] Type=oneshot -- cgit v1.2.3 From 0ae8b2f5d1fda34f1efa50de8defd127a7907576 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Fri, 8 Aug 2014 19:18:42 +0000 Subject: merge: These are the changes from the freebsd-static-networking branch. --- cloudinit/distros/freebsd.py | 83 +++++++++++++++++++++++--- tests/unittests/test_distros/test_netconfig.py | 36 +++++++++++ 2 files changed, 111 insertions(+), 8 deletions(-) diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index d98f9578..1085185b 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -26,6 +26,9 @@ from cloudinit import log as logging from cloudinit import ssh_util from cloudinit import util +from cloudinit.distros import net_util +from cloudinit.distros.parsers.resolv_conf import ResolvConf + LOG = logging.getLogger(__name__) @@ -33,6 +36,7 @@ class Distro(distros.Distro): rc_conf_fn = "/etc/rc.conf" login_conf_fn = '/etc/login.conf' login_conf_fn_bak = '/etc/login.conf.orig' + resolv_conf_fn = '/etc/resolv.conf' def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) @@ -44,30 +48,34 @@ class Distro(distros.Distro): # Updates a key in /etc/rc.conf. def updatercconf(self, key, value): - LOG.debug("updatercconf: %s => %s", key, value) + LOG.debug("Checking %s for: %s = %s", self.rc_conf_fn, key, value) conf = self.loadrcconf() config_changed = False for item in conf: if item == key and conf[item] != value: conf[item] = value - LOG.debug("[rc.conf]: Value %s for key %s needs to be changed", - value, key) + LOG.debug("Changing key in %s: %s = %s", self.rc_conf_fn, key, + value) config_changed = True if config_changed: - LOG.debug("Writing new %s file", self.rc_conf_fn) + LOG.info("Writing %s", self.rc_conf_fn) buf = StringIO() for keyval in conf.items(): - buf.write("%s=%s\n" % keyval) + buf.write('%s="%s"\n' % keyval) util.write_file(self.rc_conf_fn, buf.getvalue()) - # Load the contents of /etc/rc.conf and store all keys in a dict. + # Load the contents of /etc/rc.conf and store all keys in a dict. Make sure + # quotes are ignored: + # hostname="bla" def loadrcconf(self): conf = {} lines = util.load_file(self.rc_conf_fn).splitlines() for line in lines: tok = line.split('=') - conf[tok[0]] = tok[1].rstrip() + key = tok[0] + val = re.sub(r'^"|"$', '', tok[1].rstrip()) + conf[key] = val return conf def readrcconf(self, key): @@ -218,7 +226,66 @@ class Distro(distros.Distro): ssh_util.setup_user_keys(keys, name, options=None) def _write_network(self, settings): - return + entries = net_util.translate_network(settings) + LOG.debug("Translated network settings") + LOG.debug("\n========== UBUNTU FORMAT START ==========") + LOG.debug("%s\n========== UBUNTU FORMAT END ==========", settings) + LOG.debug("\n========== GENERIC FORMAT START ==========") + LOG.debug("%s\n========== GENERIC FORMAT END ==========", entries) + + nameservers = [] + searchdomains = [] + dev_names = entries.keys() + for (dev, info) in entries.iteritems(): + # Skip the loopback interface. + if dev == 'lo0': + continue + + LOG.info('Configuring interface %s', dev) + + if info.get('bootproto') == 'static': + LOG.debug('Configuring dev %s with %s / %s', dev, info.get('address'), info.get('netmask')) + # Configure an ipv4 address. + ifconfig = info.get('address') + ' netmask ' + info.get('netmask') + + # Configure the gateway. + self.updatercconf('defaultrouter', info.get('gateway')) + + if 'dns-nameservers' in info: + nameservers.extend(info['dns-nameservers']) + if 'dns-search' in info: + searchservers.extend(info['dns-search']) + else: + ifconfig = 'DHCP' + + self.updatercconf('ifconfig_' + dev, ifconfig) + + # Try to read the /etc/resolv.conf or just start from scratch if that + # fails. + try: + resolvconf = ResolvConf(util.load_file(self.resolv_conf_fn)) + resolvconf.parse() + except IOError: + util.logexc(LOG, "Failed to parse %s, use new empty file", self.resolv_conf_fn) + resolvconf = ResolvConf('') + resolvconf.parse() + + # Add some nameservers + for server in nameservers: + try: + resolvconf.add_nameserver(server) + except ValueError: + util.logexc(LOG, "Failed to add nameserver %s", server) + + # And add any searchdomains. + for domain in searchdomains: + try: + resolvconf.add_search_domain(domain) + except ValueError: + util.logexc(LOG, "Failed to add search domain %s", domain) + util.write_file(self.resolv_conf_fn, str(resolvconf), 0644) + + return dev_names def apply_locale(self, locale, out_fn=None): # Adjust the locals value to the new value diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 9763b14b..96379025 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -173,3 +173,39 @@ NETWORKING=yes ''' self.assertCfgEquals(expected_buf, str(write_buf)) self.assertEquals(write_buf.mode, 0644) + + def test_simple_write_freebsd(self): + fbsd_distro = self._get_distro('freebsd') + util_mock = self.mocker.replace(util.write_file, + spec=False, passthrough=False) + exists_mock = self.mocker.replace(os.path.isfile, + spec=False, passthrough=False) + + exists_mock(mocker.ARGS) + self.mocker.count(0, None) + self.mocker.result(False) + + write_bufs = {} + + def replace_write(filename, content, mode=0644, omode="wb"): + buf = WriteBuffer() + buf.mode = mode + buf.omode = omode + buf.write(content) + write_bufs[filename] = buf + + util_mock(mocker.ARGS) + self.mocker.call(replace_write) + self.mocker.replay() + fbsd_distro.apply_network(BASE_NET_CFG, False) + + self.assertIn('/etc/rc.conf', write_bufs) + write_buf = write_bufs['/etc/rc.conf'] + expected_buf = ''' +ifconfig_eth0="192.168.1.5 netmask 255.255.255.0" +ifconfig_eth1="DHCP" +defaultrouter="192.168.1.254" +''' + self.assertCfgEquals(expected_buf, str(write_buf)) + self.assertEquals(write_buf.mode, 0644) + -- cgit v1.2.3 From 019c90f07061adeda54173ea9afd7752cd11cd90 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Fri, 8 Aug 2014 19:23:56 +0000 Subject: fix: Skip lines from /etc/rc.conf not matching the pattern key=value. --- cloudinit/distros/freebsd.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index 1085185b..b4d841f8 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -72,6 +72,11 @@ class Distro(distros.Distro): conf = {} lines = util.load_file(self.rc_conf_fn).splitlines() for line in lines: + if not re.match(r'^(.+)=(.+)', line): + LOG.debug("Skipping line from /etc/rc.conf: %s", line) + continue + + # TODO: just use the matches please... tok = line.split('=') key = tok[0] val = re.sub(r'^"|"$', '', tok[1].rstrip()) -- cgit v1.2.3 From c6ca246c3fe44cb21c068ff9fe5fb134c2230ebb Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Fri, 8 Aug 2014 20:44:48 +0000 Subject: new: Config for FreeBSD. This doesn't differ much from the regular (linux) config, but currently it helps while testing and setting up fbsd cloud instances. --- config/cloud.freebsd.cfg | 107 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 config/cloud.freebsd.cfg diff --git a/config/cloud.freebsd.cfg b/config/cloud.freebsd.cfg new file mode 100644 index 00000000..bcf5adc7 --- /dev/null +++ b/config/cloud.freebsd.cfg @@ -0,0 +1,107 @@ +# The top level settings are used as module +# and system configuration. + +syslog_fix_perms: root:wheel +datasource_list: ['OpenStack'] + +# A set of users which may be applied and/or used by various modules +# when a 'default' entry is found it will reference the 'default_user' +# from the distro configuration specified below +users: + - default + +# If this is set, 'root' will not be able to ssh in and they +# will get a message to login instead as the above $user (ubuntu) +disable_root: false + +# This will cause the set+update hostname module to not operate (if true) +preserve_hostname: false + +# Example datasource config +# datasource: +# Ec2: +# metadata_urls: [ 'blah.com' ] +# timeout: 5 # (defaults to 50 seconds) +# max_wait: 10 # (defaults to 120 seconds) + +# The modules that run in the 'init' stage +cloud_init_modules: + - migrator + - seed_random + - bootcmd + - write-files + - growpart + - resizefs + - set_hostname + - update_hostname + - update_etc_hosts + - ca-certs + - rsyslog + - users-groups + - ssh + +# The modules that run in the 'config' stage +cloud_config_modules: + - disk_setup + - mounts + - ssh-import-id + - locale + - set-passwords + - package-update-upgrade-install + - landscape + - timezone + - puppet + - chef + - salt-minion + - mcollective + - disable-ec2-metadata + - runcmd + - byobu + +# The modules that run in the 'final' stage +cloud_final_modules: + - rightscale_userdata + - scripts-vendor + - scripts-per-once + - scripts-per-boot + - scripts-per-instance + - scripts-user + - ssh-authkey-fingerprints + - keys-to-console + - phone-home + - final-message + - power-state-change + +# System and/or distro specific settings +# (not accessible to handlers/transforms) +system_info: + # This will affect which distro class gets used + distro: freebsd + # Default user name + that default users groups (if added/used) + default_user: + name: beastie + lock_passwd: True + gecos: FreeBSD + groups: [wheel] + sudo: ["ALL=(ALL) NOPASSWD:ALL"] + shell: /bin/sh + # Other config here will be given to the distro class and/or path classes + paths: + cloud_dir: /var/lib/cloud/ + templates_dir: /etc/cloud/templates/ + upstart_dir: /etc/init/ + package_mirrors: + - arches: [i386, amd64] + failsafe: + primary: http://archive.ubuntu.com/ubuntu + security: http://security.ubuntu.com/ubuntu + search: + primary: + - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/ + - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/ + security: [] + - arches: [armhf, armel, default] + failsafe: + primary: http://ports.ubuntu.com/ubuntu-ports + security: http://ports.ubuntu.com/ubuntu-ports + ssh_svcname: ssh -- cgit v1.2.3 From ca1e2e17e5ae4af811826518f46631c9962f8292 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 9 Aug 2014 08:09:02 +0000 Subject: fix: The correct path end with an s. --- sysvinit/freebsd/cloudconfig | 6 +++--- sysvinit/freebsd/cloudfinal | 6 +++--- sysvinit/freebsd/cloudinit | 6 +++--- sysvinit/freebsd/cloudinitlocal | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sysvinit/freebsd/cloudconfig b/sysvinit/freebsd/cloudconfig index 15d7ab95..73877b6f 100755 --- a/sysvinit/freebsd/cloudconfig +++ b/sysvinit/freebsd/cloudconfig @@ -18,9 +18,9 @@ start_cmd="cloudconfig_start" cloudinit_override() { - # If there exist sysconfig/default variable override files use it... - if [ -f /etc/default/cloud-init ]; then - . /etc/default/cloud-init + # If there exist sysconfig/defaults variable override files use it... + if [ -f /etc/defaults/cloud-init ]; then + . /etc/defaults/cloud-init fi } diff --git a/sysvinit/freebsd/cloudfinal b/sysvinit/freebsd/cloudfinal index 49945ecd..292dd8fb 100755 --- a/sysvinit/freebsd/cloudfinal +++ b/sysvinit/freebsd/cloudfinal @@ -18,9 +18,9 @@ start_cmd="cloudfinal_start" cloudinit_override() { - # If there exist sysconfig/default variable override files use it... - if [ -f /etc/default/cloud-init ]; then - . /etc/default/cloud-init + # If there exist sysconfig/defaults variable override files use it... + if [ -f /etc/defaults/cloud-init ]; then + . /etc/defaults/cloud-init fi } diff --git a/sysvinit/freebsd/cloudinit b/sysvinit/freebsd/cloudinit index 8d5ff10e..23a1247b 100755 --- a/sysvinit/freebsd/cloudinit +++ b/sysvinit/freebsd/cloudinit @@ -18,9 +18,9 @@ start_cmd="cloudinit_start" cloudinit_override() { - # If there exist sysconfig/default variable override files use it... - if [ -f /etc/default/cloud-init ]; then - . /etc/default/cloud-init + # If there exist sysconfig/defaults variable override files use it... + if [ -f /etc/defaults/cloud-init ]; then + . /etc/defaults/cloud-init fi } diff --git a/sysvinit/freebsd/cloudinitlocal b/sysvinit/freebsd/cloudinitlocal index b55705c0..b96d1513 100755 --- a/sysvinit/freebsd/cloudinitlocal +++ b/sysvinit/freebsd/cloudinitlocal @@ -18,9 +18,9 @@ start_cmd="cloudlocal_start" cloudinit_override() { - # If there exist sysconfig/default variable override files use it... - if [ -f /etc/default/cloud-init ]; then - . /etc/default/cloud-init + # If there exist sysconfig/defaults variable override files use it... + if [ -f /etc/defaults/cloud-init ]; then + . /etc/defaults/cloud-init fi } -- cgit v1.2.3 From 9e10017821eb6f197aebd89545444cdb7bea5056 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 9 Aug 2014 08:12:35 +0000 Subject: fix: To install the new freebsd sysvinit scripts, accept a new sysvinit_freebsd argument. Specifying sysvinit will install the RH scripts, which is wrong. --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 556103b9..7b061824 100755 --- a/setup.py +++ b/setup.py @@ -63,12 +63,14 @@ def systemd_unitdir(): INITSYS_FILES = { 'sysvinit': [f for f in glob('sysvinit/redhat/*') if is_f(f)], + 'sysvinit_freebsd': [f for f in glob('sysvinit/freebsd/*') if is_f(f)], 'sysvinit_deb': [f for f in glob('sysvinit/debian/*') if is_f(f)], 'systemd': [f for f in glob('systemd/*') if is_f(f)], 'upstart': [f for f in glob('upstart/*') if is_f(f)], } INITSYS_ROOTS = { 'sysvinit': '/etc/rc.d/init.d', + 'sysvinit_freebsd': '/usr/local/etc/rc.d', 'sysvinit_deb': '/etc/init.d', 'systemd': systemd_unitdir(), 'upstart': '/etc/init/', -- cgit v1.2.3 From f72be738ba5e8556aae3bccd765754ea01555389 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 9 Aug 2014 09:58:12 +0000 Subject: change: Cloud-init config should go under /usr/local since /etc/ is reserved for base. --- sysvinit/freebsd/cloudconfig | 2 +- sysvinit/freebsd/cloudfinal | 2 +- sysvinit/freebsd/cloudinit | 2 +- sysvinit/freebsd/cloudinitlocal | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sysvinit/freebsd/cloudconfig b/sysvinit/freebsd/cloudconfig index 73877b6f..0a058ddb 100755 --- a/sysvinit/freebsd/cloudconfig +++ b/sysvinit/freebsd/cloudconfig @@ -14,7 +14,7 @@ rcvar="cloudinit_enable" start_precmd="cloudinit_override" start_cmd="cloudconfig_start" -: ${cloudinit_config:="/etc/cloud/cloud.cfg"} +: ${cloudinit_config:="/usr/local/etc/cloud/cloud.cfg"} cloudinit_override() { diff --git a/sysvinit/freebsd/cloudfinal b/sysvinit/freebsd/cloudfinal index 292dd8fb..a97e65f9 100755 --- a/sysvinit/freebsd/cloudfinal +++ b/sysvinit/freebsd/cloudfinal @@ -14,7 +14,7 @@ rcvar="cloudinit_enable" start_precmd="cloudinit_override" start_cmd="cloudfinal_start" -: ${cloudinit_config:="/etc/cloud/cloud.cfg"} +: ${cloudinit_config:="/usr/local/etc/cloud/cloud.cfg"} cloudinit_override() { diff --git a/sysvinit/freebsd/cloudinit b/sysvinit/freebsd/cloudinit index 23a1247b..f751963b 100755 --- a/sysvinit/freebsd/cloudinit +++ b/sysvinit/freebsd/cloudinit @@ -14,7 +14,7 @@ rcvar="cloudinit_enable" start_precmd="cloudinit_override" start_cmd="cloudinit_start" -: ${cloudinit_config:="/etc/cloud/cloud.cfg"} +: ${cloudinit_config:="/usr/local/etc/cloud/cloud.cfg"} cloudinit_override() { diff --git a/sysvinit/freebsd/cloudinitlocal b/sysvinit/freebsd/cloudinitlocal index b96d1513..2b5df3c9 100755 --- a/sysvinit/freebsd/cloudinitlocal +++ b/sysvinit/freebsd/cloudinitlocal @@ -14,7 +14,7 @@ rcvar="cloudinit_enable" start_precmd="cloudinit_override" start_cmd="cloudlocal_start" -: ${cloudinit_config:="/etc/cloud/cloud.cfg"} +: ${cloudinit_config:="/usr/local/etc/cloud/cloud.cfg"} cloudinit_override() { -- cgit v1.2.3 From ffc621738b172c9805c083c265eed13b415bb3f4 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 9 Aug 2014 10:11:50 +0000 Subject: fix: Command (the binary) is cloud-init, not cloud_init. --- sysvinit/freebsd/cloudfinal | 2 +- sysvinit/freebsd/cloudinit | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sysvinit/freebsd/cloudfinal b/sysvinit/freebsd/cloudfinal index a97e65f9..f044f1f3 100755 --- a/sysvinit/freebsd/cloudfinal +++ b/sysvinit/freebsd/cloudfinal @@ -7,7 +7,7 @@ . /etc/rc.subr name="cloudfinal" -command="/usr/bin/cloud_init" +command="/usr/bin/cloud-init" start_cmd="cloudfinal_start" stop_cmd=":" rcvar="cloudinit_enable" diff --git a/sysvinit/freebsd/cloudinit b/sysvinit/freebsd/cloudinit index f751963b..571b57d0 100755 --- a/sysvinit/freebsd/cloudinit +++ b/sysvinit/freebsd/cloudinit @@ -7,7 +7,7 @@ . /etc/rc.subr name="cloudinit" -command="/usr/bin/cloud_init" +command="/usr/bin/cloud-init" start_cmd="cloudinit_start" stop_cmd=":" rcvar="cloudinit_enable" -- cgit v1.2.3 From 590685594897069fe57753f197753aab5cb440e9 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 9 Aug 2014 10:14:41 +0000 Subject: change: C-i is installed in /usr/local/. --- sysvinit/freebsd/cloudconfig | 2 +- sysvinit/freebsd/cloudfinal | 2 +- sysvinit/freebsd/cloudinit | 2 +- sysvinit/freebsd/cloudinitlocal | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sysvinit/freebsd/cloudconfig b/sysvinit/freebsd/cloudconfig index 0a058ddb..b40257ab 100755 --- a/sysvinit/freebsd/cloudconfig +++ b/sysvinit/freebsd/cloudconfig @@ -7,7 +7,7 @@ . /etc/rc.subr name="cloudconfig" -command="/usr/bin/cloud-init" +command="/usr/local/bin/cloud-init" start_cmd="cloudconfig_start" stop_cmd=":" rcvar="cloudinit_enable" diff --git a/sysvinit/freebsd/cloudfinal b/sysvinit/freebsd/cloudfinal index f044f1f3..5f2c4155 100755 --- a/sysvinit/freebsd/cloudfinal +++ b/sysvinit/freebsd/cloudfinal @@ -7,7 +7,7 @@ . /etc/rc.subr name="cloudfinal" -command="/usr/bin/cloud-init" +command="/usr/local/bin/cloud-init" start_cmd="cloudfinal_start" stop_cmd=":" rcvar="cloudinit_enable" diff --git a/sysvinit/freebsd/cloudinit b/sysvinit/freebsd/cloudinit index 571b57d0..14b0257e 100755 --- a/sysvinit/freebsd/cloudinit +++ b/sysvinit/freebsd/cloudinit @@ -7,7 +7,7 @@ . /etc/rc.subr name="cloudinit" -command="/usr/bin/cloud-init" +command="/usr/local/bin/cloud-init" start_cmd="cloudinit_start" stop_cmd=":" rcvar="cloudinit_enable" diff --git a/sysvinit/freebsd/cloudinitlocal b/sysvinit/freebsd/cloudinitlocal index 2b5df3c9..fdab2a12 100755 --- a/sysvinit/freebsd/cloudinitlocal +++ b/sysvinit/freebsd/cloudinitlocal @@ -7,7 +7,7 @@ . /etc/rc.subr name="cloudinitlocal" -command="/usr/bin/cloud-init" +command="/usr/local/bin/cloud-init" start_cmd="cloudlocal_start" stop_cmd=":" rcvar="cloudinit_enable" -- cgit v1.2.3 From c08262be060f6a4e89c94f81f83d7e884b66a5cf Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 9 Aug 2014 10:24:34 +0000 Subject: fix: Use -f to properly load the configfile. --- sysvinit/freebsd/cloudconfig | 2 +- sysvinit/freebsd/cloudfinal | 2 +- sysvinit/freebsd/cloudinit | 2 +- sysvinit/freebsd/cloudinitlocal | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sysvinit/freebsd/cloudconfig b/sysvinit/freebsd/cloudconfig index b40257ab..985fb259 100755 --- a/sysvinit/freebsd/cloudconfig +++ b/sysvinit/freebsd/cloudconfig @@ -27,7 +27,7 @@ cloudinit_override() cloudconfig_start() { echo "${command} starting" - ${command} ${cloudinit_config} modules --mode config + ${command} -f ${cloudinit_config} modules --mode config } load_rc_config $name diff --git a/sysvinit/freebsd/cloudfinal b/sysvinit/freebsd/cloudfinal index 5f2c4155..dd02b081 100755 --- a/sysvinit/freebsd/cloudfinal +++ b/sysvinit/freebsd/cloudfinal @@ -27,7 +27,7 @@ cloudinit_override() cloudfinal_start() { echo -n "${command} starting" - ${command} ${cloudinit_config} modules --mode final + ${command} -f ${cloudinit_config} modules --mode final } load_rc_config $name diff --git a/sysvinit/freebsd/cloudinit b/sysvinit/freebsd/cloudinit index 14b0257e..17eaa547 100755 --- a/sysvinit/freebsd/cloudinit +++ b/sysvinit/freebsd/cloudinit @@ -27,7 +27,7 @@ cloudinit_override() cloudinit_start() { echo -n "${command} starting" - ${command} ${cloudinit_config} init + ${command} -f ${cloudinit_config} init } load_rc_config $name diff --git a/sysvinit/freebsd/cloudinitlocal b/sysvinit/freebsd/cloudinitlocal index fdab2a12..4b122ee5 100755 --- a/sysvinit/freebsd/cloudinitlocal +++ b/sysvinit/freebsd/cloudinitlocal @@ -27,7 +27,7 @@ cloudinit_override() cloudlocal_start() { echo -n "${command} starting" - ${command} ${cloudinit_config} init --local + ${command} -f ${cloudinit_config} init --local } load_rc_config $name -- cgit v1.2.3 From d2d12dda2ddd576504adc32f71944eab72d6481a Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 9 Aug 2014 10:27:28 +0000 Subject: new: Buildscript for installing on FreeBSD. This should do until a proper port is created. --- tools/build-on-freebsd | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100755 tools/build-on-freebsd diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd new file mode 100755 index 00000000..03cba7fc --- /dev/null +++ b/tools/build-on-freebsd @@ -0,0 +1,40 @@ +#!/bin/sh +# Since there is no official FreeBSD port yet, we need some way of building and +# installing cloud-init. This script takes care of building and installing. It +# will optionally make a first run at the end. + +# Check dependencies: +[ ! -f /tmp/c-i.dependencieschecked ] && pkg install py27-cheetah py27-Jinja2 py27-prettytable py27-oauth py27-serial py27-configobj py27-yaml py27-argparse py27-requests py27-six +touch /tmp/c-i.dependencieschecked + +# Required but unavailable port/pkg: py27-jsonpatch +# Luckily, the install step will take care of this by installing it from pypi... + +# Build the code and install in /usr/local/: +python setup.py build +python setup.py install -O1 --skip-build --prefix /usr/local/ --init-system sysvinit_freebsd + +# Move the configdir to /usr/local/ and use freebsd.cfg: +[ -d /usr/local/etc/cloud ] && rm -rf /usr/local/etc/cloud +mv /etc/cloud /usr/local/etc/ +mv /usr/local/etc/cloud/cloud.freebsd.cfg /usr/local/etc/cloud/cloud.cfg + +# Enable cloud-init in /etc/rc.conf: +sed -i.bak -e "/cloudinit_enable=.*/d" /etc/rc.conf +echo 'cloudinit_enable="YES"' >> /etc/rc.conf + +echo "Installation completed." + +if [ "$1" = "run" ] +then + echo "Ok, now let's see if it works." + + # Remove old metadata + rm -rf /var/lib/cloud + + # Just log everything, quick&dirty + rm /usr/local/etc/cloud/cloud.cfg.d/05_logging.cfg + + # Start: + /usr/local/etc/rc.d/cloudinit start +fi -- cgit v1.2.3 From 38c9c67ea18fefbb83aaf5c9151858c4488e2e24 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 9 Aug 2014 10:51:04 +0000 Subject: change: Add an important comment. --- config/cloud.freebsd.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/cloud.freebsd.cfg b/config/cloud.freebsd.cfg index bcf5adc7..40e2374a 100644 --- a/config/cloud.freebsd.cfg +++ b/config/cloud.freebsd.cfg @@ -2,6 +2,9 @@ # and system configuration. syslog_fix_perms: root:wheel + +# This should not be required, but leave it in place until the real cause of +# not beeing able to find -any- datasources is resolved. datasource_list: ['OpenStack'] # A set of users which may be applied and/or used by various modules -- cgit v1.2.3 From aff5c13ac9e3723e99d7414ac82f30c853f453fd Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 9 Aug 2014 10:56:13 +0000 Subject: fix: More dependencies. --- tools/build-on-freebsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index 03cba7fc..52bceb21 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -4,7 +4,7 @@ # will optionally make a first run at the end. # Check dependencies: -[ ! -f /tmp/c-i.dependencieschecked ] && pkg install py27-cheetah py27-Jinja2 py27-prettytable py27-oauth py27-serial py27-configobj py27-yaml py27-argparse py27-requests py27-six +[ ! -f /tmp/c-i.dependencieschecked ] && pkg install py27-cheetah py27-Jinja2 py27-prettytable py27-oauth py27-serial py27-configobj py27-yaml py27-argparse py27-requests py27-six py27-boto gpart sudo touch /tmp/c-i.dependencieschecked # Required but unavailable port/pkg: py27-jsonpatch -- cgit v1.2.3 From 5d96928ee2a2573aaf49727adafc91120daa1d36 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 9 Aug 2014 10:57:28 +0000 Subject: new: Some datasources (like Smartos) use dmidecode to gather some specific information, so install it. --- tools/build-on-freebsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index 52bceb21..ff59bd25 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -4,7 +4,7 @@ # will optionally make a first run at the end. # Check dependencies: -[ ! -f /tmp/c-i.dependencieschecked ] && pkg install py27-cheetah py27-Jinja2 py27-prettytable py27-oauth py27-serial py27-configobj py27-yaml py27-argparse py27-requests py27-six py27-boto gpart sudo +[ ! -f /tmp/c-i.dependencieschecked ] && pkg install py27-cheetah py27-Jinja2 py27-prettytable py27-oauth py27-serial py27-configobj py27-yaml py27-argparse py27-requests py27-six py27-boto gpart sudo dmidecode touch /tmp/c-i.dependencieschecked # Required but unavailable port/pkg: py27-jsonpatch -- cgit v1.2.3 From 33826c4e1b6cbb10a27633a5dd9627fcb88808b4 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 9 Aug 2014 11:34:46 +0000 Subject: fix: Pass -y to growfs to keep it from asking for confirmation. --- cloudinit/config/cc_resizefs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index be406034..e290efe0 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -41,7 +41,7 @@ def _resize_xfs(mount_point, devpth): # pylint: disable=W0613 def _resize_ufs(mount_point, devpth): # pylint: disable=W0613 - return ('growfs', devpth) + return ('growfs', '-y', devpth) # Do not use a dictionary as these commands should be able to be used # for multiple filesystem types if possible, e.g. one command for -- cgit v1.2.3 From f1202c1b0aeae7378589215887c74f325e6c77cd Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 9 Aug 2014 21:24:19 +0000 Subject: change: Create a symlink to python2.7, to make sure running plain python works. --- tools/build-on-freebsd | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index ff59bd25..0a545207 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -3,6 +3,9 @@ # installing cloud-init. This script takes care of building and installing. It # will optionally make a first run at the end. +# Since there is no python by default, create a symlink for convenience sake: +ln -sf /usr/local/bin/python2.7 /usr/local/bin/python + # Check dependencies: [ ! -f /tmp/c-i.dependencieschecked ] && pkg install py27-cheetah py27-Jinja2 py27-prettytable py27-oauth py27-serial py27-configobj py27-yaml py27-argparse py27-requests py27-six py27-boto gpart sudo dmidecode touch /tmp/c-i.dependencieschecked -- cgit v1.2.3 From 7e0407491345989a2640149a9a365b6f7167c7af Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 9 Aug 2014 21:25:54 +0000 Subject: change: Make note of another python lib pypi takes care of. --- tools/build-on-freebsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index 0a545207..da33e4b0 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -10,7 +10,7 @@ ln -sf /usr/local/bin/python2.7 /usr/local/bin/python [ ! -f /tmp/c-i.dependencieschecked ] && pkg install py27-cheetah py27-Jinja2 py27-prettytable py27-oauth py27-serial py27-configobj py27-yaml py27-argparse py27-requests py27-six py27-boto gpart sudo dmidecode touch /tmp/c-i.dependencieschecked -# Required but unavailable port/pkg: py27-jsonpatch +# Required but unavailable port/pkg: py27-jsonpatch py27-jsonpointer # Luckily, the install step will take care of this by installing it from pypi... # Build the code and install in /usr/local/: -- cgit v1.2.3 From 891496973520c91fd02ccfc55aa606234418accb Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sun, 10 Aug 2014 09:41:42 +0000 Subject: change: Save and restore the SSH keys between builds. --- tools/build-on-freebsd | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index da33e4b0..7d19f44c 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -32,6 +32,9 @@ if [ "$1" = "run" ] then echo "Ok, now let's see if it works." + # Backup SSH keys + mv /etc/ssh/ssh_host_* /tmp/ + # Remove old metadata rm -rf /var/lib/cloud @@ -40,4 +43,7 @@ then # Start: /usr/local/etc/rc.d/cloudinit start + + # Restore SSH keys + mv /tmp/ssh_host_* /etc/ssh/ fi -- cgit v1.2.3 From de0e832b51624603664eab189a083e612554125a Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sun, 10 Aug 2014 11:38:22 +0000 Subject: fix: Set the environment var CLOUD_CFG to specify the location of cloud.cfg since -f is not used for that. Given the importance of this file/location, it's explicitly beeing set in the initscripts instead of trusting on something in /etc/defaults. --- sysvinit/freebsd/cloudconfig | 6 +++--- sysvinit/freebsd/cloudfinal | 6 +++--- sysvinit/freebsd/cloudinit | 6 +++--- sysvinit/freebsd/cloudinitlocal | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sysvinit/freebsd/cloudconfig b/sysvinit/freebsd/cloudconfig index 985fb259..44c216b3 100755 --- a/sysvinit/freebsd/cloudconfig +++ b/sysvinit/freebsd/cloudconfig @@ -6,6 +6,8 @@ . /etc/rc.subr +export CLOUD_CFG=/usr/local/etc/cloud/cloud.cfg + name="cloudconfig" command="/usr/local/bin/cloud-init" start_cmd="cloudconfig_start" @@ -14,8 +16,6 @@ rcvar="cloudinit_enable" start_precmd="cloudinit_override" start_cmd="cloudconfig_start" -: ${cloudinit_config:="/usr/local/etc/cloud/cloud.cfg"} - cloudinit_override() { # If there exist sysconfig/defaults variable override files use it... @@ -27,7 +27,7 @@ cloudinit_override() cloudconfig_start() { echo "${command} starting" - ${command} -f ${cloudinit_config} modules --mode config + ${command} modules --mode config } load_rc_config $name diff --git a/sysvinit/freebsd/cloudfinal b/sysvinit/freebsd/cloudfinal index dd02b081..f668e036 100755 --- a/sysvinit/freebsd/cloudfinal +++ b/sysvinit/freebsd/cloudfinal @@ -6,6 +6,8 @@ . /etc/rc.subr +export CLOUD_CFG=/usr/local/etc/cloud/cloud.cfg + name="cloudfinal" command="/usr/local/bin/cloud-init" start_cmd="cloudfinal_start" @@ -14,8 +16,6 @@ rcvar="cloudinit_enable" start_precmd="cloudinit_override" start_cmd="cloudfinal_start" -: ${cloudinit_config:="/usr/local/etc/cloud/cloud.cfg"} - cloudinit_override() { # If there exist sysconfig/defaults variable override files use it... @@ -27,7 +27,7 @@ cloudinit_override() cloudfinal_start() { echo -n "${command} starting" - ${command} -f ${cloudinit_config} modules --mode final + ${command} modules --mode final } load_rc_config $name diff --git a/sysvinit/freebsd/cloudinit b/sysvinit/freebsd/cloudinit index 17eaa547..c5478678 100755 --- a/sysvinit/freebsd/cloudinit +++ b/sysvinit/freebsd/cloudinit @@ -6,6 +6,8 @@ . /etc/rc.subr +export CLOUD_CFG=/usr/local/etc/cloud/cloud.cfg + name="cloudinit" command="/usr/local/bin/cloud-init" start_cmd="cloudinit_start" @@ -14,8 +16,6 @@ rcvar="cloudinit_enable" start_precmd="cloudinit_override" start_cmd="cloudinit_start" -: ${cloudinit_config:="/usr/local/etc/cloud/cloud.cfg"} - cloudinit_override() { # If there exist sysconfig/defaults variable override files use it... @@ -27,7 +27,7 @@ cloudinit_override() cloudinit_start() { echo -n "${command} starting" - ${command} -f ${cloudinit_config} init + ${command} init } load_rc_config $name diff --git a/sysvinit/freebsd/cloudinitlocal b/sysvinit/freebsd/cloudinitlocal index 4b122ee5..c340d5d0 100755 --- a/sysvinit/freebsd/cloudinitlocal +++ b/sysvinit/freebsd/cloudinitlocal @@ -6,6 +6,8 @@ . /etc/rc.subr +export CLOUD_CFG=/usr/local/etc/cloud/cloud.cfg + name="cloudinitlocal" command="/usr/local/bin/cloud-init" start_cmd="cloudlocal_start" @@ -14,8 +16,6 @@ rcvar="cloudinit_enable" start_precmd="cloudinit_override" start_cmd="cloudlocal_start" -: ${cloudinit_config:="/usr/local/etc/cloud/cloud.cfg"} - cloudinit_override() { # If there exist sysconfig/defaults variable override files use it... @@ -27,7 +27,7 @@ cloudinit_override() cloudlocal_start() { echo -n "${command} starting" - ${command} -f ${cloudinit_config} init --local + ${command} init --local } load_rc_config $name -- cgit v1.2.3 From 833ebcba6ca333183284d1f9a0fe5f53df802712 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sun, 10 Aug 2014 11:50:19 +0000 Subject: change: Install everything in the right location on both Linux (which ofcourse already was good) and FreeBSD (which realy likes /usr/local for this). --- setup.py | 51 ++++++++++++++++++++++++++++++++++++-------------- tools/build-on-freebsd | 4 +--- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/setup.py b/setup.py index 7b061824..b09c0456 100755 --- a/setup.py +++ b/setup.py @@ -90,6 +90,42 @@ def read_requires(): return str(deps).splitlines() +# Install everything in the right location and take care of Linux (default) and +# FreeBSD systems. +def read_datafiles(): + sysname = os.uname()[0] + if sysname == 'FreeBSD': + return [ + ('/usr/local/etc/cloud', glob('config/*.cfg')), + ('/usr/local/etc/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), + ('/usr/local/etc/cloud/templates', glob('templates/*')), + ('/usr/local/share/cloud-init', []), + ('/usr/local/lib/cloud-init', + ['tools/uncloud-init', 'tools/write-ssh-key-fingerprints']), + ('/usr/local/share/doc/cloud-init', + [f for f in glob('doc/*') if is_f(f)]), + ('/usr/local/share/doc/cloud-init/examples', + [f for f in glob('doc/examples/*') if is_f(f)]), + ('/usr/local/share/doc/cloud-init/examples/seed', + [f for f in glob('doc/examples/seed/*') if is_f(f)]), + ] + else: + return [ + ('/etc/cloud', glob('config/*.cfg')), + ('/etc/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), + ('/etc/cloud/templates', glob('templates/*')), + ('/usr/share/cloud-init', []), + ('/usr/lib/cloud-init', + ['tools/uncloud-init', 'tools/write-ssh-key-fingerprints']), + ('/usr/share/doc/cloud-init', + [f for f in glob('doc/*') if is_f(f)]), + ('/usr/share/doc/cloud-init/examples', + [f for f in glob('doc/examples/*') if is_f(f)]), + ('/usr/share/doc/cloud-init/examples/seed', + [f for f in glob('doc/examples/seed/*') if is_f(f)]), + ] + + # TODO: Is there a better way to do this?? class InitsysInstallData(install): init_system = None @@ -138,20 +174,7 @@ setuptools.setup(name='cloud-init', 'tools/cloud-init-per', ], license='GPLv3', - data_files=[('/etc/cloud', glob('config/*.cfg')), - ('/etc/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), - ('/etc/cloud/templates', glob('templates/*')), - ('/usr/share/cloud-init', []), - ('/usr/lib/cloud-init', - ['tools/uncloud-init', - 'tools/write-ssh-key-fingerprints']), - ('/usr/share/doc/cloud-init', - [f for f in glob('doc/*') if is_f(f)]), - ('/usr/share/doc/cloud-init/examples', - [f for f in glob('doc/examples/*') if is_f(f)]), - ('/usr/share/doc/cloud-init/examples/seed', - [f for f in glob('doc/examples/seed/*') if is_f(f)]), - ], + data_files=read_datafiles(), install_requires=read_requires(), cmdclass={ # Use a subclass for install that handles diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index 7d19f44c..6e6ce8b6 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -17,9 +17,7 @@ touch /tmp/c-i.dependencieschecked python setup.py build python setup.py install -O1 --skip-build --prefix /usr/local/ --init-system sysvinit_freebsd -# Move the configdir to /usr/local/ and use freebsd.cfg: -[ -d /usr/local/etc/cloud ] && rm -rf /usr/local/etc/cloud -mv /etc/cloud /usr/local/etc/ +# Use the correct config file: mv /usr/local/etc/cloud/cloud.freebsd.cfg /usr/local/etc/cloud/cloud.cfg # Enable cloud-init in /etc/rc.conf: -- cgit v1.2.3 From c36d75552221e165f84493c5105318c088b44514 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sun, 10 Aug 2014 11:54:28 +0000 Subject: change: Cancel execution of several modules that are definately not tested or supported yet. --- config/cloud.freebsd.cfg | 56 +++++++++++++++--------------------------------- 1 file changed, 17 insertions(+), 39 deletions(-) diff --git a/config/cloud.freebsd.cfg b/config/cloud.freebsd.cfg index 40e2374a..2d65aac4 100644 --- a/config/cloud.freebsd.cfg +++ b/config/cloud.freebsd.cfg @@ -29,37 +29,37 @@ preserve_hostname: false # The modules that run in the 'init' stage cloud_init_modules: - - migrator +# - migrator - seed_random - bootcmd - - write-files +# - write-files - growpart - resizefs - set_hostname - update_hostname - - update_etc_hosts - - ca-certs - - rsyslog +# - update_etc_hosts +# - ca-certs +# - rsyslog - users-groups - - ssh +# - ssh # The modules that run in the 'config' stage cloud_config_modules: - - disk_setup - - mounts +# - disk_setup +# - mounts - ssh-import-id - locale - - set-passwords - - package-update-upgrade-install - - landscape - - timezone - - puppet - - chef - - salt-minion - - mcollective +# - set-passwords +# - package-update-upgrade-install +# - landscape +# - timezone +# - puppet +# - chef +# - salt-minion +# - mcollective - disable-ec2-metadata - runcmd - - byobu +# - byobu # The modules that run in the 'final' stage cloud_final_modules: @@ -78,9 +78,7 @@ cloud_final_modules: # System and/or distro specific settings # (not accessible to handlers/transforms) system_info: - # This will affect which distro class gets used distro: freebsd - # Default user name + that default users groups (if added/used) default_user: name: beastie lock_passwd: True @@ -88,23 +86,3 @@ system_info: groups: [wheel] sudo: ["ALL=(ALL) NOPASSWD:ALL"] shell: /bin/sh - # Other config here will be given to the distro class and/or path classes - paths: - cloud_dir: /var/lib/cloud/ - templates_dir: /etc/cloud/templates/ - upstart_dir: /etc/init/ - package_mirrors: - - arches: [i386, amd64] - failsafe: - primary: http://archive.ubuntu.com/ubuntu - security: http://security.ubuntu.com/ubuntu - search: - primary: - - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/ - - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/ - security: [] - - arches: [armhf, armel, default] - failsafe: - primary: http://ports.ubuntu.com/ubuntu-ports - security: http://ports.ubuntu.com/ubuntu-ports - ssh_svcname: ssh -- cgit v1.2.3 From 604ea8b8b56f5b02a365180b7cc7fc385cc5ad03 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Wed, 13 Aug 2014 16:45:24 +0000 Subject: change: Run the ssh module. --- config/cloud.freebsd.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/cloud.freebsd.cfg b/config/cloud.freebsd.cfg index 2d65aac4..bb3a4a51 100644 --- a/config/cloud.freebsd.cfg +++ b/config/cloud.freebsd.cfg @@ -41,7 +41,7 @@ cloud_init_modules: # - ca-certs # - rsyslog - users-groups -# - ssh + - ssh # The modules that run in the 'config' stage cloud_config_modules: -- cgit v1.2.3 From e8536d7ee30c0ce0c80a83ead8d475a724fd0d32 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 13 Aug 2014 13:47:58 -0400 Subject: remove un-used 'end' in result.json and status.json --- ChangeLog | 1 + bin/cloud-init | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 296a15cd..8242ff12 100644 --- a/ChangeLog +++ b/ChangeLog @@ -16,6 +16,7 @@ - SeLinuxGuard: remove invalid check that looked for stat.st_mode in os.lstat. - do not write comments in /etc/timezone (LP: #1341710) - ubuntu: provide 'ubuntu-init-switch' module to aid in systemd testing. + - status/result json: remove 'end' entry which was always null 0.7.5: - open 0.7.5 - Add a debug log message around import failures diff --git a/bin/cloud-init b/bin/cloud-init index ad0ba214..866f8ca4 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -478,7 +478,7 @@ def status_wrapper(name, args, data_d=None, link_d=None): nullstatus = { 'errors': [], 'start': None, - 'end': None, + 'finished': None, } status = {'v1': {}} for m in modes: -- cgit v1.2.3 From 57b5f63467dcc74b0b39a14b5baeb1974cdc3373 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Thu, 14 Aug 2014 16:18:15 +0000 Subject: fix: Install the python package that will install the required link to the python2.7 binary. This defaults to 2.7, which is fine. --- tools/build-on-freebsd | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index 6e6ce8b6..6a3f38ec 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -3,11 +3,8 @@ # installing cloud-init. This script takes care of building and installing. It # will optionally make a first run at the end. -# Since there is no python by default, create a symlink for convenience sake: -ln -sf /usr/local/bin/python2.7 /usr/local/bin/python - # Check dependencies: -[ ! -f /tmp/c-i.dependencieschecked ] && pkg install py27-cheetah py27-Jinja2 py27-prettytable py27-oauth py27-serial py27-configobj py27-yaml py27-argparse py27-requests py27-six py27-boto gpart sudo dmidecode +[ ! -f /tmp/c-i.dependencieschecked ] && pkg install python py27-cheetah py27-Jinja2 py27-prettytable py27-oauth py27-serial py27-configobj py27-yaml py27-argparse py27-requests py27-six py27-boto gpart sudo dmidecode touch /tmp/c-i.dependencieschecked # Required but unavailable port/pkg: py27-jsonpatch py27-jsonpointer -- cgit v1.2.3 From f181c7cbdc08222f195fa84a379f35a456d26123 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 16 Aug 2014 17:55:42 +0000 Subject: fix: Don't create a directory that will never be used anyway. --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index b09c0456..0e609d6e 100755 --- a/setup.py +++ b/setup.py @@ -99,7 +99,6 @@ def read_datafiles(): ('/usr/local/etc/cloud', glob('config/*.cfg')), ('/usr/local/etc/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), ('/usr/local/etc/cloud/templates', glob('templates/*')), - ('/usr/local/share/cloud-init', []), ('/usr/local/lib/cloud-init', ['tools/uncloud-init', 'tools/write-ssh-key-fingerprints']), ('/usr/local/share/doc/cloud-init', -- cgit v1.2.3 From a59eba66ede19aa85f53a1d2b67c3ffd8860b449 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 21 Aug 2014 13:46:41 -0400 Subject: use url_helper to combine url This seems cleaner, to avoid duplicate '/' being added. --- cloudinit/sources/DataSourceCloudStack.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 08f661e4..1bbeca59 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -78,7 +78,8 @@ class DataSourceCloudStack(sources.DataSource): (max_wait, timeout) = self._get_url_settings() - urls = [self.metadata_address + "/latest/meta-data/instance-id"] + urls = [uhelp.combine_url(self.metadata_address, + 'latest/meta-data/instance-id')] start_time = time.time() url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, timeout=timeout, status_cb=LOG.warn) -- cgit v1.2.3 From d8c8a87f9b4c314bc39f5987602c02d3861f329f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 21 Aug 2014 14:04:25 -0400 Subject: settings: fix typo resulting in OpenStack and GCE not running. LP: #1329583 --- ChangeLog | 2 ++ cloudinit/settings.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index c52d83b7..4d8d6427 100644 --- a/ChangeLog +++ b/ChangeLog @@ -19,6 +19,8 @@ - status/result json: remove 'end' entry which was always null - systemd: make cloud-init block ssh service startup to guarantee keys are generated. [Jordan Evans] (LP: #1333920) + - default settings: fix typo resulting in OpenStack and GCE not working + unless config explicitly provided (LP: #1329583) [Garrett Holmstrom]) 0.7.5: - open 0.7.5 - Add a debug log message around import failures diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 37d4958b..5efcb0b0 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -37,7 +37,7 @@ CFG_BUILTIN = { 'OVF', 'MAAS', 'GCE', - 'OpenStack' + 'OpenStack', 'Ec2', 'CloudSigma', 'CloudStack', -- cgit v1.2.3 From 589f740909b40d814a4b802263e61ff2b5c35b24 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 21 Aug 2014 16:26:43 -0400 Subject: fix rendering resolv.conf if no 'options' are provided this makes some changes to the cc_resolv_conf to make its generate_resolv_conf method more easily callable (for future test). Also sets it up so that 'options' is always defined when the template is rendered. LP: #1328953 --- ChangeLog | 1 + cloudinit/config/cc_resolv_conf.py | 29 ++++++++++++++++++----------- templates/resolv.conf.tmpl | 2 +- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/ChangeLog b/ChangeLog index 4d8d6427..4d326b31 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,6 +21,7 @@ are generated. [Jordan Evans] (LP: #1333920) - default settings: fix typo resulting in OpenStack and GCE not working unless config explicitly provided (LP: #1329583) [Garrett Holmstrom]) + - fix rendering resolv.conf if no 'options' are provided (LP: #1328953) 0.7.5: - open 0.7.5 - Add a debug log message around import failures diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py index 879b62b1..bbaa6c63 100644 --- a/cloudinit/config/cc_resolv_conf.py +++ b/cloudinit/config/cc_resolv_conf.py @@ -49,23 +49,22 @@ # +from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE from cloudinit import templater from cloudinit import util +LOG = logging.getLogger(__name__) + frequency = PER_INSTANCE distros = ['fedora', 'rhel', 'sles'] -def generate_resolv_conf(cloud, log, params): - template_fn = cloud.get_template_filename('resolv.conf') - if not template_fn: - log.warn("No template found, not rendering /etc/resolv.conf") - return - +def generate_resolv_conf(template_fn, params, target_fname="/etc/resolv.conf"): flags = [] false_flags = [] + if 'options' in params: for key, val in params['options'].iteritems(): if type(val) == bool: @@ -77,12 +76,15 @@ def generate_resolv_conf(cloud, log, params): for flag in flags + false_flags: del params['options'][flag] + if not params.get('options'): + params['options'] = {} + params['flags'] = flags - log.debug("Writing resolv.conf from template %s" % template_fn) - templater.render_to_file(template_fn, '/etc/resolv.conf', params) + LOG.debug("Writing resolv.conf from template %s" % template_fn) + templater.render_to_file(template_fn, target_fname, params) -def handle(name, cfg, _cloud, log, _args): +def handle(name, cfg, cloud, log, _args): """ Handler for resolv.conf @@ -102,8 +104,13 @@ def handle(name, cfg, _cloud, log, _args): " 'manage_resolv_conf' present but set to False"), name) return - if not "resolv_conf" in cfg: + if "resolv_conf" not in cfg: log.warn("manage_resolv_conf True but no parameters provided!") - generate_resolv_conf(_cloud, log, cfg["resolv_conf"]) + template_fn = cloud.get_template_filename('resolv.conf') + if not template_fn: + log.warn("No template found, not rendering /etc/resolv.conf") + return + + generate_resolv_conf(template_fn=template_fn, params=cfg["resolv_conf"]) return diff --git a/templates/resolv.conf.tmpl b/templates/resolv.conf.tmpl index 6f908f30..1300156c 100644 --- a/templates/resolv.conf.tmpl +++ b/templates/resolv.conf.tmpl @@ -21,7 +21,7 @@ domain {{domain}} sortlist {% for sort in sortlist %}{{sort}} {% endfor %} {% endif %} -{% if options is defined or flags is defined %} +{% if options or flags %} options {% for flag in flags %}{{flag}} {% endfor %} {% for key, value in options.iteritems() -%} -- cgit v1.2.3 From bbe4b997819f43089aea12141ea393c1a46fe84d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 22 Aug 2014 10:43:02 -0400 Subject: docs: fix disk-setup to reference 'table_type' LP: #1313114 --- ChangeLog | 1 + doc/examples/cloud-config-disk-setup.txt | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index 4d326b31..b420d037 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,6 +22,7 @@ - default settings: fix typo resulting in OpenStack and GCE not working unless config explicitly provided (LP: #1329583) [Garrett Holmstrom]) - fix rendering resolv.conf if no 'options' are provided (LP: #1328953) + - docs: fix disk-setup to reference 'table_type' [Rail Aliiev] (LP: #1313114) 0.7.5: - open 0.7.5 - Add a debug log message around import failures diff --git a/doc/examples/cloud-config-disk-setup.txt b/doc/examples/cloud-config-disk-setup.txt index 0dfef8e8..3e46a22e 100644 --- a/doc/examples/cloud-config-disk-setup.txt +++ b/doc/examples/cloud-config-disk-setup.txt @@ -7,7 +7,7 @@ disk_setup: ephmeral0: - type: 'mbr' + table_type: 'mbr' layout: True overwrite: False @@ -23,7 +23,7 @@ fs_setup: device_aliases: {'ephemeral0': '/dev/sdb'} disk_setup: ephemeral0: - type: mbr + table_type: mbr layout: True overwrite: False @@ -40,7 +40,7 @@ fs_setup: device_aliases: {'ephemeral0': '/dev/sdb'} disk_setup: ephemeral0: - type: mbr + table_type: mbr layout: False overwrite: False @@ -63,10 +63,10 @@ fs_setup: disk_setup: ephmeral0: - type: 'mbr' + table_type: 'mbr' layout: 'auto' /dev/xvdh: - type: 'mbr' + table_type: 'mbr' layout: - 33 - [33, 82] @@ -79,7 +79,7 @@ disk_setup: # The general format is: # disk_setup: # : -# type: 'mbr' +# table_type: 'mbr' # layout: # overwrite: # @@ -96,7 +96,7 @@ disk_setup: # Note: At this time, there is no handling or setup of # device mapper targets. # -# type=: Currently the following are supported: +# table_type=: Currently the following are supported: # 'mbr': default and setups a MS-DOS partition table # # Note: At this time only 'mbr' partition tables are allowed. @@ -116,7 +116,7 @@ disk_setup: # partition having a swap label, taking 1/3 of the disk space # and the remainder being used as the second partition. # /dev/xvdh': -# type: 'mbr' +# table_type: 'mbr' # layout: # - [33,82] # - 66 -- cgit v1.2.3 From d723c17b7abe5263d879d748dade763034cce476 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 22 Aug 2014 11:40:25 -0400 Subject: ec2_utils.py: get_instance_metadata explicitly add trailing / This change works around a bug in CloudStack's EC2 api implementation. That is filed upstream at [1]. The fix is safe for openstack and EC2 use cases as well. EC2 and OpenStacks' EC2 metadata service both return a list with access to either of: /latest/meta-data /latest/meta-data/ Additionally, the responses explicitly contain a trailing '/' for items that have a child. The ec2_utils code then just re-uses the trailng / there. Thus, only the top level request for 'meta-data/' needs the explicit fix. This also changes test cases. Those test cases failed without fixing them. If ever this regressed, those would fail again. -- [1] https://issues.apache.org/jira/browse/CLOUDSTACK-7405 LP: #1356855 --- cloudinit/ec2_utils.py | 4 +++- tests/unittests/test_datasource/test_openstack.py | 4 ++-- tests/unittests/test_ec2_util.py | 8 ++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index a7c9c9ab..0c751140 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -166,7 +166,9 @@ def get_instance_metadata(api_version='latest', metadata_address='http://169.254.169.254', ssl_details=None, timeout=5, retries=5): md_url = url_helper.combine_url(metadata_address, api_version) - md_url = url_helper.combine_url(md_url, 'meta-data') + # Note, 'meta-data' explicitly has trailing /. + # this is required for CloudStack (LP: #1356855) + md_url = url_helper.combine_url(md_url, 'meta-data/') caller = functools.partial(util.read_file_or_url, ssl_details=ssl_details, timeout=timeout, retries=retries) diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index c088bb55..f43cbec8 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -88,7 +88,7 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): path = uri.path.lstrip("/") if path in ec2_files: return (200, headers, ec2_files.get(path)) - if path == 'latest/meta-data': + if path == 'latest/meta-data/': buf = StringIO() for (k, v) in ec2_meta.items(): if isinstance(v, (list, tuple)): @@ -97,7 +97,7 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): buf.write("%s" % (k)) buf.write("\n") return (200, headers, buf.getvalue()) - if path.startswith('latest/meta-data'): + if path.startswith('latest/meta-data/'): value = None pieces = path.split("/") if path.endswith("/"): diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py index 700254a3..84aa002e 100644 --- a/tests/unittests/test_ec2_util.py +++ b/tests/unittests/test_ec2_util.py @@ -44,7 +44,7 @@ class TestEc2Util(helpers.HttprettyTestCase): @hp.activate def test_metadata_fetch_no_keys(self): - base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION) + base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION) hp.register_uri(hp.GET, base_url, status=200, body="\n".join(['hostname', 'instance-id', @@ -62,7 +62,7 @@ class TestEc2Util(helpers.HttprettyTestCase): @hp.activate def test_metadata_fetch_key(self): - base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION) + base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION) hp.register_uri(hp.GET, base_url, status=200, body="\n".join(['hostname', 'instance-id', @@ -83,7 +83,7 @@ class TestEc2Util(helpers.HttprettyTestCase): @hp.activate def test_metadata_fetch_with_2_keys(self): - base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION) + base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION) hp.register_uri(hp.GET, base_url, status=200, body="\n".join(['hostname', 'instance-id', @@ -108,7 +108,7 @@ class TestEc2Util(helpers.HttprettyTestCase): @hp.activate def test_metadata_fetch_bdm(self): - base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION) + base_url = 'http://169.254.169.254/%s/meta-data/' % (self.VERSION) hp.register_uri(hp.GET, base_url, status=200, body="\n".join(['hostname', 'instance-id', -- cgit v1.2.3 From 31ef77ec5a948e02def84791084ccf37ac2b6180 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 22 Aug 2014 16:43:32 -0400 Subject: setup.py: remove read_datafiles, use globals for ETC and USR --- setup.py | 57 +++++++++++++++++++++------------------------------------ 1 file changed, 21 insertions(+), 36 deletions(-) diff --git a/setup.py b/setup.py index 0e609d6e..9be88e53 100755 --- a/setup.py +++ b/setup.py @@ -77,6 +77,14 @@ INITSYS_ROOTS = { } INITSYS_TYPES = sorted(list(INITSYS_ROOTS.keys())) +# Install everything in the right location and take care of Linux (default) and +# FreeBSD systems. +USR = "/usr" +ETC = "/etc" +if os.uname()[0] == 'FreeBSD': + USR = "/usr/local" + ETC = "/usr/local/etc" + def get_version(): cmd = ['tools/read-version'] @@ -90,41 +98,6 @@ def read_requires(): return str(deps).splitlines() -# Install everything in the right location and take care of Linux (default) and -# FreeBSD systems. -def read_datafiles(): - sysname = os.uname()[0] - if sysname == 'FreeBSD': - return [ - ('/usr/local/etc/cloud', glob('config/*.cfg')), - ('/usr/local/etc/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), - ('/usr/local/etc/cloud/templates', glob('templates/*')), - ('/usr/local/lib/cloud-init', - ['tools/uncloud-init', 'tools/write-ssh-key-fingerprints']), - ('/usr/local/share/doc/cloud-init', - [f for f in glob('doc/*') if is_f(f)]), - ('/usr/local/share/doc/cloud-init/examples', - [f for f in glob('doc/examples/*') if is_f(f)]), - ('/usr/local/share/doc/cloud-init/examples/seed', - [f for f in glob('doc/examples/seed/*') if is_f(f)]), - ] - else: - return [ - ('/etc/cloud', glob('config/*.cfg')), - ('/etc/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), - ('/etc/cloud/templates', glob('templates/*')), - ('/usr/share/cloud-init', []), - ('/usr/lib/cloud-init', - ['tools/uncloud-init', 'tools/write-ssh-key-fingerprints']), - ('/usr/share/doc/cloud-init', - [f for f in glob('doc/*') if is_f(f)]), - ('/usr/share/doc/cloud-init/examples', - [f for f in glob('doc/examples/*') if is_f(f)]), - ('/usr/share/doc/cloud-init/examples/seed', - [f for f in glob('doc/examples/seed/*') if is_f(f)]), - ] - - # TODO: Is there a better way to do this?? class InitsysInstallData(install): init_system = None @@ -173,7 +146,19 @@ setuptools.setup(name='cloud-init', 'tools/cloud-init-per', ], license='GPLv3', - data_files=read_datafiles(), + data_files=[(ETC '/cloud', glob('config/*.cfg')), + (ETC '/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), + (ETC '/cloud/templates', glob('templates/*')), + (USR '/lib/cloud-init', + ['tools/uncloud-init', + 'tools/write-ssh-key-fingerprints']), + (USR '/share/doc/cloud-init', + [f for f in glob('doc/*') if is_f(f)]), + (USR '/share/doc/cloud-init/examples', + [f for f in glob('doc/examples/*') if is_f(f)]), + (USR '/share/doc/cloud-init/examples/seed', + [f for f in glob('doc/examples/seed/*') if is_f(f)]), + ], install_requires=read_requires(), cmdclass={ # Use a subclass for install that handles -- cgit v1.2.3 From 9d8628772131bbe127bfefd6337cda34025723c8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 22 Aug 2014 16:50:47 -0400 Subject: build-on-freebsd: minor cleanups/bikeshedding --- tools/build-on-freebsd | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index 6a3f38ec..66b95d68 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -3,8 +3,22 @@ # installing cloud-init. This script takes care of building and installing. It # will optionally make a first run at the end. +fail() { echo "FAILED:" "$@" 1>&2; exit 1; } + # Check dependencies: -[ ! -f /tmp/c-i.dependencieschecked ] && pkg install python py27-cheetah py27-Jinja2 py27-prettytable py27-oauth py27-serial py27-configobj py27-yaml py27-argparse py27-requests py27-six py27-boto gpart sudo dmidecode +depscheck=/tmp/c-i.dependencieschecked +pkgs=" + dmidecode + py27-argparse + py27-boto gpart sudo + py27-configobj py27-yaml + py27-Jinja2 + py27-oauth py27-serial + py27-prettytable + py27-requests py27-six + python py27-cheetah +" +[ -f "$depschecked" ] || pkg install ${pkgs} || fail "install packages" touch /tmp/c-i.dependencieschecked # Required but unavailable port/pkg: py27-jsonpatch py27-jsonpointer @@ -23,8 +37,7 @@ echo 'cloudinit_enable="YES"' >> /etc/rc.conf echo "Installation completed." -if [ "$1" = "run" ] -then +if [ "$1" = "run" ]; then echo "Ok, now let's see if it works." # Backup SSH keys -- cgit v1.2.3 From c728de2115aacbe0f48d65661c076caace5e16e7 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 23 Aug 2014 12:06:51 +0000 Subject: fix: Use the correct variable. --- tools/build-on-freebsd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index 66b95d68..b0203cb8 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -6,7 +6,7 @@ fail() { echo "FAILED:" "$@" 1>&2; exit 1; } # Check dependencies: -depscheck=/tmp/c-i.dependencieschecked +depschecked=/tmp/c-i.dependencieschecked pkgs=" dmidecode py27-argparse @@ -19,7 +19,7 @@ pkgs=" python py27-cheetah " [ -f "$depschecked" ] || pkg install ${pkgs} || fail "install packages" -touch /tmp/c-i.dependencieschecked +touch $depschecked # Required but unavailable port/pkg: py27-jsonpatch py27-jsonpointer # Luckily, the install step will take care of this by installing it from pypi... -- cgit v1.2.3 From f68dfa3fd17e0a5eb228e6c7621d6da26e962c9a Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 23 Aug 2014 12:10:30 +0000 Subject: fix: Syntax. --- setup.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 9be88e53..4d20f16c 100755 --- a/setup.py +++ b/setup.py @@ -146,17 +146,17 @@ setuptools.setup(name='cloud-init', 'tools/cloud-init-per', ], license='GPLv3', - data_files=[(ETC '/cloud', glob('config/*.cfg')), - (ETC '/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), - (ETC '/cloud/templates', glob('templates/*')), - (USR '/lib/cloud-init', + data_files=[(ETC + '/cloud', glob('config/*.cfg')), + (ETC + '/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), + (ETC + '/cloud/templates', glob('templates/*')), + (USR + '/lib/cloud-init', ['tools/uncloud-init', 'tools/write-ssh-key-fingerprints']), - (USR '/share/doc/cloud-init', + (USR + '/share/doc/cloud-init', [f for f in glob('doc/*') if is_f(f)]), - (USR '/share/doc/cloud-init/examples', + (USR + '/share/doc/cloud-init/examples', [f for f in glob('doc/examples/*') if is_f(f)]), - (USR '/share/doc/cloud-init/examples/seed', + (USR + '/share/doc/cloud-init/examples/seed', [f for f in glob('doc/examples/seed/*') if is_f(f)]), ], install_requires=read_requires(), -- cgit v1.2.3 From 9fcb3f2c7c129576f9a2175614bcb384d492f63e Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 23 Aug 2014 12:16:03 +0000 Subject: change: Rename the config file to cloud.cfg-freebsd so it doesn't get copied per default. Packaging will take care of installing this configfile on the BSD platform. --- config/cloud.cfg-freebsd | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ config/cloud.freebsd.cfg | 88 ------------------------------------------------ tools/build-on-freebsd | 4 +-- 3 files changed, 90 insertions(+), 90 deletions(-) create mode 100644 config/cloud.cfg-freebsd delete mode 100644 config/cloud.freebsd.cfg diff --git a/config/cloud.cfg-freebsd b/config/cloud.cfg-freebsd new file mode 100644 index 00000000..bb3a4a51 --- /dev/null +++ b/config/cloud.cfg-freebsd @@ -0,0 +1,88 @@ +# The top level settings are used as module +# and system configuration. + +syslog_fix_perms: root:wheel + +# This should not be required, but leave it in place until the real cause of +# not beeing able to find -any- datasources is resolved. +datasource_list: ['OpenStack'] + +# A set of users which may be applied and/or used by various modules +# when a 'default' entry is found it will reference the 'default_user' +# from the distro configuration specified below +users: + - default + +# If this is set, 'root' will not be able to ssh in and they +# will get a message to login instead as the above $user (ubuntu) +disable_root: false + +# This will cause the set+update hostname module to not operate (if true) +preserve_hostname: false + +# Example datasource config +# datasource: +# Ec2: +# metadata_urls: [ 'blah.com' ] +# timeout: 5 # (defaults to 50 seconds) +# max_wait: 10 # (defaults to 120 seconds) + +# The modules that run in the 'init' stage +cloud_init_modules: +# - migrator + - seed_random + - bootcmd +# - write-files + - growpart + - resizefs + - set_hostname + - update_hostname +# - update_etc_hosts +# - ca-certs +# - rsyslog + - users-groups + - ssh + +# The modules that run in the 'config' stage +cloud_config_modules: +# - disk_setup +# - mounts + - ssh-import-id + - locale +# - set-passwords +# - package-update-upgrade-install +# - landscape +# - timezone +# - puppet +# - chef +# - salt-minion +# - mcollective + - disable-ec2-metadata + - runcmd +# - byobu + +# The modules that run in the 'final' stage +cloud_final_modules: + - rightscale_userdata + - scripts-vendor + - scripts-per-once + - scripts-per-boot + - scripts-per-instance + - scripts-user + - ssh-authkey-fingerprints + - keys-to-console + - phone-home + - final-message + - power-state-change + +# System and/or distro specific settings +# (not accessible to handlers/transforms) +system_info: + distro: freebsd + default_user: + name: beastie + lock_passwd: True + gecos: FreeBSD + groups: [wheel] + sudo: ["ALL=(ALL) NOPASSWD:ALL"] + shell: /bin/sh diff --git a/config/cloud.freebsd.cfg b/config/cloud.freebsd.cfg deleted file mode 100644 index bb3a4a51..00000000 --- a/config/cloud.freebsd.cfg +++ /dev/null @@ -1,88 +0,0 @@ -# The top level settings are used as module -# and system configuration. - -syslog_fix_perms: root:wheel - -# This should not be required, but leave it in place until the real cause of -# not beeing able to find -any- datasources is resolved. -datasource_list: ['OpenStack'] - -# A set of users which may be applied and/or used by various modules -# when a 'default' entry is found it will reference the 'default_user' -# from the distro configuration specified below -users: - - default - -# If this is set, 'root' will not be able to ssh in and they -# will get a message to login instead as the above $user (ubuntu) -disable_root: false - -# This will cause the set+update hostname module to not operate (if true) -preserve_hostname: false - -# Example datasource config -# datasource: -# Ec2: -# metadata_urls: [ 'blah.com' ] -# timeout: 5 # (defaults to 50 seconds) -# max_wait: 10 # (defaults to 120 seconds) - -# The modules that run in the 'init' stage -cloud_init_modules: -# - migrator - - seed_random - - bootcmd -# - write-files - - growpart - - resizefs - - set_hostname - - update_hostname -# - update_etc_hosts -# - ca-certs -# - rsyslog - - users-groups - - ssh - -# The modules that run in the 'config' stage -cloud_config_modules: -# - disk_setup -# - mounts - - ssh-import-id - - locale -# - set-passwords -# - package-update-upgrade-install -# - landscape -# - timezone -# - puppet -# - chef -# - salt-minion -# - mcollective - - disable-ec2-metadata - - runcmd -# - byobu - -# The modules that run in the 'final' stage -cloud_final_modules: - - rightscale_userdata - - scripts-vendor - - scripts-per-once - - scripts-per-boot - - scripts-per-instance - - scripts-user - - ssh-authkey-fingerprints - - keys-to-console - - phone-home - - final-message - - power-state-change - -# System and/or distro specific settings -# (not accessible to handlers/transforms) -system_info: - distro: freebsd - default_user: - name: beastie - lock_passwd: True - gecos: FreeBSD - groups: [wheel] - sudo: ["ALL=(ALL) NOPASSWD:ALL"] - shell: /bin/sh diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index b0203cb8..23bdf487 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -28,8 +28,8 @@ touch $depschecked python setup.py build python setup.py install -O1 --skip-build --prefix /usr/local/ --init-system sysvinit_freebsd -# Use the correct config file: -mv /usr/local/etc/cloud/cloud.freebsd.cfg /usr/local/etc/cloud/cloud.cfg +# Install the correct config file: +cp config/cloud.cfg-freebsd /usr/local/etc/cloud/cloud.cfg # Enable cloud-init in /etc/rc.conf: sed -i.bak -e "/cloudinit_enable=.*/d" /etc/rc.conf -- cgit v1.2.3 From a0e3506c6dddf140589228c1ac74f48d2a7cde49 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Sat, 23 Aug 2014 12:19:09 +0000 Subject: fix: Drop some overly loud debug messages. --- cloudinit/distros/freebsd.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index b4d841f8..415c6ba6 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -232,12 +232,6 @@ class Distro(distros.Distro): def _write_network(self, settings): entries = net_util.translate_network(settings) - LOG.debug("Translated network settings") - LOG.debug("\n========== UBUNTU FORMAT START ==========") - LOG.debug("%s\n========== UBUNTU FORMAT END ==========", settings) - LOG.debug("\n========== GENERIC FORMAT START ==========") - LOG.debug("%s\n========== GENERIC FORMAT END ==========", entries) - nameservers = [] searchdomains = [] dev_names = entries.keys() -- cgit v1.2.3 From 190cacc430900d9d2dd4dd45c59d01e30e469720 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 26 Aug 2014 12:25:50 -0400 Subject: ssh_authkey_fingerprints: fix bug that prevented disabling the module Module intended to allow disabling by configuration, but that was broken. Now this allows: no_ssh_fingerprints = True LP: #1340903 --- ChangeLog | 2 ++ cloudinit/config/cc_ssh_authkey_fingerprints.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index b420d037..97417cb2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -23,6 +23,8 @@ unless config explicitly provided (LP: #1329583) [Garrett Holmstrom]) - fix rendering resolv.conf if no 'options' are provided (LP: #1328953) - docs: fix disk-setup to reference 'table_type' [Rail Aliiev] (LP: #1313114) + - ssh_authkey_fingerprints: fix bug that prevented disabling the module. + (LP: #1340903) [Patrick Lucas] 0.7.5: - open 0.7.5 - Add a debug log message around import failures diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index be8083db..7da62589 100644 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -92,9 +92,10 @@ def _pprint_key_entries(user, key_fn, key_entries, hash_meth='md5', def handle(name, cfg, cloud, log, _args): - if 'no_ssh_fingerprints' in cfg: + if util.is_true(cfg.get('no_ssh_fingerprints', False)): log.debug(("Skipping module named %s, " "logging of ssh fingerprints disabled"), name) + return hash_meth = util.get_cfg_option_str(cfg, "authkey_hash", "md5") (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) -- cgit v1.2.3 From 1b9b66be04aafcb39f349ccb48905afc393cfc32 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Tue, 26 Aug 2014 18:10:03 +0000 Subject: change: Use a compiled regex and use the included match groups instead of matching yet again. --- cloudinit/distros/freebsd.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index 415c6ba6..9c923480 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -69,17 +69,17 @@ class Distro(distros.Distro): # quotes are ignored: # hostname="bla" def loadrcconf(self): + RE_MATCH = re.compile(r'^(\w+)="?(\w+)"?') conf = {} lines = util.load_file(self.rc_conf_fn).splitlines() for line in lines: - if not re.match(r'^(.+)=(.+)', line): + m = RE_MATCH.match(line) + if not m: LOG.debug("Skipping line from /etc/rc.conf: %s", line) continue - # TODO: just use the matches please... - tok = line.split('=') - key = tok[0] - val = re.sub(r'^"|"$', '', tok[1].rstrip()) + key = m.group(1).rstrip() + val = m.group(2).rstrip() conf[key] = val return conf -- cgit v1.2.3 From b3216b56f3fea3259c290faa2dd496215b625904 Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Tue, 26 Aug 2014 11:50:11 -0700 Subject: fix(pep8): Fix various pep8 violations and version-lock pep8 Fixed all complaints from running "make pep8". Also version locked pep8 in test-requirements.txt to ensure that pep8 requirements don't change without an explicit commit. --- cloudinit/config/cc_apt_configure.py | 2 +- cloudinit/config/cc_byobu.py | 2 +- cloudinit/config/cc_chef.py | 3 +- cloudinit/config/cc_disk_setup.py | 4 +-- cloudinit/config/cc_grub_dpkg.py | 3 +- cloudinit/config/cc_phone_home.py | 2 +- cloudinit/config/cc_resizefs.py | 2 +- cloudinit/config/cc_rightscale_userdata.py | 34 +++++++++++----------- cloudinit/config/cc_rsyslog.py | 4 +-- cloudinit/config/cc_ssh.py | 2 +- cloudinit/config/cc_ssh_authkey_fingerprints.py | 2 +- cloudinit/config/cc_yum_add_repo.py | 4 +-- cloudinit/distros/__init__.py | 4 +-- cloudinit/mergers/m_list.py | 2 +- cloudinit/sources/DataSourceAzure.py | 4 +-- cloudinit/sources/DataSourceNoCloud.py | 2 +- cloudinit/sources/DataSourceOVF.py | 4 +-- cloudinit/sources/DataSourceSmartOS.py | 32 ++++++++++---------- cloudinit/sources/__init__.py | 2 +- cloudinit/stages.py | 2 +- cloudinit/url_helper.py | 4 +-- cloudinit/util.py | 6 ++-- test-requirements.txt | 2 +- tests/unittests/test_datasource/test_azure.py | 2 +- tests/unittests/test_datasource/test_nocloud.py | 4 +-- tests/unittests/test_distros/test_generic.py | 2 +- .../test_handler/test_handler_growpart.py | 8 ++--- .../test_handler/test_handler_power_state.py | 2 +- .../test_handler/test_handler_yum_add_repo.py | 2 +- tools/hacking.py | 2 +- 30 files changed, 76 insertions(+), 74 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 29c13a3d..f10b76a3 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -235,7 +235,7 @@ def find_apt_mirror_info(cloud, cfg): mirror = util.search_for_mirror(search) if (not mirror and - util.get_cfg_option_bool(cfg, "apt_mirror_search_dns", False)): + util.get_cfg_option_bool(cfg, "apt_mirror_search_dns", False)): mydom = "" doms = [] diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index 92d428b7..ef0ce7ab 100644 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -43,7 +43,7 @@ def handle(name, cfg, cloud, log, args): valid = ("enable-user", "enable-system", "enable", "disable-user", "disable-system", "disable") - if not value in valid: + if value not in valid: log.warn("Unknown value %s for byobu_by_default", value) mod_user = value.endswith("-user") diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 727769cd..806deed9 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -87,7 +87,8 @@ def handle(name, cfg, cloud, log, _args): # If chef is not installed, we install chef based on 'install_type' if (not os.path.isfile('/usr/bin/chef-client') or - util.get_cfg_option_bool(chef_cfg, 'force_install', default=False)): + util.get_cfg_option_bool(chef_cfg, + 'force_install', default=False)): install_type = util.get_cfg_option_str(chef_cfg, 'install_type', 'packages') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 0b970e4e..a5209268 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -271,7 +271,7 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None, return ('/dev/%s' % d['name'], False) if (d['fstype'] == fs_type and - ((label_match and d['label'] == label) or not label_match)): + ((label_match and d['label'] == label) or not label_match)): # If we find a matching device, we return that return ('/dev/%s' % d['name'], True) @@ -447,7 +447,7 @@ def get_partition_mbr_layout(size, layout): return "0," if ((len(layout) == 0 and isinstance(layout, list)) or - not isinstance(layout, list)): + not isinstance(layout, list)): raise Exception("Partition layout is invalid") last_part_num = len(layout) diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index b3ce6fb6..85716a91 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -36,7 +36,8 @@ def handle(_name, cfg, _cloud, log, _args): "grub-pc/install_devices_empty", None) if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or - (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): + (os.path.exists("/dev/xvda1") + and not os.path.exists("/dev/xvda"))): if idevs is None: idevs = "" if idevs_empty is None: diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 2e058ccd..5bc68b83 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -47,7 +47,7 @@ def handle(name, cfg, cloud, log, args): if len(args) != 0: ph_cfg = util.read_conf(args[0]) else: - if not 'phone_home' in cfg: + if 'phone_home' not in cfg: log.debug(("Skipping module named %s, " "no 'phone_home' configuration found"), name) return diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index be406034..667d5977 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -105,7 +105,7 @@ def handle(name, cfg, _cloud, log, args): container = util.is_container() if (devpth == "/dev/root" and not os.path.exists(devpth) and - not container): + not container): devpth = rootdev_from_cmdline(util.get_cmdline()) if devpth is None: log.warn("Unable to find device '/dev/root'") diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index c771728d..7d2ec10a 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -18,22 +18,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -## -## The purpose of this script is to allow cloud-init to consume -## rightscale style userdata. rightscale user data is key-value pairs -## in a url-query-string like format. -## -## for cloud-init support, there will be a key named -## 'CLOUD_INIT_REMOTE_HOOK'. -## -## This cloud-config module will -## - read the blob of data from raw user data, and parse it as key/value -## - for each key that is found, download the content to -## the local instance/scripts directory and set them executable. -## - the files in that directory will be run by the user-scripts module -## Therefore, this must run before that. -## -## +# +# The purpose of this script is to allow cloud-init to consume +# rightscale style userdata. rightscale user data is key-value pairs +# in a url-query-string like format. +# +# for cloud-init support, there will be a key named +# 'CLOUD_INIT_REMOTE_HOOK'. +# +# This cloud-config module will +# - read the blob of data from raw user data, and parse it as key/value +# - for each key that is found, download the content to +# the local instance/scripts directory and set them executable. +# - the files in that directory will be run by the user-scripts module +# Therefore, this must run before that. +# +# import os @@ -58,7 +58,7 @@ def handle(name, _cfg, cloud, log, _args): try: mdict = parse_qs(ud) - if not mdict or not MY_HOOKNAME in mdict: + if mdict or MY_HOOKNAME not in mdict: log.debug(("Skipping module %s, " "did not find %s in parsed" " raw userdata"), name, MY_HOOKNAME) diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 0c2c6880..57486edc 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -35,7 +35,7 @@ def handle(name, cfg, cloud, log, _args): # *.* @@syslogd.example.com # process 'rsyslog' - if not 'rsyslog' in cfg: + if 'rsyslog' not in cfg: log.debug(("Skipping module named %s," " no 'rsyslog' key in configuration"), name) return @@ -46,7 +46,7 @@ def handle(name, cfg, cloud, log, _args): files = [] for i, ent in enumerate(cfg['rsyslog']): if isinstance(ent, dict): - if not "content" in ent: + if "content" not in ent: log.warn("No 'content' entry in config entry %s", i + 1) continue content = ent['content'] diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 64a5e3cb..4c76581c 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -75,7 +75,7 @@ def handle(_name, cfg, cloud, log, _args): util.write_file(tgt_fn, val, tgt_perms) for (priv, pub) in PRIV_2_PUB.iteritems(): - if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: + if pub in cfg['ssh_keys'] or priv not in cfg['ssh_keys']: continue pair = (KEY_2_FILE[priv][0], KEY_2_FILE[pub][0]) cmd = ['sh', '-xc', KEY_GEN_TPL % pair] diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index 7da62589..51580633 100644 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -55,7 +55,7 @@ def _gen_fingerprint(b64_text, hash_meth='md5'): def _is_printable_key(entry): if any([entry.keytype, entry.base64, entry.comment, entry.options]): if (entry.keytype and - entry.keytype.lower().strip() in ['ssh-dss', 'ssh-rsa']): + entry.keytype.lower().strip() in ['ssh-dss', 'ssh-rsa']): return True return False diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 5c273825..0d836f28 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -42,7 +42,7 @@ def _format_repo_value(val): return val -## TODO(harlowja): move to distro? +# TODO(harlowja): move to distro? # See man yum.conf def _format_repository_config(repo_id, repo_config): to_be = configobj.ConfigObj() @@ -89,7 +89,7 @@ def handle(name, cfg, _cloud, log, _args): repo_config = n_repo_config missing_required = 0 for req_field in ['baseurl']: - if not req_field in repo_config: + if req_field not in repo_config: log.warn(("Repository %s does not contain a %s" " configuration 'required' entry"), repo_id, req_field) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 1a56dfb3..4b41220e 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -167,7 +167,7 @@ class Distro(object): def expand_osfamily(family_list): distros = [] for family in family_list: - if not family in OSFAMILIES: + if family not in OSFAMILIES: raise ValueError("No distibutions found for osfamily %s" % (family)) distros.extend(OSFAMILIES[family]) @@ -218,7 +218,7 @@ class Distro(object): fn) if (sys_hostname and prev_hostname and - sys_hostname != prev_hostname): + sys_hostname != prev_hostname): LOG.debug("%s differs from %s, assuming user maintained hostname.", prev_hostname_fn, sys_fn) diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py index 62999b4e..3b87b0fc 100644 --- a/cloudinit/mergers/m_list.py +++ b/cloudinit/mergers/m_list.py @@ -53,7 +53,7 @@ class Merger(object): def _on_list(self, value, merge_with): if (self._method == 'replace' and - not isinstance(merge_with, (tuple, list))): + not isinstance(merge_with, (tuple, list))): return merge_with # Ok we now know that what we are merging with is a list or tuple. diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index bd75e6d8..09bc196d 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -452,7 +452,7 @@ def load_azure_ovf_pubkeys(sshnode): continue if (len(child.childNodes) != 1 or - child.childNodes[0].nodeType != text_node): + child.childNodes[0].nodeType != text_node): continue cur[name] = child.childNodes[0].wholeText.strip() @@ -521,7 +521,7 @@ def read_azure_ovf(contents): simple = False value = "" if (len(child.childNodes) == 1 and - child.childNodes[0].nodeType == dom.TEXT_NODE): + child.childNodes[0].nodeType == dom.TEXT_NODE): simple = True value = child.childNodes[0].wholeText diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index a315aae0..c26a645c 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -181,7 +181,7 @@ class DataSourceNoCloud(sources.DataSource): # and the source of the seed was self.dsmode # ('local' for NoCloud, 'net' for NoCloudNet') if ('network-interfaces' in mydata['meta-data'] and - (self.dsmode in ("local", seeded_interfaces))): + (self.dsmode in ("local", seeded_interfaces))): LOG.debug("Updating network interfaces from %s", self) self.distro.apply_network( mydata['meta-data']['network-interfaces']) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 77b43e17..2f53c1ba 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -107,7 +107,7 @@ class DataSourceOVF(sources.DataSource): return True def get_public_ssh_keys(self): - if not 'public-keys' in self.metadata: + if 'public-keys' not in self.metadata: return [] pks = self.metadata['public-keys'] if isinstance(pks, (list)): @@ -205,7 +205,7 @@ def transport_iso9660(require_iso=True): fullp = os.path.join("/dev/", dev) if (fullp in mounts or - not cdmatch.match(dev) or os.path.isdir(fullp)): + not cdmatch.match(dev) or os.path.isdir(fullp)): continue try: diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 65ec0339..2733a2f6 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -41,7 +41,7 @@ import serial LOG = logging.getLogger(__name__) SMARTOS_ATTRIB_MAP = { - #Cloud-init Key : (SmartOS Key, Strip line endings) + # Cloud-init Key : (SmartOS Key, Strip line endings) 'local-hostname': ('hostname', True), 'public-keys': ('root_authorized_keys', True), 'user-script': ('user-script', False), @@ -96,21 +96,21 @@ BUILTIN_CLOUD_CONFIG = { 'device': 'ephemeral0'}], } -## builtin vendor-data is a boothook that writes a script into -## /var/lib/cloud/scripts/per-boot. *That* script then handles -## executing the 'operator-script' and 'user-script' files -## that cloud-init writes into /var/lib/cloud/instance/data/ -## if they exist. -## -## This is all very indirect, but its done like this so that at -## some point in the future, perhaps cloud-init wouldn't do it at -## all, but rather the vendor actually provide vendor-data that accomplished -## their desires. (That is the point of vendor-data). -## -## cloud-init does cheat a bit, and write the operator-script and user-script -## itself. It could have the vendor-script do that, but it seems better -## to not require the image to contain a tool (mdata-get) to read those -## keys when we have a perfectly good one inside cloud-init. +# builtin vendor-data is a boothook that writes a script into +# /var/lib/cloud/scripts/per-boot. *That* script then handles +# executing the 'operator-script' and 'user-script' files +# that cloud-init writes into /var/lib/cloud/instance/data/ +# if they exist. +# +# This is all very indirect, but its done like this so that at +# some point in the future, perhaps cloud-init wouldn't do it at +# all, but rather the vendor actually provide vendor-data that accomplished +# their desires. (That is the point of vendor-data). +# +# cloud-init does cheat a bit, and write the operator-script and user-script +# itself. It could have the vendor-script do that, but it seems better +# to not require the image to contain a tool (mdata-get) to read those +# keys when we have a perfectly good one inside cloud-init. BUILTIN_VENDOR_DATA = """\ #cloud-boothook #!/bin/sh diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index fef4d460..7d52a2e6 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -166,7 +166,7 @@ class DataSource(object): defhost = "localhost" domain = defdomain - if not self.metadata or not 'local-hostname' in self.metadata: + if self.metadata or 'local-hostname' not in self.metadata: # this is somewhat questionable really. # the cloud datasource was asked for a hostname # and didn't have one. raising error might be more appropriate diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 9e071fc4..d29d480a 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -642,7 +642,7 @@ class Modules(object): # Try the modules frequency, otherwise fallback to a known one if not freq: freq = mod.frequency - if not freq in FREQUENCIES: + if freq not in FREQUENCIES: freq = PER_INSTANCE LOG.debug("Running module %s (%s) with frequency %s", name, mod, freq) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 4a83169a..73c1fa4e 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -237,8 +237,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, return UrlResponse(r) except exceptions.RequestException as e: if (isinstance(e, (exceptions.HTTPError)) - and hasattr(e, 'response') # This appeared in v 0.10.8 - and hasattr(e.response, 'status_code')): + and hasattr(e, 'response') # This appeared in v 0.10.8 + and hasattr(e.response, 'status_code')): excps.append(UrlError(e, code=e.response.status_code, headers=e.response.headers)) else: diff --git a/cloudinit/util.py b/cloudinit/util.py index bc681f4a..0821901a 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -423,7 +423,7 @@ def get_cfg_option_list(yobj, key, default=None): @return: The configuration option as a list of strings or default if key is not found. """ - if not key in yobj: + if key not in yobj: return default if yobj[key] is None: return [] @@ -1148,7 +1148,7 @@ def chownbyname(fname, user=None, group=None): # this returns the specific 'mode' entry, cleanly formatted, with value def get_output_cfg(cfg, mode): ret = [None, None] - if not cfg or not 'output' in cfg: + if cfg or 'output' not in cfg: return ret outcfg = cfg['output'] @@ -1745,7 +1745,7 @@ def parse_mount_info(path, mountinfo_lines, log=LOG): # Ignore mount points higher than an already seen mount # point. if (match_mount_point_elements is not None and - len(match_mount_point_elements) > len(mount_point_elements)): + len(match_mount_point_elements) > len(mount_point_elements)): continue # Find the '-' which terminates a list of optional columns to diff --git a/test-requirements.txt b/test-requirements.txt index 4be0211d..47d45d47 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,6 @@ httpretty>=0.7.1 mocker nose -pep8 +pep8==1.5.7 pyflakes pylint diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 88c82d5e..e992a006 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -235,7 +235,7 @@ class TestAzureDataSource(MockerTestCase): self.assertEqual(dsrc.userdata_raw, mydata) def test_no_datasource_expected(self): - #no source should be found if no seed_dir and no devs + # no source should be found if no seed_dir and no devs data = {} dsrc = self._get_ds({}) ret = dsrc.get_data() diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 14274562..8bcc026c 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -50,7 +50,7 @@ class TestNoCloudDataSource(MockerTestCase): self.assertTrue(ret) def test_fs_label(self): - #find_devs_with should not be called ff fs_label is None + # find_devs_with should not be called ff fs_label is None ds = DataSourceNoCloud.DataSourceNoCloud class PsuedoException(Exception): @@ -74,7 +74,7 @@ class TestNoCloudDataSource(MockerTestCase): self.assertFalse(ret) def test_no_datasource_expected(self): - #no source should be found if no cmdline, config, and fs_label=None + # no source should be found if no cmdline, config, and fs_label=None sys_cfg = {'datasource': {'NoCloud': {'fs_label': None}}} ds = DataSourceNoCloud.DataSourceNoCloud diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index c24c790e..a972568f 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -193,7 +193,7 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): 'security': 'http://security-mirror2-intel'}) -#def _get_package_mirror_info(mirror_info, availability_zone=None, +# def _get_package_mirror_info(mirror_info, availability_zone=None, # mirror_filter=util.search_for_mirror): diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index f6dc4521..fa624197 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -53,7 +53,7 @@ class TestDisabled(MockerTestCase): self.handle = cc_growpart.handle def test_mode_off(self): - #Test that nothing is done if mode is off. + # Test that nothing is done if mode is off. # this really only verifies that resizer_factory isn't called config = {'growpart': {'mode': 'off'}} @@ -109,7 +109,7 @@ class TestConfig(MockerTestCase): self.assertTrue(isinstance(ret, cc_growpart.ResizeGrowPart)) def test_handle_with_no_growpart_entry(self): - #if no 'growpart' entry in config, then mode=auto should be used + # if no 'growpart' entry in config, then mode=auto should be used myresizer = object() @@ -141,7 +141,7 @@ class TestResize(MockerTestCase): self.mocker.order() def test_simple_devices(self): - #test simple device list + # test simple device list # this patches out devent2dev, os.stat, and device_part_info # so in the end, doesn't test a lot devs = ["/dev/XXda1", "/dev/YYda2"] @@ -187,7 +187,7 @@ class TestResize(MockerTestCase): find("/dev/YYda2", resized)[1]) self.assertEqual(cc_growpart.RESIZE.SKIPPED, find(enoent[0], resized)[1]) - #self.assertEqual(resize_calls, + # self.assertEqual(resize_calls, # [("/dev/XXda", "1", "/dev/XXda1"), # ("/dev/YYda", "2", "/dev/YYda2")]) finally: diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 4b7b2112..2f86b8f8 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -67,7 +67,7 @@ def check_lps_ret(psc_return, mode=None): cmd = psc_return[0] timeout = psc_return[1] - if not 'shutdown' in psc_return[0][0]: + if 'shutdown' not in psc_return[0][0]: errs.append("string 'shutdown' not in cmd") if mode is not None: diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py index 156441c7..21b89c34 100644 --- a/tests/unittests/test_handler/test_handler_yum_add_repo.py +++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py @@ -24,7 +24,7 @@ class TestConfig(helpers.FilesystemMockingTestCase): 'epel-testing': { 'name': 'Extra Packages for Enterprise Linux 5 - Testing', # Missing this should cause the repo not to be written - #'baseurl': 'http://blah.org/pub/epel/testing/5/$basearch', + # 'baseurl': 'http://blah.org/pub/epel/testing/5/$basearch', 'enabled': False, 'gpgcheck': True, 'gpgkey': 'file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL', diff --git a/tools/hacking.py b/tools/hacking.py index 26a07c53..14bd0cda 100755 --- a/tools/hacking.py +++ b/tools/hacking.py @@ -71,7 +71,7 @@ def cloud_import_alphabetical(physical_line, line_number, lines): # with or without "as y" length = [2, 4] if (len(split_line) in length and len(split_previous) in length and - split_line[0] == "import" and split_previous[0] == "import"): + split_line[0] == "import" and split_previous[0] == "import"): if split_line[1] < split_previous[1]: return (0, "N306: imports not in alphabetical order (%s, %s)" % (split_previous[1], split_line[1])) -- cgit v1.2.3 From 2e6aa38e0986ee67f5e93cd6bb7b32c52f8207e6 Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Tue, 26 Aug 2014 11:53:44 -0700 Subject: Remove pylint pylint was not passing, and per smoser in IRC pylint can/should be removed. --- HACKING.rst | 4 ++-- Makefile | 3 --- test-requirements.txt | 1 - 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/HACKING.rst b/HACKING.rst index 66bf7c90..6bfe4b4d 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -19,9 +19,9 @@ To get changes into cloud-init, the process to follow is: - ``bzr commit`` -* Check pylint and pep8 and test, and address any issues: +* Check pep8 and test, and address any issues: - - ``make test pylint pep8`` + - ``make test pep8`` * Push to launchpad to a personal branch: diff --git a/Makefile b/Makefile index c8b75e73..d96e6488 100644 --- a/Makefile +++ b/Makefile @@ -19,9 +19,6 @@ all: test check_version pep8: @$(CWD)/tools/run-pep8 $(PY_FILES) -pylint: - @$(CWD)/tools/run-pylint $(PY_FILES) - pyflakes: pyflakes $(PY_FILES) diff --git a/test-requirements.txt b/test-requirements.txt index 47d45d47..2edb8066 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,4 +3,3 @@ mocker nose pep8==1.5.7 pyflakes -pylint -- cgit v1.2.3 From 92c04cc87387e244485ad502ecacd46b3f150366 Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Tue, 26 Aug 2014 12:38:41 -0700 Subject: Upgrade configdrive version - Upgrade configdrive to use 2013-10-17 - Fix issue with vendor_data.json parsing Co-Authored-By: Paul Querna --- cloudinit/sources/DataSourceConfigDrive.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 0c35f83a..8e65fc46 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -125,7 +125,19 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): self.userdata_raw = results.get('userdata') self.version = results['version'] self.files.update(results.get('files', {})) - self.vendordata_raw = results.get('vendordata') + + # if vendordata includes 'cloud-init', then read that explicitly + # for cloud-init (for namespacing). + vd = results.get('vendordata') + if isinstance(vd, dict): + if 'cloud-init' in vd: + self.vendordata_raw = vd['cloud-init'] + else: + # TODO(pquerna): this is so wrong. + self.vendordata_raw = json.dumps(vd) + else: + self.vendordata_raw = vd + return True @@ -160,7 +172,7 @@ def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): return "net" -def read_config_drive(source_dir, version="2012-08-10"): +def read_config_drive(source_dir, version="2013-10-17"): reader = openstack.ConfigDriveReader(source_dir) finders = [ (reader.read_v2, [], {'version': version}), -- cgit v1.2.3 From 5fb6482692cfffba5ba45102858b14ba3acc5bc7 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 26 Aug 2014 15:53:41 -0400 Subject: further remove evidence of pylint. This just removes comments '# pylint:' things and other code remnents of pylint. --- Makefile | 2 +- cloudinit/config/cc_disk_setup.py | 2 +- cloudinit/config/cc_mounts.py | 2 +- cloudinit/config/cc_power_state_change.py | 4 ++-- cloudinit/config/cc_resizefs.py | 8 +++---- cloudinit/config/cc_set_passwords.py | 2 +- cloudinit/distros/parsers/resolv_conf.py | 4 ++-- cloudinit/handlers/boot_hook.py | 3 +-- cloudinit/handlers/cloud_config.py | 3 +-- cloudinit/handlers/shell_script.py | 3 +-- cloudinit/handlers/upstart_job.py | 3 +-- cloudinit/patcher.py | 2 +- cloudinit/sources/DataSourceOpenNebula.py | 2 +- cloudinit/type_utils.py | 2 -- cloudinit/url_helper.py | 13 +++++------ cloudinit/util.py | 16 ++++++------- packages/debian/control.in | 1 - pylintrc | 19 ---------------- setup.py | 2 +- tests/unittests/test__init__.py | 3 +-- tests/unittests/test_distros/test_generic.py | 4 ++-- .../test_handler/test_handler_growpart.py | 2 -- tests/unittests/test_merging.py | 2 +- tests/unittests/test_util.py | 6 ++--- tools/hacking.py | 4 ++-- tools/mock-meta.py | 8 +++---- tools/run-pep8 | 2 +- tools/run-pylint | 26 ---------------------- 28 files changed, 45 insertions(+), 105 deletions(-) delete mode 100644 pylintrc delete mode 100755 tools/run-pylint diff --git a/Makefile b/Makefile index d96e6488..009257ca 100644 --- a/Makefile +++ b/Makefile @@ -58,5 +58,5 @@ rpm: deb: ./packages/bddeb -.PHONY: test pylint pyflakes 2to3 clean pep8 rpm deb yaml check_version +.PHONY: test pyflakes 2to3 clean pep8 rpm deb yaml check_version .PHONY: pip-test-requirements pip-requirements clean_pyc diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index a5209268..1660832b 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -484,7 +484,7 @@ def get_partition_mbr_layout(size, layout): def purge_disk_ptable(device): # wipe the first and last megabyte of a disk (or file) # gpt stores partition table both at front and at end. - null = '\0' # pylint: disable=W1401 + null = '\0' start_len = 1024 * 1024 end_len = 1024 * 1024 with open(device, "rb+") as fp: diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 80590118..ba1303d1 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -18,7 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from string import whitespace # pylint: disable=W0402 +from string import whitespace import logging import os.path diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 638daef8..09d37371 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -119,7 +119,7 @@ def load_power_state(cfg): def doexit(sysexit): - os._exit(sysexit) # pylint: disable=W0212 + os._exit(sysexit) def execmd(exe_args, output=None, data_in=None): @@ -127,7 +127,7 @@ def execmd(exe_args, output=None, data_in=None): proc = subprocess.Popen(exe_args, stdin=subprocess.PIPE, stdout=output, stderr=subprocess.STDOUT) proc.communicate(data_in) - ret = proc.returncode # pylint: disable=E1101 + ret = proc.returncode except Exception: doexit(EXIT_FAIL) doexit(ret) diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 667d5977..b9655749 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -28,19 +28,19 @@ from cloudinit import util frequency = PER_ALWAYS -def _resize_btrfs(mount_point, devpth): # pylint: disable=W0613 +def _resize_btrfs(mount_point, devpth): return ('btrfs', 'filesystem', 'resize', 'max', mount_point) -def _resize_ext(mount_point, devpth): # pylint: disable=W0613 +def _resize_ext(mount_point, devpth): return ('resize2fs', devpth) -def _resize_xfs(mount_point, devpth): # pylint: disable=W0613 +def _resize_xfs(mount_point, devpth): return ('xfs_growfs', devpth) -def _resize_ufs(mount_point, devpth): # pylint: disable=W0613 +def _resize_ufs(mount_point, devpth): return ('growfs', devpth) # Do not use a dictionary as these commands should be able to be used diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 4a3b21af..24e33915 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -28,7 +28,7 @@ from cloudinit import distros as ds from cloudinit import ssh_util from cloudinit import util -from string import letters, digits # pylint: disable=W0402 +from string import letters, digits # We are removing certain 'painful' letters/numbers PW_SET = (letters.translate(None, 'loLOI') + diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py index 1be9d46b..5733c25a 100644 --- a/cloudinit/distros/parsers/resolv_conf.py +++ b/cloudinit/distros/parsers/resolv_conf.py @@ -137,8 +137,8 @@ class ResolvConf(object): self._contents.append(('option', ['search', s_list, ''])) return flat_sds - @local_domain.setter # pl51222 pylint: disable=E1101 - def local_domain(self, domain): # pl51222 pylint: disable=E0102 + @local_domain.setter + def local_domain(self, domain): self.parse() self._remove_option('domain') self._contents.append(('option', ['domain', str(domain), ''])) diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index 1848ce2c..3a50cf87 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -53,8 +53,7 @@ class BootHookPartHandler(handlers.Handler): util.write_file(filepath, contents.lstrip(), 0700) return filepath - def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 - payload, frequency): # pylint: disable=W0613 + def handle_part(self, data, ctype, filename, payload, frequency): if ctype in handlers.CONTENT_SIGNALS: return diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 4232700f..bf994e33 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -138,8 +138,7 @@ class CloudConfigPartHandler(handlers.Handler): self.file_names = [] self.cloud_buf = None - def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 - payload, _frequency, headers): # pylint: disable=W0613 + def handle_part(self, data, ctype, filename, payload, frequency, headers): if ctype == handlers.CONTENT_START: self._reset() return diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py index 30c1ed89..9755ab05 100644 --- a/cloudinit/handlers/shell_script.py +++ b/cloudinit/handlers/shell_script.py @@ -44,8 +44,7 @@ class ShellScriptPartHandler(handlers.Handler): handlers.type_from_starts_with(SHELL_PREFIX), ] - def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 - payload, frequency): # pylint: disable=W0613 + def handle_part(self, data, ctype, filename, payload, frequency): if ctype in handlers.CONTENT_SIGNALS: # TODO(harlowja): maybe delete existing things here return diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index bac4cad2..50d193c4 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -44,8 +44,7 @@ class UpstartJobPartHandler(handlers.Handler): handlers.type_from_starts_with(UPSTART_PREFIX), ] - def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 - payload, frequency): + def handle_part(self, data, ctype, filename, payload, frequency): if ctype in handlers.CONTENT_SIGNALS: return diff --git a/cloudinit/patcher.py b/cloudinit/patcher.py index 0f3c034e..f6609d6f 100644 --- a/cloudinit/patcher.py +++ b/cloudinit/patcher.py @@ -41,7 +41,7 @@ def _patch_logging(): fallback_handler = QuietStreamHandler(sys.stderr) fallback_handler.setFormatter(logging.Formatter(FALL_FORMAT)) - def handleError(self, record): # pylint: disable=W0613 + def handleError(self, record): try: fallback_handler.handle(record) fallback_handler.flush() diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 34557f8b..e2469f6e 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -28,7 +28,7 @@ import base64 import os import pwd import re -import string # pylint: disable=W0402 +import string from cloudinit import log as logging from cloudinit import sources diff --git a/cloudinit/type_utils.py b/cloudinit/type_utils.py index 2decbfc5..cc3d9495 100644 --- a/cloudinit/type_utils.py +++ b/cloudinit/type_utils.py @@ -19,8 +19,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# -# pylint: disable=C0302 import types diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 73c1fa4e..3074dd08 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -44,7 +44,7 @@ try: from distutils.version import LooseVersion import pkg_resources _REQ = pkg_resources.get_distribution('requests') - _REQ_VER = LooseVersion(_REQ.version) # pylint: disable=E1103 + _REQ_VER = LooseVersion(_REQ.version) if _REQ_VER >= LooseVersion('0.8.8'): SSL_ENABLED = True if _REQ_VER >= LooseVersion('0.7.0') and _REQ_VER < LooseVersion('1.0.0'): @@ -54,7 +54,7 @@ except: def _cleanurl(url): - parsed_url = list(urlparse(url, scheme='http')) # pylint: disable=E1123 + parsed_url = list(urlparse(url, scheme='http')) if not parsed_url[1] and parsed_url[2]: # Swap these since this seems to be a common # occurrence when given urls like 'www.google.com' @@ -90,7 +90,7 @@ class StringResponse(object): self.contents = contents self.url = None - def ok(self, *args, **kwargs): # pylint: disable=W0613 + def ok(self, *args, **kwargs): if self.code != 200: return False return True @@ -150,7 +150,7 @@ class UrlError(IOError): def _get_ssl_args(url, ssl_details): ssl_args = {} - scheme = urlparse(url).scheme # pylint: disable=E1101 + scheme = urlparse(url).scheme if scheme == 'https' and ssl_details: if not SSL_ENABLED: LOG.warn("SSL is not supported in requests v%s, " @@ -227,10 +227,9 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, r = requests.request(**req_args) if check_status: - r.raise_for_status() # pylint: disable=E1103 + r.raise_for_status() LOG.debug("Read from %s (%s, %sb) after %s attempts", url, - r.status_code, len(r.content), # pylint: disable=E1103 - (i + 1)) + r.status_code, len(r.content), (i + 1)) # Doesn't seem like we can make it use a different # subclass for responses, so add our own backward-compat # attrs diff --git a/cloudinit/util.py b/cloudinit/util.py index 0821901a..bdb0f268 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -19,8 +19,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# -# pylint: disable=C0302 from StringIO import StringIO @@ -42,7 +40,7 @@ import re import shutil import socket import stat -import string # pylint: disable=W0402 +import string import subprocess import sys import tempfile @@ -198,11 +196,11 @@ def fork_cb(child_cb, *args): if fid == 0: try: child_cb(*args) - os._exit(0) # pylint: disable=W0212 + os._exit(0) except: logexc(LOG, "Failed forking and calling callback %s", type_utils.obj_name(child_cb)) - os._exit(1) # pylint: disable=W0212 + os._exit(1) else: LOG.debug("Forked child %s who will run callback %s", fid, type_utils.obj_name(child_cb)) @@ -487,7 +485,7 @@ def redirect_output(outfmt, errfmt, o_out=None, o_err=None): new_fp = open(arg, owith) elif mode == "|": proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin # pylint: disable=E1101 + new_fp = proc.stdin else: raise TypeError("Invalid type for output format: %s" % outfmt) @@ -509,7 +507,7 @@ def redirect_output(outfmt, errfmt, o_out=None, o_err=None): new_fp = open(arg, owith) elif mode == "|": proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin # pylint: disable=E1101 + new_fp = proc.stdin else: raise TypeError("Invalid type for error format: %s" % errfmt) @@ -937,7 +935,7 @@ def is_resolvable(name): should also not exist. The random entry will be resolved inside the search list. """ - global _DNS_REDIRECT_IP # pylint: disable=W0603 + global _DNS_REDIRECT_IP if _DNS_REDIRECT_IP is None: badips = set() badnames = ("does-not-exist.example.com.", "example.invalid.", @@ -1532,7 +1530,7 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, (out, err) = sp.communicate(data) except OSError as e: raise ProcessExecutionError(cmd=args, reason=e) - rc = sp.returncode # pylint: disable=E1101 + rc = sp.returncode if rc not in rcs: raise ProcessExecutionError(stdout=out, stderr=err, exit_code=rc, diff --git a/packages/debian/control.in b/packages/debian/control.in index c892747c..9207e5f4 100644 --- a/packages/debian/control.in +++ b/packages/debian/control.in @@ -9,7 +9,6 @@ Build-Depends: debhelper (>= 9), python (>= 2.6.6-3~), python-nose, pyflakes, - pylint, python-setuptools, python-selinux, python-cheetah, diff --git a/pylintrc b/pylintrc deleted file mode 100644 index ee886510..00000000 --- a/pylintrc +++ /dev/null @@ -1,19 +0,0 @@ -[General] -init-hook='import sys; sys.path.append("tests/")' - -[MESSAGES CONTROL] -# See: http://pylint-messages.wikidot.com/all-codes -# W0142: *args and **kwargs are fine. -# W0511: TODOs in code comments are fine. -# W0702: No exception type(s) specified -# W0703: Catch "Exception" -# C0103: Invalid name -# C0111: Missing docstring -disable=W0142,W0511,W0702,W0703,C0103,C0111 - -[REPORTS] -reports=no -include-ids=yes - -[FORMAT] -max-line-length=79 diff --git a/setup.py b/setup.py index 556103b9..2fb191c0 100755 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ def tiny_p(cmd, capture=True): sp = subprocess.Popen(cmd, stdout=stdout, stderr=stderr, stdin=None) (out, err) = sp.communicate() - ret = sp.returncode # pylint: disable=E1101 + ret = sp.returncode if ret not in [0]: raise RuntimeError("Failed running %s [rc=%s] (%s, %s)" % (cmd, ret, out, err)) diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index 03065c8b..17965488 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -18,8 +18,7 @@ class FakeModule(handlers.Handler): def list_types(self): return self.types - def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 - payload, frequency): + def handle_part(self, data, ctype, filename, payload, frequency): pass diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index a972568f..db6aa0e8 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -26,8 +26,8 @@ package_mirrors = [ unknown_arch_info ] -gpmi = distros._get_package_mirror_info # pylint: disable=W0212 -gapmi = distros._get_arch_package_mirror_info # pylint: disable=W0212 +gpmi = distros._get_package_mirror_info +gapmi = distros._get_arch_package_mirror_info class TestGenericDistro(helpers.FilesystemMockingTestCase): diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index fa624197..5d0636d1 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -203,8 +203,6 @@ def simple_device_part_info(devpath): class Bunch(object): - st_mode = None # fix pylint complaint - def __init__(self, **kwds): self.__dict__.update(kwds) diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 17704f8e..07b610f7 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -11,7 +11,7 @@ import glob import os import random import re -import string # pylint: disable=W0402 +import string SOURCE_PAT = "source*.*yaml" EXPECTED_PAT = "expected%s.yaml" diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 0cb41520..35e92445 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -1,5 +1,3 @@ -# pylint: disable=C0301 -# the mountinfo data lines are too long import os import stat import yaml @@ -18,7 +16,7 @@ class FakeSelinux(object): self.match_what = match_what self.restored = [] - def matchpathcon(self, path, mode): # pylint: disable=W0613 + def matchpathcon(self, path, mode): if path == self.match_what: return else: @@ -27,7 +25,7 @@ class FakeSelinux(object): def is_selinux_enabled(self): return True - def restorecon(self, path, recursive): # pylint: disable=W0613 + def restorecon(self, path, recursive): self.restored.append(path) diff --git a/tools/hacking.py b/tools/hacking.py index 14bd0cda..e7797564 100755 --- a/tools/hacking.py +++ b/tools/hacking.py @@ -154,7 +154,7 @@ def add_cloud(): if not inspect.isfunction(function): continue if name.startswith("cloud_"): - exec("pep8.%s = %s" % (name, name)) # pylint: disable=W0122 + exec("pep8.%s = %s" % (name, name)) if __name__ == "__main__": # NOVA based 'hacking.py' error codes start with an N @@ -163,7 +163,7 @@ if __name__ == "__main__": pep8.current_file = current_file pep8.readlines = readlines try: - pep8._main() # pylint: disable=W0212 + pep8._main() finally: if len(_missingImport) > 0: print >> sys.stderr, ("%i imports missing in this test environment" diff --git a/tools/mock-meta.py b/tools/mock-meta.py index c79f0598..dfbc2a71 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -23,7 +23,7 @@ import json import logging import os import random -import string # pylint: disable=W0402 +import string import sys import yaml @@ -306,7 +306,7 @@ class UserDataHandler(object): blob = "\n".join(lines) return blob.strip() - def get_data(self, params, who, **kwargs): # pylint: disable=W0613 + def get_data(self, params, who, **kwargs): if not params: return self._get_user_blob(who=who) return NOT_IMPL_RESPONSE @@ -427,8 +427,8 @@ def extract_opts(): def setup_fetchers(opts): - global meta_fetcher # pylint: disable=W0603 - global user_fetcher # pylint: disable=W0603 + global meta_fetcher + global user_fetcher meta_fetcher = MetaDataHandler(opts) user_fetcher = UserDataHandler(opts) diff --git a/tools/run-pep8 b/tools/run-pep8 index cfce5edd..d0a131f6 100755 --- a/tools/run-pep8 +++ b/tools/run-pep8 @@ -13,7 +13,7 @@ else base=`pwd`/tools/ fi -IGNORE="E501" # Line too long (these are caught by pylint) +IGNORE="" # King Arthur: Be quiet! ... Be Quiet! I Order You to Be Quiet. IGNORE="$IGNORE,E121" # Continuation line indentation is not a multiple of four diff --git a/tools/run-pylint b/tools/run-pylint deleted file mode 100755 index 0fe0c64a..00000000 --- a/tools/run-pylint +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -if [ $# -eq 0 ]; then - files=( bin/cloud-init $(find * -name "*.py" -type f) ) -else - files=( "$@" ); -fi - -RC_FILE="pylintrc" -if [ ! -f $RC_FILE ]; then - RC_FILE="../pylintrc" -fi - -cmd=( - pylint - --rcfile=$RC_FILE - --disable=R - --disable=I - --dummy-variables-rgx="_" - "${files[@]}" -) - -echo -e "\nRunning pylint:" -echo "${cmd[@]}" -"${cmd[@]}" - -- cgit v1.2.3 From 5fc46193d3e65d0eaaaa45bcf41c5b35b4e80df7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 26 Aug 2014 13:41:46 -0700 Subject: Optimize away most of the path_exists checks --- cloudinit/sources/helpers/openstack.py | 101 +++++++++++++++++---------------- 1 file changed, 52 insertions(+), 49 deletions(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 0fac0335..361c9994 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -149,10 +149,6 @@ class BaseReader(object): def _path_join(self, base, *add_ons): pass - @abc.abstractmethod - def _path_exists(self, path): - pass - @abc.abstractmethod def _path_read(self, path): pass @@ -170,22 +166,9 @@ class BaseReader(object): path = self._path_join(self.base_path, "openstack", *path_pieces) return self._path_read(path) + @abc.abstractmethod def _find_working_version(self, version): - search_versions = [version] + list(OS_VERSIONS) - for potential_version in search_versions: - if not potential_version: - continue - path = self._path_join(self.base_path, "openstack", - potential_version) - if self._path_exists(path): - if potential_version != version: - LOG.debug("Version '%s' not available, attempting to use" - " version '%s' instead", version, - potential_version) - return potential_version - LOG.debug("Version '%s' not available, attempting to use '%s'" - " instead", version, OS_LATEST) - return OS_LATEST + pass def read_v2(self, version=None): """Reads a version 2 formatted location. @@ -228,15 +211,14 @@ class BaseReader(object): path = self._path_join(self.base_path, path) data = None found = False - if self._path_exists(path): - try: - data = self._path_read(path) - except IOError: - raise NonReadable("Failed to read: %s" % path) - found = True + try: + data = self._path_read(path) + except IOError: + pass else: - if required: - raise NonReadable("Missing mandatory path: %s" % path) + found = True + if required and not found: + raise NonReadable("Missing mandatory path: %s" % path) if found and translator: try: data = translator(data) @@ -315,6 +297,23 @@ class ConfigDriveReader(BaseReader): def _path_read(self, path): return util.load_file(path) + def _find_working_version(self, version): + search_versions = [version] + list(OS_VERSIONS) + for potential_version in search_versions: + if not potential_version: + continue + path = self._path_join(self.base_path, "openstack", + potential_version) + if self._path_exists(path): + if potential_version != version: + LOG.debug("Version '%s' not available, attempting to use" + " version '%s' instead", version, + potential_version) + return potential_version + LOG.debug("Version '%s' not available, attempting to use '%s'" + " instead", version, OS_LATEST) + return OS_LATEST + def _read_ec2_metadata(self): path = self._path_join(self.base_path, 'ec2', 'latest', 'meta-data.json') @@ -401,6 +400,32 @@ class MetadataReader(BaseReader): self.timeout = float(timeout) self.retries = int(retries) + def _find_working_version(self, version): + search_versions = [version] + list(OS_VERSIONS) + version_path = self._path_join(self.base_path, "openstack") + versions_available = [] + try: + versions = self._path_read(version_path) + except IOError: + pass + else: + for line in versions.splitlines(): + line = line.strip() + if not line: + continue + versions_available.append(line) + for potential_version in search_versions: + if potential_version not in versions_available: + continue + if potential_version != version: + LOG.debug("Version '%s' not available, attempting to use" + " version '%s' instead", version, + potential_version) + return potential_version + LOG.debug("Version '%s' not available, searched %s, attempting to" + " use '%s' instead", version, search_versions, OS_LATEST) + return OS_LATEST + def _path_read(self, path): response = url_helper.readurl(path, retries=self.retries, @@ -408,28 +433,6 @@ class MetadataReader(BaseReader): timeout=self.timeout) return response.contents - def _path_exists(self, path): - - def should_retry_cb(request, cause): - try: - code = int(cause.code) - if code >= 400: - return False - except (TypeError, ValueError): - # Older versions of requests didn't have a code. - pass - return True - - try: - response = url_helper.readurl(path, - retries=self.retries, - ssl_details=self.ssl_details, - timeout=self.timeout, - exception_cb=should_retry_cb) - return response.ok() - except IOError: - return False - def _path_join(self, base, *add_ons): return url_helper.combine_url(base, *add_ons) -- cgit v1.2.3 From df7b6c5564e6ef8f5ceae9de8ff79410a4894f5f Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Tue, 26 Aug 2014 14:45:08 -0700 Subject: Add failback for older Openstack configdrive versions - Also utilizing the constants defined in cloudinit/sources/helpers/openstack.py for configdrive versions --- cloudinit/sources/DataSourceConfigDrive.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 8e65fc46..c55f7d4a 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -172,10 +172,12 @@ def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): return "net" -def read_config_drive(source_dir, version="2013-10-17"): +def read_config_drive(source_dir, version=openstack.OS_HAVANA): reader = openstack.ConfigDriveReader(source_dir) finders = [ (reader.read_v2, [], {'version': version}), + (reader.read_v2, [], {'version': openstack.OS_GRIZZLY}), + (reader.read_v2, [], {'version': openstack.OS_FOLSOM}), (reader.read_v1, [], {}), ] excps = [] -- cgit v1.2.3 From 1e019cc001a8f31fa978d76b86fbe7813b2a076f Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Tue, 26 Aug 2014 15:12:35 -0700 Subject: Refactor vendor_data handling vendor_data is guaranteed to be a dict if it exists; if it doesn't exist ensure it's represented by an empty dict to avoid checking it to see if it's a dict. --- cloudinit/sources/DataSourceConfigDrive.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index c55f7d4a..65477184 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -126,15 +126,12 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): self.version = results['version'] self.files.update(results.get('files', {})) + # If there is no vendordata, set vd to an empty dict instead of None + vd = results.get('vendordata', {}) # if vendordata includes 'cloud-init', then read that explicitly # for cloud-init (for namespacing). - vd = results.get('vendordata') - if isinstance(vd, dict): - if 'cloud-init' in vd: - self.vendordata_raw = vd['cloud-init'] - else: - # TODO(pquerna): this is so wrong. - self.vendordata_raw = json.dumps(vd) + if 'cloud-init' in vd: + self.vendordata_raw = vd['cloud-init'] else: self.vendordata_raw = vd -- cgit v1.2.3 From cc9e3af6c95b3263a49d4590d9dd176bdc570c99 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 26 Aug 2014 23:02:58 -0700 Subject: Add logging around failure to read optional/mandatory paths --- cloudinit/sources/helpers/openstack.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 361c9994..e5a38de0 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -213,8 +213,12 @@ class BaseReader(object): found = False try: data = self._path_read(path) - except IOError: - pass + except IOError as e: + if not required: + LOG.debug("Failed reading optional path %s due" + " to: %s", path, e) + else: + LOG.exception("Failed reading mandatory path %s", path) else: found = True if required and not found: -- cgit v1.2.3 From 6a6d3a499c2327b03993bbaea2b9d0df5dc7eb64 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 26 Aug 2014 23:04:56 -0700 Subject: Just use os.path.exists directly --- cloudinit/sources/helpers/openstack.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index e5a38de0..418ab4d1 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -295,9 +295,6 @@ class ConfigDriveReader(BaseReader): components = [base] + list(add_ons) return os.path.join(*components) - def _path_exists(self, path): - return os.path.exists(path) - def _path_read(self, path): return util.load_file(path) @@ -308,7 +305,7 @@ class ConfigDriveReader(BaseReader): continue path = self._path_join(self.base_path, "openstack", potential_version) - if self._path_exists(path): + if os.path.exists(path): if potential_version != version: LOG.debug("Version '%s' not available, attempting to use" " version '%s' instead", version, @@ -321,7 +318,7 @@ class ConfigDriveReader(BaseReader): def _read_ec2_metadata(self): path = self._path_join(self.base_path, 'ec2', 'latest', 'meta-data.json') - if not self._path_exists(path): + if not os.path.exists(path): return {} else: try: @@ -341,7 +338,7 @@ class ConfigDriveReader(BaseReader): found = {} for name in FILES_V1.keys(): path = self._path_join(self.base_path, name) - if self._path_exists(path): + if os.path.exists(path): found[name] = path if len(found) == 0: raise NonReadable("%s: no files found" % (self.base_path)) -- cgit v1.2.3 From 9a7587f35f327b9f8cafce8687832e9e77c1afde Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 26 Aug 2014 23:08:14 -0700 Subject: Log a warning when unable to fetch the openstack versions --- cloudinit/sources/helpers/openstack.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 418ab4d1..56986a94 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -407,8 +407,9 @@ class MetadataReader(BaseReader): versions_available = [] try: versions = self._path_read(version_path) - except IOError: - pass + except IOError as e: + LOG.warn("Unable to read openstack versions from %s due" + " to: %s", version_path, e) else: for line in versions.splitlines(): line = line.strip() -- cgit v1.2.3 From fb482ce4a36d9b4be75a8bf5b428189548a205d9 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 26 Aug 2014 23:10:50 -0700 Subject: Show the available versions in the debug log message --- cloudinit/sources/helpers/openstack.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 56986a94..3ceec837 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -424,8 +424,9 @@ class MetadataReader(BaseReader): " version '%s' instead", version, potential_version) return potential_version - LOG.debug("Version '%s' not available, searched %s, attempting to" - " use '%s' instead", version, search_versions, OS_LATEST) + LOG.debug("Version '%s' not available, searched for %s (with available" + " versions being %s), attempting to use '%s' instead", + version, search_versions, versions_available, OS_LATEST) return OS_LATEST def _path_read(self, path): -- cgit v1.2.3 From d701035265c765bc42cb3bc358f2bfd0b41f484b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 27 Aug 2014 12:30:23 -0700 Subject: Fixed more of the slowness around fetching and retrying --- cloudinit/sources/helpers/openstack.py | 112 +++++++++++++--------- tests/unittests/test_datasource/test_openstack.py | 23 ++++- 2 files changed, 84 insertions(+), 51 deletions(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 3ceec837..2d0fc70e 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -21,6 +21,7 @@ import abc import base64 import copy +import httplib import os from cloudinit import ec2_utils @@ -153,10 +154,31 @@ class BaseReader(object): def _path_read(self, path): pass + @abc.abstractmethod + def _fetch_available_versions(self): + pass + @abc.abstractmethod def _read_ec2_metadata(self): pass + def _find_working_version(self, version): + search_versions = [version] + list(OS_VERSIONS) + available_versions = self._fetch_available_versions() + for potential_version in search_versions: + if not potential_version: + continue + if potential_version not in available_versions: + continue + if potential_version != version: + LOG.debug("Version '%s' not available, attempting to use" + " version '%s' instead", version, + potential_version) + return potential_version + LOG.debug("Version '%s' not available, attempting to use '%s'" + " instead", version, OS_LATEST) + return OS_LATEST + def _read_content_path(self, item): path = item.get('content_path', '').lstrip("/") path_pieces = path.split("/") @@ -166,10 +188,6 @@ class BaseReader(object): path = self._path_join(self.base_path, "openstack", *path_pieces) return self._path_read(path) - @abc.abstractmethod - def _find_working_version(self, version): - pass - def read_v2(self, version=None): """Reads a version 2 formatted location. @@ -290,6 +308,7 @@ class BaseReader(object): class ConfigDriveReader(BaseReader): def __init__(self, base_path): super(ConfigDriveReader, self).__init__(base_path) + self._versions = None def _path_join(self, base, *add_ons): components = [base] + list(add_ons) @@ -298,22 +317,21 @@ class ConfigDriveReader(BaseReader): def _path_read(self, path): return util.load_file(path) - def _find_working_version(self, version): - search_versions = [version] + list(OS_VERSIONS) - for potential_version in search_versions: - if not potential_version: - continue - path = self._path_join(self.base_path, "openstack", - potential_version) - if os.path.exists(path): - if potential_version != version: - LOG.debug("Version '%s' not available, attempting to use" - " version '%s' instead", version, - potential_version) - return potential_version - LOG.debug("Version '%s' not available, attempting to use '%s'" - " instead", version, OS_LATEST) - return OS_LATEST + def _fetch_available_versions(self): + if self._versions is not None: + return self._versions + else: + versions_available = [] + path = self._path_join(self.base_path, 'openstack') + try: + for child in os.listdir(path): + child_path = os.path.join(path, child) + if os.path.isdir(child_path): + versions_available.append(child) + except (OSError, IOError): + pass + self._versions = tuple(versions_available) + return self._versions def _read_ec2_metadata(self): path = self._path_join(self.base_path, @@ -400,40 +418,40 @@ class MetadataReader(BaseReader): self.ssl_details = ssl_details self.timeout = float(timeout) self.retries = int(retries) + self._versions = None - def _find_working_version(self, version): - search_versions = [version] + list(OS_VERSIONS) - version_path = self._path_join(self.base_path, "openstack") - versions_available = [] - try: - versions = self._path_read(version_path) - except IOError as e: - LOG.warn("Unable to read openstack versions from %s due" - " to: %s", version_path, e) + def _fetch_available_versions(self): + if self._versions is not None: + return self._versions else: - for line in versions.splitlines(): - line = line.strip() - if not line: - continue - versions_available.append(line) - for potential_version in search_versions: - if potential_version not in versions_available: - continue - if potential_version != version: - LOG.debug("Version '%s' not available, attempting to use" - " version '%s' instead", version, - potential_version) - return potential_version - LOG.debug("Version '%s' not available, searched for %s (with available" - " versions being %s), attempting to use '%s' instead", - version, search_versions, versions_available, OS_LATEST) - return OS_LATEST + path = self._path_join(self.base_path, "openstack") + versions_available = [] + try: + versions = self._path_read(path) + except IOError as e: + LOG.warn("Unable to read openstack versions from %s due" + " to: %s", path, e) + else: + for line in versions.splitlines(): + line = line.strip() + if not line: + continue + versions_available.append(line) + self._versions = tuple(versions_available) + return self._versions def _path_read(self, path): + + def should_retry(_request_args, cause): + if cause.code == httplib.NOT_FOUND: + return False + return True + response = url_helper.readurl(path, retries=self.retries, ssl_details=self.ssl_details, - timeout=self.timeout) + timeout=self.timeout, + exception_cb=should_retry) return response.contents def _path_join(self, base, *add_ons): diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index f43cbec8..412ae5a4 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -67,8 +67,8 @@ OSTACK_META = { CONTENT_0 = 'This is contents of /etc/foo.cfg\n' CONTENT_1 = '# this is /etc/bar/bar.cfg\n' OS_FILES = { - 'openstack/2012-08-10/meta_data.json': json.dumps(OSTACK_META), - 'openstack/2012-08-10/user_data': USER_DATA, + 'openstack/latest/meta_data.json': json.dumps(OSTACK_META), + 'openstack/latest/user_data': USER_DATA, 'openstack/content/0000': CONTENT_0, 'openstack/content/0001': CONTENT_1, 'openstack/latest/meta_data.json': json.dumps(OSTACK_META), @@ -78,6 +78,9 @@ OS_FILES = { EC2_FILES = { 'latest/user-data': USER_DATA, } +EC2_VERSIONS = [ + 'lastest', +] def _register_uris(version, ec2_files, ec2_meta, os_files): @@ -85,6 +88,9 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): same data returned by the openstack metadata service (and ec2 service).""" def match_ec2_url(uri, headers): + path = uri.path.strip("/") + if len(path) == 0: + return (200, headers, "\n".join(EC2_VERSIONS)) path = uri.path.lstrip("/") if path in ec2_files: return (200, headers, ec2_files.get(path)) @@ -110,11 +116,20 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): return (200, headers, str(value)) return (404, headers, '') - def get_request_callback(method, uri, headers): - uri = urlparse(uri) + def match_os_uri(uri, headers): + path = uri.path.strip("/") + if path == 'openstack': + return (200, headers, "\n".join([openstack.OS_LATEST])) path = uri.path.lstrip("/") if path in os_files: return (200, headers, os_files.get(path)) + return (404, headers, '') + + def get_request_callback(method, uri, headers): + uri = urlparse(uri) + path = uri.path.lstrip("/").split("/") + if path[0] == 'openstack': + return match_os_uri(uri, headers) return match_ec2_url(uri, headers) hp.register_uri(hp.GET, re.compile(r'http://169.254.169.254/.*'), -- cgit v1.2.3 From 984d36fb8c5feb82c31121ffd5d6a72b4f593499 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 27 Aug 2014 12:56:50 -0700 Subject: Fix retry cb to reflect what used to exist --- cloudinit/sources/helpers/openstack.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 2d0fc70e..025a2404 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -21,7 +21,6 @@ import abc import base64 import copy -import httplib import os from cloudinit import ec2_utils @@ -442,16 +441,21 @@ class MetadataReader(BaseReader): def _path_read(self, path): - def should_retry(_request_args, cause): - if cause.code == httplib.NOT_FOUND: - return False + def should_retry_cb(_request_args, cause): + try: + code = int(cause.code) + if code >= 400: + return False + except (TypeError, ValueError): + # Older versions of requests didn't have a code. + pass return True response = url_helper.readurl(path, retries=self.retries, ssl_details=self.ssl_details, timeout=self.timeout, - exception_cb=should_retry) + exception_cb=should_retry_cb) return response.contents def _path_join(self, base, *add_ons): -- cgit v1.2.3 From 419e0caab7e005827485460372c7f0d54ac0e9c9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 27 Aug 2014 17:03:43 -0400 Subject: no functional changes, but some minor changes. --- cloudinit/sources/helpers/openstack.py | 75 ++++++++++------------- tests/unittests/test_datasource/test_openstack.py | 2 +- 2 files changed, 34 insertions(+), 43 deletions(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 025a2404..3c6bb6aa 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -162,21 +162,25 @@ class BaseReader(object): pass def _find_working_version(self, version): + try: + versions_available = self._fetch_available_versions(self) + except Exception as e: + LOG.warn("Unable to read openstack versions from %s due to: %s", + self.base_path, e) + versions_available = [] + search_versions = [version] + list(OS_VERSIONS) - available_versions = self._fetch_available_versions() + selected_version = OS_LATEST for potential_version in search_versions: - if not potential_version: + if potential_version not in versions_available: continue - if potential_version not in available_versions: - continue - if potential_version != version: - LOG.debug("Version '%s' not available, attempting to use" - " version '%s' instead", version, - potential_version) - return potential_version - LOG.debug("Version '%s' not available, attempting to use '%s'" - " instead", version, OS_LATEST) - return OS_LATEST + selected_version = potential_version + break + + if selected_version != version: + LOG.warn("Version '%s' not available, attempting to use" + " version '%s' instead", version, selected_version) + return selected_version def _read_content_path(self, item): path = item.get('content_path', '').lstrip("/") @@ -317,20 +321,12 @@ class ConfigDriveReader(BaseReader): return util.load_file(path) def _fetch_available_versions(self): - if self._versions is not None: - return self._versions - else: - versions_available = [] + if self._versions is None: path = self._path_join(self.base_path, 'openstack') - try: - for child in os.listdir(path): - child_path = os.path.join(path, child) - if os.path.isdir(child_path): - versions_available.append(child) - except (OSError, IOError): - pass - self._versions = tuple(versions_available) - return self._versions + found = [d for d in os.listdir(path) + if os.path.isdir(os.path.join(path))] + self._versions = tuple(found) + return self._versions def _read_ec2_metadata(self): path = self._path_join(self.base_path, @@ -420,24 +416,19 @@ class MetadataReader(BaseReader): self._versions = None def _fetch_available_versions(self): + # /openstack/ returns a newline separated list of versions if self._versions is not None: - return self._versions - else: - path = self._path_join(self.base_path, "openstack") - versions_available = [] - try: - versions = self._path_read(path) - except IOError as e: - LOG.warn("Unable to read openstack versions from %s due" - " to: %s", path, e) - else: - for line in versions.splitlines(): - line = line.strip() - if not line: - continue - versions_available.append(line) - self._versions = tuple(versions_available) - return self._versions + return self.os_versions + found = [] + content = self._path_read(version_path) + for line in content.splitlines(): + line = line.strip() + if not line: + continue + found.append(line) + self._versions = tuple(found) + return self._versions + def _path_read(self, path): diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 412ae5a4..530fba20 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -79,7 +79,7 @@ EC2_FILES = { 'latest/user-data': USER_DATA, } EC2_VERSIONS = [ - 'lastest', + 'latest', ] -- cgit v1.2.3 From 5fbdb88b843bfa3b5fb2c454219779b9889cf787 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Thu, 28 Aug 2014 17:50:24 +0000 Subject: fix: Make sure this freebsd test succeeds on all platforms (harlowja). --- tests/unittests/test_distros/test_netconfig.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 96379025..ed997a1d 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -180,12 +180,17 @@ NETWORKING=yes spec=False, passthrough=False) exists_mock = self.mocker.replace(os.path.isfile, spec=False, passthrough=False) + load_mock = self.mocker.replace(util.load_file, + spec=False, passthrough=False) exists_mock(mocker.ARGS) self.mocker.count(0, None) self.mocker.result(False) write_bufs = {} + read_bufs = { + '/etc/rc.conf': '', + } def replace_write(filename, content, mode=0644, omode="wb"): buf = WriteBuffer() @@ -194,8 +199,24 @@ NETWORKING=yes buf.write(content) write_bufs[filename] = buf + def replace_read(fname, read_cb=None, quiet=False): + if fname not in read_bufs: + if fname in write_bufs: + return str(write_bufs[fname]) + raise IOError("%s not found" % fname) + else: + if fname in write_bufs: + return str(write_bufs[fname]) + return read_bufs[fname] + util_mock(mocker.ARGS) self.mocker.call(replace_write) + self.mocker.count(0, None) + + load_mock(mocker.ARGS) + self.mocker.call(replace_read) + self.mocker.count(0, None) + self.mocker.replay() fbsd_distro.apply_network(BASE_NET_CFG, False) -- cgit v1.2.3 From fbfd3789ede42ad50e0b79f202ab18b65537af52 Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Thu, 28 Aug 2014 17:54:00 +0000 Subject: fix: The original regex was a little harsh, the rest of the bits regarding keys and values from /etc/rc.conf is tweaked as well (harlowja). --- cloudinit/distros/freebsd.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index 9c923480..42ef2290 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -51,12 +51,18 @@ class Distro(distros.Distro): LOG.debug("Checking %s for: %s = %s", self.rc_conf_fn, key, value) conf = self.loadrcconf() config_changed = False - for item in conf: - if item == key and conf[item] != value: - conf[item] = value - LOG.debug("Changing key in %s: %s = %s", self.rc_conf_fn, key, - value) - config_changed = True + if key not in conf: + LOG.debug("Adding key in %s: %s = %s", self.rc_conf_fn, key, + value) + conf[key] = value + config_changed = True + else: + for item in conf.keys(): + if item == key and conf[item] != value: + conf[item] = value + LOG.debug("Changing key in %s: %s = %s", self.rc_conf_fn, + key, value) + config_changed = True if config_changed: LOG.info("Writing %s", self.rc_conf_fn) @@ -69,7 +75,7 @@ class Distro(distros.Distro): # quotes are ignored: # hostname="bla" def loadrcconf(self): - RE_MATCH = re.compile(r'^(\w+)="?(\w+)"?') + RE_MATCH = re.compile(r'^(\w+)\s*=\s*(.*)\s*') conf = {} lines = util.load_file(self.rc_conf_fn).splitlines() for line in lines: @@ -77,9 +83,17 @@ class Distro(distros.Distro): if not m: LOG.debug("Skipping line from /etc/rc.conf: %s", line) continue - key = m.group(1).rstrip() val = m.group(2).rstrip() + # Kill them quotes (not completely correct, aka won't handle + # quoted values, but should be ok ...) + if val[0] in ('"', "'"): + val = val[1:] + if val[-1] in ('"', "'"): + val = val[0:-1] + if len(val) == 0: + LOG.debug("Skipping empty value from /etc/rc.conf: %s", line) + continue conf[key] = val return conf @@ -237,7 +251,7 @@ class Distro(distros.Distro): dev_names = entries.keys() for (dev, info) in entries.iteritems(): # Skip the loopback interface. - if dev == 'lo0': + if dev.startswith('lo'): continue LOG.info('Configuring interface %s', dev) -- cgit v1.2.3 From 141caf7f3b224c0265c7bb0014b96ca08aa67193 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 2 Sep 2014 13:31:18 -0700 Subject: Remove/adjust the verbose 'failed at attempted import of' log Instead of using this log (which really isn't a failure) we should instead of just return the looked up locations and then if there really is an error the caller can handle the usage of the looked up locations as they choose fit. --- cloudinit/distros/__init__.py | 8 +++----- cloudinit/importer.py | 21 ++++++++------------- cloudinit/mergers/__init__.py | 10 ++++++---- cloudinit/sources/__init__.py | 6 +++--- cloudinit/stages.py | 18 +++++++++--------- 5 files changed, 29 insertions(+), 34 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 4b41220e..9c9211fe 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -847,12 +847,10 @@ def extract_default(users, default_name=None, default_config=None): def fetch(name): - locs = importer.find_module(name, - ['', __name__], - ['Distro']) + locs, looked_locs = importer.find_module(name, ['', __name__], ['Distro']) if not locs: - raise ImportError("No distribution found for distro %s" - % (name)) + raise ImportError("No distribution found for distro %s (searched %s)" + % (name, looked_locs)) mod = importer.import_module(locs[0]) cls = getattr(mod, 'Distro') return cls diff --git a/cloudinit/importer.py b/cloudinit/importer.py index a1929137..fb57253c 100644 --- a/cloudinit/importer.py +++ b/cloudinit/importer.py @@ -22,10 +22,6 @@ import sys -from cloudinit import log as logging - -LOG = logging.getLogger(__name__) - def import_module(module_name): __import__(module_name) @@ -33,25 +29,24 @@ def import_module(module_name): def find_module(base_name, search_paths, required_attrs=None): - found_places = [] if not required_attrs: required_attrs = [] # NOTE(harlowja): translate the search paths to include the base name. - real_paths = [] + lookup_paths = [] for path in search_paths: real_path = [] if path: real_path.extend(path.split(".")) real_path.append(base_name) full_path = '.'.join(real_path) - real_paths.append(full_path) - for full_path in real_paths: + lookup_paths.append(full_path) + found_paths = [] + for full_path in lookup_paths: mod = None try: mod = import_module(full_path) - except ImportError as e: - LOG.debug("Failed at attempted import of '%s' due to: %s", - full_path, e) + except ImportError: + pass if not mod: continue found_attrs = 0 @@ -59,5 +54,5 @@ def find_module(base_name, search_paths, required_attrs=None): if hasattr(mod, attr): found_attrs += 1 if found_attrs == len(required_attrs): - found_places.append(full_path) - return found_places + found_paths.append(full_path) + return (found_paths, lookup_paths) diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py index 650b42a9..03aa1ee1 100644 --- a/cloudinit/mergers/__init__.py +++ b/cloudinit/mergers/__init__.py @@ -143,12 +143,14 @@ def construct(parsed_mergers): for (m_name, m_ops) in parsed_mergers: if not m_name.startswith(MERGER_PREFIX): m_name = MERGER_PREFIX + str(m_name) - merger_locs = importer.find_module(m_name, - [__name__], - [MERGER_ATTR]) + merger_locs, looked_locs = importer.find_module(m_name, + [__name__], + [MERGER_ATTR]) if not merger_locs: msg = ("Could not find merger module named '%s' " - "with attribute '%s'") % (m_name, MERGER_ATTR) + "with attribute '%s' (searched %s)") % (m_name, + MERGER_ATTR, + looked_locs) raise ImportError(msg) else: mod = importer.import_module(merger_locs[0]) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 7d52a2e6..5d58bebd 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -272,9 +272,9 @@ def list_sources(cfg_list, depends, pkg_list): for ds_name in cfg_list: if not ds_name.startswith(DS_PREFIX): ds_name = '%s%s' % (DS_PREFIX, ds_name) - m_locs = importer.find_module(ds_name, - pkg_list, - ['get_datasource_list']) + m_locs, _looked_locs = importer.find_module(ds_name, + pkg_list, + ['get_datasource_list']) for m_loc in m_locs: mod = importer.import_module(m_loc) lister = getattr(mod, "get_datasource_list") diff --git a/cloudinit/stages.py b/cloudinit/stages.py index d29d480a..67f467f7 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -386,12 +386,12 @@ class Init(object): potential_handlers = util.find_modules(path) for (fname, mod_name) in potential_handlers.iteritems(): try: - mod_locs = importer.find_module(mod_name, [''], - ['list_types', - 'handle_part']) + mod_locs, looked_locs = importer.find_module( + mod_name, [''], ['list_types', 'handle_part']) if not mod_locs: - LOG.warn(("Could not find a valid user-data handler" - " named %s in file %s"), mod_name, fname) + LOG.warn("Could not find a valid user-data handler" + " named %s in file %s (searched %s)", + mod_name, fname, looked_locs) continue mod = importer.import_module(mod_locs[0]) mod = handlers.fixup_handler(mod) @@ -621,11 +621,11 @@ class Modules(object): " has an unknown frequency %s"), raw_name, freq) # Reset it so when ran it will get set to a known value freq = None - mod_locs = importer.find_module(mod_name, - ['', type_utils.obj_name(config)], - ['handle']) + mod_locs, looked_locs = importer.find_module( + mod_name, ['', type_utils.obj_name(config)], ['handle']) if not mod_locs: - LOG.warn("Could not find module named %s", mod_name) + LOG.warn("Could not find module named %s (searched %s)", + mod_name, looked_locs) continue mod = config.fixup_module(importer.import_module(mod_locs[0])) mostly_mods.append([mod, raw_name, freq, run_args]) -- cgit v1.2.3 From 7dade55ae5e429e0abb1687fb94799226c211e18 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 2 Sep 2014 14:17:38 -0700 Subject: Fix logic statement and pep8 issue --- cloudinit/sources/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 7d52a2e6..a2a8f492 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -66,7 +66,7 @@ class DataSource(object): name = name[0:-3] self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, - ("datasource", name), {}) + ("datasource", name), {}) if not ud_proc: self.ud_proc = ud.UserDataProcessor(self.paths) else: @@ -166,7 +166,7 @@ class DataSource(object): defhost = "localhost" domain = defdomain - if self.metadata or 'local-hostname' not in self.metadata: + if not self.metadata or 'local-hostname' not in self.metadata: # this is somewhat questionable really. # the cloud datasource was asked for a hostname # and didn't have one. raising error might be more appropriate -- cgit v1.2.3 From 439f9c95f0bd024bbc47e01531c50e45f2d68c2c Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Wed, 3 Sep 2014 17:19:37 +0000 Subject: fix: Write sudo rules upon user creation. --- cloudinit/distros/freebsd.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index 42ef2290..849834eb 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -37,6 +37,7 @@ class Distro(distros.Distro): login_conf_fn = '/etc/login.conf' login_conf_fn_bak = '/etc/login.conf.orig' resolv_conf_fn = '/etc/resolv.conf' + ci_sudoers_fn = '/usr/local/etc/sudoers.d/90-cloud-init-users' def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) @@ -219,10 +220,6 @@ class Distro(distros.Distro): util.logexc(LOG, "Failed to lock user %s", name) raise e - # TODO: - def write_sudo_rules(self, name, rules, sudo_file=None): - LOG.debug("[write_sudo_rules] Name: %s", name) - def create_user(self, name, **kwargs): self.add_user(name, **kwargs) -- cgit v1.2.3 From 3bec4688a7001a3ce9bbfd1a82e691ea94884a9e Mon Sep 17 00:00:00 2001 From: Harm Weites Date: Wed, 3 Sep 2014 17:20:43 +0000 Subject: fix: Since these now actually exist in ports, install them properly (with pkg). --- tools/build-on-freebsd | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index 23bdf487..65d783f7 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -17,6 +17,7 @@ pkgs=" py27-prettytable py27-requests py27-six python py27-cheetah + py27-jsonpointer py27-jsonpatch " [ -f "$depschecked" ] || pkg install ${pkgs} || fail "install packages" touch $depschecked -- cgit v1.2.3 From 6f8bd54650e78bd383b29a59bee95a887ff13c81 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 4 Sep 2014 10:33:37 -0700 Subject: Reduce logging levels and fix broken call to _fetch_available_versions --- cloudinit/sources/helpers/openstack.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 3c6bb6aa..61c61570 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -163,10 +163,10 @@ class BaseReader(object): def _find_working_version(self, version): try: - versions_available = self._fetch_available_versions(self) + versions_available = self._fetch_available_versions() except Exception as e: - LOG.warn("Unable to read openstack versions from %s due to: %s", - self.base_path, e) + LOG.debug("Unable to read openstack versions from %s due to: %s", + self.base_path, e) versions_available = [] search_versions = [version] + list(OS_VERSIONS) @@ -178,8 +178,8 @@ class BaseReader(object): break if selected_version != version: - LOG.warn("Version '%s' not available, attempting to use" - " version '%s' instead", version, selected_version) + LOG.debug("Version '%s' not available, attempting to use" + " version '%s' instead", version, selected_version) return selected_version def _read_content_path(self, item): @@ -239,7 +239,8 @@ class BaseReader(object): LOG.debug("Failed reading optional path %s due" " to: %s", path, e) else: - LOG.exception("Failed reading mandatory path %s", path) + LOG.debug("Failed reading mandatory path %s due" + " to: %s", path, e) else: found = True if required and not found: -- cgit v1.2.3 From 67efb99bc8e4628e86aef76fed3411c1cc763571 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 4 Sep 2014 12:33:41 -0700 Subject: Fix a couple more cases of exceptional logs --- cloudinit/sources/DataSourceOpenStack.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 0970d07b..221759e1 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -64,13 +64,13 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): try: max_wait = int(self.ds_cfg.get("max_wait", max_wait)) - except Exception: - util.logexc(LOG, "Failed to get max wait. using %s", max_wait) + except Exception as e: + LOG.debug("Failed to get max wait. using %s: %s", max_wait, e) try: timeout = max(0, int(self.ds_cfg.get("timeout", timeout))) - except Exception: - util.logexc(LOG, "Failed to get timeout, using %s", timeout) + except Exception as e: + LOG.debug("Failed to get timeout, using %s: %s", timeout, e) return (max_wait, timeout) def wait_for_metadata_service(self): @@ -82,7 +82,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): if len(filtered): urls = filtered else: - LOG.warn("Empty metadata url list! using default list") + LOG.debug("Empty metadata url list! using default list") urls = [DEF_MD_URL] md_urls = [] @@ -123,9 +123,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'version': openstack.OS_HAVANA}) except openstack.NonReadable: return False - except (openstack.BrokenMetadata, IOError): - util.logexc(LOG, "Broken metadata address %s", - self.metadata_address) + except (openstack.BrokenMetadata, IOError) as e: + LOG.debug("Broken metadata address %s: %s", + self.metadata_address, e) return False user_dsmode = results.get('dsmode', None) -- cgit v1.2.3 From df6085227b7d6603f00c39a7808401faf81942b7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 5 Sep 2014 10:24:19 -0700 Subject: Allow customized metadata leaf decoding --- cloudinit/ec2_utils.py | 58 +++++++++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 0c751140..e69d06ff 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -28,23 +28,44 @@ LOG = logging.getLogger(__name__) SKIP_USERDATA_CODES = frozenset([httplib.NOT_FOUND]) -def maybe_json_object(text): - if not text: +class MetadataLeafDecoder(object): + """Decodes a leaf blob into something meaningful.""" + + def _maybe_json_object(self, text): + if not text: + return False + text = text.strip() + if text.startswith("{") and text.endswith("}"): + return True return False - text = text.strip() - if text.startswith("{") and text.endswith("}"): - return True - return False + + def __call__(self, field, blob): + if not blob: + return blob + if self._maybe_json_object(blob): + try: + # Assume it's json, unless it fails parsing... + return json.loads(blob) + except (ValueError, TypeError) as e: + LOG.warn("Field %s looked like a json object, but it was" + " not: %s", field, e) + if blob.find("\n") != -1: + return blob.splitlines() + return blob # See: http://bit.ly/TyoUQs # class MetadataMaterializer(object): - def __init__(self, blob, base_url, caller): + def __init__(self, blob, base_url, caller, leaf_decoder=None): self._blob = blob self._md = None self._base_url = base_url self._caller = caller + if leaf_decoder is None: + self._leaf_decoder = MetadataLeafDecoder() + else: + self._leaf_decoder = leaf_decoder def _parse(self, blob): leaves = {} @@ -90,20 +111,6 @@ class MetadataMaterializer(object): self._md = self._materialize(self._blob, self._base_url) return self._md - def _decode_leaf_blob(self, field, blob): - if not blob: - return blob - if maybe_json_object(blob): - try: - # Assume it's json, unless it fails parsing... - return json.loads(blob) - except (ValueError, TypeError) as e: - LOG.warn("Field %s looked like a json object, but it was" - " not: %s", field, e) - if blob.find("\n") != -1: - return blob.splitlines() - return blob - def _materialize(self, blob, base_url): (leaves, children) = self._parse(blob) child_contents = {} @@ -117,7 +124,7 @@ class MetadataMaterializer(object): for (field, resource) in leaves.items(): leaf_url = url_helper.combine_url(base_url, resource) leaf_blob = str(self._caller(leaf_url)) - leaf_contents[field] = self._decode_leaf_blob(field, leaf_blob) + leaf_contents[field] = self._leaf_decoder(field, leaf_blob) joined = {} joined.update(child_contents) for field in leaf_contents.keys(): @@ -164,7 +171,8 @@ def get_instance_userdata(api_version='latest', def get_instance_metadata(api_version='latest', metadata_address='http://169.254.169.254', - ssl_details=None, timeout=5, retries=5): + ssl_details=None, timeout=5, retries=5, + leaf_decoder=None): md_url = url_helper.combine_url(metadata_address, api_version) # Note, 'meta-data' explicitly has trailing /. # this is required for CloudStack (LP: #1356855) @@ -175,7 +183,9 @@ def get_instance_metadata(api_version='latest', try: response = caller(md_url) - materializer = MetadataMaterializer(str(response), md_url, caller) + materializer = MetadataMaterializer(str(response), + md_url, caller, + leaf_decoder=leaf_decoder) md = materializer.materialize() if not isinstance(md, (dict)): md = {} -- cgit v1.2.3 From e17823ebeb397110c2163629d9f0f94dbea0c5e4 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 8 Sep 2014 12:50:22 -0400 Subject: resizefs: first check if device is writable before attempting In a container the device nodes may exist but not be writable. I'm seeing this on trusty host with trusty containers, the root device ends up looking like it is to /dev/loop0. LP: #1366891 --- ChangeLog | 1 + cloudinit/config/cc_resizefs.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index dbd0715a..01ecb53c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -31,6 +31,7 @@ - Datasource: fix broken logic to provide hostname if datasource does not provide one - Improved and less verbose logging. + - resizefs: first check that device is writable. 0.7.5: - open 0.7.5 - Add a debug log message around import failures diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index b9655749..7e796228 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -98,12 +98,12 @@ def handle(name, cfg, _cloud, log, args): (devpth, fs_type, mount_point) = result - # Ensure the path is a block device. info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what) log.debug("resize_info: %s" % info) container = util.is_container() + # Ensure the path is a block device. if (devpth == "/dev/root" and not os.path.exists(devpth) and not container): devpth = rootdev_from_cmdline(util.get_cmdline()) @@ -117,14 +117,22 @@ def handle(name, cfg, _cloud, log, args): except OSError as exc: if container and exc.errno == errno.ENOENT: log.debug("Device '%s' did not exist in container. " - "cannot resize: %s" % (devpth, info)) + "cannot resize: %s", devpth, info) elif exc.errno == errno.ENOENT: - log.warn("Device '%s' did not exist. cannot resize: %s" % - (devpth, info)) + log.warn("Device '%s' did not exist. cannot resize: %s", + devpth, info) else: raise exc return + if not os.access(devpth, os.W_OK): + if container: + log.debug("'%s' not writable in container. cannot resize: %s", + devpth, info) + else: + log.warn("'%s' not writable. cannot resize: %s", devpth, info) + return + if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode): if container: log.debug("device '%s' not a block device in container." -- cgit v1.2.3 From 822000955100179f9f5944e0b2403c95fac39458 Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Mon, 8 Sep 2014 21:32:21 -0700 Subject: Update read_config_drive to use OS_VERSIONS tuple for readers Updated read_config_drive: removed the unused version kwarg, used the OS_VERSIONS tuple from the openstack helper to avoid hardcoding versions. Added a comment to the tuple in helpers/openstack.py asking for it to be kept in chronological order. --- cloudinit/sources/DataSourceConfigDrive.py | 18 ++++++++++-------- cloudinit/sources/helpers/openstack.py | 1 + 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 65477184..2456bc2e 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -169,15 +169,17 @@ def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): return "net" -def read_config_drive(source_dir, version=openstack.OS_HAVANA): - reader = openstack.ConfigDriveReader(source_dir) - finders = [ - (reader.read_v2, [], {'version': version}), - (reader.read_v2, [], {'version': openstack.OS_GRIZZLY}), - (reader.read_v2, [], {'version': openstack.OS_FOLSOM}), - (reader.read_v1, [], {}), - ] +def read_config_drive(source_dir): excps = [] + finders = [] + reader = openstack.ConfigDriveReader(source_dir) + + # openstack.OS_VERSIONS is stored in chronological order, so to check the + # newest first, use reversed() + for version in reversed(openstack.OS_VERSIONS): + finders.append((reader.read_v2, [], {'version': version})) + finders.append((reader.read_v1, [], {})) + for (functor, args, kwargs) in finders: try: return functor(*args, **kwargs) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 0fac0335..ed102c4c 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -48,6 +48,7 @@ OS_LATEST = 'latest' OS_FOLSOM = '2012-08-10' OS_GRIZZLY = '2013-04-04' OS_HAVANA = '2013-10-17' +# keep this in chronological order by time: add new entries to the end OS_VERSIONS = ( OS_FOLSOM, OS_GRIZZLY, -- cgit v1.2.3 From 6386accbf4933ecb368d2f32b7db95583e03b525 Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Wed, 10 Sep 2014 09:01:42 -0700 Subject: Only use vendordata under cloud-init key for ConfigDrive This data will be treated the same as vendordata from other sources. --- cloudinit/sources/DataSourceConfigDrive.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 2456bc2e..83cc6b25 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -132,8 +132,6 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): # for cloud-init (for namespacing). if 'cloud-init' in vd: self.vendordata_raw = vd['cloud-init'] - else: - self.vendordata_raw = vd return True -- cgit v1.2.3 From 1b975098ea2cc82a168203c720ef936db4864467 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 10 Sep 2014 14:39:59 -0400 Subject: log as warn if things are obviously wrong If something is broken as in a built in config, or code just broken, then logging warning during search for metadata is ok. --- cloudinit/sources/DataSourceOpenStack.py | 16 ++++++++-------- cloudinit/sources/helpers/openstack.py | 6 ++++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 221759e1..0970d07b 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -64,13 +64,13 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): try: max_wait = int(self.ds_cfg.get("max_wait", max_wait)) - except Exception as e: - LOG.debug("Failed to get max wait. using %s: %s", max_wait, e) + except Exception: + util.logexc(LOG, "Failed to get max wait. using %s", max_wait) try: timeout = max(0, int(self.ds_cfg.get("timeout", timeout))) - except Exception as e: - LOG.debug("Failed to get timeout, using %s: %s", timeout, e) + except Exception: + util.logexc(LOG, "Failed to get timeout, using %s", timeout) return (max_wait, timeout) def wait_for_metadata_service(self): @@ -82,7 +82,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): if len(filtered): urls = filtered else: - LOG.debug("Empty metadata url list! using default list") + LOG.warn("Empty metadata url list! using default list") urls = [DEF_MD_URL] md_urls = [] @@ -123,9 +123,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'version': openstack.OS_HAVANA}) except openstack.NonReadable: return False - except (openstack.BrokenMetadata, IOError) as e: - LOG.debug("Broken metadata address %s: %s", - self.metadata_address, e) + except (openstack.BrokenMetadata, IOError): + util.logexc(LOG, "Broken metadata address %s", + self.metadata_address) return False user_dsmode = results.get('dsmode', None) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 7b27621c..0c6b9b5a 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -178,8 +178,10 @@ class BaseReader(object): break if selected_version != version: - LOG.debug("Version '%s' not available, attempting to use" - " version '%s' instead", version, selected_version) + LOG.warn("Version '%s' not available, attempting to use " + "version '%s' instead", version, selected_version) + else: + LOG.debug("Version '%s' was available.", version) return selected_version def _read_content_path(self, item): -- cgit v1.2.3 From 64f4a63c174a4f7c21e6e372927906099103d64c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 10 Sep 2014 14:42:33 -0400 Subject: fix a bug, use a list instead of tuple for _versions using tuple for _versions was just not necessary. fix reference to undefined os_versions. --- cloudinit/sources/helpers/openstack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 0c6b9b5a..3c3de7e4 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -328,7 +328,7 @@ class ConfigDriveReader(BaseReader): path = self._path_join(self.base_path, 'openstack') found = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path))] - self._versions = tuple(found) + self._versions = found return self._versions def _read_ec2_metadata(self): @@ -421,7 +421,7 @@ class MetadataReader(BaseReader): def _fetch_available_versions(self): # /openstack/ returns a newline separated list of versions if self._versions is not None: - return self.os_versions + return self._versions found = [] version_path = self._path_join(self.base_path, "openstack") content = self._path_read(version_path) @@ -430,7 +430,7 @@ class MetadataReader(BaseReader): if not line: continue found.append(line) - self._versions = tuple(found) + self._versions = found return self._versions -- cgit v1.2.3 From c4b09a27239bc88bcf6b4ec536410bcc02cdf11c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 10 Sep 2014 15:37:09 -0400 Subject: make BaseReader select latest supported version --- cloudinit/sources/DataSourceConfigDrive.py | 16 ++++++---------- cloudinit/sources/DataSourceOpenStack.py | 3 +-- cloudinit/sources/helpers/openstack.py | 19 +++++++++++++------ 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 83cc6b25..b8c16361 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -167,17 +167,13 @@ def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): return "net" -def read_config_drive(source_dir): - excps = [] - finders = [] +def read_config_drive(source_dir, version=None): reader = openstack.ConfigDriveReader(source_dir) - - # openstack.OS_VERSIONS is stored in chronological order, so to check the - # newest first, use reversed() - for version in reversed(openstack.OS_VERSIONS): - finders.append((reader.read_v2, [], {'version': version})) - finders.append((reader.read_v1, [], {})) - + finders = [ + (reader.read_v2, [], {'version': version}), + (reader.read_v1, [], {}), + ] + excps = [] for (functor, args, kwargs) in finders: try: return functor(*args, **kwargs) diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 0970d07b..ce8e8364 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -119,8 +119,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'Crawl of openstack metadata service', read_metadata_service, args=[self.metadata_address], - kwargs={'ssl_details': self.ssl_details, - 'version': openstack.OS_HAVANA}) + kwargs={'ssl_details': self.ssl_details}) except openstack.NonReadable: return False except (openstack.BrokenMetadata, IOError): diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index ed91a55c..ed9cdbaa 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -48,7 +48,7 @@ OS_LATEST = 'latest' OS_FOLSOM = '2012-08-10' OS_GRIZZLY = '2013-04-04' OS_HAVANA = '2013-10-17' -# keep this in chronological order by time: add new entries to the end +# keep this in chronological order. new supported versions go at the end. OS_VERSIONS = ( OS_FOLSOM, OS_GRIZZLY, @@ -162,7 +162,7 @@ class BaseReader(object): def _read_ec2_metadata(self): pass - def _find_working_version(self, version): + def _find_working_version(self, version=None): try: versions_available = self._fetch_available_versions() except Exception as e: @@ -170,7 +170,14 @@ class BaseReader(object): self.base_path, e) versions_available = [] - search_versions = [version] + list(OS_VERSIONS) + # openstack.OS_VERSIONS is stored in chronological order, so + # reverse it to check newest first. + supported = [v for v in reversed(list(OS_VERSIONS))] + if version is not None: + search_versions = [version] + supported + else: + search_versions = supported + selected_version = OS_LATEST for potential_version in search_versions: if potential_version not in versions_available: @@ -178,11 +185,12 @@ class BaseReader(object): selected_version = potential_version break - if selected_version != version: + if version is not None and selected_version != version: LOG.warn("Version '%s' not available, attempting to use " "version '%s' instead", version, selected_version) else: - LOG.debug("Version '%s' was available.", version) + LOG.debug("Selected version '%s' from %s", version, + versions_available) return selected_version def _read_content_path(self, item): @@ -434,7 +442,6 @@ class MetadataReader(BaseReader): self._versions = found return self._versions - def _path_read(self, path): def should_retry_cb(_request_args, cause): -- cgit v1.2.3 From c548fdf3201519c1c30c815ba9feec643b87e0bf Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 10 Sep 2014 15:40:30 -0400 Subject: fix log message --- cloudinit/sources/helpers/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index ed9cdbaa..3d903a3c 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -189,7 +189,7 @@ class BaseReader(object): LOG.warn("Version '%s' not available, attempting to use " "version '%s' instead", version, selected_version) else: - LOG.debug("Selected version '%s' from %s", version, + LOG.debug("Selected version '%s' from %s", selected_version, versions_available) return selected_version -- cgit v1.2.3 From 44113217719ebee756325b40a5d14045ba8f3a3a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 10 Sep 2014 16:00:00 -0400 Subject: drop version= from readers instead of taking a version that they should look for, the readers now just select the highest supported version. definitely a use case later for having version= but nothing is using it now. --- cloudinit/sources/DataSourceConfigDrive.py | 4 ++-- cloudinit/sources/DataSourceOpenStack.py | 4 ++-- cloudinit/sources/helpers/openstack.py | 23 +++++++---------------- tests/unittests/test_datasource/test_openstack.py | 16 ++++++++-------- 4 files changed, 19 insertions(+), 28 deletions(-) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index b8c16361..a27d07fb 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -167,10 +167,10 @@ def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): return "net" -def read_config_drive(source_dir, version=None): +def read_config_drive(source_dir): reader = openstack.ConfigDriveReader(source_dir) finders = [ - (reader.read_v2, [], {'version': version}), + (reader.read_v2, [], {}), (reader.read_v1, [], {}), ] excps = [] diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index ce8e8364..466de8f4 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -153,9 +153,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): return True -def read_metadata_service(base_url, version=None, ssl_details=None): +def read_metadata_service(base_url, ssl_details=None): reader = openstack.MetadataReader(base_url, ssl_details=ssl_details) - return reader.read_v2(version=version) + return reader.read_v2() # Used to match classes to dependencies diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 3d903a3c..a7dd05df 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -162,7 +162,7 @@ class BaseReader(object): def _read_ec2_metadata(self): pass - def _find_working_version(self, version=None): + def _find_working_version(self): try: versions_available = self._fetch_available_versions() except Exception as e: @@ -173,24 +173,16 @@ class BaseReader(object): # openstack.OS_VERSIONS is stored in chronological order, so # reverse it to check newest first. supported = [v for v in reversed(list(OS_VERSIONS))] - if version is not None: - search_versions = [version] + supported - else: - search_versions = supported - selected_version = OS_LATEST - for potential_version in search_versions: + + for potential_version in supported: if potential_version not in versions_available: continue selected_version = potential_version break - if version is not None and selected_version != version: - LOG.warn("Version '%s' not available, attempting to use " - "version '%s' instead", version, selected_version) - else: - LOG.debug("Selected version '%s' from %s", selected_version, - versions_available) + LOG.debug("Selected version '%s' from %s", selected_version, + versions_available) return selected_version def _read_content_path(self, item): @@ -202,7 +194,7 @@ class BaseReader(object): path = self._path_join(self.base_path, "openstack", *path_pieces) return self._path_read(path) - def read_v2(self, version=None): + def read_v2(self): """Reads a version 2 formatted location. Return a dict with metadata, userdata, ec2-metadata, dsmode, @@ -233,12 +225,11 @@ class BaseReader(object): ) return files - version = self._find_working_version(version) results = { 'userdata': '', 'version': 2, } - data = datafiles(version) + data = datafiles(self._find_working_version()) for (name, (path, required, translator)) in data.iteritems(): path = self._path_join(self.base_path, path) data = None diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 530fba20..6809823e 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -142,7 +142,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): @hp.activate def test_successful(self): _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) - f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + f = ds.read_metadata_service(BASE_URL) self.assertEquals(VENDOR_DATA, f.get('vendordata')) self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) @@ -164,7 +164,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): @hp.activate def test_no_ec2(self): _register_uris(self.VERSION, {}, {}, OS_FILES) - f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + f = ds.read_metadata_service(BASE_URL) self.assertEquals(VENDOR_DATA, f.get('vendordata')) self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) @@ -180,7 +180,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) self.assertRaises(openstack.NonReadable, ds.read_metadata_service, - BASE_URL, version=self.VERSION) + BASE_URL) @hp.activate def test_bad_uuid(self): @@ -192,7 +192,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): os_files[k] = json.dumps(os_meta) _register_uris(self.VERSION, {}, {}, os_files) self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL, version=self.VERSION) + BASE_URL) @hp.activate def test_userdata_empty(self): @@ -201,7 +201,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('user_data'): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) - f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + f = ds.read_metadata_service(BASE_URL) self.assertEquals(VENDOR_DATA, f.get('vendordata')) self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) @@ -214,7 +214,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('vendor_data.json'): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) - f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + f = ds.read_metadata_service(BASE_URL) self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertFalse(f.get('vendordata')) @@ -227,7 +227,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): os_files[k] = '{' # some invalid json _register_uris(self.VERSION, {}, {}, os_files) self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL, version=self.VERSION) + BASE_URL) @hp.activate def test_metadata_invalid(self): @@ -237,7 +237,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): os_files[k] = '{' # some invalid json _register_uris(self.VERSION, {}, {}, os_files) self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL, version=self.VERSION) + BASE_URL) @hp.activate def test_datasource(self): -- cgit v1.2.3 From 2755274f34ef11651a5ee57a31955f3e413cdfc4 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 10 Sep 2014 16:46:11 -0400 Subject: OpenStack: search less urls to determine if MD service is there. We were checking for presense of meta_data.json for each supported metadata version. Instead just check that /openstack is there. This reduces the time to check on EC2 or any other cloud. --- cloudinit/sources/DataSourceOpenStack.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 466de8f4..765137c6 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -88,11 +88,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): md_urls = [] url2base = {} for url in urls: - for version in openstack.OS_VERSIONS + (openstack.OS_LATEST,): - md_url = url_helper.combine_url(url, 'openstack', - version, 'meta_data.json') - md_urls.append(md_url) - url2base[md_url] = url + md_url = url_helper.combine_url(url, 'openstack') + md_urls.append(md_url) + url2base[md_url] = url (max_wait, timeout) = self._get_url_settings() start_time = time.time() -- cgit v1.2.3 From 932c073db79e667623d27174c55e5b16ea439578 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 10 Sep 2014 21:17:40 -0400 Subject: Openstack: Vendor data cleanup For now, this vendor data handling is just added to openstack. However, in an effort to allow sanely handling of multi-part vendor-data that is namespaced, we add openstack.convert_vendordata_json . That basically takes whatever was loaded from vendordata and takes the 'cloud-init' key if it is a dict. This way the author can namespace cloud-init, basically telling it to ignore everything else. --- cloudinit/sources/DataSourceConfigDrive.py | 13 +++++---- cloudinit/sources/DataSourceOpenStack.py | 12 ++++---- cloudinit/sources/helpers/openstack.py | 25 ++++++++++++++++ tests/unittests/test_datasource/test_openstack.py | 35 ++++++++++++++++++++++- 4 files changed, 72 insertions(+), 13 deletions(-) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index a27d07fb..4e5d90de 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -126,12 +126,13 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): self.version = results['version'] self.files.update(results.get('files', {})) - # If there is no vendordata, set vd to an empty dict instead of None - vd = results.get('vendordata', {}) - # if vendordata includes 'cloud-init', then read that explicitly - # for cloud-init (for namespacing). - if 'cloud-init' in vd: - self.vendordata_raw = vd['cloud-init'] + vd = results.get('vendordata') + self.vendordata_pure = vd + try: + self.vendordata_raw = openstack.convert_vendordata_json(vd) + except ValueError as e: + LOG.warn("Invalid content in vendor-data: %s", e) + self.vendordata_raw = None return True diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 765137c6..469c2e2a 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -140,13 +140,13 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.version = results['version'] self.files.update(results.get('files', {})) - # if vendordata includes 'cloud-init', then read that explicitly - # for cloud-init (for namespacing). vd = results.get('vendordata') - if isinstance(vd, dict) and 'cloud-init' in vd: - self.vendordata_raw = vd['cloud-init'] - else: - self.vendordata_raw = vd + self.vendordata_pure = vd + try: + self.vendordata_raw = openstack.convert_vendordata_json(vd) + except ValueError as e: + LOG.warn("Invalid content in vendor-data: %s", e) + self.vendordata_raw = None return True diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index a7dd05df..9718b0be 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -459,3 +459,28 @@ class MetadataReader(BaseReader): return ec2_utils.get_instance_metadata(ssl_details=self.ssl_details, timeout=self.timeout, retries=self.retries) + + +def convert_vendordata_json(data, recurse=True): + """ data: a loaded json *object* (strings, arrays, dicts). + return something suitable for cloudinit vendordata_raw. + + if data is: + None: return None + string: return string + list: return data + the list is then processed in UserDataProcessor + dict: return convert_vendordata_json(data.get('cloud-init')) + """ + if not data: + return None + if isinstance(data, (str, unicode, basestring)): + return data + if isinstance(data, list): + return copy.deepcopy(data) + if isinstance(data, dict): + if recurse is True: + return convert_vendordata_json(data.get('cloud-init'), + recurse=False) + raise ValueError("vendordata['cloud-init'] cannot be dict") + raise ValueError("Unknown data type for vendordata: %s" % type(data)) diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 6809823e..7b4e651a 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -19,6 +19,7 @@ import copy import json import re +import unittest from StringIO import StringIO @@ -256,7 +257,8 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): self.assertEquals(EC2_META, ds_os.ec2_metadata) self.assertEquals(USER_DATA, ds_os.userdata_raw) self.assertEquals(2, len(ds_os.files)) - self.assertEquals(VENDOR_DATA, ds_os.vendordata_raw) + self.assertEquals(VENDOR_DATA, ds_os.vendordata_pure) + self.assertEquals(ds_os.vendordata_raw, None) @hp.activate def test_bad_datasource_meta(self): @@ -314,3 +316,34 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): found = ds_os.get_data() self.assertFalse(found) self.assertIsNone(ds_os.version) + + +class TestVendorDataLoading(unittest.TestCase): + def cvj(self, data): + return openstack.convert_vendordata_json(data) + + def test_vd_load_none(self): + # non-existant vendor-data should return none + self.assertIsNone(self.cvj(None)) + + def test_vd_load_string(self): + self.assertEqual(self.cvj("foobar"), "foobar") + + def test_vd_load_list(self): + data = [{'foo': 'bar'}, 'mystring', list(['another', 'list'])] + self.assertEqual(self.cvj(data), data) + + def test_vd_load_dict_no_ci(self): + self.assertEqual(self.cvj({'foo': 'bar'}), None) + + def test_vd_load_dict_ci_dict(self): + self.assertRaises(ValueError, self.cvj, + {'foo': 'bar', 'cloud-init': {'x': 1}}) + + def test_vd_load_dict_ci_string(self): + data = {'foo': 'bar', 'cloud-init': 'VENDOR_DATA'} + self.assertEqual(self.cvj(data), data['cloud-init']) + + def test_vd_load_dict_ci_list(self): + data = {'foo': 'bar', 'cloud-init': ['VD_1', 'VD_2']} + self.assertEqual(self.cvj(data), data['cloud-init']) -- cgit v1.2.3 From b2ba18dcc48b4721b097dc108f5a5eceba6b87ab Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 11 Sep 2014 10:41:10 -0400 Subject: when loading vendordata allow it to be string or list --- cloudinit/sources/helpers/openstack.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 9718b0be..b7e19314 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -21,6 +21,7 @@ import abc import base64 import copy +import functools import os from cloudinit import ec2_utils @@ -203,6 +204,9 @@ class BaseReader(object): If not a valid location, raise a NonReadable exception. """ + load_json_anytype = functools.partial( + util.load_json, root_types=(dict, basestring, list)) + def datafiles(version): files = {} files['metadata'] = ( @@ -221,7 +225,7 @@ class BaseReader(object): files['vendordata'] = ( self._path_join("openstack", version, 'vendor_data.json'), False, - util.load_json, + load_json_anytype, ) return files -- cgit v1.2.3 From 26c1cf9eeeb39e9bfcc1a37743679e5c76ab97b0 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 11 Sep 2014 10:50:04 -0400 Subject: cloud-init-local depends on /run. reflect that in upstart job. With the writing of cloud-init status, cloud-init-local needs to have /run mounted. The issue we were seeing was a race where: cloud-init-local creates /run/cloud-init /run is mounted cloud-init-local tries to link a file into /run/cloud-init that directory was no longer visisable as /run was mounted over the top. LP: #1353008 --- upstart/cloud-init-local.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upstart/cloud-init-local.conf b/upstart/cloud-init-local.conf index 061fe406..713b36d9 100644 --- a/upstart/cloud-init-local.conf +++ b/upstart/cloud-init-local.conf @@ -1,6 +1,6 @@ # cloud-init - the initial cloud-init job # crawls metadata service, emits cloud-config -start on mounted MOUNTPOINT=/ +start on mounted MOUNTPOINT=/ and mounted MOUNTPOINT=/run task -- cgit v1.2.3 From b09b12bc0130b6787bfee0332addd055459b9629 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 12 Sep 2014 13:03:44 -0400 Subject: upstart: add blocknet which blocks net until after cloud-init-local This makes it so networking wont start to come up until after cloud-init-local has had a chance to search local datasources and set /etc/network/interfaces. The changes most likely need to still be done for systemd. LP: #1368861 --- upstart/cloud-init-blocknet.conf | 83 ++++++++++++++++++++++++++++++++++++++++ upstart/cloud-init-local.conf | 9 ++++- upstart/cloud-init-nonet.conf | 5 +-- 3 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 upstart/cloud-init-blocknet.conf diff --git a/upstart/cloud-init-blocknet.conf b/upstart/cloud-init-blocknet.conf new file mode 100644 index 00000000..82d5ffbc --- /dev/null +++ b/upstart/cloud-init-blocknet.conf @@ -0,0 +1,83 @@ +# cloud-init-blocknet +# the purpose of this job is +# * to block networking from coming up until cloud-init-nonet has run +# * timeout if they all do not come up in a reasonable amount of time +description "block networking until cloud-init-local" +start on (starting network-interface + or starting network-manager + or starting networking) +stop on stopped cloud-init-local + +instance $JOB${INTERFACE:+/}${INTERFACE:-} +export INTERFACE +task + +script + set +e # you cannot trap TERM reliably with 'set -e' + SLEEP_CHILD="" + LOG="/run/${UPSTART_JOB}.log" + + static_network_up() { + local emitted="/run/network/static-network-up-emitted" + # /run/network/static-network-up-emitted is written by + # upstart (via /etc/network/if-up.d/upstart). its presense would + # indicate that static-network-up has already fired. + [ -e "$emitted" -o -e "/var/$emitted" ] + } + msg() { + local uptime="" idle="" msg="" + if [ -r /proc/uptime ]; then + read uptime idle < /proc/uptime + fi + msg="${UPSTART_INSTANCE}${uptime:+[${uptime}]}: $*" + echo "$msg" >> "$LOG" + echo "$msg" + } + + handle_sigterm() { + # if we received sigterm and static networking is up then it probably + # came from upstart as a result of 'stop on static-network-up' + msg "got sigterm" + if [ -n "$SLEEP_CHILD" ]; then + if ! kill $SLEEP_CHILD 2>/dev/null; then + [ ! -d "/proc/$SLEEP_CHILD" ] || + msg "hm.. failed to kill sleep pid $SLEEP_CHILD" + fi + fi + msg "stopped" + exit 0 + } + + dowait() { + msg "blocking $1 seconds" + # all this 'exec -a' does is get me a nicely named process in 'ps' + # ie, 'sleep-block-network-interface.eth1' + if [ -x /bin/bash ]; then + bash -c 'exec -a sleep-block-$1 sleep $2' -- "$UPSTART_INSTANCE" "$1" & + else + sleep "$1" & + fi + SLEEP_CHILD=$! + msg "sleepchild=$SLEEP_CHILD" + wait $SLEEP_CHILD + SLEEP_CHILD="" + } + + trap handle_sigterm TERM + + if [ -n "$INTERFACE" -a "${INTERFACE#lo}" != "${INTERFACE}" ]; then + msg "ignoring interface ${INTERFACE}"; + exit 0; + fi + + # static_network_up already occurred + static_network_up && { msg "static_network_up already"; exit 0; } + + # local-finished cloud-init-local success or failure + lfin="/run/cloud-init/local-finished" + [ -f "$lfin" ] && { msg "$lfin found"; exit 0; } + + dowait 120 + msg "gave up waiting for $lfin" + exit 1 +end script diff --git a/upstart/cloud-init-local.conf b/upstart/cloud-init-local.conf index 713b36d9..5def043d 100644 --- a/upstart/cloud-init-local.conf +++ b/upstart/cloud-init-local.conf @@ -6,4 +6,11 @@ task console output -exec /usr/bin/cloud-init init --local +script + lfin=/run/cloud-init/local-finished + ret=0 + cloud-init init --local || ret=$? + [ -r /proc/uptime ] && read up idle < /proc/uptime || up="N/A" + echo "$ret up $up" > "$lfin" + exit $ret +end script diff --git a/upstart/cloud-init-nonet.conf b/upstart/cloud-init-nonet.conf index e8ebee96..40059ad6 100644 --- a/upstart/cloud-init-nonet.conf +++ b/upstart/cloud-init-nonet.conf @@ -58,9 +58,8 @@ script # static_network_up already occurred static_network_up && exit 0 - # obj.pkl comes from cloud-init-local (or previous boot and - # manual_cache_clean) - [ -f /var/lib/cloud/instance/obj.pkl ] && exit 0 + # local-finished comes from cloud-init-local + [ -f /run/cloud-init/local-finished ] && exit 0 dowait 10 dowait 120 -- cgit v1.2.3 From 8eb2f6ea588775d3ddd55a62630cab310497190e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 12 Sep 2014 15:38:56 -0400 Subject: allow local disabling of blocknet --- upstart/cloud-init-blocknet.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/upstart/cloud-init-blocknet.conf b/upstart/cloud-init-blocknet.conf index 82d5ffbc..dd4f1798 100644 --- a/upstart/cloud-init-blocknet.conf +++ b/upstart/cloud-init-blocknet.conf @@ -75,7 +75,9 @@ script # local-finished cloud-init-local success or failure lfin="/run/cloud-init/local-finished" + disable="/etc/cloud/no-blocknet" [ -f "$lfin" ] && { msg "$lfin found"; exit 0; } + [ -f "$disable" ] && { msg "$disable found"; exit 0; } dowait 120 msg "gave up waiting for $lfin" -- cgit v1.2.3 From 78dd5e2783dbf06c0e2d6569423d1bbb24dcfa89 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 12 Sep 2014 17:22:29 -0400 Subject: netinfo: log error on failure of route info --- cloudinit/netinfo.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index 1bdca9f7..8d4df342 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -21,10 +21,13 @@ # along with this program. If not, see . import cloudinit.util as util +from cloudinit.log import logging import re from prettytable import PrettyTable +LOG = logging.getLogger() + def netdev_info(empty=""): fields = ("hwaddr", "addr", "bcast", "mask") @@ -168,8 +171,9 @@ def route_pformat(): lines = [] try: routes = route_info() - except Exception: + except Exception as e: lines.append(util.center('Route info failed', '!', 80)) + util.logexc(LOG, "Route info failed: %s" % e) routes = None if routes is not None: fields = ['Route', 'Destination', 'Gateway', -- cgit v1.2.3 From 41a52ef99a9dddd3fef9bdc883dfca55770ef2a5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 12 Sep 2014 17:45:20 -0400 Subject: upstart/cloud-init-nonet.conf: do not exit on existance of obj.pkl not sure why this was here. the intent must of have been to allow for a local datasource to continue booting and not annoyingly block waiting for network information (if ithere was no network information). However, that seems wrong. If the datasource wipes /etc/network/interfaces and there are no network interfaces then we're probably breaking that use case here. However we're fixing the other more common case. --- upstart/cloud-init-nonet.conf | 3 --- 1 file changed, 3 deletions(-) diff --git a/upstart/cloud-init-nonet.conf b/upstart/cloud-init-nonet.conf index 40059ad6..38630b39 100644 --- a/upstart/cloud-init-nonet.conf +++ b/upstart/cloud-init-nonet.conf @@ -58,9 +58,6 @@ script # static_network_up already occurred static_network_up && exit 0 - # local-finished comes from cloud-init-local - [ -f /run/cloud-init/local-finished ] && exit 0 - dowait 10 dowait 120 msg "gave up waiting for a network device." -- cgit v1.2.3 From f5be74dc84aac7bb9a634e5518f5aac6ae80098a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 12 Sep 2014 17:59:52 -0400 Subject: upstart/cloud-init-nonet.conf: only mention wait if larger than 5 seconds silently wait 5 seconds for networking to come up. We started seeing the message more now, as we are now blocking the networking from coming up until cloud-init-local is done. Previously that would have happened in parallel, and we were less likely to see that message. --- upstart/cloud-init-nonet.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/upstart/cloud-init-nonet.conf b/upstart/cloud-init-nonet.conf index 38630b39..6abf6573 100644 --- a/upstart/cloud-init-nonet.conf +++ b/upstart/cloud-init-nonet.conf @@ -46,7 +46,7 @@ script } dowait() { - msg "waiting $1 seconds for network device" + [ $# -eq 2 ] || msg "waiting $1 seconds for network device" sleep "$1" & SLEEP_CHILD=$! wait $SLEEP_CHILD @@ -58,8 +58,9 @@ script # static_network_up already occurred static_network_up && exit 0 + dowait 5 silent dowait 10 - dowait 120 + dowait 115 msg "gave up waiting for a network device." : > /var/lib/cloud/data/no-net end script -- cgit v1.2.3 From 6093b8b2733814b9265494c47f4268167c9491ab Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 15 Sep 2014 09:40:29 -0400 Subject: cloud-init-blocknet: remove debug code going to /run/cloud-init-blocknet --- upstart/cloud-init-blocknet.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/upstart/cloud-init-blocknet.conf b/upstart/cloud-init-blocknet.conf index dd4f1798..be09e7d8 100644 --- a/upstart/cloud-init-blocknet.conf +++ b/upstart/cloud-init-blocknet.conf @@ -15,7 +15,6 @@ task script set +e # you cannot trap TERM reliably with 'set -e' SLEEP_CHILD="" - LOG="/run/${UPSTART_JOB}.log" static_network_up() { local emitted="/run/network/static-network-up-emitted" @@ -30,7 +29,6 @@ script read uptime idle < /proc/uptime fi msg="${UPSTART_INSTANCE}${uptime:+[${uptime}]}: $*" - echo "$msg" >> "$LOG" echo "$msg" } -- cgit v1.2.3 From fa7b60d5269cb76c4205b4709bd3048e14b9a0c7 Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Mon, 15 Sep 2014 14:39:57 -0700 Subject: Fix bug 1338614 util.log_time()'s return value was what was being sent to fork_cb. This means the resize ran in parallel and the call to fork_cb threw a traceback (trying to call Nonetype). By permitting fork_cb to take kwargs, and using the correct method syntax, this now forks and resizes in the background as appropriate. --- cloudinit/config/cc_resizefs.py | 4 ++-- cloudinit/util.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 667d5977..a6280e6c 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -154,8 +154,8 @@ def handle(name, cfg, _cloud, log, args): # Fork to a child that will run # the resize command util.fork_cb( - util.log_time(logfunc=log.debug, msg="backgrounded Resizing", - func=do_resize, args=(resize_cmd, log))) + util.log_time, logfunc=log.debug, msg="backgrounded Resizing", + func=do_resize, args=(resize_cmd, log)) else: util.log_time(logfunc=log.debug, msg="Resizing", func=do_resize, args=(resize_cmd, log)) diff --git a/cloudinit/util.py b/cloudinit/util.py index 0821901a..f0fa7a19 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -193,11 +193,11 @@ def ExtendedTemporaryFile(**kwargs): return fh -def fork_cb(child_cb, *args): +def fork_cb(child_cb, *args, **kwargs): fid = os.fork() if fid == 0: try: - child_cb(*args) + child_cb(*args, **kwargs) os._exit(0) # pylint: disable=W0212 except: logexc(LOG, "Failed forking and calling callback %s", -- cgit v1.2.3 From 668919511625f7b6a8922e4504e224e915f7be22 Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Mon, 15 Sep 2014 15:10:17 -0700 Subject: Remove pylint: lines --- cloudinit/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index f0fa7a19..9bbb6b3c 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -198,11 +198,11 @@ def fork_cb(child_cb, *args, **kwargs): if fid == 0: try: child_cb(*args, **kwargs) - os._exit(0) # pylint: disable=W0212 + os._exit(0) except: logexc(LOG, "Failed forking and calling callback %s", type_utils.obj_name(child_cb)) - os._exit(1) # pylint: disable=W0212 + os._exit(1) else: LOG.debug("Forked child %s who will run callback %s", fid, type_utils.obj_name(child_cb)) -- cgit v1.2.3