summaryrefslogtreecommitdiff
path: root/cloudinit
diff options
context:
space:
mode:
Diffstat (limited to 'cloudinit')
-rw-r--r--cloudinit/config/cc_apt_pipelining.py2
-rw-r--r--cloudinit/config/cc_disk_setup.py3
-rw-r--r--cloudinit/config/cc_snappy.py6
-rw-r--r--cloudinit/distros/__init__.py15
-rw-r--r--cloudinit/handlers/__init__.py11
-rw-r--r--cloudinit/sources/DataSourceAzure.py132
-rw-r--r--cloudinit/sources/DataSourceGCE.py92
-rw-r--r--cloudinit/sources/DataSourceOpenNebula.py1
-rw-r--r--cloudinit/user_data.py9
-rw-r--r--cloudinit/util.py16
10 files changed, 159 insertions, 128 deletions
diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py
index e5629175..40c32c84 100644
--- a/cloudinit/config/cc_apt_pipelining.py
+++ b/cloudinit/config/cc_apt_pipelining.py
@@ -43,7 +43,7 @@ def handle(_name, cfg, _cloud, log, _args):
write_apt_snippet("0", log, DEFAULT_FILE)
elif apt_pipe_value_s in ("none", "unchanged", "os"):
return
- elif apt_pipe_value_s in [str(b) for b in xrange(0, 6)]:
+ elif apt_pipe_value_s in [str(b) for b in range(0, 6)]:
write_apt_snippet(apt_pipe_value_s, log, DEFAULT_FILE)
else:
log.warn("Invalid option for apt_pipeling: %s", apt_pipe_value)
diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py
index f899210b..e2ce6db4 100644
--- a/cloudinit/config/cc_disk_setup.py
+++ b/cloudinit/config/cc_disk_setup.py
@@ -304,8 +304,7 @@ def is_disk_used(device):
# If the child count is higher 1, then there are child nodes
# such as partition or device mapper nodes
- use_count = [x for x in enumerate_disk(device)]
- if len(use_count.splitlines()) > 1:
+ if len(list(enumerate_disk(device))) > 1:
return True
# If we see a file system, then its used
diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py
index 6a7ae09b..7aaec94a 100644
--- a/cloudinit/config/cc_snappy.py
+++ b/cloudinit/config/cc_snappy.py
@@ -42,12 +42,10 @@ Example config:
"""
from cloudinit import log as logging
-from cloudinit import templater
from cloudinit import util
from cloudinit.settings import PER_INSTANCE
import glob
-import six
import tempfile
import os
@@ -72,7 +70,7 @@ def parse_filename(fname):
name = fname_noext.partition("_")[0]
shortname = name.partition(".")[0]
return(name, shortname, fname_noext)
-
+
def get_fs_package_ops(fspath):
if not fspath:
@@ -98,7 +96,7 @@ def makeop(op, name, config=None, path=None, cfgfile=None):
def get_package_config(configs, name):
# load the package's config from the configs dict.
- # prefer full-name entry (config-example.canonical)
+ # prefer full-name entry (config-example.canonical)
# over short name entry (config-example)
if name in configs:
return configs[name]
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index ab874b45..05721922 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -208,6 +208,15 @@ class Distro(object):
and sys_hostname != hostname):
update_files.append(sys_fn)
+ # If something else has changed the hostname after we set it
+ # initially, we should not overwrite those changes (we should
+ # only be setting the hostname once per instance)
+ if (sys_hostname and prev_hostname and
+ sys_hostname != prev_hostname):
+ LOG.info("%s differs from %s, assuming user maintained hostname.",
+ prev_hostname_fn, sys_fn)
+ return
+
# Remove duplicates (incase the previous config filename)
# is the same as the system config filename, don't bother
# doing it twice
@@ -222,11 +231,6 @@ class Distro(object):
util.logexc(LOG, "Failed to write hostname %s to %s", hostname,
fn)
- if (sys_hostname and prev_hostname and
- sys_hostname != prev_hostname):
- LOG.debug("%s differs from %s, assuming user maintained hostname.",
- prev_hostname_fn, sys_fn)
-
# If the system hostname file name was provided set the
# non-fqdn as the transient hostname.
if sys_fn in update_files:
@@ -318,6 +322,7 @@ class Distro(object):
"gecos": '--comment',
"homedir": '--home',
"primary_group": '--gid',
+ "uid": '--uid',
"groups": '--groups',
"passwd": '--password',
"shell": '--shell',
diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py
index d62fcd19..53d5604a 100644
--- a/cloudinit/handlers/__init__.py
+++ b/cloudinit/handlers/__init__.py
@@ -170,12 +170,12 @@ def _extract_first_or_bytes(blob, size):
start = blob.split("\n", 1)[0]
else:
# We want to avoid decoding the whole blob (it might be huge)
- # By taking 4*size bytes we have a guarantee to decode size utf8 chars
- start = blob[:4*size].decode(errors='ignore').split("\n", 1)[0]
+ # By taking 4*size bytes we guarantee to decode size utf8 chars
+ start = blob[:4 * size].decode(errors='ignore').split("\n", 1)[0]
if len(start) >= size:
start = start[:size]
except UnicodeDecodeError:
- # Bytes array doesn't contain a text object -- return chunk of raw bytes
+ # Bytes array doesn't contain text so return chunk of raw bytes
start = blob[0:size]
return start
@@ -263,7 +263,10 @@ def fixup_handler(mod, def_freq=PER_INSTANCE):
def type_from_starts_with(payload, default=None):
- payload_lc = payload.lower()
+ try:
+ payload_lc = util.decode_binary(payload).lower()
+ except UnicodeDecodeError:
+ return default
payload_lc = payload_lc.lstrip()
for text in INCLUSION_SRCH:
if payload_lc.startswith(text):
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 6e030217..a19d9ca2 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -17,6 +17,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import base64
+import contextlib
import crypt
import fnmatch
import os
@@ -66,6 +67,36 @@ DS_CFG_PATH = ['datasource', DS_NAME]
DEF_EPHEMERAL_LABEL = 'Temporary Storage'
+def get_hostname(hostname_command='hostname'):
+ return util.subp(hostname_command, capture=True)[0].strip()
+
+
+def set_hostname(hostname, hostname_command='hostname'):
+ util.subp([hostname_command, hostname])
+
+
+@contextlib.contextmanager
+def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'):
+ """
+ Set a temporary hostname, restoring the previous hostname on exit.
+
+ Will have the value of the previous hostname when used as a context
+ manager, or None if the hostname was not changed.
+ """
+ policy = cfg['hostname_bounce']['policy']
+ previous_hostname = get_hostname(hostname_command)
+ if (not util.is_true(cfg.get('set_hostname'))
+ or util.is_false(policy)
+ or (previous_hostname == temp_hostname and policy != 'force')):
+ yield None
+ return
+ set_hostname(temp_hostname, hostname_command)
+ try:
+ yield previous_hostname
+ finally:
+ set_hostname(previous_hostname, hostname_command)
+
+
class DataSourceAzureNet(sources.DataSource):
def __init__(self, sys_cfg, distro, paths):
sources.DataSource.__init__(self, sys_cfg, distro, paths)
@@ -154,33 +185,40 @@ class DataSourceAzureNet(sources.DataSource):
# the directory to be protected.
write_files(ddir, files, dirmode=0o700)
- # handle the hostname 'publishing'
- try:
- handle_set_hostname(mycfg.get('set_hostname'),
- self.metadata.get('local-hostname'),
- mycfg['hostname_bounce'])
- except Exception as e:
- LOG.warn("Failed publishing hostname: %s", e)
- util.logexc(LOG, "handling set_hostname failed")
-
- try:
- invoke_agent(mycfg['agent_command'])
- except util.ProcessExecutionError:
- # claim the datasource even if the command failed
- util.logexc(LOG, "agent command '%s' failed.",
- mycfg['agent_command'])
-
- shcfgxml = os.path.join(ddir, "SharedConfig.xml")
- wait_for = [shcfgxml]
-
- fp_files = []
- for pk in self.cfg.get('_pubkeys', []):
- bname = str(pk['fingerprint'] + ".crt")
- fp_files += [os.path.join(ddir, bname)]
+ temp_hostname = self.metadata.get('local-hostname')
+ hostname_command = mycfg['hostname_bounce']['hostname_command']
+ with temporary_hostname(temp_hostname, mycfg,
+ hostname_command=hostname_command) \
+ as previous_hostname:
+ if (previous_hostname is not None
+ and util.is_true(mycfg.get('set_hostname'))):
+ cfg = mycfg['hostname_bounce']
+ try:
+ perform_hostname_bounce(hostname=temp_hostname,
+ cfg=cfg,
+ prev_hostname=previous_hostname)
+ except Exception as e:
+ LOG.warn("Failed publishing hostname: %s", e)
+ util.logexc(LOG, "handling set_hostname failed")
- missing = util.log_time(logfunc=LOG.debug, msg="waiting for files",
- func=wait_for_files,
- args=(wait_for + fp_files,))
+ try:
+ invoke_agent(mycfg['agent_command'])
+ except util.ProcessExecutionError:
+ # claim the datasource even if the command failed
+ util.logexc(LOG, "agent command '%s' failed.",
+ mycfg['agent_command'])
+
+ shcfgxml = os.path.join(ddir, "SharedConfig.xml")
+ wait_for = [shcfgxml]
+
+ fp_files = []
+ for pk in self.cfg.get('_pubkeys', []):
+ bname = str(pk['fingerprint'] + ".crt")
+ fp_files += [os.path.join(ddir, bname)]
+
+ missing = util.log_time(logfunc=LOG.debug, msg="waiting for files",
+ func=wait_for_files,
+ args=(wait_for + fp_files,))
if len(missing):
LOG.warn("Did not find files, but going on: %s", missing)
@@ -299,39 +337,15 @@ def support_new_ephemeral(cfg):
return mod_list
-def handle_set_hostname(enabled, hostname, cfg):
- if not util.is_true(enabled):
- return
-
- if not hostname:
- LOG.warn("set_hostname was true but no local-hostname")
- return
-
- apply_hostname_bounce(hostname=hostname, policy=cfg['policy'],
- interface=cfg['interface'],
- command=cfg['command'],
- hostname_command=cfg['hostname_command'])
-
-
-def apply_hostname_bounce(hostname, policy, interface, command,
- hostname_command="hostname"):
+def perform_hostname_bounce(hostname, cfg, prev_hostname):
# set the hostname to 'hostname' if it is not already set to that.
# then, if policy is not off, bounce the interface using command
- prev_hostname = util.subp(hostname_command, capture=True)[0].strip()
-
- util.subp([hostname_command, hostname])
-
- msg = ("phostname=%s hostname=%s policy=%s interface=%s" %
- (prev_hostname, hostname, policy, interface))
-
- if util.is_false(policy):
- LOG.debug("pubhname: policy false, skipping [%s]", msg)
- return
-
- if prev_hostname == hostname and policy != "force":
- LOG.debug("pubhname: no change, policy != force. skipping. [%s]", msg)
- return
+ command = cfg['command']
+ interface = cfg['interface']
+ policy = cfg['policy']
+ msg = ("hostname=%s policy=%s interface=%s" %
+ (hostname, policy, interface))
env = os.environ.copy()
env['interface'] = interface
env['hostname'] = hostname
@@ -344,9 +358,9 @@ def apply_hostname_bounce(hostname, policy, interface, command,
shell = not isinstance(command, (list, tuple))
# capture=False, see comments in bug 1202758 and bug 1206164.
util.log_time(logfunc=LOG.debug, msg="publishing hostname",
- get_uptime=True, func=util.subp,
- kwargs={'args': command, 'shell': shell, 'capture': False,
- 'env': env})
+ get_uptime=True, func=util.subp,
+ kwargs={'args': command, 'shell': shell, 'capture': False,
+ 'env': env})
def crtfile_to_pubkey(fname):
diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py
index 608c07f1..f4ed915d 100644
--- a/cloudinit/sources/DataSourceGCE.py
+++ b/cloudinit/sources/DataSourceGCE.py
@@ -30,6 +30,31 @@ BUILTIN_DS_CONFIG = {
REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname')
+class GoogleMetadataFetcher(object):
+ headers = {'X-Google-Metadata-Request': True}
+
+ def __init__(self, metadata_address):
+ self.metadata_address = metadata_address
+
+ def get_value(self, path, is_text):
+ value = None
+ try:
+ resp = url_helper.readurl(url=self.metadata_address + path,
+ headers=self.headers)
+ except url_helper.UrlError as exc:
+ msg = "url %s raised exception %s"
+ LOG.debug(msg, path, exc)
+ else:
+ if resp.code == 200:
+ if is_text:
+ value = util.decode_binary(resp.contents)
+ else:
+ value = resp.contents
+ else:
+ LOG.debug("url %s returned code %s", path, resp.code)
+ return value
+
+
class DataSourceGCE(sources.DataSource):
def __init__(self, sys_cfg, distro, paths):
sources.DataSource.__init__(self, sys_cfg, distro, paths)
@@ -50,17 +75,15 @@ class DataSourceGCE(sources.DataSource):
return public_key
def get_data(self):
- # GCE metadata server requires a custom header since v1
- headers = {'X-Google-Metadata-Request': True}
-
# url_map: (our-key, path, required, is_text)
url_map = [
- ('instance-id', 'instance/id', True, True),
- ('availability-zone', 'instance/zone', True, True),
- ('local-hostname', 'instance/hostname', True, True),
- ('public-keys', 'project/attributes/sshKeys', False, True),
- ('user-data', 'instance/attributes/user-data', False, False),
- ('user-data-encoding', 'instance/attributes/user-data-encoding',
+ ('instance-id', ('instance/id',), True, True),
+ ('availability-zone', ('instance/zone',), True, True),
+ ('local-hostname', ('instance/hostname',), True, True),
+ ('public-keys', ('project/attributes/sshKeys',
+ 'instance/attributes/sshKeys'), False, True),
+ ('user-data', ('instance/attributes/user-data',), False, False),
+ ('user-data-encoding', ('instance/attributes/user-data-encoding',),
False, True),
]
@@ -69,40 +92,25 @@ class DataSourceGCE(sources.DataSource):
LOG.debug("%s is not resolvable", self.metadata_address)
return False
+ metadata_fetcher = GoogleMetadataFetcher(self.metadata_address)
# iterate over url_map keys to get metadata items
- found = False
- for (mkey, path, required, is_text) in url_map:
- try:
- resp = url_helper.readurl(url=self.metadata_address + path,
- headers=headers)
- if resp.code == 200:
- found = True
- if is_text:
- self.metadata[mkey] = util.decode_binary(resp.contents)
- else:
- self.metadata[mkey] = resp.contents
+ running_on_gce = False
+ for (mkey, paths, required, is_text) in url_map:
+ value = None
+ for path in paths:
+ new_value = metadata_fetcher.get_value(path, is_text)
+ if new_value is not None:
+ value = new_value
+ if value:
+ running_on_gce = True
+ if required and value is None:
+ msg = "required key %s returned nothing. not GCE"
+ if not running_on_gce:
+ LOG.debug(msg, mkey)
else:
- if required:
- msg = "required url %s returned code %s. not GCE"
- if not found:
- LOG.debug(msg, path, resp.code)
- else:
- LOG.warn(msg, path, resp.code)
- return False
- else:
- self.metadata[mkey] = None
- except url_helper.UrlError as e:
- if required:
- msg = "required url %s raised exception %s. not GCE"
- if not found:
- LOG.debug(msg, path, e)
- else:
- LOG.warn(msg, path, e)
- return False
- msg = "Failed to get %s metadata item: %s."
- LOG.debug(msg, path, e)
-
- self.metadata[mkey] = None
+ LOG.warn(msg, mkey)
+ return False
+ self.metadata[mkey] = value
if self.metadata['public-keys']:
lines = self.metadata['public-keys'].splitlines()
@@ -116,7 +124,7 @@ class DataSourceGCE(sources.DataSource):
else:
LOG.warn('unknown user-data-encoding: %s, ignoring', encoding)
- return found
+ return running_on_gce
@property
def launch_index(self):
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 61709c1b..ac2c3b45 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -24,7 +24,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-import base64
import os
import pwd
import re
diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py
index eb3c7336..f7c5787c 100644
--- a/cloudinit/user_data.py
+++ b/cloudinit/user_data.py
@@ -49,6 +49,7 @@ INCLUDE_TYPES = ['text/x-include-url', 'text/x-include-once-url']
ARCHIVE_TYPES = ["text/cloud-config-archive"]
UNDEF_TYPE = "text/plain"
ARCHIVE_UNDEF_TYPE = "text/cloud-config"
+ARCHIVE_UNDEF_BINARY_TYPE = "application/octet-stream"
# This seems to hit most of the gzip possible content types.
DECOMP_TYPES = [
@@ -265,11 +266,15 @@ class UserDataProcessor(object):
content = ent.get('content', '')
mtype = ent.get('type')
if not mtype:
- mtype = handlers.type_from_starts_with(content,
- ARCHIVE_UNDEF_TYPE)
+ default = ARCHIVE_UNDEF_TYPE
+ if isinstance(content, six.binary_type):
+ default = ARCHIVE_UNDEF_BINARY_TYPE
+ mtype = handlers.type_from_starts_with(content, default)
maintype, subtype = mtype.split('/', 1)
if maintype == "text":
+ if isinstance(content, six.binary_type):
+ content = content.decode()
msg = MIMEText(content, _subtype=subtype)
else:
msg = MIMEBase(maintype, subtype)
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 971c1c2d..db4e02b8 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -121,8 +121,12 @@ def fully_decoded_payload(part):
if (six.PY3 and
part.get_content_maintype() == 'text' and
isinstance(cte_payload, bytes)):
- charset = part.get_charset() or 'utf-8'
- return cte_payload.decode(charset, errors='surrogateescape')
+ charset = part.get_charset()
+ if charset and charset.input_codec:
+ encoding = charset.input_codec
+ else:
+ encoding = 'utf-8'
+ return cte_payload.decode(encoding, errors='surrogateescape')
return cte_payload
@@ -762,10 +766,6 @@ def fetch_ssl_details(paths=None):
return ssl_details
-def load_tfile_or_url(*args, **kwargs):
- return(decode_binary(read_file_or_url(*args, **kwargs).contents))
-
-
def read_file_or_url(url, timeout=5, retries=10,
headers=None, data=None, sec_between=1, ssl_details=None,
headers_cb=None, exception_cb=None):
@@ -833,10 +833,10 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0):
ud_url = "%s%s%s" % (base, "user-data", ext)
md_url = "%s%s%s" % (base, "meta-data", ext)
- md_resp = load_tfile_or_url(md_url, timeout, retries, file_retries)
+ md_resp = read_file_or_url(md_url, timeout, retries, file_retries)
md = None
if md_resp.ok():
- md = load_yaml(md_resp.contents, default={})
+ md = load_yaml(decode_binary(md_resp.contents), default={})
ud_resp = read_file_or_url(ud_url, timeout, retries, file_retries)
ud = None
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-13 02:53:47 +0000