ec2: Use instance-identity doc for region and instance-id

The instance identity document is a better source for region information,
partly because region isn't actually in meta-data at all, only
availability-zone, which happens to be named similarly.

Reviewed-by: Ethan Faust <efaust@amazon.com>
Reviewed-by: Cyle Riggs <cyler@amazon.com>
Reviewed-by: Tom Kirchner <tjk@amazon.com>
Reviewed-by: Matt Nierzwicki <nierzwic@amazon.com>
[ajorgens@amazon.com: rebase onto 0.7.9]
[ajorgens@amazon.com: changes per merge proposal discussions]

4 files changed, 78 insertions, 22 deletions

diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 99e60e7a..55260eae 100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -45,6 +45,10 @@ OSFAMILIES = {
 
 LOG = logging.getLogger(__name__)
 
+# This is a best guess regex, based on current EC2 AZs on 2017-12-11.
+# It could break when Amazon adds new regions and new AZs.
+_EC2_AZ_RE = re.compile('^[a-z][a-z]-(?:[a-z]+-)+[0-9][a-z]$')
+
 
 @six.add_metaclass(abc.ABCMeta)
 class Distro(object):
@@ -683,18 +687,13 @@ def _get_package_mirror_info(mirror_info, data_source=None,
     if not mirror_info:
         mirror_info = {}
 
-    # ec2 availability zones are named cc-direction-[0-9][a-d] (us-east-1b)
-    # the region is us-east-1. so region = az[0:-1]
-    directions_re = '|'.join([
-        'central', 'east', 'north', 'northeast', 'northwest',
-        'south', 'southeast', 'southwest', 'west'])
-    ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" % directions_re)
-
     subst = {}
     if data_source and data_source.availability_zone:
         subst['availability_zone'] = data_source.availability_zone
 
-        if re.match(ec2_az_re, data_source.availability_zone):
+        # ec2 availability zones are named cc-direction-[0-9][a-d] (us-east-1b)
+        # the region is us-east-1. so region = az[0:-1]
+        if _EC2_AZ_RE.match(data_source.availability_zone):
             subst['ec2_region'] = "%s" % data_source.availability_zone[0:-1]
 
     if data_source and data_source.region:
diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py
index 723d6bd6..d6c61e4c 100644
--- a/cloudinit/ec2_utils.py
+++ b/cloudinit/ec2_utils.py
@@ -1,6 +1,8 @@
 # Copyright (C) 2012 Yahoo! Inc.
+# Copyright (C) 2014 Amazon.com, Inc. or its affiliates.
 #
 # Author: Joshua Harlow <harlowja@yahoo-inc.com>
+# Author: Andrew Jorgensen <ajorgens@amazon.com>
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
@@ -164,14 +166,11 @@ def get_instance_userdata(api_version='latest',
     return user_data
 
 
-def get_instance_metadata(api_version='latest',
-                          metadata_address='http://169.254.169.254',
-                          ssl_details=None, timeout=5, retries=5,
-                          leaf_decoder=None):
-    md_url = url_helper.combine_url(metadata_address, api_version)
-    # Note, 'meta-data' explicitly has trailing /.
-    # this is required for CloudStack (LP: #1356855)
-    md_url = url_helper.combine_url(md_url, 'meta-data/')
+def _get_instance_metadata(tree, api_version='latest',
+                           metadata_address='http://169.254.169.254',
+                           ssl_details=None, timeout=5, retries=5,
+                           leaf_decoder=None):
+    md_url = url_helper.combine_url(metadata_address, api_version, tree)
     caller = functools.partial(util.read_file_or_url,
                                ssl_details=ssl_details, timeout=timeout,
                                retries=retries)
@@ -189,7 +188,29 @@ def get_instance_metadata(api_version='latest',
             md = {}
         return md
     except Exception:
-        util.logexc(LOG, "Failed fetching metadata from url %s", md_url)
+        util.logexc(LOG, "Failed fetching %s from url %s", tree, md_url)
         return {}
 
+
+def get_instance_metadata(api_version='latest',
+                          metadata_address='http://169.254.169.254',
+                          ssl_details=None, timeout=5, retries=5,
+                          leaf_decoder=None):
+    # Note, 'meta-data' explicitly has trailing /.
+    # this is required for CloudStack (LP: #1356855)
+    return _get_instance_metadata(tree='meta-data/', api_version=api_version,
+                                  metadata_address=metadata_address,
+                                  ssl_details=ssl_details, timeout=timeout,
+                                  retries=retries, leaf_decoder=leaf_decoder)
+
+
+def get_instance_identity(api_version='latest',
+                          metadata_address='http://169.254.169.254',
+                          ssl_details=None, timeout=5, retries=5,
+                          leaf_decoder=None):
+    return _get_instance_metadata(tree='dynamic/instance-identity',
+                                  api_version=api_version,
+                                  metadata_address=metadata_address,
+                                  ssl_details=ssl_details, timeout=timeout,
+                                  retries=retries, leaf_decoder=leaf_decoder)
 # vi: ts=4 expandtab
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index e5c88334..0f89f34d 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -154,7 +154,12 @@ class DataSourceEc2(sources.DataSource):
         return self.min_metadata_version
 
     def get_instance_id(self):
-        return self.metadata['instance-id']
+        if self.cloud_platform == Platforms.AWS:
+            # Prefer the ID from the instance identity document, but fall back
+            return self.identity.get(
+                'instanceId', self.metadata['instance-id'])
+        else:
+            return self.metadata['instance-id']
 
     def _get_url_settings(self):
         mcfg = self.ds_cfg
@@ -268,15 +273,27 @@ class DataSourceEc2(sources.DataSource):
     @property
     def availability_zone(self):
         try:
-            return self.metadata['placement']['availability-zone']
+            if self.cloud_platform == Platforms.AWS:
+                return self.identity.get(
+                    'availabilityZone',
+                    self.metadata['placement']['availability-zone'])
+            else:
+                return self.metadata['placement']['availability-zone']
         except KeyError:
             return None
 
     @property
     def region(self):
-        az = self.availability_zone
-        if az is not None:
-            return az[:-1]
+        if self.cloud_platform == Platforms.AWS:
+            region = self.identity.get('region')
+            # Fallback to trimming the availability zone if region is missing
+            if self.availability_zone and not region:
+                region = self.availability_zone[:-1]
+            return region
+        else:
+            az = self.availability_zone
+            if az is not None:
+                return az[:-1]
         return None
 
     @property
@@ -357,6 +374,9 @@ class DataSourceEc2(sources.DataSource):
                 api_version, self.metadata_address)
             self.metadata = ec2.get_instance_metadata(
                 api_version, self.metadata_address)
+            if self.cloud_platform == Platforms.AWS:
+                self.identity = ec2.get_instance_identity(
+                    api_version, self.metadata_address).get('document', {})
         except Exception:
             util.logexc(
                 LOG, "Failed reading from metadata address %s",
diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py
index 714f5dac..4fa9616b 100644
--- a/tests/unittests/test_datasource/test_aliyun.py
+++ b/tests/unittests/test_datasource/test_aliyun.py
@@ -47,6 +47,9 @@ def register_mock_metaserver(base_url, data):
         elif isinstance(body, list):
             register(base_url.rstrip('/'), '\n'.join(body) + '\n')
         elif isinstance(body, dict):
+            if not body:
+                register(base_url.rstrip('/') + '/', 'not found',
+                         status_code=404)
             vals = []
             for k, v in body.items():
                 if isinstance(v, (str, list)):
@@ -91,9 +94,22 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
             self.metadata_address,
             self.ds.min_metadata_version, 'user-data')
 
+    # EC2 provides an instance-identity document which must return 404 here
+    # for this test to pass.
+    @property
+    def default_identity(self):
+        return {}
+
+    @property
+    def identity_url(self):
+        return os.path.join(self.metadata_address,
+                            self.ds.min_metadata_version,
+                            'dynamic', 'instance-identity')
+
     def regist_default_server(self):
         register_mock_metaserver(self.metadata_url, self.default_metadata)
         register_mock_metaserver(self.userdata_url, self.default_userdata)
+        register_mock_metaserver(self.identity_url, self.default_identity)
 
     def _test_get_data(self):
         self.assertEqual(self.ds.metadata, self.default_metadata)