From e8a10a41d22876d555084def823817337d9c2a80 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 20 Sep 2012 17:56:22 -0700 Subject: Use only util methods for reading/loading/appending/peeking at files since it is likely soon that we will add a new way of adjusting the root of files read, also it is useful for debugging to track what is being read/written in a central fashion. --- cloudinit/sources/DataSourceMAAS.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'cloudinit/sources/DataSourceMAAS.py') diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index c568d365..d166e9e3 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -301,9 +301,7 @@ if __name__ == "__main__": 'token_secret': args.tsec, 'consumer_secret': args.csec} if args.config: - import yaml - with open(args.config) as fp: - cfg = yaml.safe_load(fp) + cfg = util.read_conf(args.config) if 'datasource' in cfg: cfg = cfg['datasource']['MAAS'] for key in creds.keys(): @@ -312,7 +310,7 @@ if __name__ == "__main__": def geturl(url, headers_cb): req = urllib2.Request(url, data=None, headers=headers_cb(url)) - return(urllib2.urlopen(req).read()) + return (urllib2.urlopen(req).read()) def printurl(url, headers_cb): print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) -- cgit v1.2.3 From 70cc7536f45a8d7052617ad88e2816291db0a309 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 24 Sep 2012 17:13:38 -0400 Subject: DataSourceMAAS: if a oauth request fails due to 403 try updating local time In the event of a 403 (Unauthorized) in oauth, try set a 'oauth_clockskew' variable. In future headers, use a time created by 'time.time() + self.oauth_clockskew'. The idea here is that if the local time is bad (or even if the server time is bad) we will essentially use something that should be similar to the remote clock. This fixes LP: #978127. LP: #978127 --- cloudinit/sources/DataSourceMAAS.py | 43 +++++++++++++++++++++++++++++++++---- cloudinit/url_helper.py | 11 ++++++++-- 2 files changed, 48 insertions(+), 6 deletions(-) (limited to 'cloudinit/sources/DataSourceMAAS.py') diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index c568d365..581e9a4b 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -18,6 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from email.utils import parsedate import errno import oauth.oauth as oauth import os @@ -46,6 +47,7 @@ class DataSourceMAAS(sources.DataSource): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.base_url = None self.seed_dir = os.path.join(paths.seed_dir, 'maas') + self.oauth_clockskew = None def __str__(self): return "%s [%s]" % (util.obj_name(self), self.base_url) @@ -95,11 +97,17 @@ class DataSourceMAAS(sources.DataSource): return {} consumer_secret = mcfg.get('consumer_secret', "") + + timestamp = None + if self.oauth_clockskew: + timestamp = int(time.time()) + self.oauth_clockskew + return oauth_headers(url=url, consumer_key=mcfg['consumer_key'], token_key=mcfg['token_key'], token_secret=mcfg['token_secret'], - consumer_secret=consumer_secret) + consumer_secret=consumer_secret, + timestamp=timestamp) def wait_for_metadata_service(self, url): mcfg = self.ds_cfg @@ -124,7 +132,7 @@ class DataSourceMAAS(sources.DataSource): check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) urls = [check_url] url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn, + timeout=timeout, exception_cb=self._except_cb, headers_cb=self.md_headers) if url: @@ -135,6 +143,26 @@ class DataSourceMAAS(sources.DataSource): return bool(url) + def _except_cb(self, msg, exception): + if not (isinstance(exception, urllib2.HTTPError) and + exception.code == 403): + return + if 'date' not in exception.headers: + LOG.warn("date field not in 403 headers") + return + + date = exception.headers['date'] + + try: + ret_time = time.mktime(parsedate(date)) + except: + LOG.warn("failed to convert datetime '%s'") + return + + self.oauth_clockskew = int(ret_time - time.time()) + LOG.warn("set oauth clockskew to %d" % self.oauth_clockskew) + return + def read_maas_seed_dir(seed_d): """ @@ -229,13 +257,20 @@ def check_seed_contents(content, seed): return (userdata, md) -def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): +def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret, + timestamp=None): consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) token = oauth.OAuthToken(token_key, token_secret) + + if timestamp is None: + ts = int(time.time()) + else: + ts = timestamp + params = { 'oauth_version': "1.0", 'oauth_nonce': oauth.generate_nonce(), - 'oauth_timestamp': int(time.time()), + 'oauth_timestamp': ts, 'oauth_token': token.key, 'oauth_consumer_key': consumer.key, } diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 732d6aec..f3e3fd7e 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -136,7 +136,8 @@ def readurl(url, data=None, timeout=None, def wait_for_url(urls, max_wait=None, timeout=None, - status_cb=None, headers_cb=None, sleep_time=1): + status_cb=None, headers_cb=None, sleep_time=1, + exception_cb=None): """ urls: a list of urls to try max_wait: roughly the maximum time to wait before giving up @@ -146,6 +147,8 @@ def wait_for_url(urls, max_wait=None, timeout=None, status_cb: call method with string message when a url is not available headers_cb: call method with single argument of url to get headers for request. + exception_cb: call method with 2 arguments 'msg' (per status_cb) and + 'exception', the exception that occurred. the idea of this routine is to wait for the EC2 metdata service to come up. On both Eucalyptus and EC2 we have seen the case where @@ -164,7 +167,7 @@ def wait_for_url(urls, max_wait=None, timeout=None, """ start_time = time.time() - def log_status_cb(msg): + def log_status_cb(msg, exc=None): LOG.debug(msg) if status_cb is None: @@ -196,8 +199,10 @@ def wait_for_url(urls, max_wait=None, timeout=None, resp = readurl(url, headers=headers, timeout=timeout) if not resp.contents: reason = "empty response [%s]" % (resp.code) + e = ValueError(reason) elif not resp.ok(): reason = "bad status code [%s]" % (resp.code) + e = ValueError(reason) else: return url except urllib2.HTTPError as e: @@ -214,6 +219,8 @@ def wait_for_url(urls, max_wait=None, timeout=None, time_taken, max_wait, reason) status_cb(status_msg) + if exception_cb: + exception_cb(msg=status_msg, exception=e) if timeup(max_wait, start_time): break -- cgit v1.2.3 From d285a0463b6d16487eb5859373ccfd27eaec8b90 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 28 Sep 2012 16:54:22 -0400 Subject: make DataSourceMAAS 'main()' use load_yaml --- cloudinit/sources/DataSourceMAAS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/sources/DataSourceMAAS.py') diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 581e9a4b..c172150b 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -338,7 +338,7 @@ if __name__ == "__main__": if args.config: import yaml with open(args.config) as fp: - cfg = yaml.safe_load(fp) + cfg = util.load_yaml(fp.read()) if 'datasource' in cfg: cfg = cfg['datasource']['MAAS'] for key in creds.keys(): -- cgit v1.2.3 From a28d7fe46cf8e3277a13c35c5dd0185f65ab1d0c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sun, 30 Sep 2012 09:20:59 -0400 Subject: [pylint]: remove unused import --- cloudinit/sources/DataSourceMAAS.py | 1 - 1 file changed, 1 deletion(-) (limited to 'cloudinit/sources/DataSourceMAAS.py') diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index c172150b..ec52d775 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -336,7 +336,6 @@ if __name__ == "__main__": 'token_secret': args.tsec, 'consumer_secret': args.csec} if args.config: - import yaml with open(args.config) as fp: cfg = util.load_yaml(fp.read()) if 'datasource' in cfg: -- cgit v1.2.3 From f8b23b39bdf8753986df9ecf5948ffd8e8fdee74 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 1 Oct 2012 11:50:48 -0400 Subject: fix oauth time skew. actual implementation was returning 401 not 403. This fixes (tested) bug 978127. The server was actually returning a 401 not a 403. As such, the fix here was insufficient. This will now take either of those 2 error codes. I've also tested it by changing the clock in the cloud-init upstart job with a stanza like below, and verifying that we do see the problem and then it resolve itself: pre-start script offset="10 minutes ago" past=$(date -R --date "$offset") date --set "$past" && echo ===== "set date to $past [$offset]" ===== || echo ===== "failed to set date to $past [$offset]" ==== end script LP: #978127 --- ChangeLog | 4 ++++ cloudinit/sources/DataSourceMAAS.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources/DataSourceMAAS.py') diff --git a/ChangeLog b/ChangeLog index c5dcd418..cbfba6d0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,8 @@ 0.7.0: + - add a 'exception_cb' argument to 'wait_for_url'. If provided, this + method will be called back with the exception received and the message. + - utilize the 'exception_cb' above to modify the oauth timestamp in + DataSourceMAAS requests if a 401 or 403 is received. (LP: #978127) - catch signals and exit rather than stack tracing - if logging fails, enable a fallback logger by patching the logging module - do not 'start networking' in cloud-init-nonet, but add diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index ec52d775..e187aec9 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -145,10 +145,10 @@ class DataSourceMAAS(sources.DataSource): def _except_cb(self, msg, exception): if not (isinstance(exception, urllib2.HTTPError) and - exception.code == 403): + (exception.code == 403 or exception.code == 401)): return if 'date' not in exception.headers: - LOG.warn("date field not in 403 headers") + LOG.warn("date field not in %d headers" % exception.code) return date = exception.headers['date'] -- cgit v1.2.3