1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
|
# Author: Julien Castets <castets.j@gmail.com>
#
# This file is part of cloud-init. See LICENSE file for license information.
# Scaleway API:
# https://developer.scaleway.com/#metadata
import json
import os
import socket
import time
import requests
# pylint fails to import the two modules below.
# These are imported via requests.packages rather than urllib3 because:
# a.) the provider of the requests package should ensure that urllib3
# contained in it is consistent/correct.
# b.) cloud-init does not specifically have a dependency on urllib3
#
# For future reference, see:
# https://github.com/kennethreitz/requests/pull/2375
# https://github.com/requests/requests/issues/4104
# pylint: disable=E0401
from requests.packages.urllib3.connection import HTTPConnection
from requests.packages.urllib3.poolmanager import PoolManager
from cloudinit import log as logging
from cloudinit import sources
from cloudinit import url_helper
from cloudinit import util
LOG = logging.getLogger(__name__)
DS_BASE_URL = 'http://169.254.42.42'
BUILTIN_DS_CONFIG = {
'metadata_url': DS_BASE_URL + '/conf?format=json',
'userdata_url': DS_BASE_URL + '/user_data/cloud-init',
'vendordata_url': DS_BASE_URL + '/vendor_data/cloud-init'
}
DEF_MD_RETRIES = 5
DEF_MD_TIMEOUT = 10
def on_scaleway():
"""
There are three ways to detect if you are on Scaleway:
* check DMI data: not yet implemented by Scaleway, but the check is made to
be future-proof.
* the initrd created the file /var/run/scaleway.
* "scaleway" is in the kernel cmdline.
"""
vendor_name = util.read_dmi_data('system-manufacturer')
if vendor_name == 'Scaleway':
return True
if os.path.exists('/var/run/scaleway'):
return True
cmdline = util.get_cmdline()
if 'scaleway' in cmdline:
return True
return False
class SourceAddressAdapter(requests.adapters.HTTPAdapter):
"""
Adapter for requests to choose the local address to bind to.
"""
def __init__(self, source_address, **kwargs):
self.source_address = source_address
super(SourceAddressAdapter, self).__init__(**kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
socket_options = HTTPConnection.default_socket_options + [
(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
]
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
source_address=self.source_address,
socket_options=socket_options)
def query_data_api_once(api_address, timeout, requests_session):
"""
Retrieve user data or vendor data.
Scaleway user/vendor data API returns HTTP/404 if user/vendor data is not
set.
This function calls `url_helper.readurl` but instead of considering
HTTP/404 as an error that requires a retry, it considers it as empty
user/vendor data.
Also, be aware the user data/vendor API requires the source port to be
below 1024 to ensure the client is root (since non-root users can't bind
ports below 1024). If requests raises ConnectionError (EADDRINUSE), the
caller should retry to call this function on an other port.
"""
try:
resp = url_helper.readurl(
api_address,
data=None,
timeout=timeout,
# It's the caller's responsability to recall this function in case
# of exception. Don't let url_helper.readurl() retry by itself.
retries=0,
session=requests_session,
# If the error is a HTTP/404 or a ConnectionError, go into raise
# block below.
exception_cb=lambda _, exc: exc.code == 404 or (
isinstance(exc.cause, requests.exceptions.ConnectionError)
)
)
return util.decode_binary(resp.contents)
except url_helper.UrlError as exc:
# Empty user data.
if exc.code == 404:
return None
raise
def query_data_api(api_type, api_address, retries, timeout):
"""Get user or vendor data.
Handle the retrying logic in case the source port is used.
Scaleway metadata service requires the source port of the client to
be a privileged port (<1024). This is done to ensure that only a
privileged user on the system can access the metadata service.
"""
# Query user/vendor data. Try to make a request on the first privileged
# port available.
for port in range(1, max(retries, 2)):
try:
LOG.debug(
'Trying to get %s data (bind on port %d)...',
api_type, port
)
requests_session = requests.Session()
requests_session.mount(
'http://',
SourceAddressAdapter(source_address=('0.0.0.0', port))
)
data = query_data_api_once(
api_address,
timeout=timeout,
requests_session=requests_session
)
LOG.debug('%s-data downloaded', api_type)
return data
except url_helper.UrlError as exc:
# Local port already in use or HTTP/429.
LOG.warning('Error while trying to get %s data: %s', api_type, exc)
time.sleep(5)
last_exc = exc
continue
# Max number of retries reached.
raise last_exc
class DataSourceScaleway(sources.DataSource):
dsname = "Scaleway"
def __init__(self, sys_cfg, distro, paths):
super(DataSourceScaleway, self).__init__(sys_cfg, distro, paths)
self.ds_cfg = util.mergemanydict([
util.get_cfg_by_path(sys_cfg, ["datasource", "Scaleway"], {}),
BUILTIN_DS_CONFIG
])
self.metadata_address = self.ds_cfg['metadata_url']
self.userdata_address = self.ds_cfg['userdata_url']
self.vendordata_address = self.ds_cfg['vendordata_url']
self.retries = int(self.ds_cfg.get('retries', DEF_MD_RETRIES))
self.timeout = int(self.ds_cfg.get('timeout', DEF_MD_TIMEOUT))
def _get_data(self):
if not on_scaleway():
return False
resp = url_helper.readurl(self.metadata_address,
timeout=self.timeout,
retries=self.retries)
self.metadata = json.loads(util.decode_binary(resp.contents))
self.userdata_raw = query_data_api(
'user-data', self.userdata_address,
self.retries, self.timeout
)
self.vendordata_raw = query_data_api(
'vendor-data', self.vendordata_address,
self.retries, self.timeout
)
return True
@property
def launch_index(self):
return None
def get_instance_id(self):
return self.metadata['id']
def get_public_ssh_keys(self):
return [key['key'] for key in self.metadata['ssh_public_keys']]
def get_hostname(self, fqdn=False, resolve_ip=False):
return self.metadata['hostname']
@property
def availability_zone(self):
return None
@property
def region(self):
return None
datasources = [
(DataSourceScaleway, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
]
def get_datasource_list(depends):
return sources.list_from_depends(depends, datasources)
|
