diff options
Diffstat (limited to 'tests/integration_tests')
68 files changed, 4953 insertions, 589 deletions
diff --git a/tests/integration_tests/__init__.py b/tests/integration_tests/__init__.py new file mode 100644 index 00000000..81f9b02f --- /dev/null +++ b/tests/integration_tests/__init__.py @@ -0,0 +1,14 @@ +import random + + +def random_mac_address() -> str: + """Generate a random MAC address. + + The MAC address will have a 1 in its least significant bit, indicating it + to be a locally administered address. + """ + return "02:00:00:%02x:%02x:%02x" % ( + random.randint(0, 255), + random.randint(0, 255), + random.randint(0, 255), + ) diff --git a/tests/integration_tests/assets/keys/id_rsa.test1 b/tests/integration_tests/assets/keys/id_rsa.test1 new file mode 100644 index 00000000..bd4c822e --- /dev/null +++ b/tests/integration_tests/assets/keys/id_rsa.test1 @@ -0,0 +1,38 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn +NhAAAAAwEAAQAAAYEAtRlG96aJ23URvAgO/bBsuLl+lquc350aSwV98/i8vlvOn5GVcHye +t/rXQg4lZ4s0owG3kWyQFY8nvTk+G+UNU8fN0anAzBDi+4MzsejkF9scjTMFmXVrIpICqV +3bYQNjPv6r+ubQdkD01du3eB9t5/zl84gtshp0hBdofyz8u1/A25s7fVU67GyI7PdKvaS+ +yvJSInZnb2e9VQzfJC+qAnN7gUZatBKjdgUtJeiUUeDaVnaS17b0aoT9iBO0sIcQtOTBlY +lCjFt1TAMLZ64Hj3SfGZB7Yj0Z+LzFB2IWX1zzsjI68YkYPKOSL/NYhQU9e55kJQ7WnngN +HY/2n/A7dNKSFDmgM5c9IWgeZ7fjpsfIYAoJ/CAxFIND+PEHd1gCS6xoEhaUVyh5WH/Xkw +Kv1nx4AiZ2BFCE+75kySRLZUJ+5y0r3DU5ktMXeURzVIP7pu0R8DCul+GU+M/+THyWtAEO +geaNJ6fYpo2ipDhbmTYt3kk2lMIapRxGBFs+37sdAAAFgGGJssNhibLDAAAAB3NzaC1yc2 +EAAAGBALUZRvemidt1EbwIDv2wbLi5fparnN+dGksFffP4vL5bzp+RlXB8nrf610IOJWeL +NKMBt5FskBWPJ705PhvlDVPHzdGpwMwQ4vuDM7Ho5BfbHI0zBZl1ayKSAqld22EDYz7+q/ +rm0HZA9NXbt3gfbef85fOILbIadIQXaH8s/LtfwNubO31VOuxsiOz3Sr2kvsryUiJ2Z29n +vVUM3yQvqgJze4FGWrQSo3YFLSXolFHg2lZ2kte29GqE/YgTtLCHELTkwZWJQoxbdUwDC2 +euB490nxmQe2I9Gfi8xQdiFl9c87IyOvGJGDyjki/zWIUFPXueZCUO1p54DR2P9p/wO3TS +khQ5oDOXPSFoHme346bHyGAKCfwgMRSDQ/jxB3dYAkusaBIWlFcoeVh/15MCr9Z8eAImdg +RQhPu+ZMkkS2VCfuctK9w1OZLTF3lEc1SD+6btEfAwrpfhlPjP/kx8lrQBDoHmjSen2KaN +oqQ4W5k2Ld5JNpTCGqUcRgRbPt+7HQAAAAMBAAEAAAGBAJJCTOd70AC2ptEGbR0EHHqADT +Wgefy7A94tHFEqxTy0JscGq/uCGimaY7kMdbcPXT59B4VieWeAC2cuUPP0ZHQSfS5ke7oT +tU3N47U+0uBVbNS4rUAH7bOo2o9wptnOA5x/z+O+AARRZ6tEXQOd1oSy4gByLf2Wkh2QTi +vP6Hln1vlFgKEzcXg6G8fN3MYWxKRhWmZM3DLERMvorlqqSBLcs5VvfZfLKcsKWTExioAq +KgwEjYm8T9+rcpsw1xBus3j9k7wCI1Sus6PCDjq0pcYKLMYM7p8ygnU2tRYrOztdIxgWRA +w/1oenm1Mqq2tV5xJcBCwCLOGe6SFwkIRywOYc57j5McH98Xhhg9cViyyBdXy/baF0mro+ +qPhOsWDxqwD4VKZ9UmQ6O8kPNKcc7QcIpFJhcO0g9zbp/MT0KueaWYrTKs8y4lUkTT7Xz6 ++MzlR122/JwlAbBo6Y2kWtB+y+XwBZ0BfyJsm2czDhKm7OI5KfuBNhq0tFfKwOlYBq4QAA +AMAyvUof1R8LLISkdO3EFTKn5RGNkPPoBJmGs6LwvU7NSjjLj/wPQe4jsIBc585tvbrddp +60h72HgkZ5tqOfdeBYOKqX0qQQBHUEvI6M+NeQTQRev8bCHMLXQ21vzpClnrwNzlja359E +uTRfiPRwIlyPLhOUiClBDSAnBI9h82Hkk3zzsQ/xGfsPB7iOjRbW69bMRSVCRpeweCVmWC +77DTsEOq69V2TdljhQNIXE5OcOWonIlfgPiI74cdd+dLhzc/AAAADBAO1/JXd2kYiRyNkZ +aXTLcwiSgBQIYbobqVP3OEtTclr0P1JAvby3Y4cCaEhkenx+fBqgXAku5lKM+U1Q9AEsMk +cjIhaDpb43rU7GPjMn4zHwgGsEKd5pC1yIQ2PlK+cHanAdsDjIg+6RR+fuvid/mBeBOYXb +Py0sa3HyekLJmCdx4UEyNASoiNaGFLQVAqo+RACsXy6VMxFH5dqDYlvwrfUQLwxJmse9Vb +GEuuPAsklNugZqssC2XOIujFVUpslduQAAAMEAwzVHQVtsc3icCSzEAARpDTUdTbI29OhB +/FMBnjzS9/3SWfLuBOSm9heNCHs2jdGNb8cPdKZuY7S9Fx6KuVUPyTbSSYkjj0F4fTeC9g +0ym4p4UWYdF67WSWwLORkaG8K0d+G/CXkz8hvKUg6gcZWKBHAE1ROrHu1nsc8v7mkiKq4I +bnTw5Q9TgjbWcQWtgPq0wXyyl/K8S1SFdkMCTOHDD0RQ+jTV2WNGVwFTodIRHenX+Rw2g4 +CHbTWbsFrHR1qFAAAACmphbWVzQG5ld3Q= +-----END OPENSSH PRIVATE KEY----- diff --git a/tests/integration_tests/assets/keys/id_rsa.test1.pub b/tests/integration_tests/assets/keys/id_rsa.test1.pub new file mode 100644 index 00000000..3d2e26e1 --- /dev/null +++ b/tests/integration_tests/assets/keys/id_rsa.test1.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC1GUb3ponbdRG8CA79sGy4uX6Wq5zfnRpLBX3z+Ly+W86fkZVwfJ63+tdCDiVnizSjAbeRbJAVjye9OT4b5Q1Tx83RqcDMEOL7gzOx6OQX2xyNMwWZdWsikgKpXdthA2M+/qv65tB2QPTV27d4H23n/OXziC2yGnSEF2h/LPy7X8Dbmzt9VTrsbIjs90q9pL7K8lIidmdvZ71VDN8kL6oCc3uBRlq0EqN2BS0l6JRR4NpWdpLXtvRqhP2IE7SwhxC05MGViUKMW3VMAwtnrgePdJ8ZkHtiPRn4vMUHYhZfXPOyMjrxiRg8o5Iv81iFBT17nmQlDtaeeA0dj/af8Dt00pIUOaAzlz0haB5nt+Omx8hgCgn8IDEUg0P48Qd3WAJLrGgSFpRXKHlYf9eTAq/WfHgCJnYEUIT7vmTJJEtlQn7nLSvcNTmS0xd5RHNUg/um7RHwMK6X4ZT4z/5MfJa0AQ6B5o0np9imjaKkOFuZNi3eSTaUwhqlHEYEWz7fux0= test1@host diff --git a/tests/integration_tests/assets/keys/id_rsa.test2 b/tests/integration_tests/assets/keys/id_rsa.test2 new file mode 100644 index 00000000..5854d901 --- /dev/null +++ b/tests/integration_tests/assets/keys/id_rsa.test2 @@ -0,0 +1,38 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn +NhAAAAAwEAAQAAAYEAvK50D2PWOc4ikyHVRJS6tDhqzjL5cKiivID4p1X8BYCVw83XAEGO +LnItUyVXHNADlh6fpVq1NY6A2JVtygoPF6ZFx8ph7IWMmnhDdnxLLyGsbhd1M1tiXJD/R+ +3WnGHRJ4PKrQavMLgqHRrieV3QVVfjFSeo6jX/4TruP6ZmvITMZWJrXaGphxJ/pPykEdkO +i8AmKU9FNviojyPS2nNtj9B/635IdgWvrd7Vf5Ycsw9MR55LWSidwa856RH62Yl6LpEGTH +m1lJiMk1u88JPSqvohhaUkLKkFpcQwcB0m76W1KOyllJsmX8bNXrlZsI+WiiYI7Xl5vQm2 +17DEuNeavtPAtDMxu8HmTg2UJ55Naxehbfe2lx2k5kYGGw3i1O1OVN2pZ2/OB71LucYd/5 +qxPaz03wswcGOJYGPkNc40vdES/Scc7Yt8HsnZuzqkyOgzn0HiUCzoYUYLYTpLf+yGmwxS +yAEY056aOfkCsboKHOKiOmlJxNaZZFQkX1evep4DAAAFgC7HMbUuxzG1AAAAB3NzaC1yc2 +EAAAGBALyudA9j1jnOIpMh1USUurQ4as4y+XCooryA+KdV/AWAlcPN1wBBji5yLVMlVxzQ +A5Yen6VatTWOgNiVbcoKDxemRcfKYeyFjJp4Q3Z8Sy8hrG4XdTNbYlyQ/0ft1pxh0SeDyq +0GrzC4Kh0a4nld0FVX4xUnqOo1/+E67j+mZryEzGVia12hqYcSf6T8pBHZDovAJilPRTb4 +qI8j0tpzbY/Qf+t+SHYFr63e1X+WHLMPTEeeS1koncGvOekR+tmJei6RBkx5tZSYjJNbvP +CT0qr6IYWlJCypBaXEMHAdJu+ltSjspZSbJl/GzV65WbCPloomCO15eb0JttewxLjXmr7T +wLQzMbvB5k4NlCeeTWsXoW33tpcdpOZGBhsN4tTtTlTdqWdvzge9S7nGHf+asT2s9N8LMH +BjiWBj5DXONL3REv0nHO2LfB7J2bs6pMjoM59B4lAs6GFGC2E6S3/shpsMUsgBGNOemjn5 +ArG6ChziojppScTWmWRUJF9Xr3qeAwAAAAMBAAEAAAGASj/kkEHbhbfmxzujL2/P4Sfqb+ +aDXqAeGkwujbs6h/fH99vC5ejmSMTJrVSeaUo6fxLiBDIj6UWA0rpLEBzRP59BCpRL4MXV +RNxav/+9nniD4Hb+ug0WMhMlQmsH71ZW9lPYqCpfOq7ec8GmqdgPKeaCCEspH7HMVhfYtd +eHylwAC02lrpz1l5/h900sS5G9NaWR3uPA+xbzThDs4uZVkSidjlCNt1QZhDSSk7jA5n34 +qJ5UTGu9WQDZqyxWKND+RIyQuFAPGQyoyCC1FayHO2sEhT5qHuumL14Mn81XpzoXFoKyql +rhBDe+pHhKArBYt92Evch0k1ABKblFxtxLXcvk4Fs7pHi+8k4+Cnazej2kcsu1kURlMZJB +w2QT/8BV4uImbH05LtyscQuwGzpIoxqrnHrvg5VbohStmhoOjYybzqqW3/M0qhkn5JgTiy +dJcHRJisRnAcmbmEchYtLDi6RW1e022H4I9AFXQqyr5HylBq6ugtWcFCsrcX8ibZ8xAAAA +wQCAOPgwae6yZLkrYzRfbxZtGKNmhpI0EtNSDCHYuQQapFZJe7EFENs/VAaIiiut0yajGj +c3aoKcwGIoT8TUM8E3GSNW6+WidUOC7H6W+/6N2OYZHRBACGz820xO+UBCl2oSk+dLBlfr +IQzBGUWn5uVYCs0/2nxfCdFyHtMK8dMF/ypbdG+o1rXz5y9b7PVG6Mn+o1Rjsdkq7VERmy +Pukd8hwATOIJqoKl3TuFyBeYFLqe+0e7uTeswQFw17PF31VjAAAADBAOpJRQb8c6qWqsvv +vkve0uMuL0DfWW0G6+SxjPLcV6aTWL5xu0Grd8uBxDkkHU/CDrAwpchXyuLsvbw21Eje/u +U5k9nLEscWZwcX7odxlK+EfAY2Bf5+Hd9bH5HMzTRJH8KkWK1EppOLPyiDxz4LZGzPLVyv +/1PgSuvXkSWk1KIE4SvSemyxGX2tPVI6uO+URqevfnPOS1tMB7BMQlgkR6eh4bugx9UYx9 +mwlXonNa4dN0iQxZ7N4rKFBbT/uyB2bQAAAMEAzisnkD8k9Tn8uyhxpWLHwb03X4ZUUHDV +zu15e4a8dZ+mM8nHO986913Xz5JujlJKkGwFTvgWkIiR2zqTEauZHARH7gANpaweTm6lPd +E4p2S0M3ulY7xtp9lCFIrDhMPPkGq8SFZB6qhgucHcZSRLq6ZDou3S2IdNOzDTpBtkhRCS +0zFcdTLh3zZweoy8HGbW36bwB6s1CIL76Pd4F64i0Ms9CCCU6b+E5ArFhYQIsXiDbgHWbD +tZRSm2GEgnDGAvAAAACmphbWVzQG5ld3Q= +-----END OPENSSH PRIVATE KEY----- diff --git a/tests/integration_tests/assets/keys/id_rsa.test2.pub b/tests/integration_tests/assets/keys/id_rsa.test2.pub new file mode 100644 index 00000000..f3831a57 --- /dev/null +++ b/tests/integration_tests/assets/keys/id_rsa.test2.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC8rnQPY9Y5ziKTIdVElLq0OGrOMvlwqKK8gPinVfwFgJXDzdcAQY4uci1TJVcc0AOWHp+lWrU1joDYlW3KCg8XpkXHymHshYyaeEN2fEsvIaxuF3UzW2JckP9H7dacYdEng8qtBq8wuCodGuJ5XdBVV+MVJ6jqNf/hOu4/pma8hMxlYmtdoamHEn+k/KQR2Q6LwCYpT0U2+KiPI9Lac22P0H/rfkh2Ba+t3tV/lhyzD0xHnktZKJ3BrznpEfrZiXoukQZMebWUmIyTW7zwk9Kq+iGFpSQsqQWlxDBwHSbvpbUo7KWUmyZfxs1euVmwj5aKJgjteXm9CbbXsMS415q+08C0MzG7weZODZQnnk1rF6Ft97aXHaTmRgYbDeLU7U5U3alnb84HvUu5xh3/mrE9rPTfCzBwY4lgY+Q1zjS90RL9Jxzti3weydm7OqTI6DOfQeJQLOhhRgthOkt/7IabDFLIARjTnpo5+QKxugoc4qI6aUnE1plkVCRfV696ngM= test2@host diff --git a/tests/integration_tests/assets/keys/id_rsa.test3 b/tests/integration_tests/assets/keys/id_rsa.test3 new file mode 100644 index 00000000..2596c762 --- /dev/null +++ b/tests/integration_tests/assets/keys/id_rsa.test3 @@ -0,0 +1,38 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn +NhAAAAAwEAAQAAAYEApPG4MdkYQKD57/qreFrh9GRC22y66qZOWZWRjC887rrbvBzO69hV +yJpTIXleJEvpWiHYcjMR5G6NNFsnNtZ4fxDqmSc4vcFj53JsE/XNqLKq6psXadCb5vkNpG +bxA+Z5bJlzJ969PgJIIEbgc86sei4kgR2MuPWqtZbY5GkpNCTqWuLYeFK+14oFruA2nyWH +9MOIRDHK/d597psHy+LTMtymO7ZPhO571abKw6jvvwiSeDxVE9kV7KAQIuM9/S3gftvgQQ +ron3GL34pgmIabdSGdbfHqGDooryJhlbquJZELBN236KgRNTCAjVvUzjjQr1eRP3xssGwV +O6ECBGCQLl/aYogAgtwnwj9iXqtfiLK3EwlgjquU4+JQ0CVtLhG3gIZB+qoMThco0pmHTr +jtfQCwrztsBBFunSa2/CstuV1mQ5O5ZrZ6ACo9yPRBNkns6+CiKdtMtCtzi3k2RDz9jpYm +Pcak03Lr7IkdC1Tp6+jA+//yPHSO1o4CqW89IQzNAAAFgEUd7lZFHe5WAAAAB3NzaC1yc2 +EAAAGBAKTxuDHZGECg+e/6q3ha4fRkQttsuuqmTlmVkYwvPO6627wczuvYVciaUyF5XiRL +6Voh2HIzEeRujTRbJzbWeH8Q6pknOL3BY+dybBP1zaiyquqbF2nQm+b5DaRm8QPmeWyZcy +fevT4CSCBG4HPOrHouJIEdjLj1qrWW2ORpKTQk6lri2HhSvteKBa7gNp8lh/TDiEQxyv3e +fe6bB8vi0zLcpju2T4Tue9WmysOo778Ikng8VRPZFeygECLjPf0t4H7b4EEK6J9xi9+KYJ +iGm3UhnW3x6hg6KK8iYZW6riWRCwTdt+ioETUwgI1b1M440K9XkT98bLBsFTuhAgRgkC5f +2mKIAILcJ8I/Yl6rX4iytxMJYI6rlOPiUNAlbS4Rt4CGQfqqDE4XKNKZh0647X0AsK87bA +QRbp0mtvwrLbldZkOTuWa2egAqPcj0QTZJ7OvgoinbTLQrc4t5NkQ8/Y6WJj3GpNNy6+yJ +HQtU6evowPv/8jx0jtaOAqlvPSEMzQAAAAMBAAEAAAGAGaqbdPZJNdVWzyb8g6/wtSzc0n +Qq6dSTIJGLonq/So69HpqFAGIbhymsger24UMGvsXBfpO/1wH06w68HWZmPa+OMeLOi4iK +WTuO4dQ/+l5DBlq32/lgKSLcIpb6LhcxEdsW9j9Mx1dnjc45owun/yMq/wRwH1/q/nLIsV +JD3R9ZcGcYNDD8DWIm3D17gmw+qbG7hJES+0oh4n0xS2KyZpm7LFOEMDVEA8z+hE/HbryQ +vjD1NC91n+qQWD1wKfN3WZDRwip3z1I5VHMpvXrA/spHpa9gzHK5qXNmZSz3/dfA1zHjCR +2dHjJnrIUH8nyPfw8t+COC+sQBL3Nr0KUWEFPRM08cOcQm4ctzg17aDIZBONjlZGKlReR8 +1zfAw84Q70q2spLWLBLXSFblHkaOfijEbejIbaz2UUEQT27WD7RHAORdQlkx7eitk66T9d +DzIq/cpYhm5Fs8KZsh3PLldp9nsHbD2Oa9J9LJyI4ryuIW0mVwRdvPSiiYi3K+mDCpAAAA +wBe+ugEEJ+V7orb1f4Zez0Bd4FNkEc52WZL4CWbaCtM+ZBg5KnQ6xW14JdC8IS9cNi/I5P +yLsBvG4bWPLGgQruuKY6oLueD6BFnKjqF6ACUCiSQldh4BAW1nYc2U48+FFvo3ZQyudFSy +QEFlhHmcaNMDo0AIJY5Xnq2BG3nEX7AqdtZ8hhenHwLCRQJatDwSYBHDpSDdh9vpTnGp/2 +0jBz25Ko4UANzvSAc3sA4yN3jfpoM366TgdNf8x3g1v7yljQAAAMEA0HSQjzH5nhEwB58k +mYYxnBYp1wb86zIuVhAyjZaeinvBQSTmLow8sXIHcCVuD3CgBezlU2SX5d9YuvRU9rcthi +uzn4wWnbnzYy4SwzkMJXchUAkumFVD8Hq5TNPh2Z+033rLLE08EhYypSeVpuzdpFoStaS9 +3DUZA2bR/zLZI9MOVZRUcYImNegqIjOYHY8Sbj3/0QPV6+WpUJFMPvvedWhfaOsRMTA6nr +VLG4pxkrieVl0UtuRGbzD/exXhXVi7AAAAwQDKkJj4ez/+KZFYlZQKiV0BrfUFcgS6ElFM +2CZIEagCtu8eedrwkNqx2FUX33uxdvUTr4c9I3NvWeEEGTB9pgD4lh1x/nxfuhyGXtimFM +GnznGV9oyz0DmKlKiKSEGwWf5G+/NiiCwwVJ7wsQQm7TqNtkQ9b8MhWWXC7xlXKUs7dmTa +e8AqAndCCMEnbS1UQFO/R5PNcZXkFWDggLQ/eWRYKlrXgdnUgH6h0saOcViKpNJBUXb3+x +eauhOY52PS/BcAAAAKamFtZXNAbmV3dAE= +-----END OPENSSH PRIVATE KEY----- diff --git a/tests/integration_tests/assets/keys/id_rsa.test3.pub b/tests/integration_tests/assets/keys/id_rsa.test3.pub new file mode 100644 index 00000000..057db632 --- /dev/null +++ b/tests/integration_tests/assets/keys/id_rsa.test3.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCk8bgx2RhAoPnv+qt4WuH0ZELbbLrqpk5ZlZGMLzzuutu8HM7r2FXImlMheV4kS+laIdhyMxHkbo00Wyc21nh/EOqZJzi9wWPncmwT9c2osqrqmxdp0Jvm+Q2kZvED5nlsmXMn3r0+AkggRuBzzqx6LiSBHYy49aq1ltjkaSk0JOpa4th4Ur7XigWu4DafJYf0w4hEMcr93n3umwfL4tMy3KY7tk+E7nvVpsrDqO+/CJJ4PFUT2RXsoBAi4z39LeB+2+BBCuifcYvfimCYhpt1IZ1t8eoYOiivImGVuq4lkQsE3bfoqBE1MICNW9TOONCvV5E/fGywbBU7oQIEYJAuX9piiACC3CfCP2Jeq1+IsrcTCWCOq5Tj4lDQJW0uEbeAhkH6qgxOFyjSmYdOuO19ALCvO2wEEW6dJrb8Ky25XWZDk7lmtnoAKj3I9EE2Sezr4KIp20y0K3OLeTZEPP2OliY9xqTTcuvsiR0LVOnr6MD7//I8dI7WjgKpbz0hDM0= test3@host diff --git a/tests/integration_tests/assets/test_version_change.pkl b/tests/integration_tests/assets/test_version_change.pkl Binary files differnew file mode 100644 index 00000000..65ae93e5 --- /dev/null +++ b/tests/integration_tests/assets/test_version_change.pkl diff --git a/tests/integration_tests/assets/trusty_with_mime.pkl b/tests/integration_tests/assets/trusty_with_mime.pkl new file mode 100644 index 00000000..a4089ecf --- /dev/null +++ b/tests/integration_tests/assets/trusty_with_mime.pkl @@ -0,0 +1,572 @@ +ccopy_reg +_reconstructor +p1 +(ccloudinit.sources.DataSourceNoCloud +DataSourceNoCloudNet +p2 +c__builtin__ +object +p3 +NtRp4 +(dp5 +S'paths' +p6 +g1 +(ccloudinit.helpers +Paths +p7 +g3 +NtRp8 +(dp9 +S'lookups' +p10 +(dp11 +S'cloud_config' +p12 +S'cloud-config.txt' +p13 +sS'userdata' +p14 +S'user-data.txt.i' +p15 +sS'vendordata' +p16 +S'vendor-data.txt.i' +p17 +sS'userdata_raw' +p18 +S'user-data.txt' +p19 +sS'boothooks' +p20 +g20 +sS'scripts' +p21 +g21 +sS'sem' +p22 +g22 +sS'data' +p23 +g23 +sS'vendor_scripts' +p24 +S'scripts/vendor' +p25 +sS'handlers' +p26 +g26 +sS'obj_pkl' +p27 +S'obj.pkl' +p28 +sS'vendordata_raw' +p29 +S'vendor-data.txt' +p30 +sS'vendor_cloud_config' +p31 +S'vendor-cloud-config.txt' +p32 +ssS'template_tpl' +p33 +S'/etc/cloud/templates/%s.tmpl' +p34 +sS'cfgs' +p35 +(dp36 +S'cloud_dir' +p37 +S'/var/lib/cloud/' +p38 +sS'templates_dir' +p39 +S'/etc/cloud/templates/' +p40 +sS'upstart_dir' +p41 +S'/etc/init/' +p42 +ssS'cloud_dir' +p43 +g38 +sS'datasource' +p44 +NsS'upstart_conf_d' +p45 +g42 +sS'boot_finished' +p46 +S'/var/lib/cloud/instance/boot-finished' +p47 +sS'instance_link' +p48 +S'/var/lib/cloud/instance' +p49 +sS'seed_dir' +p50 +S'/var/lib/cloud/seed' +p51 +sbsS'supported_seed_starts' +p52 +(S'http://' +p53 +S'https://' +p54 +S'ftp://' +p55 +tp56 +sS'sys_cfg' +p57 +(dp58 +S'output' +p59 +(dp60 +S'all' +p61 +S'| tee -a /var/log/cloud-init-output.log' +p62 +ssS'users' +p63 +(lp64 +S'default' +p65 +asS'def_log_file' +p66 +S'/var/log/cloud-init.log' +p67 +sS'cloud_final_modules' +p68 +(lp69 +S'rightscale_userdata' +p70 +aS'scripts-vendor' +p71 +aS'scripts-per-once' +p72 +aS'scripts-per-boot' +p73 +aS'scripts-per-instance' +p74 +aS'scripts-user' +p75 +aS'ssh-authkey-fingerprints' +p76 +aS'keys-to-console' +p77 +aS'phone-home' +p78 +aS'final-message' +p79 +aS'power-state-change' +p80 +asS'disable_root' +p81 +I01 +sS'syslog_fix_perms' +p82 +S'syslog:adm' +p83 +sS'log_cfgs' +p84 +(lp85 +(lp86 +S'[loggers]\nkeys=root,cloudinit\n\n[handlers]\nkeys=consoleHandler,cloudLogHandler\n\n[formatters]\nkeys=simpleFormatter,arg0Formatter\n\n[logger_root]\nlevel=DEBUG\nhandlers=consoleHandler,cloudLogHandler\n\n[logger_cloudinit]\nlevel=DEBUG\nqualname=cloudinit\nhandlers=\npropagate=1\n\n[handler_consoleHandler]\nclass=StreamHandler\nlevel=WARNING\nformatter=arg0Formatter\nargs=(sys.stderr,)\n\n[formatter_arg0Formatter]\nformat=%(asctime)s - %(filename)s[%(levelname)s]: %(message)s\n\n[formatter_simpleFormatter]\nformat=[CLOUDINIT] %(filename)s[%(levelname)s]: %(message)s\n' +p87 +aS'[handler_cloudLogHandler]\nclass=handlers.SysLogHandler\nlevel=DEBUG\nformatter=simpleFormatter\nargs=("/dev/log", handlers.SysLogHandler.LOG_USER)\n' +p88 +aa(lp89 +g87 +aS"[handler_cloudLogHandler]\nclass=FileHandler\nlevel=DEBUG\nformatter=arg0Formatter\nargs=('/var/log/cloud-init.log',)\n" +p90 +aasS'cloud_init_modules' +p91 +(lp92 +S'migrator' +p93 +aS'seed_random' +p94 +aS'bootcmd' +p95 +aS'write-files' +p96 +aS'growpart' +p97 +aS'resizefs' +p98 +aS'set_hostname' +p99 +aS'update_hostname' +p100 +aS'update_etc_hosts' +p101 +aS'ca-certs' +p102 +aS'rsyslog' +p103 +aS'users-groups' +p104 +aS'ssh' +p105 +asS'preserve_hostname' +p106 +I00 +sS'_log' +p107 +(lp108 +g87 +ag90 +ag88 +asS'datasource_list' +p109 +(lp110 +S'NoCloud' +p111 +aS'ConfigDrive' +p112 +aS'OpenNebula' +p113 +aS'Azure' +p114 +aS'AltCloud' +p115 +aS'OVF' +p116 +aS'MAAS' +p117 +aS'GCE' +p118 +aS'OpenStack' +p119 +aS'CloudSigma' +p120 +aS'Ec2' +p121 +aS'CloudStack' +p122 +aS'SmartOS' +p123 +aS'None' +p124 +asS'vendor_data' +p125 +(dp126 +S'prefix' +p127 +(lp128 +sS'enabled' +p129 +I01 +ssS'cloud_config_modules' +p130 +(lp131 +S'emit_upstart' +p132 +aS'disk_setup' +p133 +aS'mounts' +p134 +aS'ssh-import-id' +p135 +aS'locale' +p136 +aS'set-passwords' +p137 +aS'grub-dpkg' +p138 +aS'apt-pipelining' +p139 +aS'apt-configure' +p140 +aS'package-update-upgrade-install' +p141 +aS'landscape' +p142 +aS'timezone' +p143 +aS'puppet' +p144 +aS'chef' +p145 +aS'salt-minion' +p146 +aS'mcollective' +p147 +aS'disable-ec2-metadata' +p148 +aS'runcmd' +p149 +aS'byobu' +p150 +assg14 +(iemail.mime.multipart +MIMEMultipart +p151 +(dp152 +S'_headers' +p153 +(lp154 +(S'Content-Type' +p155 +S'multipart/mixed; boundary="===============4291038100093149247=="' +tp156 +a(S'MIME-Version' +p157 +S'1.0' +p158 +tp159 +a(S'Number-Attachments' +p160 +S'1' +tp161 +asS'_payload' +p162 +(lp163 +(iemail.mime.base +MIMEBase +p164 +(dp165 +g153 +(lp166 +(g157 +g158 +tp167 +a(S'Content-Type' +p168 +S'text/x-not-multipart' +tp169 +a(S'Content-Disposition' +p170 +S'attachment; filename="part-001"' +tp171 +asg162 +S'' +sS'_charset' +p172 +NsS'_default_type' +p173 +S'text/plain' +p174 +sS'preamble' +p175 +NsS'defects' +p176 +(lp177 +sS'_unixfrom' +p178 +NsS'epilogue' +p179 +Nsbasg172 +Nsg173 +g174 +sg175 +Nsg176 +(lp180 +sg178 +Nsg179 +Nsbsg16 +S'#cloud-config\n{}\n\n' +p181 +sg18 +S'Content-Type: multipart/mixed; boundary="===============1378281702283945349=="\nMIME-Version: 1.0\n\n--===============1378281702283945349==\nContent-Type: text/x-shellscript; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nContent-Disposition: attachment; filename="script1.sh"\n\nIyEvYmluL3NoCgplY2hvICdoaScgPiAvdmFyL3RtcC9oaQo=\n\n--===============1378281702283945349==\nContent-Type: text/x-shellscript; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nContent-Disposition: attachment; filename="script2.sh"\n\nIyEvYmluL2Jhc2gKCmVjaG8gJ2hpMicgPiAvdmFyL3RtcC9oaTIK\n\n--===============1378281702283945349==--\n\n#cloud-config\n# final_message: |\n# This is my final message!\n# $version\n# $timestamp\n# $datasource\n# $uptime\n# updates:\n# network:\n# when: [\'hotplug\']\n' +p182 +sg29 +NsS'dsmode' +p183 +S'net' +p184 +sS'seed' +p185 +S'/var/lib/cloud/seed/nocloud-net' +p186 +sS'cmdline_id' +p187 +S'ds=nocloud-net' +p188 +sS'ud_proc' +p189 +g1 +(ccloudinit.user_data +UserDataProcessor +p190 +g3 +NtRp191 +(dp192 +g6 +g8 +sS'ssl_details' +p193 +(dp194 +sbsg50 +g186 +sS'ds_cfg' +p195 +(dp196 +sS'distro' +p197 +g1 +(ccloudinit.distros.ubuntu +Distro +p198 +g3 +NtRp199 +(dp200 +S'osfamily' +p201 +S'debian' +p202 +sS'_paths' +p203 +g8 +sS'name' +p204 +S'ubuntu' +p205 +sS'_runner' +p206 +g1 +(ccloudinit.helpers +Runners +p207 +g3 +NtRp208 +(dp209 +g6 +g8 +sS'sems' +p210 +(dp211 +sbsS'_cfg' +p212 +(dp213 +S'paths' +p214 +(dp215 +g37 +g38 +sg39 +g40 +sg41 +g42 +ssS'default_user' +p216 +(dp217 +S'shell' +p218 +S'/bin/bash' +p219 +sS'name' +p220 +S'ubuntu' +p221 +sS'sudo' +p222 +(lp223 +S'ALL=(ALL) NOPASSWD:ALL' +p224 +asS'lock_passwd' +p225 +I01 +sS'gecos' +p226 +S'Ubuntu' +p227 +sS'groups' +p228 +(lp229 +S'adm' +p230 +aS'audio' +p231 +aS'cdrom' +p232 +aS'dialout' +p233 +aS'dip' +p234 +aS'floppy' +p235 +aS'netdev' +p236 +aS'plugdev' +p237 +aS'sudo' +p238 +aS'video' +p239 +assS'package_mirrors' +p240 +(lp241 +(dp242 +S'arches' +p243 +(lp244 +S'i386' +p245 +aS'amd64' +p246 +asS'failsafe' +p247 +(dp248 +S'security' +p249 +S'http://security.ubuntu.com/ubuntu' +p250 +sS'primary' +p251 +S'http://archive.ubuntu.com/ubuntu' +p252 +ssS'search' +p253 +(dp254 +S'security' +p255 +(lp256 +sS'primary' +p257 +(lp258 +S'http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/' +p259 +aS'http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/' +p260 +aS'http://%(region)s.clouds.archive.ubuntu.com/ubuntu/' +p261 +assa(dp262 +S'arches' +p263 +(lp264 +S'armhf' +p265 +aS'armel' +p266 +aS'default' +p267 +asS'failsafe' +p268 +(dp269 +S'security' +p270 +S'http://ports.ubuntu.com/ubuntu-ports' +p271 +sS'primary' +p272 +S'http://ports.ubuntu.com/ubuntu-ports' +p273 +ssasS'ssh_svcname' +p274 +S'ssh' +p275 +ssbsS'metadata' +p276 +(dp277 +g183 +g184 +sS'local-hostname' +p278 +S'me' +p279 +sS'instance-id' +p280 +S'me' +p281 +ssb.
\ No newline at end of file diff --git a/tests/integration_tests/bugs/test_gh570.py b/tests/integration_tests/bugs/test_gh570.py new file mode 100644 index 00000000..e98ab5d0 --- /dev/null +++ b/tests/integration_tests/bugs/test_gh570.py @@ -0,0 +1,39 @@ +"""Integration test for #570. + +Test that we can add optional vendor-data to the seedfrom file in a +NoCloud environment +""" + +import pytest + +from tests.integration_tests.instances import IntegrationInstance + +VENDOR_DATA = """\ +#cloud-config +runcmd: + - touch /var/tmp/seeded_vendordata_test_file +""" + + +# Only running on LXD because we need NoCloud for this test +@pytest.mark.lxd_container +@pytest.mark.lxd_vm +def test_nocloud_seedfrom_vendordata(client: IntegrationInstance): + seed_dir = "/var/tmp/test_seed_dir" + result = client.execute( + "mkdir {seed_dir} && " + "touch {seed_dir}/user-data && " + "touch {seed_dir}/meta-data && " + "echo 'seedfrom: {seed_dir}/' > " + "/var/lib/cloud/seed/nocloud-net/meta-data".format(seed_dir=seed_dir) + ) + assert result.return_code == 0 + + client.write_to_file( + "{}/vendor-data".format(seed_dir), + VENDOR_DATA, + ) + client.execute("cloud-init clean --logs") + client.restart() + assert client.execute("cloud-init status").ok + assert "seeded_vendordata_test_file" in client.execute("ls /var/tmp") diff --git a/tests/integration_tests/bugs/test_gh626.py b/tests/integration_tests/bugs/test_gh626.py new file mode 100644 index 00000000..b80b677a --- /dev/null +++ b/tests/integration_tests/bugs/test_gh626.py @@ -0,0 +1,43 @@ +"""Integration test for gh-626. + +Ensure if wakeonlan is specified in the network config that it is rendered +in the /etc/network/interfaces or netplan config. +""" + +import pytest +import yaml + +from tests.integration_tests import random_mac_address +from tests.integration_tests.instances import IntegrationInstance + +MAC_ADDRESS = random_mac_address() +NETWORK_CONFIG = """\ +version: 2 +ethernets: + eth0: + dhcp4: true + wakeonlan: true + match: + macaddress: {} +""".format( + MAC_ADDRESS +) + +EXPECTED_ENI_END = """\ +iface eth0 inet dhcp + ethernet-wol g""" + + +@pytest.mark.lxd_container +@pytest.mark.lxd_vm +@pytest.mark.lxd_config_dict( + { + "user.network-config": NETWORK_CONFIG, + "volatile.eth0.hwaddr": MAC_ADDRESS, + } +) +def test_wakeonlan(client: IntegrationInstance): + netplan_cfg = client.execute("cat /etc/netplan/50-cloud-init.yaml") + netplan_yaml = yaml.safe_load(netplan_cfg) + assert "wakeonlan" in netplan_yaml["network"]["ethernets"]["eth0"] + assert netplan_yaml["network"]["ethernets"]["eth0"]["wakeonlan"] is True diff --git a/tests/integration_tests/bugs/test_gh632.py b/tests/integration_tests/bugs/test_gh632.py new file mode 100644 index 00000000..c7a897c6 --- /dev/null +++ b/tests/integration_tests/bugs/test_gh632.py @@ -0,0 +1,33 @@ +"""Integration test for gh-632. + +Verify that if cloud-init is using DataSourceRbxCloud, there is +no traceback if the metadata disk cannot be found. +""" +import pytest + +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import verify_clean_log + + +# With some datasource hacking, we can run this on a NoCloud instance +@pytest.mark.lxd_container +@pytest.mark.lxd_vm +def test_datasource_rbx_no_stacktrace(client: IntegrationInstance): + client.write_to_file( + "/etc/cloud/cloud.cfg.d/90_dpkg.cfg", + "datasource_list: [ RbxCloud, NoCloud ]\n", + ) + client.write_to_file( + "/etc/cloud/ds-identify.cfg", + "policy: enabled\n", + ) + client.execute("cloud-init clean --logs") + client.restart() + + log = client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log) + assert "Failed to load metadata and userdata" not in log + assert ( + "Getting data from <class 'cloudinit.sources.DataSourceRbxCloud." + "DataSourceRbxCloud'> failed" not in log + ) diff --git a/tests/integration_tests/bugs/test_gh668.py b/tests/integration_tests/bugs/test_gh668.py new file mode 100644 index 00000000..95edb48d --- /dev/null +++ b/tests/integration_tests/bugs/test_gh668.py @@ -0,0 +1,46 @@ +"""Integration test for gh-668. + +Ensure that static route to host is working correctly. +The original problem is specific to the ENI renderer but that test is suitable +for all network configuration outputs. +""" + +import pytest + +from tests.integration_tests import random_mac_address +from tests.integration_tests.instances import IntegrationInstance + +DESTINATION_IP = "172.16.0.10" +GATEWAY_IP = "10.0.0.100" +MAC_ADDRESS = random_mac_address() + +NETWORK_CONFIG = """\ +version: 2 +ethernets: + eth0: + addresses: [10.0.0.10/8] + dhcp4: false + routes: + - to: {}/32 + via: {} + match: + macaddress: {} +""".format( + DESTINATION_IP, GATEWAY_IP, MAC_ADDRESS +) + +EXPECTED_ROUTE = "{} via {}".format(DESTINATION_IP, GATEWAY_IP) + + +@pytest.mark.lxd_container +@pytest.mark.lxd_vm +@pytest.mark.lxd_config_dict( + { + "user.network-config": NETWORK_CONFIG, + "volatile.eth0.hwaddr": MAC_ADDRESS, + } +) +@pytest.mark.lxd_use_exec +def test_static_route_to_host(client: IntegrationInstance): + route = client.execute("ip route | grep {}".format(DESTINATION_IP)) + assert route.startswith(EXPECTED_ROUTE) diff --git a/tests/integration_tests/bugs/test_gh671.py b/tests/integration_tests/bugs/test_gh671.py new file mode 100644 index 00000000..2d7c8118 --- /dev/null +++ b/tests/integration_tests/bugs/test_gh671.py @@ -0,0 +1,53 @@ +"""Integration test for gh-671. + +Verify that on Azure that if a default user and password are specified +through the Azure API that a change in the default password overwrites +the old password +""" + +import crypt + +import pytest + +from tests.integration_tests.clouds import IntegrationCloud + +OLD_PASSWORD = "DoIM33tTheComplexityRequirements!??" +NEW_PASSWORD = "DoIM33tTheComplexityRequirementsNow!??" + + +def _check_password(instance, unhashed_password): + shadow_password = instance.execute("getent shadow ubuntu").split(":")[1] + salt = shadow_password.rsplit("$", 1)[0] + hashed_password = crypt.crypt(unhashed_password, salt) + assert shadow_password == hashed_password + + +@pytest.mark.azure +def test_update_default_password(setup_image, session_cloud: IntegrationCloud): + os_profile = { + "os_profile": { + "admin_password": "", + "linux_configuration": {"disable_password_authentication": False}, + } + } + os_profile["os_profile"]["admin_password"] = OLD_PASSWORD + instance1 = session_cloud.launch(launch_kwargs={"vm_params": os_profile}) + + _check_password(instance1, OLD_PASSWORD) + + snapshot_id = instance1.cloud.cloud_instance.snapshot( + instance1.instance, delete_provisioned_user=False + ) + + os_profile["os_profile"]["admin_password"] = NEW_PASSWORD + try: + with session_cloud.launch( + launch_kwargs={ + "image_id": snapshot_id, + "vm_params": os_profile, + } + ) as instance2: + _check_password(instance2, NEW_PASSWORD) + finally: + session_cloud.cloud_instance.delete_image(snapshot_id) + instance1.destroy() diff --git a/tests/integration_tests/bugs/test_gh868.py b/tests/integration_tests/bugs/test_gh868.py new file mode 100644 index 00000000..a62e8b36 --- /dev/null +++ b/tests/integration_tests/bugs/test_gh868.py @@ -0,0 +1,27 @@ +"""Ensure no Traceback when 'chef_license' is set""" +import pytest + +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import verify_clean_log + +USERDATA = """\ +#cloud-config +chef: + install_type: omnibus + chef_license: accept + server_url: https://chef.yourorg.invalid + validation_name: some-validator +""" + + +@pytest.mark.adhoc # Can't be regularly reaching out to chef install script +@pytest.mark.ec2 +@pytest.mark.gce +@pytest.mark.azure +@pytest.mark.oci +@pytest.mark.lxd_container +@pytest.mark.lxd_vm +@pytest.mark.user_data(USERDATA) +def test_chef_license(client: IntegrationInstance): + log = client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log) diff --git a/tests/integration_tests/bugs/test_lp1813396.py b/tests/integration_tests/bugs/test_lp1813396.py new file mode 100644 index 00000000..ddae02f5 --- /dev/null +++ b/tests/integration_tests/bugs/test_lp1813396.py @@ -0,0 +1,31 @@ +"""Integration test for lp-1813396 + +Ensure gpg is called with no tty flag. +""" + +import pytest + +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import verify_ordered_items_in_text + +USER_DATA = """\ +#cloud-config +apt: + sources: + cloudinit: + source: 'deb [arch=amd64] http://ppa.launchpad.net/cloud-init-dev/daily/ubuntu focal main' + keyserver: keyserver.ubuntu.com + keyid: E4D304DF +""" # noqa: E501 + + +@pytest.mark.user_data(USER_DATA) +def test_gpg_no_tty(client: IntegrationInstance): + log = client.read_from_file("/var/log/cloud-init.log") + to_verify = [ + "Running command ['gpg', '--no-tty', " + "'--keyserver=keyserver.ubuntu.com', '--recv-keys', 'E4D304DF'] " + "with allowed return codes [0] (shell=False, capture=True)", + "Imported key 'E4D304DF' from keyserver 'keyserver.ubuntu.com'", + ] + verify_ordered_items_in_text(to_verify, log) diff --git a/tests/integration_tests/bugs/test_lp1835584.py b/tests/integration_tests/bugs/test_lp1835584.py new file mode 100644 index 00000000..765d73ef --- /dev/null +++ b/tests/integration_tests/bugs/test_lp1835584.py @@ -0,0 +1,101 @@ +""" Integration test for LP #1835584 + +Upstream linux kernels prior to 4.15 provide DMI product_uuid in uppercase. +More recent kernels switched to lowercase for DMI product_uuid. Azure +datasource uses this product_uuid as the instance-id for cloud-init. + +The linux-azure-fips kernel installed in PRO FIPs images, that product UUID is +uppercase whereas the linux-azure cloud-optimized kernel reports the UUID as +lowercase. + +In cases where product_uuid changes case, ensure cloud-init doesn't +recreate ssh hostkeys across reboot (due to detecting an instance_id change). + +This currently only affects linux-azure-fips -> linux-azure on Bionic. +This test won't run on Xenial because both linux-azure-fips and linux-azure +report uppercase product_uuids. + +The test will launch a specific Bionic Ubuntu PRO FIPS image which has a +linux-azure-fips kernel known to report product_uuid as uppercase. Then upgrade +and reboot into linux-azure kernel which is known to report product_uuid as +lowercase. + +Across the reboot, assert that we didn't re-run config_ssh by virtue of +seeing only one semaphore creation log entry of type: + + Writing to /var/lib/cloud/instances/<UUID>/sem/config_ssh - + +https://bugs.launchpad.net/cloud-init/+bug/1835584 +""" +import re + +import pytest + +from tests.integration_tests.clouds import ImageSpecification, IntegrationCloud +from tests.integration_tests.conftest import get_validated_source +from tests.integration_tests.instances import IntegrationInstance + +IMG_AZURE_UBUNTU_PRO_FIPS_BIONIC = ( + "Canonical:0001-com-ubuntu-pro-bionic-fips:pro-fips-18_04:18.04.202010201" +) + + +def _check_iid_insensitive_across_kernel_upgrade( + instance: IntegrationInstance, +): + uuid = instance.read_from_file("/sys/class/dmi/id/product_uuid") + assert ( + uuid.isupper() + ), "Expected uppercase UUID on Ubuntu FIPS image {}".format(uuid) + orig_kernel = instance.execute("uname -r").strip() + assert "azure-fips" in orig_kernel + result = instance.execute("apt-get update") + # Install a 5.4+ kernel which provides lowercase product_uuid + result = instance.execute("apt-get install linux-azure --assume-yes") + if not result.ok: + pytest.fail("Unable to install linux-azure kernel: {}".format(result)) + # Remove ubuntu-azure-fips metapkg which mandates FIPS-flavour kernel + result = instance.execute("ua disable fips --assume-yes") + assert result.ok, "Unable to disable fips: {}".format(result) + instance.restart() + new_kernel = instance.execute("uname -r").strip() + assert orig_kernel != new_kernel + assert "azure-fips" not in new_kernel + assert "azure" in new_kernel + new_uuid = instance.read_from_file("/sys/class/dmi/id/product_uuid") + assert ( + uuid.lower() == new_uuid + ), "Expected UUID on linux-azure to be lowercase of FIPS: {}".format(uuid) + log = instance.read_from_file("/var/log/cloud-init.log") + RE_CONFIG_SSH_SEMAPHORE = r"Writing.*sem/config_ssh " + ssh_runs = len(re.findall(RE_CONFIG_SSH_SEMAPHORE, log)) + assert 1 == ssh_runs, "config_ssh ran too many times {}".format(ssh_runs) + + +@pytest.mark.azure +def test_azure_kernel_upgrade_case_insensitive_uuid( + session_cloud: IntegrationCloud, +): + cfg_image_spec = ImageSpecification.from_os_image() + if (cfg_image_spec.os, cfg_image_spec.release) != ("ubuntu", "bionic"): + pytest.skip( + "Test only supports ubuntu:bionic not {0.os}:{0.release}".format( + cfg_image_spec + ) + ) + source = get_validated_source(session_cloud) + if not source.installs_new_version(): + pytest.skip( + "Provide CLOUD_INIT_SOURCE to install expected working cloud-init" + ) + image_id = IMG_AZURE_UBUNTU_PRO_FIPS_BIONIC + with session_cloud.launch( + launch_kwargs={"image_id": image_id} + ) as instance: + # We can't use setup_image fixture here because we want to avoid + # taking a snapshot or cleaning the booted machine after cloud-init + # upgrade. + instance.install_new_cloud_init( + source, take_snapshot=False, clean=False + ) + _check_iid_insensitive_across_kernel_upgrade(instance) diff --git a/tests/integration_tests/bugs/test_lp1886531.py b/tests/integration_tests/bugs/test_lp1886531.py index 058ea8bb..d56ca320 100644 --- a/tests/integration_tests/bugs/test_lp1886531.py +++ b/tests/integration_tests/bugs/test_lp1886531.py @@ -11,6 +11,7 @@ https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/1886531 """ import pytest +from tests.integration_tests.util import verify_clean_log USER_DATA = """\ #cloud-config @@ -20,8 +21,7 @@ bootcmd: class TestLp1886531: - @pytest.mark.user_data(USER_DATA) def test_lp1886531(self, client): log_content = client.read_from_file("/var/log/cloud-init.log") - assert "WARNING" not in log_content + verify_clean_log(log_content) diff --git a/tests/integration_tests/bugs/test_lp1897099.py b/tests/integration_tests/bugs/test_lp1897099.py index 27c8927f..1f5030ce 100644 --- a/tests/integration_tests/bugs/test_lp1897099.py +++ b/tests/integration_tests/bugs/test_lp1897099.py @@ -7,7 +7,6 @@ https://bugs.launchpad.net/cloud-init/+bug/1897099 import pytest - USER_DATA = """\ #cloud-config bootcmd: @@ -19,13 +18,12 @@ swap: """ -@pytest.mark.sru_2020_11 @pytest.mark.user_data(USER_DATA) -@pytest.mark.no_container('Containers cannot configure swap') +@pytest.mark.no_container("Containers cannot configure swap") def test_fallocate_fallback(client): - log = client.read_from_file('/var/log/cloud-init.log') - assert '/swap.img' in client.execute('cat /proc/swaps') - assert '/swap.img' in client.execute('cat /etc/fstab') - assert 'fallocate swap creation failed, will attempt with dd' in log + log = client.read_from_file("/var/log/cloud-init.log") + assert "/swap.img" in client.execute("cat /proc/swaps") + assert "/swap.img" in client.execute("cat /etc/fstab") + assert "fallocate swap creation failed, will attempt with dd" in log assert "Running command ['dd', 'if=/dev/zero', 'of=/swap.img'" in log - assert 'SUCCESS: config-mounts ran successfully' in log + assert "SUCCESS: config-mounts ran successfully" in log diff --git a/tests/integration_tests/bugs/test_lp1898997.py b/tests/integration_tests/bugs/test_lp1898997.py new file mode 100644 index 00000000..d8ea54c3 --- /dev/null +++ b/tests/integration_tests/bugs/test_lp1898997.py @@ -0,0 +1,77 @@ +"""Integration test for LP: #1898997 + +cloud-init was incorrectly excluding Open vSwitch bridge members from its list +of interfaces. This meant that instances which had only one interface which +was in an Open vSwitch bridge would not boot correctly: cloud-init would not +find the expected physical interfaces, so would not apply network config. + +This test checks that cloud-init believes it has successfully applied the +network configuration, and confirms that the bridge can be used to ping the +default gateway. +""" +import pytest + +from tests.integration_tests import random_mac_address +from tests.integration_tests.util import verify_clean_log + +MAC_ADDRESS = random_mac_address() + + +NETWORK_CONFIG = """\ +bridges: + ovs-br: + dhcp4: true + interfaces: + - enp5s0 + macaddress: 52:54:00:d9:08:1c + mtu: 1500 + openvswitch: {{}} +ethernets: + enp5s0: + mtu: 1500 + set-name: enp5s0 + match: + macaddress: {} +version: 2 +""".format( + MAC_ADDRESS +) + + +@pytest.mark.lxd_config_dict( + { + "user.network-config": NETWORK_CONFIG, + "volatile.eth0.hwaddr": MAC_ADDRESS, + } +) +@pytest.mark.lxd_vm +@pytest.mark.lxd_use_exec +@pytest.mark.not_bionic +@pytest.mark.ubuntu +class TestInterfaceListingWithOpenvSwitch: + def test_ovs_member_interfaces_not_excluded(self, client): + # We need to install openvswitch for our provided network configuration + # to apply (on next boot), so DHCP on our default interface to fetch it + client.execute("dhclient enp5s0") + client.execute("apt update -qqy") + client.execute("apt-get install -qqy openvswitch-switch") + + # Now our networking config should successfully apply on a clean reboot + client.execute("cloud-init clean --logs") + client.restart() + + cloudinit_output = client.read_from_file("/var/log/cloud-init.log") + + # Confirm that the network configuration was applied successfully + verify_clean_log(cloudinit_output) + # Confirm that the applied network config created the OVS bridge + assert "ovs-br" in client.execute("ip addr") + + # Test that we can ping our gateway using our bridge + gateway = client.execute( + "ip -4 route show default | awk '{ print $3 }'" + ) + ping_result = client.execute( + "ping -c 1 -W 1 -I ovs-br {}".format(gateway) + ) + assert ping_result.ok diff --git a/tests/integration_tests/bugs/test_lp1900837.py b/tests/integration_tests/bugs/test_lp1900837.py index 3fe7d0d0..d9ef18aa 100644 --- a/tests/integration_tests/bugs/test_lp1900837.py +++ b/tests/integration_tests/bugs/test_lp1900837.py @@ -4,14 +4,12 @@ This test mirrors the reproducing steps from the reported bug: it changes the permissions on cloud-init.log to 600 and confirms that they remain 600 after a reboot. """ -import pytest def _get_log_perms(client): return client.execute("stat -c %a /var/log/cloud-init.log") -@pytest.mark.sru_2020_11 class TestLogPermissionsNotResetOnReboot: def test_permissions_unchanged(self, client): # Confirm that the current permissions aren't 600 @@ -22,7 +20,8 @@ class TestLogPermissionsNotResetOnReboot: assert "600" == _get_log_perms(client) # Reboot - client.instance.restart() + client.restart() + assert client.execute("cloud-init status").ok # Check that permissions are not reset on reboot assert "600" == _get_log_perms(client) diff --git a/tests/integration_tests/bugs/test_lp1901011.py b/tests/integration_tests/bugs/test_lp1901011.py new file mode 100644 index 00000000..7de8bd77 --- /dev/null +++ b/tests/integration_tests/bugs/test_lp1901011.py @@ -0,0 +1,67 @@ +"""Integration test for LP: #1901011 + +Ensure an ephemeral disk exists after boot. + +See https://github.com/canonical/cloud-init/pull/800 +""" +import pytest + +from tests.integration_tests.clouds import IntegrationCloud + + +@pytest.mark.azure +@pytest.mark.parametrize( + "instance_type,is_ephemeral", + [ + ("Standard_DS1_v2", True), + ("Standard_D2s_v4", False), + ], +) +def test_ephemeral( + instance_type, is_ephemeral, session_cloud: IntegrationCloud, setup_image +): + if is_ephemeral: + expected_log = ( + "Ephemeral resource disk '/dev/disk/cloud/azure_resource' exists. " + "Merging default Azure cloud ephemeral disk configs." + ) + else: + expected_log = ( + "Ephemeral resource disk '/dev/disk/cloud/azure_resource' does " + "not exist. Not merging default Azure cloud ephemeral disk " + "configs." + ) + + with session_cloud.launch( + launch_kwargs={"instance_type": instance_type} + ) as client: + # Verify log file + log = client.read_from_file("/var/log/cloud-init.log") + assert expected_log in log + + # Verify devices + dev_links = client.execute("ls /dev/disk/cloud") + assert "azure_root" in dev_links + assert "azure_root-part1" in dev_links + if is_ephemeral: + assert "azure_resource" in dev_links + assert "azure_resource-part1" in dev_links + + # Verify mounts + blks = client.execute("lsblk -pPo NAME,TYPE,MOUNTPOINT") + root_device = client.execute( + "realpath /dev/disk/cloud/azure_root-part1" + ) + assert ( + 'NAME="{}" TYPE="part" MOUNTPOINT="/"'.format(root_device) in blks + ) + if is_ephemeral: + ephemeral_device = client.execute( + "realpath /dev/disk/cloud/azure_resource-part1" + ) + assert ( + 'NAME="{}" TYPE="part" MOUNTPOINT="/mnt"'.format( + ephemeral_device + ) + in blks + ) diff --git a/tests/integration_tests/bugs/test_lp1910835.py b/tests/integration_tests/bugs/test_lp1910835.py new file mode 100644 index 00000000..1844594c --- /dev/null +++ b/tests/integration_tests/bugs/test_lp1910835.py @@ -0,0 +1,64 @@ +"""Integration test for LP: #1910835. + +If users do not provide an SSH key and instead ask Azure to generate a key for +them, the key material available in the IMDS may include CRLF sequences. Prior +to e56b55452549cb037da0a4165154ffa494e9678a, the Azure datasource handled keys +via a certificate, the tooling for which removed these sequences. This test +ensures that cloud-init does not regress support for this Azure behaviour. + +This test provides the SSH key configured for tests to the instance in two +ways: firstly, with CRLFs to mimic the generated keys, via the Azure API; +secondly, as user-data in unmodified form. This means that even on systems +which exhibit the bug fetching the platform's metadata, we can SSH into the SUT +to confirm this (instead of having to assert SSH failure; there are lots of +reasons SSH might fail). + +Once SSH'd in, we check that the two keys in .ssh/authorized_keys have the same +material: if the Azure datasource has removed the CRLFs correctly, then they +will match. +""" +import pytest + +USER_DATA_TMPL = """\ +#cloud-config +ssh_authorized_keys: + - {}""" + + +@pytest.mark.azure +def test_crlf_in_azure_metadata_ssh_keys(session_cloud, setup_image): + authorized_keys_path = "/home/{}/.ssh/authorized_keys".format( + session_cloud.cloud_instance.username + ) + # Pass in user-data to allow us to access the instance when the normal + # path fails + key_data = session_cloud.cloud_instance.key_pair.public_key_content + user_data = USER_DATA_TMPL.format(key_data) + # Throw a CRLF into the otherwise good key data, to emulate Azure's + # behaviour for generated keys + key_data = key_data[:20] + "\r\n" + key_data[20:] + vm_params = { + "os_profile": { + "linux_configuration": { + "ssh": { + "public_keys": [ + {"path": authorized_keys_path, "key_data": key_data} + ] + } + } + } + } + with session_cloud.launch( + launch_kwargs={"vm_params": vm_params, "user_data": user_data} + ) as client: + authorized_keys = ( + client.read_from_file(authorized_keys_path).strip().splitlines() + ) + # We expect one key from the cloud, one from user-data + assert 2 == len(authorized_keys) + # And those two keys should be the same, except for a possible key + # comment, which Azure strips out + assert ( + authorized_keys[0].rsplit(" ")[:2] + == authorized_keys[1].split(" ")[:2] + ) diff --git a/tests/integration_tests/bugs/test_lp1912844.py b/tests/integration_tests/bugs/test_lp1912844.py new file mode 100644 index 00000000..55511ed2 --- /dev/null +++ b/tests/integration_tests/bugs/test_lp1912844.py @@ -0,0 +1,105 @@ +"""Integration test for LP: #1912844 + +cloud-init should ignore OVS-internal interfaces when performing its own +interface determination: these interfaces are handled fully by OVS, so +cloud-init should never need to touch them. + +This test is a semi-synthetic reproducer for the bug. It uses a similar +network configuration, tweaked slightly to DHCP in a way that will succeed even +on "failed" boots. The exact bug doesn't reproduce with the NoCloud +datasource, because it runs at init-local time (whereas the MAAS datasource, +from the report, runs only at init (network) time): this means that the +networking code runs before OVS creates its interfaces (which happens after +init-local but, of course, before networking is up), and so doesn't generate +the traceback that they cause. We work around this by calling +``get_interfaces_by_mac` directly in the test code. +""" +import pytest + +from tests.integration_tests import random_mac_address + +MAC_ADDRESS = random_mac_address() + +NETWORK_CONFIG = """\ +bonds: + bond0: + interfaces: + - enp5s0 + macaddress: {0} + mtu: 1500 +bridges: + ovs-br: + interfaces: + - bond0 + macaddress: {0} + mtu: 1500 + openvswitch: {{}} + dhcp4: true +ethernets: + enp5s0: + mtu: 1500 + set-name: enp5s0 + match: + macaddress: {0} +version: 2 +vlans: + ovs-br.100: + id: 100 + link: ovs-br + mtu: 1500 + ovs-br.200: + id: 200 + link: ovs-br + mtu: 1500 +""".format( + MAC_ADDRESS +) + + +SETUP_USER_DATA = """\ +#cloud-config +packages: +- openvswitch-switch +""" + + +@pytest.fixture +def ovs_enabled_session_cloud(session_cloud): + """A session_cloud wrapper, to use an OVS-enabled image for tests. + + This implementation is complicated by wanting to use ``session_cloud``s + snapshot cleanup/retention logic, to avoid having to reimplement that here. + """ + old_snapshot_id = session_cloud.snapshot_id + with session_cloud.launch( + user_data=SETUP_USER_DATA, + ) as instance: + instance.instance.clean() + session_cloud.snapshot_id = instance.snapshot() + + yield session_cloud + + try: + session_cloud.delete_snapshot() + finally: + session_cloud.snapshot_id = old_snapshot_id + + +@pytest.mark.lxd_vm +def test_get_interfaces_by_mac_doesnt_traceback(ovs_enabled_session_cloud): + """Launch our OVS-enabled image and confirm the bug doesn't reproduce.""" + launch_kwargs = { + "config_dict": { + "user.network-config": NETWORK_CONFIG, + "volatile.eth0.hwaddr": MAC_ADDRESS, + }, + } + with ovs_enabled_session_cloud.launch( + launch_kwargs=launch_kwargs, + ) as client: + result = client.execute( + "python3 -c" + "'from cloudinit.net import get_interfaces_by_mac;" + "get_interfaces_by_mac()'" + ) + assert result.ok diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py index 88ac4408..83bc6af6 100644 --- a/tests/integration_tests/clouds.py +++ b/tests/integration_tests/clouds.py @@ -1,38 +1,107 @@ # This file is part of cloud-init. See LICENSE file for license information. -from abc import ABC, abstractmethod +import datetime import logging - -from pycloudlib import EC2, GCE, Azure, OCI, LXDContainer, LXDVirtualMachine +import os.path +import random +import string +from abc import ABC, abstractmethod +from typing import Optional, Type +from uuid import UUID + +from pycloudlib import ( + EC2, + GCE, + OCI, + Azure, + LXDContainer, + LXDVirtualMachine, + Openstack, +) +from pycloudlib.cloud import BaseCloud +from pycloudlib.lxd.cloud import _BaseLXD from pycloudlib.lxd.instance import LXDInstance import cloudinit -from cloudinit.subp import subp +from cloudinit.subp import ProcessExecutionError, subp from tests.integration_tests import integration_settings -from tests.integration_tests.instances import ( - IntegrationEc2Instance, - IntegrationGceInstance, - IntegrationAzureInstance, IntegrationInstance, - IntegrationOciInstance, - IntegrationLxdInstance, -) +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import emit_dots_on_travis -try: - from typing import Optional -except ImportError: - pass +log = logging.getLogger("integration_testing") -log = logging.getLogger('integration_testing') +def _get_ubuntu_series() -> list: + """Use distro-info-data's ubuntu.csv to get a list of Ubuntu series""" + out = "" + try: + out, _err = subp(["ubuntu-distro-info", "-a"]) + except ProcessExecutionError: + log.info( + "ubuntu-distro-info (from the distro-info package) must be" + " installed to guess Ubuntu os/release" + ) + return out.splitlines() + + +class ImageSpecification: + """A specification of an image to launch for testing. + + If either of ``os`` and ``release`` are not specified, an attempt will be + made to infer the correct values for these on instantiation. + + :param image_id: + The image identifier used by the rest of the codebase to launch this + image. + :param os: + An optional string describing the operating system this image is for + (e.g. "ubuntu", "rhel", "freebsd"). + :param release: + A optional string describing the operating system release (e.g. + "focal", "8"; the exact values here will depend on the OS). + """ + + def __init__( + self, + image_id: str, + os: Optional[str] = None, + release: Optional[str] = None, + ): + if image_id in _get_ubuntu_series(): + if os is None: + os = "ubuntu" + if release is None: + release = image_id + + self.image_id = image_id + self.os = os + self.release = release + log.info( + "Detected image: image_id=%s os=%s release=%s", + self.image_id, + self.os, + self.release, + ) + + @classmethod + def from_os_image(cls): + """Return an ImageSpecification for integration_settings.OS_IMAGE.""" + parts = integration_settings.OS_IMAGE.split("::", 2) + return cls(*parts) class IntegrationCloud(ABC): - datasource = None # type: Optional[str] - integration_instance_cls = IntegrationInstance + datasource: str + cloud_instance: BaseCloud def __init__(self, settings=integration_settings): self.settings = settings - self.cloud_instance = self._get_cloud_instance() - self.image_id = self._get_initial_image() + self.cloud_instance: BaseCloud = self._get_cloud_instance() + self.initial_image_id = self._get_initial_image() + self.snapshot_id = None + + @property + def image_id(self): + return self.snapshot_id or self.initial_image_id def emit_settings_to_log(self) -> None: log.info( @@ -50,49 +119,62 @@ class IntegrationCloud(ABC): raise NotImplementedError def _get_initial_image(self): - image_id = self.settings.OS_IMAGE + image = ImageSpecification.from_os_image() try: - image_id = self.cloud_instance.released_image( - self.settings.OS_IMAGE) + return self.cloud_instance.daily_image(image.image_id) except (ValueError, IndexError): - pass - return image_id + return image.image_id - def _perform_launch(self, launch_kwargs): + def _perform_launch(self, launch_kwargs, **kwargs): pycloudlib_instance = self.cloud_instance.launch(**launch_kwargs) - pycloudlib_instance.wait(raise_on_cloudinit_failure=False) return pycloudlib_instance - def launch(self, user_data=None, launch_kwargs=None, - settings=integration_settings): + def launch( + self, + user_data=None, + launch_kwargs=None, + settings=integration_settings, + **kwargs, + ) -> IntegrationInstance: + if launch_kwargs is None: + launch_kwargs = {} if self.settings.EXISTING_INSTANCE_ID: log.info( - 'Not launching instance due to EXISTING_INSTANCE_ID. ' - 'Instance id: %s', self.settings.EXISTING_INSTANCE_ID) + "Not launching instance due to EXISTING_INSTANCE_ID. " + "Instance id: %s", + self.settings.EXISTING_INSTANCE_ID, + ) self.instance = self.cloud_instance.get_instance( self.settings.EXISTING_INSTANCE_ID ) - return - kwargs = { - 'image_id': self.image_id, - 'user_data': user_data, - 'wait': False, + return self.instance + default_launch_kwargs = { + "image_id": self.image_id, + "user_data": user_data, } - if launch_kwargs: - kwargs.update(launch_kwargs) + launch_kwargs = {**default_launch_kwargs, **launch_kwargs} log.info( - "Launching instance with launch_kwargs:\n{}".format( - "\n".join("{}={}".format(*item) for item in kwargs.items()) - ) + "Launching instance with launch_kwargs:\n%s", + "\n".join("{}={}".format(*item) for item in launch_kwargs.items()), ) - pycloudlib_instance = self._perform_launch(kwargs) - - log.info('Launched instance: %s', pycloudlib_instance) - return self.get_instance(pycloudlib_instance, settings) + with emit_dots_on_travis(): + pycloudlib_instance = self._perform_launch(launch_kwargs, **kwargs) + log.info("Launched instance: %s", pycloudlib_instance) + instance = self.get_instance(pycloudlib_instance, settings) + if launch_kwargs.get("wait", True): + # If we aren't waiting, we can't rely on command execution here + log.info( + "cloud-init version: %s", + instance.execute("cloud-init --version"), + ) + serial = instance.execute("grep serial /etc/cloud/build.info") + if serial: + log.info("image serial: %s", serial.split()[1]) + return instance def get_instance(self, cloud_instance, settings=integration_settings): - return self.integration_instance_cls(self, cloud_instance, settings) + return IntegrationInstance(self, cloud_instance, settings) def destroy(self): pass @@ -100,52 +182,69 @@ class IntegrationCloud(ABC): def snapshot(self, instance): return self.cloud_instance.snapshot(instance, clean=True) + def delete_snapshot(self): + if self.snapshot_id: + if self.settings.KEEP_IMAGE: + log.info( + "NOT deleting snapshot image created for this testrun " + "because KEEP_IMAGE is True: %s", + self.snapshot_id, + ) + else: + log.info( + "Deleting snapshot image created for this testrun: %s", + self.snapshot_id, + ) + self.cloud_instance.delete_image(self.snapshot_id) + class Ec2Cloud(IntegrationCloud): - datasource = 'ec2' - integration_instance_cls = IntegrationEc2Instance + datasource = "ec2" def _get_cloud_instance(self): - return EC2(tag='ec2-integration-test') + return EC2(tag="ec2-integration-test") class GceCloud(IntegrationCloud): - datasource = 'gce' - integration_instance_cls = IntegrationGceInstance + datasource = "gce" def _get_cloud_instance(self): return GCE( - tag='gce-integration-test', - project=self.settings.GCE_PROJECT, - region=self.settings.GCE_REGION, - zone=self.settings.GCE_ZONE, + tag="gce-integration-test", ) class AzureCloud(IntegrationCloud): - datasource = 'azure' - integration_instance_cls = IntegrationAzureInstance + datasource = "azure" + cloud_instance: Azure def _get_cloud_instance(self): - return Azure(tag='azure-integration-test') + return Azure(tag="azure-integration-test") def destroy(self): - self.cloud_instance.delete_resource_group() + if self.settings.KEEP_INSTANCE: + log.info( + "NOT deleting resource group because KEEP_INSTANCE is true " + "and deleting resource group would also delete instance. " + "Instance and resource group must both be manually deleted." + ) + else: + self.cloud_instance.delete_resource_group() class OciCloud(IntegrationCloud): - datasource = 'oci' - integration_instance_cls = IntegrationOciInstance + datasource = "oci" def _get_cloud_instance(self): return OCI( - tag='oci-integration-test', - compartment_id=self.settings.OCI_COMPARTMENT_ID + tag="oci-integration-test", ) class _LxdIntegrationCloud(IntegrationCloud): - integration_instance_cls = IntegrationLxdInstance + pycloudlib_instance_cls: Type[_BaseLXD] + instance_tag: str + cloud_instance: _BaseLXD def _get_cloud_instance(self): return self.pycloudlib_instance_cls(tag=self.instance_tag) @@ -156,60 +255,102 @@ class _LxdIntegrationCloud(IntegrationCloud): @staticmethod def _mount_source(instance: LXDInstance): - target_path = '/usr/lib/python3/dist-packages/cloudinit' - format_variables = { - 'name': instance.name, - 'source_path': cloudinit.__path__[0], - 'container_path': target_path, - } - log.info( - 'Mounting source {source_path} directly onto LXD container/vm ' - 'named {name} at {container_path}'.format(**format_variables)) - command = ( - 'lxc config device add {name} host-cloud-init disk ' - 'source={source_path} ' - 'path={container_path}' - ).format(**format_variables) - subp(command.split()) - - def _perform_launch(self, launch_kwargs): - launch_kwargs['inst_type'] = launch_kwargs.pop('instance_type', None) - launch_kwargs.pop('wait') - release = launch_kwargs.pop('image_id') + cloudinit_path = cloudinit.__path__[0] + mounts = [ + (cloudinit_path, "/usr/lib/python3/dist-packages/cloudinit"), + ( + os.path.join(cloudinit_path, "..", "templates"), + "/etc/cloud/templates", + ), + ] + for (n, (source_path, target_path)) in enumerate(mounts): + format_variables = { + "name": instance.name, + "source_path": os.path.realpath(source_path), + "container_path": target_path, + "idx": n, + } + log.info( + "Mounting source %(source_path)s directly onto LXD" + " container/VM named %(name)s at %(container_path)s", + format_variables, + ) + command = ( + "lxc config device add {name} host-cloud-init-{idx} disk " + "source={source_path} " + "path={container_path}" + ).format(**format_variables) + subp(command.split()) + + def _perform_launch(self, launch_kwargs, **kwargs): + launch_kwargs["inst_type"] = launch_kwargs.pop("instance_type", None) + wait = launch_kwargs.pop("wait", True) + release = launch_kwargs.pop("image_id") try: - profile_list = launch_kwargs['profile_list'] + profile_list = launch_kwargs["profile_list"] except KeyError: profile_list = self._get_or_set_profile_list(release) + prefix = datetime.datetime.utcnow().strftime("cloudinit-%m%d-%H%M%S") + default_name = prefix + "".join( + random.choices(string.ascii_lowercase + string.digits, k=8) + ) pycloudlib_instance = self.cloud_instance.init( - launch_kwargs.pop('name', None), + launch_kwargs.pop("name", default_name), release, profile_list=profile_list, - **launch_kwargs + **launch_kwargs, ) - if self.settings.CLOUD_INIT_SOURCE == 'IN_PLACE': + if self.settings.CLOUD_INIT_SOURCE == "IN_PLACE": self._mount_source(pycloudlib_instance) - pycloudlib_instance.start(wait=False) - pycloudlib_instance.wait(raise_on_cloudinit_failure=False) + if "lxd_setup" in kwargs: + log.info("Running callback specified by 'lxd_setup' mark") + kwargs["lxd_setup"](pycloudlib_instance) + pycloudlib_instance.start(wait=wait) return pycloudlib_instance class LxdContainerCloud(_LxdIntegrationCloud): - datasource = 'lxd_container' + datasource = "lxd_container" + cloud_instance: LXDContainer pycloudlib_instance_cls = LXDContainer - instance_tag = 'lxd-container-integration-test' + instance_tag = "lxd-container-integration-test" class LxdVmCloud(_LxdIntegrationCloud): - datasource = 'lxd_vm' + datasource = "lxd_vm" + cloud_instance: LXDVirtualMachine pycloudlib_instance_cls = LXDVirtualMachine - instance_tag = 'lxd-vm-integration-test' + instance_tag = "lxd-vm-integration-test" _profile_list = None def _get_or_set_profile_list(self, release): if self._profile_list: return self._profile_list self._profile_list = self.cloud_instance.build_necessary_profiles( - release) + release + ) return self._profile_list + + +class OpenstackCloud(IntegrationCloud): + datasource = "openstack" + + def _get_cloud_instance(self): + return Openstack( + tag="openstack-integration-test", + ) + + def _get_initial_image(self): + image = ImageSpecification.from_os_image() + try: + UUID(image.image_id) + except ValueError as e: + raise Exception( + "When using Openstack, `OS_IMAGE` MUST be specified with " + "a 36-character UUID image ID. Passing in a release name is " + "not valid here.\n" + "OS image id: {}".format(image.image_id) + ) from e + return image.image_id diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index 73b44bfc..a90a5d49 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -1,33 +1,51 @@ # This file is part of cloud-init. See LICENSE file for license information. +import datetime +import functools import logging import os -import pytest import sys from contextlib import contextmanager +from pathlib import Path +from tarfile import TarFile +from typing import Dict, Type + +import pytest +from pycloudlib.lxd.instance import LXDInstance from tests.integration_tests import integration_settings from tests.integration_tests.clouds import ( + AzureCloud, Ec2Cloud, GceCloud, - AzureCloud, - OciCloud, + ImageSpecification, + IntegrationCloud, LxdContainerCloud, LxdVmCloud, + OciCloud, + OpenstackCloud, + _LxdIntegrationCloud, +) +from tests.integration_tests.instances import ( + CloudInitSource, + IntegrationInstance, ) - -log = logging.getLogger('integration_testing') +log = logging.getLogger("integration_testing") log.addHandler(logging.StreamHandler(sys.stdout)) log.setLevel(logging.INFO) -platforms = { - 'ec2': Ec2Cloud, - 'gce': GceCloud, - 'azure': AzureCloud, - 'oci': OciCloud, - 'lxd_container': LxdContainerCloud, - 'lxd_vm': LxdVmCloud, +platforms: Dict[str, Type[IntegrationCloud]] = { + "ec2": Ec2Cloud, + "gce": GceCloud, + "azure": AzureCloud, + "oci": OciCloud, + "lxd_container": LxdContainerCloud, + "lxd_vm": LxdVmCloud, + "openstack": OpenstackCloud, } +os_list = ["ubuntu"] + +session_start_time = datetime.datetime.now().strftime("%y%m%d%H%M%S") def pytest_runtest_setup(item): @@ -42,18 +60,30 @@ def pytest_runtest_setup(item): test_marks = [mark.name for mark in item.iter_markers()] supported_platforms = set(all_platforms).intersection(test_marks) current_platform = integration_settings.PLATFORM - unsupported_message = 'Cannot run on platform {}'.format(current_platform) - if 'no_container' in test_marks: - if 'lxd_container' in test_marks: + unsupported_message = "Cannot run on platform {}".format(current_platform) + if "no_container" in test_marks: + if "lxd_container" in test_marks: raise Exception( - 'lxd_container and no_container marks simultaneously set ' - 'on test' + "lxd_container and no_container marks simultaneously set " + "on test" ) - if current_platform == 'lxd_container': + if current_platform == "lxd_container": pytest.skip(unsupported_message) if supported_platforms and current_platform not in supported_platforms: pytest.skip(unsupported_message) + image = ImageSpecification.from_os_image() + current_os = image.os + supported_os_set = set(os_list).intersection(test_marks) + if current_os and supported_os_set and current_os not in supported_os_set: + pytest.skip("Cannot run on OS {}".format(current_os)) + if "unstable" in test_marks and not integration_settings.RUN_UNSTABLE: + pytest.skip("Test marked unstable. Manually remove mark to run it") + + current_release = image.release + if "not_{}".format(current_release) in test_marks: + pytest.skip("Cannot run on release {}".format(current_release)) + # disable_subp_usage is defined at a higher level, but we don't # want it applied here @@ -62,7 +92,7 @@ def disable_subp_usage(request): pass -@pytest.yield_fixture(scope='session') +@pytest.fixture(scope="session") def session_cloud(): if integration_settings.PLATFORM not in platforms.keys(): raise ValueError( @@ -74,83 +104,185 @@ def session_cloud(): cloud = platforms[integration_settings.PLATFORM]() cloud.emit_settings_to_log() + yield cloud + cloud.destroy() -@pytest.fixture(scope='session', autouse=True) -def setup_image(session_cloud): +def get_validated_source( + session_cloud: IntegrationCloud, + source=integration_settings.CLOUD_INIT_SOURCE, +) -> CloudInitSource: + if source == "NONE": + return CloudInitSource.NONE + elif source == "IN_PLACE": + if session_cloud.datasource not in ["lxd_container", "lxd_vm"]: + raise ValueError( + "IN_PLACE as CLOUD_INIT_SOURCE only works for LXD" + ) + return CloudInitSource.IN_PLACE + elif source == "PROPOSED": + return CloudInitSource.PROPOSED + elif source.startswith("ppa:"): + return CloudInitSource.PPA + elif os.path.isfile(str(source)): + return CloudInitSource.DEB_PACKAGE + elif source == "UPGRADE": + return CloudInitSource.UPGRADE + raise ValueError( + "Invalid value for CLOUD_INIT_SOURCE setting: {}".format(source) + ) + + +@pytest.fixture(scope="session") +def setup_image(session_cloud: IntegrationCloud, request): """Setup the target environment with the correct version of cloud-init. So we can launch instances / run tests with the correct image """ - client = None - log.info('Setting up environment for %s', session_cloud.datasource) - if integration_settings.CLOUD_INIT_SOURCE == 'NONE': - pass # that was easy - elif integration_settings.CLOUD_INIT_SOURCE == 'IN_PLACE': - if session_cloud.datasource not in ['lxd_container', 'lxd_vm']: - raise ValueError( - 'IN_PLACE as CLOUD_INIT_SOURCE only works for LXD') - # The mount needs to happen after the instance is created, so - # no further action needed here - elif integration_settings.CLOUD_INIT_SOURCE == 'PROPOSED': - client = session_cloud.launch() - client.install_proposed_image() - elif integration_settings.CLOUD_INIT_SOURCE.startswith('ppa:'): - client = session_cloud.launch() - client.install_ppa(integration_settings.CLOUD_INIT_SOURCE) - elif os.path.isfile(str(integration_settings.CLOUD_INIT_SOURCE)): - client = session_cloud.launch() - client.install_deb() - else: - raise ValueError( - 'Invalid value for CLOUD_INIT_SOURCE setting: {}'.format( - integration_settings.CLOUD_INIT_SOURCE)) - if client: - # Even if we're keeping instances, we don't want to keep this - # one around as it was just for image creation - client.destroy() - log.info('Done with environment setup') + + source = get_validated_source(session_cloud) + if not source.installs_new_version(): + return + log.info("Setting up environment for %s", session_cloud.datasource) + client = session_cloud.launch() + client.install_new_cloud_init(source) + # Even if we're keeping instances, we don't want to keep this + # one around as it was just for image creation + client.destroy() + log.info("Done with environment setup") + + # For some reason a yield here raises a + # ValueError: setup_image did not yield a value + # during setup so use a finalizer instead. + request.addfinalizer(session_cloud.delete_snapshot) + + +def _collect_logs( + instance: IntegrationInstance, node_id: str, test_failed: bool +): + """Collect logs from remote instance. + + Args: + instance: The current IntegrationInstance to collect logs from + node_id: The pytest representation of this test, E.g.: + tests/integration_tests/test_example.py::TestExample.test_example + test_failed: If test failed or not + """ + if any( + [ + integration_settings.COLLECT_LOGS == "NEVER", + integration_settings.COLLECT_LOGS == "ON_ERROR" + and not test_failed, + ] + ): + return + instance.execute( + "cloud-init collect-logs -u -t /var/tmp/cloud-init.tar.gz" + ) + node_id_path = Path( + node_id.replace( + ".py", "" + ) # Having a directory with '.py' would be weird + .replace("::", os.path.sep) # Turn classes/tests into paths + .replace("[", "-") # For parametrized names + .replace("]", "") # For parameterized names + ) + log_dir = ( + Path(integration_settings.LOCAL_LOG_PATH) + / session_start_time + / node_id_path + ) + log.info("Writing logs to %s", log_dir) + + if not log_dir.exists(): + log_dir.mkdir(parents=True) + + # Add a symlink to the latest log output directory + last_symlink = Path(integration_settings.LOCAL_LOG_PATH) / "last" + if os.path.islink(last_symlink): + os.unlink(last_symlink) + os.symlink(log_dir.parent, last_symlink) + + tarball_path = log_dir / "cloud-init.tar.gz" + try: + instance.pull_file("/var/tmp/cloud-init.tar.gz", tarball_path) + except Exception as e: + log.error("Failed to pull logs: %s", e) + return + + tarball = TarFile.open(str(tarball_path)) + tarball.extractall(path=str(log_dir)) + tarball_path.unlink() @contextmanager -def _client(request, fixture_utils, session_cloud): +def _client(request, fixture_utils, session_cloud: IntegrationCloud): """Fixture implementation for the client fixtures. Launch the dynamic IntegrationClient instance using any provided userdata, yield to the test, then cleanup """ - user_data = fixture_utils.closest_marker_first_arg_or( - request, 'user_data', None) - name = fixture_utils.closest_marker_first_arg_or( - request, 'instance_name', None + getter = functools.partial( + fixture_utils.closest_marker_first_arg_or, request, default=None + ) + user_data = getter("user_data") + name = getter("instance_name") + lxd_config_dict = getter("lxd_config_dict") + lxd_setup = getter("lxd_setup") + lxd_use_exec = fixture_utils.closest_marker_args_or( + request, "lxd_use_exec", None ) + launch_kwargs = {} if name is not None: - launch_kwargs = {"name": name} + launch_kwargs["name"] = name + if lxd_config_dict is not None: + if not isinstance(session_cloud, _LxdIntegrationCloud): + pytest.skip("lxd_config_dict requires LXD") + launch_kwargs["config_dict"] = lxd_config_dict + if lxd_use_exec is not None: + if not isinstance(session_cloud, _LxdIntegrationCloud): + pytest.skip("lxd_use_exec requires LXD") + launch_kwargs["execute_via_ssh"] = False + local_launch_kwargs = {} + if lxd_setup is not None: + if not isinstance(session_cloud, _LxdIntegrationCloud): + pytest.skip("lxd_setup requires LXD") + local_launch_kwargs["lxd_setup"] = lxd_setup + with session_cloud.launch( - user_data=user_data, launch_kwargs=launch_kwargs + user_data=user_data, launch_kwargs=launch_kwargs, **local_launch_kwargs ) as instance: + if lxd_use_exec is not None and isinstance( + instance.instance, LXDInstance + ): + # Existing instances are not affected by the launch kwargs, so + # ensure it here; we still need the launch kwarg so waiting works + instance.instance.execute_via_ssh = False + previous_failures = request.session.testsfailed yield instance + test_failed = request.session.testsfailed - previous_failures > 0 + _collect_logs(instance, request.node.nodeid, test_failed) -@pytest.yield_fixture -def client(request, fixture_utils, session_cloud): +@pytest.fixture +def client(request, fixture_utils, session_cloud, setup_image): """Provide a client that runs for every test.""" with _client(request, fixture_utils, session_cloud) as client: yield client -@pytest.yield_fixture(scope='module') -def module_client(request, fixture_utils, session_cloud): +@pytest.fixture(scope="module") +def module_client(request, fixture_utils, session_cloud, setup_image): """Provide a client that runs once per module.""" with _client(request, fixture_utils, session_cloud) as client: yield client -@pytest.yield_fixture(scope='class') -def class_client(request, fixture_utils, session_cloud): +@pytest.fixture(scope="class") +def class_client(request, fixture_utils, session_cloud, setup_image): """Provide a client that runs once per class.""" with _client(request, fixture_utils, session_cloud) as client: yield client @@ -180,3 +312,20 @@ def pytest_assertrepr_compare(op, left, right): '"{}" not in cloud-init.log string; unexpectedly found on' " these lines:".format(left) ] + found_lines + + +def pytest_configure(config): + """Perform initial configuration, before the test runs start. + + This hook is only called if integration tests are being executed, so we can + use it to configure defaults for integration testing that differ from the + rest of the tests in the codebase. + + See + https://docs.pytest.org/en/latest/reference.html#_pytest.hookspec.pytest_configure + for pytest's documentation. + """ + if "log_cli_level" in config.option and not config.option.log_cli_level: + # If log_cli_level is available in this version of pytest and not set + # to anything, set it to INFO. + config.option.log_cli_level = "INFO" diff --git a/tests/integration_tests/datasources/test_lxd_discovery.py b/tests/integration_tests/datasources/test_lxd_discovery.py new file mode 100644 index 00000000..eb2a4cf2 --- /dev/null +++ b/tests/integration_tests/datasources/test_lxd_discovery.py @@ -0,0 +1,90 @@ +import json + +import pytest +import yaml + +from tests.integration_tests.clouds import ImageSpecification +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import verify_clean_log + + +def _customize_envionment(client: IntegrationInstance): + client.write_to_file( + "/etc/cloud/cloud.cfg.d/99-detect-lxd.cfg", + "datasource_list: [LXD]\n", + ) + client.execute("cloud-init clean --logs") + client.restart() + + +# This test should be able to work on any cloud whose datasource specifies +# a NETWORK dependency +@pytest.mark.lxd_container +@pytest.mark.lxd_vm +@pytest.mark.ubuntu # Because netplan +def test_lxd_datasource_discovery(client: IntegrationInstance): + """Test that DataSourceLXD is detected instead of NoCloud.""" + _customize_envionment(client) + nic_dev = "enp5s0" if client.settings.PLATFORM == "lxd_vm" else "eth0" + result = client.execute("cloud-init status --long") + if not result.ok: + raise AssertionError("cloud-init failed:\n%s", result.stderr) + if "DataSourceLXD" not in result.stdout: + raise AssertionError( + "cloud-init did not discover DataSourceLXD", result.stdout + ) + netplan_yaml = client.execute("cat /etc/netplan/50-cloud-init.yaml") + netplan_cfg = yaml.safe_load(netplan_yaml) + assert { + "network": {"ethernets": {nic_dev: {"dhcp4": True}}, "version": 2} + } == netplan_cfg + log = client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log) + result = client.execute("cloud-id") + if result.stdout != "lxd": + raise AssertionError( + "cloud-id didn't report lxd. Result: %s", result.stdout + ) + # Validate config instance data represented + data = json.loads( + client.read_from_file("/run/cloud-init/instance-data.json") + ) + v1 = data["v1"] + ds_cfg = data["ds"] + assert "lxd" == v1["platform"] + assert "LXD socket API v. 1.0 (/dev/lxd/sock)" == v1["subplatform"] + ds_cfg = json.loads(client.execute("cloud-init query ds").stdout) + assert ["_doc", "_metadata_api_version", "config", "meta-data"] == sorted( + list(ds_cfg.keys()) + ) + if ( + client.settings.PLATFORM == "lxd_vm" + and ImageSpecification.from_os_image().release == "bionic" + ): + # pycloudlib injects user.vendor_data for lxd_vm on bionic + # to start the lxd-agent. + # https://github.com/canonical/pycloudlib/blob/main/pycloudlib/\ + # lxd/defaults.py#L13-L27 + # Underscore-delimited aliases exist for any keys containing hyphens or + # dots. + lxd_config_keys = ["user.meta-data", "user.vendor-data"] + else: + lxd_config_keys = ["user.meta-data"] + assert "1.0" == ds_cfg["_metadata_api_version"] + assert lxd_config_keys == list(ds_cfg["config"].keys()) + assert {"public-keys": v1["public_ssh_keys"][0]} == ( + yaml.safe_load(ds_cfg["config"]["user.meta-data"]) + ) + assert "#cloud-config\ninstance-id" in ds_cfg["meta-data"] + # Assert NoCloud seed data is still present in cloud image metadata + # This will start failing if we redact metadata templates from + # https://cloud-images.ubuntu.com/daily/server/jammy/current/\ + # jammy-server-cloudimg-amd64-lxd.tar.xz + nocloud_metadata = yaml.safe_load( + client.read_from_file("/var/lib/cloud/seed/nocloud-net/meta-data") + ) + assert client.instance.name == nocloud_metadata["instance-id"] + assert ( + nocloud_metadata["instance-id"] == nocloud_metadata["local-hostname"] + ) + assert v1["public_ssh_keys"][0] == nocloud_metadata["public-keys"] diff --git a/tests/integration_tests/datasources/test_network_dependency.py b/tests/integration_tests/datasources/test_network_dependency.py new file mode 100644 index 00000000..32ac7053 --- /dev/null +++ b/tests/integration_tests/datasources/test_network_dependency.py @@ -0,0 +1,33 @@ +import pytest + +from tests.integration_tests.instances import IntegrationInstance + + +def _customize_envionment(client: IntegrationInstance): + # Insert our "disable_network_activation" file here + client.write_to_file( + "/etc/cloud/cloud.cfg.d/99-disable-network-activation.cfg", + "disable_network_activation: true\n", + ) + client.execute("cloud-init clean --logs") + client.restart() + + +# This test should be able to work on any cloud whose datasource specifies +# a NETWORK dependency +@pytest.mark.gce +@pytest.mark.ubuntu # Because netplan +def test_network_activation_disabled(client: IntegrationInstance): + """Test that the network is not activated during init mode.""" + _customize_envionment(client) + result = client.execute("systemctl status google-guest-agent.service") + if not result.ok: + raise AssertionError( + "google-guest-agent is not active:\n%s", result.stdout + ) + log = client.read_from_file("/var/log/cloud-init.log") + + assert "Running command ['netplan', 'apply']" not in log + + assert "Not bringing up newly configured network interfaces" in log + assert "Bringing up newly configured network interfaces" not in log diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py index 9b13288c..e26ee233 100644 --- a/tests/integration_tests/instances.py +++ b/tests/integration_tests/instances.py @@ -2,34 +2,61 @@ import logging import os import uuid +from enum import Enum from tempfile import NamedTemporaryFile from pycloudlib.instance import BaseInstance from pycloudlib.result import Result from tests.integration_tests import integration_settings +from tests.integration_tests.util import retry try: from typing import TYPE_CHECKING + if TYPE_CHECKING: - from tests.integration_tests.clouds import IntegrationCloud + from tests.integration_tests.clouds import ( # noqa: F401 + IntegrationCloud, + ) except ImportError: pass -log = logging.getLogger('integration_testing') +log = logging.getLogger("integration_testing") def _get_tmp_path(): tmp_filename = str(uuid.uuid4()) - return '/var/tmp/{}.tmp'.format(tmp_filename) + return "/var/tmp/{}.tmp".format(tmp_filename) -class IntegrationInstance: - use_sudo = True +class CloudInitSource(Enum): + """Represents the cloud-init image source setting as a defined value. + + Values here represent all possible values for CLOUD_INIT_SOURCE in + tests/integration_tests/integration_settings.py. See that file for an + explanation of these values. If the value set there can't be parsed into + one of these values, an exception will be raised + """ + + NONE = 1 + IN_PLACE = 2 + PROPOSED = 3 + PPA = 4 + DEB_PACKAGE = 5 + UPGRADE = 6 - def __init__(self, cloud: 'IntegrationCloud', instance: BaseInstance, - settings=integration_settings): + def installs_new_version(self): + return self.name not in [self.NONE.name, self.IN_PLACE.name] + + +class IntegrationInstance: + def __init__( + self, + cloud: "IntegrationCloud", + instance: BaseInstance, + settings=integration_settings, + ): self.cloud = cloud self.instance = instance self.settings = settings @@ -37,44 +64,53 @@ class IntegrationInstance: def destroy(self): self.instance.delete() - def execute(self, command, *, use_sudo=None) -> Result: - if self.instance.username == 'root' and use_sudo is False: - raise Exception('Root user cannot run unprivileged') - if use_sudo is None: - use_sudo = self.use_sudo + def restart(self): + """Restart this instance (via cloud mechanism) and wait for boot. + + This wraps pycloudlib's `BaseInstance.restart` + """ + log.info("Restarting instance and waiting for boot") + self.instance.restart() + + def execute(self, command, *, use_sudo=True) -> Result: + if self.instance.username == "root" and use_sudo is False: + raise Exception("Root user cannot run unprivileged") return self.instance.execute(command, use_sudo=use_sudo) def pull_file(self, remote_path, local_path): # First copy to a temporary directory because of permissions issues tmp_path = _get_tmp_path() - self.instance.execute('cp {} {}'.format(remote_path, tmp_path)) - self.instance.pull_file(tmp_path, local_path) + self.instance.execute("cp {} {}".format(str(remote_path), tmp_path)) + self.instance.pull_file(tmp_path, str(local_path)) def push_file(self, local_path, remote_path): # First push to a temporary directory because of permissions issues tmp_path = _get_tmp_path() - self.instance.push_file(local_path, tmp_path) - self.execute('mv {} {}'.format(tmp_path, remote_path)) + self.instance.push_file(str(local_path), tmp_path) + assert self.execute("mv {} {}".format(tmp_path, str(remote_path))).ok def read_from_file(self, remote_path) -> str: - result = self.execute('cat {}'.format(remote_path)) + result = self.execute("cat {}".format(remote_path)) if result.failed: # TODO: Raise here whatever pycloudlib raises when it has # a consistent error response raise IOError( - 'Failed reading remote file via cat: {}\n' - 'Return code: {}\n' - 'Stderr: {}\n' - 'Stdout: {}'.format( - remote_path, result.return_code, - result.stderr, result.stdout) + "Failed reading remote file via cat: {}\n" + "Return code: {}\n" + "Stderr: {}\n" + "Stdout: {}".format( + remote_path, + result.return_code, + result.stderr, + result.stdout, + ) ) return result.stdout def write_to_file(self, remote_path, contents: str): # Writes file locally and then pushes it rather # than writing the file directly on the instance - with NamedTemporaryFile('w', delete=False) as tmp_file: + with NamedTemporaryFile("w", delete=False) as tmp_file: tmp_file.write(contents) try: @@ -83,48 +119,79 @@ class IntegrationInstance: os.unlink(tmp_file.name) def snapshot(self): - return self.cloud.snapshot(self.instance) - - def _install_new_cloud_init(self, remote_script): - self.execute(remote_script) - version = self.execute('cloud-init -v').split()[-1] - log.info('Installed cloud-init version: %s', version) - self.instance.clean() - image_id = self.snapshot() - log.info('Created new image: %s', image_id) - self.cloud.image_id = image_id - + image_id = self.cloud.snapshot(self.instance) + log.info("Created new image: %s", image_id) + return image_id + + def install_new_cloud_init( + self, + source: CloudInitSource, + take_snapshot=True, + clean=True, + ): + if source == CloudInitSource.DEB_PACKAGE: + self.install_deb() + elif source == CloudInitSource.PPA: + self.install_ppa() + elif source == CloudInitSource.PROPOSED: + self.install_proposed_image() + elif source == CloudInitSource.UPGRADE: + self.upgrade_cloud_init() + else: + raise Exception( + "Specified to install {} which isn't supported here".format( + source + ) + ) + version = self.execute("cloud-init -v").split()[-1] + log.info("Installed cloud-init version: %s", version) + if clean: + self.instance.clean() + if take_snapshot: + snapshot_id = self.snapshot() + self.cloud.snapshot_id = snapshot_id + + # assert with retry because we can compete with apt already running in the + # background and get: E: Could not get lock /var/lib/apt/lists/lock - open + # (11: Resource temporarily unavailable) + + @retry(tries=30, delay=1) def install_proposed_image(self): - log.info('Installing proposed image') - remote_script = ( - '{sudo} echo deb "http://archive.ubuntu.com/ubuntu ' - '$(lsb_release -sc)-proposed main" | ' - '{sudo} tee /etc/apt/sources.list.d/proposed.list\n' - '{sudo} apt-get update -q\n' - '{sudo} apt-get install -qy cloud-init' - ).format(sudo='sudo' if self.use_sudo else '') - self._install_new_cloud_init(remote_script) - - def install_ppa(self, repo): - log.info('Installing PPA') - remote_script = ( - '{sudo} add-apt-repository {repo} -y && ' - '{sudo} apt-get update -q && ' - '{sudo} apt-get install -qy cloud-init' - ).format(sudo='sudo' if self.use_sudo else '', repo=repo) - self._install_new_cloud_init(remote_script) - + log.info("Installing proposed image") + assert self.execute( + 'echo deb "http://archive.ubuntu.com/ubuntu ' + '$(lsb_release -sc)-proposed main" >> ' + "/etc/apt/sources.list.d/proposed.list" + ).ok + assert self.execute("apt-get update -q").ok + assert self.execute("apt-get install -qy cloud-init").ok + + @retry(tries=30, delay=1) + def install_ppa(self): + log.info("Installing PPA") + assert self.execute( + "add-apt-repository {} -y".format(self.settings.CLOUD_INIT_SOURCE) + ).ok + assert self.execute("apt-get update -q").ok + assert self.execute("apt-get install -qy cloud-init").ok + + @retry(tries=30, delay=1) def install_deb(self): - log.info('Installing deb package') + log.info("Installing deb package") deb_path = integration_settings.CLOUD_INIT_SOURCE deb_name = os.path.basename(deb_path) - remote_path = '/var/tmp/{}'.format(deb_name) + remote_path = "/var/tmp/{}".format(deb_name) self.push_file( local_path=integration_settings.CLOUD_INIT_SOURCE, - remote_path=remote_path) - remote_script = '{sudo} dpkg -i {path}'.format( - sudo='sudo' if self.use_sudo else '', path=remote_path) - self._install_new_cloud_init(remote_script) + remote_path=remote_path, + ) + assert self.execute("dpkg -i {path}".format(path=remote_path)).ok + + @retry(tries=30, delay=1) + def upgrade_cloud_init(self): + log.info("Upgrading cloud-init to latest version in archive") + assert self.execute("apt-get update -q").ok + assert self.execute("apt-get install -qy cloud-init").ok def __enter__(self): return self @@ -132,23 +199,3 @@ class IntegrationInstance: def __exit__(self, exc_type, exc_val, exc_tb): if not self.settings.KEEP_INSTANCE: self.destroy() - - -class IntegrationEc2Instance(IntegrationInstance): - pass - - -class IntegrationGceInstance(IntegrationInstance): - pass - - -class IntegrationAzureInstance(IntegrationInstance): - pass - - -class IntegrationOciInstance(IntegrationInstance): - pass - - -class IntegrationLxdInstance(IntegrationInstance): - use_sudo = False diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py index a0609f7e..f27e4f12 100644 --- a/tests/integration_tests/integration_settings.py +++ b/tests/integration_tests/integration_settings.py @@ -1,29 +1,40 @@ # This file is part of cloud-init. See LICENSE file for license information. import os +from cloudinit.util import is_false, is_true + ################################################################## # LAUNCH SETTINGS ################################################################## # Keep instance (mostly for debugging) when test is finished KEEP_INSTANCE = False +# Keep snapshot image (mostly for debugging) when test is finished +KEEP_IMAGE = False +# Run tests marked as unstable. Expect failures and dragons. +RUN_UNSTABLE = False # One of: # lxd_container +# lxd_vm # azure # ec2 # gce # oci -PLATFORM = 'lxd_container' +# openstack +PLATFORM = "lxd_container" # The cloud-specific instance type to run. E.g., a1.medium on AWS # If the pycloudlib instance provides a default, this can be left None INSTANCE_TYPE = None # Determines the base image to use or generate new images from. -# Can be the name of the OS if running a stock image, -# otherwise the id of the image being used if using a custom image -OS_IMAGE = 'focal' +# +# This can be the name of an Ubuntu release, or in the format +# <image_id>[::<os>[::<release>]]. If given, os and release should describe +# the image specified by image_id. (Ubuntu releases are converted to this +# format internally; in this case, to "focal::ubuntu::focal".) +OS_IMAGE = "focal" # Populate if you want to use a pre-launched instance instead of # creating a new one. The exact contents will be platform dependent @@ -49,35 +60,30 @@ EXISTING_INSTANCE_ID = None # code. # PROPOSED # Install from the Ubuntu proposed repo +# UPGRADE +# Upgrade cloud-init to the version in the Ubuntu archive # <ppa repo>, e.g., ppa:cloud-init-dev/proposed # Install from a PPA. It MUST start with 'ppa:' # <file path> # A path to a valid package to be uploaded and installed -CLOUD_INIT_SOURCE = 'NONE' - -################################################################## -# GCE SPECIFIC SETTINGS -################################################################## -# Required for GCE -GCE_PROJECT = None +CLOUD_INIT_SOURCE = "NONE" -# You probably want to override these -GCE_REGION = 'us-central1' -GCE_ZONE = 'a' - -################################################################## -# OCI SPECIFIC SETTINGS -################################################################## -# Compartment-id found at -# https://console.us-phoenix-1.oraclecloud.com/a/identity/compartments -# Required for Oracle -OCI_COMPARTMENT_ID = None +# Before an instance is torn down, we run `cloud-init collect-logs` +# and transfer them locally. These settings specify when to collect these +# logs and where to put them on the local filesystem +# One of: +# 'ALWAYS' +# 'ON_ERROR' +# 'NEVER' +COLLECT_LOGS = "ON_ERROR" +LOCAL_LOG_PATH = "/tmp/cloud_init_test_logs" ################################################################## # USER SETTINGS OVERRIDES ################################################################## # Bring in any user-file defined settings try: + # pylint: disable=wildcard-import,unused-wildcard-import from tests.integration_tests.user_settings import * # noqa except ImportError: pass @@ -91,6 +97,13 @@ except ImportError: # Perhaps a bit too hacky, but it works :) current_settings = [var for var in locals() if var.isupper()] for setting in current_settings: - globals()[setting] = os.getenv( - 'CLOUD_INIT_{}'.format(setting), globals()[setting] + env_setting = os.getenv( + "CLOUD_INIT_{}".format(setting), globals()[setting] ) + if isinstance(env_setting, str): + env_setting = env_setting.strip() + if is_true(env_setting): + env_setting = True + elif is_false(env_setting): + env_setting = False + globals()[setting] = env_setting diff --git a/tests/integration_tests/modules/test_apt.py b/tests/integration_tests/modules/test_apt.py new file mode 100644 index 00000000..adab46a8 --- /dev/null +++ b/tests/integration_tests/modules/test_apt.py @@ -0,0 +1,354 @@ +"""Series of integration tests covering apt functionality.""" +import re + +import pytest + +from cloudinit import gpg +from cloudinit.config import cc_apt_configure +from tests.integration_tests.clouds import ImageSpecification +from tests.integration_tests.instances import IntegrationInstance + +USER_DATA = """\ +#cloud-config +apt: + conf: | + APT { + Get { + Assume-Yes "true"; + Fix-Broken "true"; + } + } + primary: + - arches: [default] + uri: http://badarchive.ubuntu.com/ubuntu + security: + - arches: [default] + uri: http://badsecurity.ubuntu.com/ubuntu + sources_list: | + deb $MIRROR $RELEASE main restricted + deb-src $MIRROR $RELEASE main restricted + deb $PRIMARY $RELEASE universe restricted + deb-src $PRIMARY $RELEASE universe restricted + deb $SECURITY $RELEASE-security multiverse + deb-src $SECURITY $RELEASE-security multiverse + sources: + test_keyserver: + keyid: 110E21D8B0E2A1F0243AF6820856F197B892ACEA + keyserver: keyserver.ubuntu.com + source: "deb http://ppa.launchpad.net/canonical-kernel-team/ppa/ubuntu $RELEASE main" + test_ppa: + keyid: 441614D8 + keyserver: keyserver.ubuntu.com + source: "ppa:simplestreams-dev/trunk" + test_signed_by: + keyid: A2EB2DEC0BD7519B7B38BE38376A290EC8068B11 + keyserver: keyserver.ubuntu.com + source: "deb [signed-by=$KEY_FILE] http://ppa.launchpad.net/juju/stable/ubuntu $RELEASE main" + test_bad_key: + key: "" + source: "deb $MIRROR $RELEASE main" + test_key: + source: "deb http://ppa.launchpad.net/cloud-init-dev/test-archive/ubuntu $RELEASE main" + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: SKS 1.1.6 + Comment: Hostname: keyserver.ubuntu.com + + mQINBFbZRUIBEAC+A0PIKYBP9kLC4hQtRrffRS11uLo8/BdtmOdrlW0hpPHzCfKnjR3tvSEI + lqPHG1QrrjAXKZDnZMRz+h/px7lUztvytGzHPSJd5ARUzAyjyRezUhoJ3VSCxrPqx62avuWf + RfoJaIeHfDehL5/dTVkyiWxfVZ369ZX6JN2AgLsQTeybTQ75+2z0xPrrhnGmgh6g0qTYcAaq + M5ONOGiqeSBX/Smjh6ALy5XkhUiFGLsI7Yluf6XSICY/x7gd6RAfgSIQrUTNMoS1sqhT4aot + +xvOfQy8ySkfAK4NddXql6E/+ZqTmBY/Lr0YklFBy8jGT+UysfiIznPMIwbmgq5Li7BtDDtX + b8Uyi4edPpjtextezfXYn4NVIpPL5dPZS/FXh4HpzyH0pYCfrH4QDGA7i52AGmhpiOFjJMo6 + N33sdjZHOH/2Vyp+QZaQnsdUAi1N4M6c33tQbpIScn1SY+El8z5JDA4PBzkw8HpLCi1gGoa6 + V4kfbWqXXbGAJFkLkP/vc4+pY9axOlmCkJg7xCPwhI75y1cONgovhz+BEXOzolh5KZuGbGbj + xe0wva5DLBeIg7EQFf+99pOS7Syby3Xpm6ZbswEFV0cllK4jf/QMjtfInxobuMoI0GV0bE5l + WlRtPCK5FnbHwxi0wPNzB/5fwzJ77r6HgPrR0OkT0lWmbUyoOQARAQABtC1MYXVuY2hwYWQg + UFBBIGZvciBjbG91ZCBpbml0IGRldmVsb3BtZW50IHRlYW2JAjgEEwECACIFAlbZRUICGwMG + CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEAg9Bvvk0wTfHfcP/REK5N2s1JYc69qEa9ZN + o6oi+A7l6AYw+ZY88O5TJe7F9otv5VXCIKSUT0Vsepjgf0mtXAgf/sb2lsJn/jp7tzgov3YH + vSrkTkRydz8xcA87gwQKePuvTLxQpftF4flrBxgSueIn5O/tPrBOxLz7EVYBc78SKg9aj9L2 + yUp+YuNevlwfZCTYeBb9r3FHaab2HcgkwqYch66+nKYfwiLuQ9NzXXm0Wn0JcEQ6pWvJscbj + C9BdawWovfvMK5/YLfI6Btm7F4mIpQBdhSOUp/YXKmdvHpmwxMCN2QhqYK49SM7qE9aUDbJL + arppSEBtlCLWhRBZYLTUna+BkuQ1bHz4St++XTR49Qd7vDERALpApDjB2dxPfMiBzCMwQQyq + uy13exU8o2ETLg+dZSLfDTzrBNsBFmXlw8WW17nTISYdKeGKL+QdlUjpzdwUMMzHhAO8SmMH + zjeSlDSRMXBJFAFSbCl7EwmMKa3yVX0zInT91fNllZ3iatAmtVdqVH/BFQfTIMH2ET7A8WzJ + ZzVSuMRhqoKdr5AMcHuJGPUoVkVJHQA+NNvEiXSysF3faL7jmKapmUwrhpYYX2H8pf+VMu2e + cLflKTI28dl+ZQ4Pl/aVsxrti/pzhdYy05Sn5ddtySyIkvo8L1cU5MWpbvSlFPkTstBUDLBf + pb0uBy+g0oxJQg15 + =uy53 + -----END PGP PUBLIC KEY BLOCK----- +apt_pipelining: os +""" # noqa: E501 + +EXPECTED_REGEXES = [ + r"deb http://badarchive.ubuntu.com/ubuntu [a-z]+ main restricted", + r"deb-src http://badarchive.ubuntu.com/ubuntu [a-z]+ main restricted", + r"deb http://badarchive.ubuntu.com/ubuntu [a-z]+ universe restricted", + r"deb-src http://badarchive.ubuntu.com/ubuntu [a-z]+ universe restricted", + r"deb http://badsecurity.ubuntu.com/ubuntu [a-z]+-security multiverse", + r"deb-src http://badsecurity.ubuntu.com/ubuntu [a-z]+-security multiverse", +] + +TEST_KEYSERVER_KEY = "110E 21D8 B0E2 A1F0 243A F682 0856 F197 B892 ACEA" +TEST_PPA_KEY = "3552 C902 B4DD F7BD 3842 1821 015D 28D7 4416 14D8" +TEST_KEY = "1FF0 D853 5EF7 E719 E5C8 1B9C 083D 06FB E4D3 04DF" +TEST_SIGNED_BY_KEY = "A2EB 2DEC 0BD7 519B 7B38 BE38 376A 290E C806 8B11" + + +@pytest.mark.ubuntu +@pytest.mark.user_data(USER_DATA) +class TestApt: + def get_keys(self, class_client: IntegrationInstance): + """Return all keys in /etc/apt/trusted.gpg.d/ and /etc/apt/trusted.gpg + in human readable format. Mimics the output of apt-key finger + """ + list_cmd = " ".join(gpg.GPG_LIST) + " " + keys = class_client.execute(list_cmd + cc_apt_configure.APT_LOCAL_KEYS) + print(keys) + files = class_client.execute( + "ls " + cc_apt_configure.APT_TRUSTED_GPG_DIR + ) + for file in files.split(): + path = cc_apt_configure.APT_TRUSTED_GPG_DIR + file + keys += class_client.execute(list_cmd + path) or "" + return keys + + def test_sources_list(self, class_client: IntegrationInstance): + """Integration test for the apt module's `sources_list` functionality. + + This test specifies a ``sources_list`` and then checks that (a) the + expected number of sources.list entries is present, and (b) that each + expected line appears in the file. + + (This is ported from + `tests/cloud_tests/testcases/modules/apt_configure_sources_list.yaml`.) + """ + sources_list = class_client.read_from_file("/etc/apt/sources.list") + assert 6 == len(sources_list.rstrip().split("\n")) + + for expected_re in EXPECTED_REGEXES: + assert re.search(expected_re, sources_list) is not None + + def test_apt_conf(self, class_client: IntegrationInstance): + """Test the apt conf functionality. + + Ported from tests/cloud_tests/testcases/modules/apt_configure_conf.py + """ + apt_config = class_client.read_from_file( + "/etc/apt/apt.conf.d/94cloud-init-config" + ) + assert 'Assume-Yes "true";' in apt_config + assert 'Fix-Broken "true";' in apt_config + + def test_ppa_source(self, class_client: IntegrationInstance): + """Test the apt ppa functionality. + + Ported from + tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.py + """ + release = ImageSpecification.from_os_image().release + ppa_path_contents = class_client.read_from_file( + "/etc/apt/sources.list.d/" + "simplestreams-dev-ubuntu-trunk-{}.list".format(release) + ) + + assert ( + "http://ppa.launchpad.net/simplestreams-dev/trunk/ubuntu" + in ppa_path_contents + ) + + assert TEST_PPA_KEY in self.get_keys(class_client) + + def test_signed_by(self, class_client: IntegrationInstance): + """Test the apt signed-by functionality.""" + release = ImageSpecification.from_os_image().release + source = ( + "deb [signed-by=/etc/apt/cloud-init.gpg.d/test_signed_by.gpg] " + "http://ppa.launchpad.net/juju/stable/ubuntu" + " {} main".format(release) + ) + path_contents = class_client.read_from_file( + "/etc/apt/sources.list.d/test_signed_by.list" + ) + assert path_contents == source + + key = class_client.execute( + "gpg --no-default-keyring --with-fingerprint --list-keys " + "--keyring /etc/apt/cloud-init.gpg.d/test_signed_by.gpg" + ) + + assert TEST_SIGNED_BY_KEY in key + + def test_bad_key(self, class_client: IntegrationInstance): + """Test the apt signed-by functionality.""" + with pytest.raises(OSError): + class_client.read_from_file( + "/etc/apt/trusted.list.d/test_bad_key.gpg" + ) + + def test_key(self, class_client: IntegrationInstance): + """Test the apt key functionality. + + Ported from + tests/cloud_tests/testcases/modules/apt_configure_sources_key.py + """ + test_archive_contents = class_client.read_from_file( + "/etc/apt/sources.list.d/test_key.list" + ) + + assert ( + "http://ppa.launchpad.net/cloud-init-dev/test-archive/ubuntu" + in test_archive_contents + ) + assert TEST_KEY in self.get_keys(class_client) + + def test_keyserver(self, class_client: IntegrationInstance): + """Test the apt keyserver functionality. + + Ported from + tests/cloud_tests/testcases/modules/apt_configure_sources_keyserver.py + """ + test_keyserver_contents = class_client.read_from_file( + "/etc/apt/sources.list.d/test_keyserver.list" + ) + + assert ( + "http://ppa.launchpad.net/canonical-kernel-team/ppa/ubuntu" + in test_keyserver_contents + ) + + assert TEST_KEYSERVER_KEY in self.get_keys(class_client) + + def test_os_pipelining(self, class_client: IntegrationInstance): + """Test 'os' settings does not write apt config file. + + Ported from tests/cloud_tests/testcases/modules/apt_pipelining_os.py + """ + conf_exists = class_client.execute( + "test -f /etc/apt/apt.conf.d/90cloud-init-pipelining" + ).ok + assert conf_exists is False + + +_DEFAULT_DATA = """\ +#cloud-config +apt: + primary: + - arches: + - default + {uri} + security: + - arches: + - default +""" +DEFAULT_DATA = _DEFAULT_DATA.format(uri="") + + +@pytest.mark.ubuntu +@pytest.mark.user_data(DEFAULT_DATA) +class TestDefaults: + @pytest.mark.openstack + def test_primary_on_openstack(self, class_client: IntegrationInstance): + """Test apt default primary source on openstack. + + When no uri is provided. + """ + zone = class_client.execute("cloud-init query v1.availability_zone") + sources_list = class_client.read_from_file("/etc/apt/sources.list") + assert "{}.clouds.archive.ubuntu.com".format(zone) in sources_list + + def test_security(self, class_client: IntegrationInstance): + """Test apt default security sources. + + Ported from + tests/cloud_tests/testcases/modules/apt_configure_security.py + """ + sources_list = class_client.read_from_file("/etc/apt/sources.list") + + # 3 lines from main, universe, and multiverse + sec_url = "deb http://security.ubuntu.com/ubuntu" + if class_client.settings.PLATFORM == "azure": + sec_url = ( + "deb http://azure.archive.ubuntu.com/ubuntu/ jammy-security" + ) + sec_src_url = sec_url.replace("deb ", "# deb-src ") + assert 3 == sources_list.count(sec_url) + assert 3 == sources_list.count(sec_src_url) + + +DEFAULT_DATA_WITH_URI = _DEFAULT_DATA.format( + uri='uri: "http://something.random.invalid/ubuntu"' +) + + +@pytest.mark.user_data(DEFAULT_DATA_WITH_URI) +def test_default_primary_with_uri(client: IntegrationInstance): + """Test apt default primary sources. + + Ported from + tests/cloud_tests/testcases/modules/apt_configure_primary.py + """ + sources_list = client.read_from_file("/etc/apt/sources.list") + assert "archive.ubuntu.com" not in sources_list + + assert "something.random.invalid" in sources_list + + +DISABLED_DATA = """\ +#cloud-config +apt: + disable_suites: + - $RELEASE + - $RELEASE-updates + - $RELEASE-backports + - $RELEASE-security +apt_pipelining: false +""" + + +@pytest.mark.ubuntu +@pytest.mark.user_data(DISABLED_DATA) +class TestDisabled: + def test_disable_suites(self, class_client: IntegrationInstance): + """Test disabling of apt suites. + + Ported from + tests/cloud_tests/testcases/modules/apt_configure_disable_suites.py + """ + sources_list = class_client.execute( + "cat /etc/apt/sources.list | grep -v '^#'" + ).strip() + assert "" == sources_list + + def test_disable_apt_pipelining(self, class_client: IntegrationInstance): + """Test disabling of apt pipelining. + + Ported from + tests/cloud_tests/testcases/modules/apt_pipelining_disable.py + """ + conf = class_client.read_from_file( + "/etc/apt/apt.conf.d/90cloud-init-pipelining" + ) + assert 'Acquire::http::Pipeline-Depth "0";' in conf + + +APT_PROXY_DATA = """\ +#cloud-config +apt: + proxy: "http://proxy.internal:3128" + http_proxy: "http://squid.internal:3128" + ftp_proxy: "ftp://squid.internal:3128" + https_proxy: "https://squid.internal:3128" +""" + + +@pytest.mark.ubuntu +@pytest.mark.user_data(APT_PROXY_DATA) +def test_apt_proxy(client: IntegrationInstance): + """Test the apt proxy data gets written correctly.""" + out = client.read_from_file("/etc/apt/apt.conf.d/90cloud-init-aptproxy") + assert 'Acquire::http::Proxy "http://proxy.internal:3128";' in out + assert 'Acquire::http::Proxy "http://squid.internal:3128";' in out + assert 'Acquire::ftp::Proxy "ftp://squid.internal:3128";' in out + assert 'Acquire::https::Proxy "https://squid.internal:3128";' in out diff --git a/tests/integration_tests/modules/test_apt_configure_sources_list.py b/tests/integration_tests/modules/test_apt_configure_sources_list.py deleted file mode 100644 index d2bcc61a..00000000 --- a/tests/integration_tests/modules/test_apt_configure_sources_list.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Integration test for the apt module's ``sources_list`` functionality. - -This test specifies a ``sources_list`` and then checks that (a) the expected -number of sources.list entries is present, and (b) that each expected line -appears in the file. - -(This is ported from -``tests/cloud_tests/testcases/modules/apt_configure_sources_list.yaml``.)""" -import re - -import pytest - - -USER_DATA = """\ -#cloud-config -apt: - primary: - - arches: [default] - uri: http://archive.ubuntu.com/ubuntu - security: - - arches: [default] - uri: http://security.ubuntu.com/ubuntu - sources_list: | - deb $MIRROR $RELEASE main restricted - deb-src $MIRROR $RELEASE main restricted - deb $PRIMARY $RELEASE universe restricted - deb-src $PRIMARY $RELEASE universe restricted - deb $SECURITY $RELEASE-security multiverse - deb-src $SECURITY $RELEASE-security multiverse -""" - -EXPECTED_REGEXES = [ - r"deb http://archive.ubuntu.com/ubuntu [a-z].* main restricted", - r"deb-src http://archive.ubuntu.com/ubuntu [a-z].* main restricted", - r"deb http://archive.ubuntu.com/ubuntu [a-z].* universe restricted", - r"deb-src http://archive.ubuntu.com/ubuntu [a-z].* universe restricted", - r"deb http://security.ubuntu.com/ubuntu [a-z].*security multiverse", - r"deb-src http://security.ubuntu.com/ubuntu [a-z].*security multiverse", -] - - -@pytest.mark.ci -class TestAptConfigureSourcesList: - - @pytest.mark.user_data(USER_DATA) - def test_sources_list(self, client): - sources_list = client.read_from_file("/etc/apt/sources.list") - assert 6 == len(sources_list.rstrip().split('\n')) - - for expected_re in EXPECTED_REGEXES: - assert re.search(expected_re, sources_list) is not None diff --git a/tests/integration_tests/modules/test_ca_certs.py b/tests/integration_tests/modules/test_ca_certs.py new file mode 100644 index 00000000..7247fd7d --- /dev/null +++ b/tests/integration_tests/modules/test_ca_certs.py @@ -0,0 +1,90 @@ +"""Integration tests for cc_ca_certs. + +(This is ported from ``tests/cloud_tests//testcases/modules/ca_certs.yaml``.) + +TODO: +* Mark this as running on Debian and Alpine (once we have marks for that) +* Implement testing for the RHEL-specific paths +""" +import os.path + +import pytest + +USER_DATA = """\ +#cloud-config +ca_certs: + remove_defaults: true + trusted: + - | + -----BEGIN CERTIFICATE----- + MIIGJzCCBA+gAwIBAgIBATANBgkqhkiG9w0BAQUFADCBsjELMAkGA1UEBhMCRlIx + DzANBgNVBAgMBkFsc2FjZTETMBEGA1UEBwwKU3RyYXNib3VyZzEYMBYGA1UECgwP + d3d3LmZyZWVsYW4ub3JnMRAwDgYDVQQLDAdmcmVlbGFuMS0wKwYDVQQDDCRGcmVl + bGFuIFNhbXBsZSBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxIjAgBgkqhkiG9w0BCQEW + E2NvbnRhY3RAZnJlZWxhbi5vcmcwHhcNMTIwNDI3MTAzMTE4WhcNMjIwNDI1MTAz + MTE4WjB+MQswCQYDVQQGEwJGUjEPMA0GA1UECAwGQWxzYWNlMRgwFgYDVQQKDA93 + d3cuZnJlZWxhbi5vcmcxEDAOBgNVBAsMB2ZyZWVsYW4xDjAMBgNVBAMMBWFsaWNl + MSIwIAYJKoZIhvcNAQkBFhNjb250YWN0QGZyZWVsYW4ub3JnMIICIjANBgkqhkiG + 9w0BAQEFAAOCAg8AMIICCgKCAgEA3W29+ID6194bH6ejLrIC4hb2Ugo8v6ZC+Mrc + k2dNYMNPjcOKABvxxEtBamnSaeU/IY7FC/giN622LEtV/3oDcrua0+yWuVafyxmZ + yTKUb4/GUgafRQPf/eiX9urWurtIK7XgNGFNUjYPq4dSJQPPhwCHE/LKAykWnZBX + RrX0Dq4XyApNku0IpjIjEXH+8ixE12wH8wt7DEvdO7T3N3CfUbaITl1qBX+Nm2Z6 + q4Ag/u5rl8NJfXg71ZmXA3XOj7zFvpyapRIZcPmkvZYn7SMCp8dXyXHPdpSiIWL2 + uB3KiO4JrUYvt2GzLBUThp+lNSZaZ/Q3yOaAAUkOx+1h08285Pi+P8lO+H2Xic4S + vMq1xtLg2bNoPC5KnbRfuFPuUD2/3dSiiragJ6uYDLOyWJDivKGt/72OVTEPAL9o + 6T2pGZrwbQuiFGrGTMZOvWMSpQtNl+tCCXlT4mWqJDRwuMGrI4DnnGzt3IKqNwS4 + Qyo9KqjMIPwnXZAmWPm3FOKe4sFwc5fpawKO01JZewDsYTDxVj+cwXwFxbE2yBiF + z2FAHwfopwaH35p3C6lkcgP2k/zgAlnBluzACUI+MKJ/G0gv/uAhj1OHJQ3L6kn1 + SpvQ41/ueBjlunExqQSYD7GtZ1Kg8uOcq2r+WISE3Qc9MpQFFkUVllmgWGwYDuN3 + Zsez95kCAwEAAaN7MHkwCQYDVR0TBAIwADAsBglghkgBhvhCAQ0EHxYdT3BlblNT + TCBHZW5lcmF0ZWQgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFFlfyRO6G8y5qEFKikl5 + ajb2fT7XMB8GA1UdIwQYMBaAFCNsLT0+KV14uGw+quK7Lh5sh/JTMA0GCSqGSIb3 + DQEBBQUAA4ICAQAT5wJFPqervbja5+90iKxi1d0QVtVGB+z6aoAMuWK+qgi0vgvr + mu9ot2lvTSCSnRhjeiP0SIdqFMORmBtOCFk/kYDp9M/91b+vS+S9eAlxrNCB5VOf + PqxEPp/wv1rBcE4GBO/c6HcFon3F+oBYCsUQbZDKSSZxhDm3mj7pb67FNbZbJIzJ + 70HDsRe2O04oiTx+h6g6pW3cOQMgIAvFgKN5Ex727K4230B0NIdGkzuj4KSML0NM + slSAcXZ41OoSKNjy44BVEZv0ZdxTDrRM4EwJtNyggFzmtTuV02nkUj1bYYYC5f0L + ADr6s0XMyaNk8twlWYlYDZ5uKDpVRVBfiGcq0uJIzIvemhuTrofh8pBQQNkPRDFT + Rq1iTo1Ihhl3/Fl1kXk1WR3jTjNb4jHX7lIoXwpwp767HAPKGhjQ9cFbnHMEtkro + RlJYdtRq5mccDtwT0GFyoJLLBZdHHMHJz0F9H7FNk2tTQQMhK5MVYwg+LIaee586 + CQVqfbscp7evlgjLW98H+5zylRHAgoH2G79aHljNKMp9BOuq6SnEglEsiWGVtu2l + hnx8SB3sVJZHeer8f/UQQwqbAO+Kdy70NmbSaqaVtp8jOxLiidWkwSyRTsuU6D8i + DiH5uEqBXExjrj0FslxcVKdVj5glVcSmkLwZKbEU1OKwleT/iXFhvooWhQ== + -----END CERTIFICATE----- +""" + + +@pytest.mark.ubuntu +@pytest.mark.user_data(USER_DATA) +class TestCaCerts: + def test_certs_updated(self, class_client): + """Test that /etc/ssl/certs is updated as we expect.""" + root = "/etc/ssl/certs" + filenames = class_client.execute(["ls", "-1", root]).splitlines() + unlinked_files = [] + links = {} + for filename in filenames: + full_path = os.path.join(root, filename) + symlink_target = class_client.execute(["readlink", full_path]) + is_symlink = symlink_target.ok + if is_symlink: + links[filename] = symlink_target + else: + unlinked_files.append(filename) + + assert ["ca-certificates.crt"] == unlinked_files + assert "cloud-init-ca-certs.pem" == links["a535c1f3.0"] + assert ( + "/usr/share/ca-certificates/cloud-init-ca-certs.crt" + == links["cloud-init-ca-certs.pem"] + ) + + def test_cert_installed(self, class_client): + """Test that our specified cert has been installed""" + checksum = class_client.execute( + "sha256sum /etc/ssl/certs/ca-certificates.crt" + ) + assert ( + "78e875f18c73c1aab9167ae0bd323391e52222cc2dbcda42d129537219300062" + in checksum + ) diff --git a/tests/integration_tests/modules/test_cli.py b/tests/integration_tests/modules/test_cli.py new file mode 100644 index 00000000..baaa7567 --- /dev/null +++ b/tests/integration_tests/modules/test_cli.py @@ -0,0 +1,81 @@ +"""Integration tests for CLI functionality + +These would be for behavior manually invoked by user from the command line +""" + +import pytest + +from tests.integration_tests.instances import IntegrationInstance + +VALID_USER_DATA = """\ +#cloud-config +runcmd: + - echo 'hi' > /var/tmp/test +""" + +INVALID_USER_DATA_HEADER = """\ +runcmd: + - echo 'hi' > /var/tmp/test +""" + +INVALID_USER_DATA_SCHEMA = """\ +#cloud-config +updates: + notnetwork: -1 +apt_pipelining: bogus +""" + + +@pytest.mark.user_data(VALID_USER_DATA) +def test_valid_userdata(client: IntegrationInstance): + """Test `cloud-init devel schema` with valid userdata. + + PR #575 + """ + result = client.execute("cloud-init devel schema --system") + assert result.ok + assert "Valid cloud-config: system userdata" == result.stdout.strip() + result = client.execute("cloud-init status --long") + if not result.ok: + raise AssertionError( + f"Unexpected error from cloud-init status: {result}" + ) + + +@pytest.mark.user_data(INVALID_USER_DATA_HEADER) +def test_invalid_userdata(client: IntegrationInstance): + """Test `cloud-init devel schema` with invalid userdata. + + PR #575 + """ + result = client.execute("cloud-init devel schema --system") + assert not result.ok + assert "Cloud config schema errors" in result.stderr + assert 'needs to begin with "#cloud-config"' in result.stderr + result = client.execute("cloud-init status --long") + if not result.ok: + raise AssertionError( + f"Unexpected error from cloud-init status: {result}" + ) + + +@pytest.mark.user_data(INVALID_USER_DATA_SCHEMA) +def test_invalid_userdata_schema(client: IntegrationInstance): + """Test invalid schema represented as Warnings, not fatal + + PR #1175 + """ + result = client.execute("cloud-init status --long") + assert result.ok + log = client.read_from_file("/var/log/cloud-init.log") + warning = ( + "[WARNING]: Invalid cloud-config provided:\napt_pipelining: 'bogus'" + " is not valid under any of the given schemas\nupdates: Additional" + " properties are not allowed ('notnetwork' was unexpected)" + ) + assert warning in log + result = client.execute("cloud-init status --long") + if not result.ok: + raise AssertionError( + f"Unexpected error from cloud-init status: {result}" + ) diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py new file mode 100644 index 00000000..7a9a6e27 --- /dev/null +++ b/tests/integration_tests/modules/test_combined.py @@ -0,0 +1,342 @@ +# This file is part of cloud-init. See LICENSE file for license information. +"""A set of somewhat unrelated tests that can be combined into a single +instance launch. Generally tests should only be added here if a failure +of the test would be unlikely to affect the running of another test using +the same instance launch. Most independent module coherence tests can go +here. +""" +import json +import re + +import pytest + +from tests.integration_tests.clouds import ImageSpecification +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import ( + retry, + verify_clean_log, + verify_ordered_items_in_text, +) + +USER_DATA = """\ +#cloud-config +apt: + primary: + - arches: [default] + uri: http://us.archive.ubuntu.com/ubuntu/ +byobu_by_default: enable +final_message: | + This is my final message! + $version + $timestamp + $datasource + $uptime +locale: en_GB.UTF-8 +locale_configfile: /etc/default/locale +ntp: + servers: ['ntp.ubuntu.com'] +package_update: true +random_seed: + data: 'MYUb34023nD:LFDK10913jk;dfnk:Df' + encoding: raw + file: /root/seed +rsyslog: + configs: + - "*.* @@127.0.0.1" + - filename: 0-basic-config.conf + content: | + module(load="imtcp") + input(type="imtcp" port="514") + $template RemoteLogs,"/var/tmp/rsyslog.log" + *.* ?RemoteLogs + & ~ + remotes: + me: "127.0.0.1" +runcmd: + - echo 'hello world' > /var/tmp/runcmd_output + + - # + - logger "My test log" +snap: + squashfuse_in_container: true + commands: + - snap install hello-world +ssh_import_id: + - gh:powersj + - lp:smoser +timezone: US/Aleutian +""" + + +@pytest.mark.ci +@pytest.mark.user_data(USER_DATA) +class TestCombined: + def test_final_message(self, class_client: IntegrationInstance): + """Test that final_message module works as expected. + + Also tests LP 1511485: final_message is silent. + """ + client = class_client + log = client.read_from_file("/var/log/cloud-init.log") + expected = ( + "This is my final message!\n" + r"\d+\.\d+.*\n" + r"\w{3}, \d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2} \+\d{4}\n" # Datetime + "DataSource.*\n" + r"\d+\.\d+" + ) + + assert re.search(expected, log) + + def test_ntp_with_apt(self, class_client: IntegrationInstance): + """LP #1628337. + + cloud-init tries to install NTP before even + configuring the archives. + """ + client = class_client + log = client.read_from_file("/var/log/cloud-init.log") + assert "W: Failed to fetch" not in log + assert "W: Some index files failed to download" not in log + assert "E: Unable to locate package ntp" not in log + + def test_byobu(self, class_client: IntegrationInstance): + """Test byobu configured as enabled by default.""" + client = class_client + assert client.execute('test -e "/etc/byobu/autolaunch"').ok + + def test_configured_locale(self, class_client: IntegrationInstance): + """Test locale can be configured correctly.""" + client = class_client + default_locale = client.read_from_file("/etc/default/locale") + assert "LANG=en_GB.UTF-8" in default_locale + + locale_a = client.execute("locale -a") + verify_ordered_items_in_text(["en_GB.utf8", "en_US.utf8"], locale_a) + + locale_gen = client.execute( + "cat /etc/locale.gen | grep -v '^#' | uniq" + ) + verify_ordered_items_in_text( + ["en_GB.UTF-8", "en_US.UTF-8"], locale_gen + ) + + def test_random_seed_data(self, class_client: IntegrationInstance): + """Integration test for the random seed module. + + This test specifies a command to be executed by the ``seed_random`` + module, by providing a different data to be used as seed data. We will + then check if that seed data was actually used. + """ + client = class_client + + # Only read the first 31 characters, because the rest could be + # binary data + result = client.execute("head -c 31 < /root/seed") + assert result.startswith("MYUb34023nD:LFDK10913jk;dfnk:Df") + + def test_rsyslog(self, class_client: IntegrationInstance): + """Test rsyslog is configured correctly.""" + client = class_client + assert "My test log" in client.read_from_file("/var/tmp/rsyslog.log") + + def test_runcmd(self, class_client: IntegrationInstance): + """Test runcmd works as expected""" + client = class_client + assert "hello world" == client.read_from_file("/var/tmp/runcmd_output") + + @retry(tries=30, delay=1) + def test_ssh_import_id(self, class_client: IntegrationInstance): + """Integration test for the ssh_import_id module. + + This test specifies ssh keys to be imported by the ``ssh_import_id`` + module and then checks that if the ssh keys were successfully imported. + + TODO: + * This test assumes that SSH keys will be imported into the + /home/ubuntu; this will need modification to run on other OSes. + """ + client = class_client + ssh_output = client.read_from_file("/home/ubuntu/.ssh/authorized_keys") + + assert "# ssh-import-id gh:powersj" in ssh_output + assert "# ssh-import-id lp:smoser" in ssh_output + + def test_snap(self, class_client: IntegrationInstance): + """Integration test for the snap module. + + This test specifies a command to be executed by the ``snap`` module + and then checks that if that command was executed during boot. + """ + client = class_client + snap_output = client.execute("snap list") + assert "core " in snap_output + assert "hello-world " in snap_output + + def test_timezone(self, class_client: IntegrationInstance): + """Integration test for the timezone module. + + This test specifies a timezone to be used by the ``timezone`` module + and then checks that if that timezone was respected during boot. + """ + client = class_client + timezone_output = client.execute( + 'date "+%Z" --date="Thu, 03 Nov 2016 00:47:00 -0400"' + ) + assert timezone_output.strip() == "HDT" + + def test_no_problems(self, class_client: IntegrationInstance): + """Test no errors, warnings, or tracebacks""" + client = class_client + status_file = client.read_from_file("/run/cloud-init/status.json") + status_json = json.loads(status_file)["v1"] + for stage in ("init", "init-local", "modules-config", "modules-final"): + assert status_json[stage]["errors"] == [] + result_file = client.read_from_file("/run/cloud-init/result.json") + result_json = json.loads(result_file)["v1"] + assert result_json["errors"] == [] + + log = client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log) + + def test_correct_datasource_detected( + self, class_client: IntegrationInstance + ): + """Test datasource is detected at the proper boot stage.""" + client = class_client + status_file = client.read_from_file("/run/cloud-init/status.json") + parsed_datasource = json.loads(status_file)["v1"]["datasource"] + + if client.settings.PLATFORM in ["lxd_container", "lxd_vm"]: + assert parsed_datasource.startswith("DataSourceNoCloud") + else: + platform_datasources = { + "azure": "DataSourceAzure [seed=/dev/sr0]", + "ec2": "DataSourceEc2Local", + "gce": "DataSourceGCELocal", + "oci": "DataSourceOracle", + "openstack": "DataSourceOpenStackLocal [net,ver=2]", + } + assert ( + platform_datasources[client.settings.PLATFORM] + == parsed_datasource + ) + + def test_cloud_id_file_symlink(self, class_client: IntegrationInstance): + cloud_id = class_client.execute("cloud-id").stdout + expected_link_output = ( + "'/run/cloud-init/cloud-id' -> " + f"'/run/cloud-init/cloud-id-{cloud_id}'" + ) + assert expected_link_output == str( + class_client.execute("stat -c %N /run/cloud-init/cloud-id") + ) + + def _check_common_metadata(self, data): + assert data["base64_encoded_keys"] == [] + assert data["merged_cfg"] == "redacted for non-root user" + + image_spec = ImageSpecification.from_os_image() + assert data["sys_info"]["dist"][0] == image_spec.os + + v1_data = data["v1"] + assert re.match(r"\d\.\d+\.\d+-\d+", v1_data["kernel_release"]) + assert v1_data["variant"] == image_spec.os + assert v1_data["distro"] == image_spec.os + assert v1_data["distro_release"] == image_spec.release + assert v1_data["machine"] == "x86_64" + assert re.match(r"3.\d\.\d", v1_data["python_version"]) + + @pytest.mark.lxd_container + def test_instance_json_lxd(self, class_client: IntegrationInstance): + client = class_client + instance_json_file = client.read_from_file( + "/run/cloud-init/instance-data.json" + ) + + data = json.loads(instance_json_file) + self._check_common_metadata(data) + v1_data = data["v1"] + assert v1_data["cloud_name"] == "unknown" + assert v1_data["platform"] == "lxd" + assert v1_data["cloud_id"] == "lxd" + assert f"{v1_data['cloud_id']}" == client.read_from_file( + "/run/cloud-init/cloud-id-lxd" + ) + assert ( + v1_data["subplatform"] + == "seed-dir (/var/lib/cloud/seed/nocloud-net)" + ) + assert v1_data["availability_zone"] is None + assert v1_data["instance_id"] == client.instance.name + assert v1_data["local_hostname"] == client.instance.name + assert v1_data["region"] is None + + @pytest.mark.lxd_vm + def test_instance_json_lxd_vm(self, class_client: IntegrationInstance): + client = class_client + instance_json_file = client.read_from_file( + "/run/cloud-init/instance-data.json" + ) + + data = json.loads(instance_json_file) + self._check_common_metadata(data) + v1_data = data["v1"] + assert v1_data["cloud_name"] == "unknown" + assert v1_data["platform"] == "lxd" + assert v1_data["cloud_id"] == "lxd" + assert f"{v1_data['cloud_id']}" == client.read_from_file( + "/run/cloud-init/cloud-id-lxd" + ) + assert any( + [ + "/var/lib/cloud/seed/nocloud-net" in v1_data["subplatform"], + "/dev/sr0" in v1_data["subplatform"], + ] + ) + assert v1_data["availability_zone"] is None + assert v1_data["instance_id"] == client.instance.name + assert v1_data["local_hostname"] == client.instance.name + assert v1_data["region"] is None + + @pytest.mark.ec2 + def test_instance_json_ec2(self, class_client: IntegrationInstance): + client = class_client + instance_json_file = client.read_from_file( + "/run/cloud-init/instance-data.json" + ) + data = json.loads(instance_json_file) + v1_data = data["v1"] + assert v1_data["cloud_name"] == "aws" + assert v1_data["platform"] == "ec2" + # Different regions will show up as ec2-(gov|china) + assert v1_data["cloud_id"].startswith("ec2") + assert f"{v1_data['cloud_id']}" == client.read_from_file( + "/run/cloud-init/cloud-id-ec2" + ) + assert v1_data["subplatform"].startswith("metadata") + assert ( + v1_data["availability_zone"] == client.instance.availability_zone + ) + assert v1_data["instance_id"] == client.instance.name + assert v1_data["local_hostname"].startswith("ip-") + assert v1_data["region"] == client.cloud.cloud_instance.region + + @pytest.mark.gce + def test_instance_json_gce(self, class_client: IntegrationInstance): + client = class_client + instance_json_file = client.read_from_file( + "/run/cloud-init/instance-data.json" + ) + data = json.loads(instance_json_file) + self._check_common_metadata(data) + v1_data = data["v1"] + assert v1_data["cloud_name"] == "gce" + assert v1_data["platform"] == "gce" + assert f"{v1_data['cloud_id']}" == client.read_from_file( + "/run/cloud-init/cloud-id-gce" + ) + assert v1_data["subplatform"].startswith("metadata") + assert v1_data["availability_zone"] == client.instance.zone + assert v1_data["instance_id"] == client.instance.instance_id + assert v1_data["local_hostname"] == client.instance.name diff --git a/tests/integration_tests/modules/test_command_output.py b/tests/integration_tests/modules/test_command_output.py new file mode 100644 index 00000000..96525cac --- /dev/null +++ b/tests/integration_tests/modules/test_command_output.py @@ -0,0 +1,21 @@ +"""Integration test for output redirection. + +This test redirects the output of a command to a file and then checks the file. + +(This is ported from +``tests/cloud_tests/testcases/main/command_output_simple.yaml``.)""" +import pytest + +from tests.integration_tests.instances import IntegrationInstance + +USER_DATA = """\ +#cloud-config +output: { all: "| tee -a /var/log/cloud-init-test-output" } +final_message: "should be last line in cloud-init-test-output file" +""" + + +@pytest.mark.user_data(USER_DATA) +def test_runcmd(client: IntegrationInstance): + log = client.read_from_file("/var/log/cloud-init-test-output") + assert "should be last line in cloud-init-test-output file" in log diff --git a/tests/integration_tests/modules/test_disk_setup.py b/tests/integration_tests/modules/test_disk_setup.py new file mode 100644 index 00000000..7aaba7db --- /dev/null +++ b/tests/integration_tests/modules/test_disk_setup.py @@ -0,0 +1,212 @@ +import json +import os +from uuid import uuid4 + +import pytest +from pycloudlib.lxd.instance import LXDInstance + +from cloudinit.subp import subp +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import verify_clean_log + +DISK_PATH = "/tmp/test_disk_setup_{}".format(uuid4()) + + +def setup_and_mount_lxd_disk(instance: LXDInstance): + subp( + "lxc config device add {} test-disk-setup-disk disk source={}".format( + instance.name, DISK_PATH + ).split() + ) + + +@pytest.fixture +def create_disk(): + # 640k should be enough for anybody + subp("dd if=/dev/zero of={} bs=1k count=640".format(DISK_PATH).split()) + yield + os.remove(DISK_PATH) + + +ALIAS_USERDATA = """\ +#cloud-config +device_aliases: + my_alias: /dev/sdb +disk_setup: + my_alias: + table_type: mbr + layout: [50, 50] + overwrite: True +fs_setup: +- label: fs1 + device: my_alias.1 + filesystem: ext4 +- label: fs2 + device: my_alias.2 + filesystem: ext4 +mounts: +- ["my_alias.1", "/mnt1"] +- ["my_alias.2", "/mnt2"] +""" + + +@pytest.mark.user_data(ALIAS_USERDATA) +@pytest.mark.lxd_setup.with_args(setup_and_mount_lxd_disk) +@pytest.mark.ubuntu +@pytest.mark.lxd_vm +class TestDeviceAliases: + """Test devices aliases work on disk setup/mount""" + + def test_device_alias(self, create_disk, client: IntegrationInstance): + log = client.read_from_file("/var/log/cloud-init.log") + assert ( + "updated disk_setup device entry 'my_alias' to '/dev/sdb'" in log + ) + assert "changed my_alias.1 => /dev/sdb1" in log + assert "changed my_alias.2 => /dev/sdb2" in log + verify_clean_log(log) + + lsblk = json.loads(client.execute("lsblk --json")) + sdb = [x for x in lsblk["blockdevices"] if x["name"] == "sdb"][0] + assert len(sdb["children"]) == 2 + assert sdb["children"][0]["name"] == "sdb1" + assert sdb["children"][1]["name"] == "sdb2" + if "mountpoint" in sdb["children"][0]: + assert sdb["children"][0]["mountpoint"] == "/mnt1" + assert sdb["children"][1]["mountpoint"] == "/mnt2" + else: + assert sdb["children"][0]["mountpoints"] == ["/mnt1"] + assert sdb["children"][1]["mountpoints"] == ["/mnt2"] + result = client.execute("mount -a") + assert result.return_code == 0 + assert result.stdout.strip() == "" + assert result.stderr.strip() == "" + result = client.execute("findmnt -J /mnt1") + assert result.return_code == 0 + result = client.execute("findmnt -J /mnt2") + assert result.return_code == 0 + + +PARTPROBE_USERDATA = """\ +#cloud-config +disk_setup: + /dev/sdb: + table_type: mbr + layout: [50, 50] + overwrite: True +fs_setup: + - label: test + device: /dev/sdb1 + filesystem: ext4 + - label: test2 + device: /dev/sdb2 + filesystem: ext4 +mounts: +- ["/dev/sdb1", "/mnt1"] +- ["/dev/sdb2", "/mnt2"] +""" + +UPDATED_PARTPROBE_USERDATA = """\ +#cloud-config +disk_setup: + /dev/sdb: + table_type: mbr + layout: [100] + overwrite: True +fs_setup: + - label: test3 + device: /dev/sdb1 + filesystem: ext4 +mounts: +- ["/dev/sdb1", "/mnt3"] +""" + + +@pytest.mark.user_data(PARTPROBE_USERDATA) +@pytest.mark.lxd_setup.with_args(setup_and_mount_lxd_disk) +@pytest.mark.ubuntu +@pytest.mark.lxd_vm +class TestPartProbeAvailability: + """Test disk setup works with partprobe + + Disk setup can run successfully on a mounted partition when + partprobe is being used. + + lp-1920939 + """ + + def _verify_first_disk_setup(self, client, log): + verify_clean_log(log) + lsblk = json.loads(client.execute("lsblk --json")) + sdb = [x for x in lsblk["blockdevices"] if x["name"] == "sdb"][0] + assert len(sdb["children"]) == 2 + assert sdb["children"][0]["name"] == "sdb1" + assert sdb["children"][1]["name"] == "sdb2" + if "mountpoint" in sdb["children"][0]: + assert sdb["children"][0]["mountpoint"] == "/mnt1" + assert sdb["children"][1]["mountpoint"] == "/mnt2" + else: + assert sdb["children"][0]["mountpoints"] == ["/mnt1"] + assert sdb["children"][1]["mountpoints"] == ["/mnt2"] + + # Not bionic because the LXD agent gets in the way of us + # changing the userdata + @pytest.mark.not_bionic + def test_disk_setup_when_mounted( + self, create_disk, client: IntegrationInstance + ): + """Test lp-1920939. + + We insert an extra disk into our VM, format it to have two partitions, + modify our cloud config to mount devices before disk setup, and modify + our userdata to setup a single partition on the disk. + + This allows cloud-init to attempt disk setup on a mounted partition. + When blockdev is in use, it will fail with + "blockdev: ioctl error on BLKRRPART: Device or resource busy" along + with a warning and a traceback. When partprobe is in use, everything + should work successfully. + """ + log = client.read_from_file("/var/log/cloud-init.log") + self._verify_first_disk_setup(client, log) + + # Update our userdata and cloud.cfg to mount then perform new disk + # setup + client.write_to_file( + "/var/lib/cloud/seed/nocloud-net/user-data", + UPDATED_PARTPROBE_USERDATA, + ) + client.execute( + "sed -i 's/write-files/write-files\\n - mounts/' " + "/etc/cloud/cloud.cfg" + ) + + client.execute("cloud-init clean --logs") + client.restart() + + # Assert new setup works as expected + verify_clean_log(log) + + lsblk = json.loads(client.execute("lsblk --json")) + sdb = [x for x in lsblk["blockdevices"] if x["name"] == "sdb"][0] + assert len(sdb["children"]) == 1 + assert sdb["children"][0]["name"] == "sdb1" + if "mountpoint" in sdb["children"][0]: + assert sdb["children"][0]["mountpoint"] == "/mnt3" + else: + assert sdb["children"][0]["mountpoints"] == ["/mnt3"] + + def test_disk_setup_no_partprobe( + self, create_disk, client: IntegrationInstance + ): + """Ensure disk setup still works as expected without partprobe.""" + # We can't do this part in a bootcmd because the path has already + # been found by the time we get to the bootcmd + client.execute("rm $(which partprobe)") + client.execute("cloud-init clean --logs") + client.restart() + + log = client.read_from_file("/var/log/cloud-init.log") + self._verify_first_disk_setup(client, log) + + assert "partprobe" not in log diff --git a/tests/integration_tests/modules/test_growpart.py b/tests/integration_tests/modules/test_growpart.py new file mode 100644 index 00000000..67251817 --- /dev/null +++ b/tests/integration_tests/modules/test_growpart.py @@ -0,0 +1,68 @@ +import json +import os +import pathlib +from uuid import uuid4 + +import pytest +from pycloudlib.lxd.instance import LXDInstance + +from cloudinit.subp import subp +from tests.integration_tests.instances import IntegrationInstance + +DISK_PATH = "/tmp/test_disk_setup_{}".format(uuid4()) + + +def setup_and_mount_lxd_disk(instance: LXDInstance): + subp( + "lxc config device add {} test-disk-setup-disk disk source={}".format( + instance.name, DISK_PATH + ).split() + ) + + +@pytest.fixture(scope="class", autouse=True) +def create_disk(): + """Create 16M sparse file""" + pathlib.Path(DISK_PATH).touch() + os.truncate(DISK_PATH, 1 << 24) + yield + os.remove(DISK_PATH) + + +# Create undersized partition in bootcmd +ALIAS_USERDATA = """\ +#cloud-config +bootcmd: + - parted /dev/sdb --script \ + mklabel gpt \ + mkpart primary 0 1MiB + - parted /dev/sdb --script print +growpart: + devices: + - "/" + - "/dev/sdb1" +runcmd: + - parted /dev/sdb --script print +""" + + +@pytest.mark.user_data(ALIAS_USERDATA) +@pytest.mark.lxd_setup.with_args(setup_and_mount_lxd_disk) +@pytest.mark.ubuntu +@pytest.mark.lxd_vm +class TestGrowPart: + """Test growpart""" + + def test_grow_part(self, client: IntegrationInstance): + """Verify""" + log = client.read_from_file("/var/log/cloud-init.log") + assert ( + "cc_growpart.py[INFO]: '/dev/sdb1' resized:" + " changed (/dev/sdb, 1) from" in log + ) + + lsblk = json.loads(client.execute("lsblk --json")) + sdb = [x for x in lsblk["blockdevices"] if x["name"] == "sdb"][0] + assert len(sdb["children"]) == 1 + assert sdb["children"][0]["name"] == "sdb1" + assert sdb["size"] == "16M" diff --git a/tests/integration_tests/modules/test_hotplug.py b/tests/integration_tests/modules/test_hotplug.py new file mode 100644 index 00000000..0bad761e --- /dev/null +++ b/tests/integration_tests/modules/test_hotplug.py @@ -0,0 +1,112 @@ +import time +from collections import namedtuple + +import pytest +import yaml + +from tests.integration_tests.instances import IntegrationInstance + +USER_DATA = """\ +#cloud-config +updates: + network: + when: ['hotplug'] +""" + +ip_addr = namedtuple("ip_addr", "interface state ip4 ip6") + + +def _wait_till_hotplug_complete(client, expected_runs=1): + for _ in range(60): + log = client.read_from_file("/var/log/cloud-init.log") + if log.count("Exiting hotplug handler") == expected_runs: + return log + time.sleep(1) + raise Exception("Waiting for hotplug handler failed") + + +def _get_ip_addr(client): + ips = [] + lines = client.execute("ip --brief addr").split("\n") + for line in lines: + attributes = line.split() + interface, state = attributes[0], attributes[1] + ip4_cidr = attributes[2] if len(attributes) > 2 else None + ip6_cidr = attributes[3] if len(attributes) > 3 else None + ip4 = ip4_cidr.split("/")[0] if ip4_cidr else None + ip6 = ip6_cidr.split("/")[0] if ip6_cidr else None + ip = ip_addr(interface, state, ip4, ip6) + ips.append(ip) + return ips + + +@pytest.mark.openstack +# On Bionic, we traceback when attempting to detect the hotplugged +# device in the updated metadata. This is because Bionic is specifically +# configured not to provide network metadata. +@pytest.mark.not_bionic +@pytest.mark.user_data(USER_DATA) +def test_hotplug_add_remove(client: IntegrationInstance): + ips_before = _get_ip_addr(client) + log = client.read_from_file("/var/log/cloud-init.log") + assert "Exiting hotplug handler" not in log + assert client.execute( + "test -f /etc/udev/rules.d/10-cloud-init-hook-hotplug.rules" + ).ok + + # Add new NIC + added_ip = client.instance.add_network_interface() + _wait_till_hotplug_complete(client, expected_runs=1) + ips_after_add = _get_ip_addr(client) + new_addition = [ip for ip in ips_after_add if ip.ip4 == added_ip][0] + + assert len(ips_after_add) == len(ips_before) + 1 + assert added_ip not in [ip.ip4 for ip in ips_before] + assert added_ip in [ip.ip4 for ip in ips_after_add] + assert new_addition.state == "UP" + + netplan_cfg = client.read_from_file("/etc/netplan/50-cloud-init.yaml") + config = yaml.safe_load(netplan_cfg) + assert new_addition.interface in config["network"]["ethernets"] + + # Remove new NIC + client.instance.remove_network_interface(added_ip) + _wait_till_hotplug_complete(client, expected_runs=2) + ips_after_remove = _get_ip_addr(client) + assert len(ips_after_remove) == len(ips_before) + assert added_ip not in [ip.ip4 for ip in ips_after_remove] + + netplan_cfg = client.read_from_file("/etc/netplan/50-cloud-init.yaml") + config = yaml.safe_load(netplan_cfg) + assert new_addition.interface not in config["network"]["ethernets"] + + assert "enabled" == client.execute( + "cloud-init devel hotplug-hook -s net query" + ) + + +@pytest.mark.openstack +def test_no_hotplug_in_userdata(client: IntegrationInstance): + ips_before = _get_ip_addr(client) + log = client.read_from_file("/var/log/cloud-init.log") + assert "Exiting hotplug handler" not in log + assert client.execute( + "test -f /etc/udev/rules.d/10-cloud-init-hook-hotplug.rules" + ).failed + + # Add new NIC + client.instance.add_network_interface() + log = client.read_from_file("/var/log/cloud-init.log") + assert "hotplug-hook" not in log + + ips_after_add = _get_ip_addr(client) + if len(ips_after_add) == len(ips_before) + 1: + # We can see the device, but it should not have been brought up + new_ip = [ip for ip in ips_after_add if ip not in ips_before][0] + assert new_ip.state == "DOWN" + else: + assert len(ips_after_add) == len(ips_before) + + assert "disabled" == client.execute( + "cloud-init devel hotplug-hook -s net query" + ) diff --git a/tests/integration_tests/modules/test_jinja_templating.py b/tests/integration_tests/modules/test_jinja_templating.py new file mode 100644 index 00000000..7788c6f0 --- /dev/null +++ b/tests/integration_tests/modules/test_jinja_templating.py @@ -0,0 +1,33 @@ +# This file is part of cloud-init. See LICENSE file for license information. +import pytest + +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import verify_ordered_items_in_text + +USER_DATA = """\ +## template: jinja +#cloud-config +runcmd: + - echo {{v1.local_hostname}} > /var/tmp/runcmd_output + - echo {{merged_cfg._doc}} >> /var/tmp/runcmd_output + - echo {{v1['local-hostname']}} >> /var/tmp/runcmd_output +""" + + +@pytest.mark.user_data(USER_DATA) +def test_runcmd_with_variable_substitution(client: IntegrationInstance): + """Test jinja substitution. + + Ensure underscore-delimited aliases exist for hyphenated key and + we can also substitute variables from instance-data-sensitive + LP: #1931392. + """ + hostname = client.execute("hostname").stdout.strip() + expected = [ + hostname, + "Merged cloud-init system config from /etc/cloud/cloud.cfg and " + "/etc/cloud/cloud.cfg.d/", + hostname, + ] + output = client.read_from_file("/var/tmp/runcmd_output") + verify_ordered_items_in_text(expected, output) diff --git a/tests/integration_tests/modules/test_keyboard.py b/tests/integration_tests/modules/test_keyboard.py new file mode 100644 index 00000000..7db35014 --- /dev/null +++ b/tests/integration_tests/modules/test_keyboard.py @@ -0,0 +1,17 @@ +import pytest + +USER_DATA = """\ +#cloud-config +keyboard: + layout: de + model: pc105 + variant: nodeadkeys + options: compose:rwin +""" + + +class TestKeyboard: + @pytest.mark.user_data(USER_DATA) + def test_keyboard(self, client): + lc = client.execute("localectl") + assert "X11 Layout: de" in lc diff --git a/tests/integration_tests/modules/test_keys_to_console.py b/tests/integration_tests/modules/test_keys_to_console.py new file mode 100644 index 00000000..50899982 --- /dev/null +++ b/tests/integration_tests/modules/test_keys_to_console.py @@ -0,0 +1,113 @@ +"""Integration tests for the cc_keys_to_console module. + +(This is ported from +``tests/cloud_tests/testcases/modules/keys_to_console.yaml``.)""" +import pytest + +from tests.integration_tests.util import retry + +BLACKLIST_USER_DATA = """\ +#cloud-config +ssh_fp_console_blacklist: [ssh-dss, ssh-dsa, ecdsa-sha2-nistp256] +ssh_key_console_blacklist: [ssh-dss, ssh-dsa, ecdsa-sha2-nistp256] +""" + +BLACKLIST_ALL_KEYS_USER_DATA = """\ +#cloud-config +ssh_fp_console_blacklist: [ssh-dsa, ssh-ecdsa, ssh-ed25519, ssh-rsa, ssh-dss, ecdsa-sha2-nistp256] +""" # noqa: E501 + +DISABLED_USER_DATA = """\ +#cloud-config +ssh: + emit_keys_to_console: false +""" + +ENABLE_KEYS_TO_CONSOLE_USER_DATA = """\ +#cloud-config +ssh: + emit_keys_to_console: true +users: + - default + - name: barfoo +""" + + +@pytest.mark.user_data(BLACKLIST_USER_DATA) +class TestKeysToConsoleBlacklist: + """Test that the blacklist options work as expected.""" + + @pytest.mark.parametrize("key_type", ["DSA", "ECDSA"]) + def test_excluded_keys(self, class_client, key_type): + syslog = class_client.read_from_file("/var/log/syslog") + assert "({})".format(key_type) not in syslog + + # retry decorator here because it can take some time to be reflected + # in syslog + @retry(tries=30, delay=1) + @pytest.mark.parametrize("key_type", ["ED25519", "RSA"]) + def test_included_keys(self, class_client, key_type): + syslog = class_client.read_from_file("/var/log/syslog") + assert "({})".format(key_type) in syslog + + +@pytest.mark.user_data(BLACKLIST_ALL_KEYS_USER_DATA) +class TestAllKeysToConsoleBlacklist: + """Test that when key blacklist contains all key types that + no header/footer are output. + """ + + def test_header_excluded(self, class_client): + syslog = class_client.read_from_file("/var/log/syslog") + assert "BEGIN SSH HOST KEY FINGERPRINTS" not in syslog + + def test_footer_excluded(self, class_client): + syslog = class_client.read_from_file("/var/log/syslog") + assert "END SSH HOST KEY FINGERPRINTS" not in syslog + + +@pytest.mark.user_data(DISABLED_USER_DATA) +class TestKeysToConsoleDisabled: + """Test that output can be fully disabled.""" + + @pytest.mark.parametrize("key_type", ["DSA", "ECDSA", "ED25519", "RSA"]) + def test_keys_excluded(self, class_client, key_type): + syslog = class_client.read_from_file("/var/log/syslog") + assert "({})".format(key_type) not in syslog + + def test_header_excluded(self, class_client): + syslog = class_client.read_from_file("/var/log/syslog") + assert "BEGIN SSH HOST KEY FINGERPRINTS" not in syslog + + def test_footer_excluded(self, class_client): + syslog = class_client.read_from_file("/var/log/syslog") + assert "END SSH HOST KEY FINGERPRINTS" not in syslog + + +@pytest.mark.user_data(ENABLE_KEYS_TO_CONSOLE_USER_DATA) +@pytest.mark.ec2 +@pytest.mark.lxd_container +@pytest.mark.oci +@pytest.mark.openstack +class TestKeysToConsoleEnabled: + """Test that output can be enabled disabled.""" + + def test_duplicate_messaging_console_log(self, class_client): + class_client.execute("cloud-init status --wait --long").ok + try: + console_log = class_client.instance.console_log() + except NotImplementedError: + # Assume that an exception here means that we can't use the console + # log + pytest.skip("NotImplementedError when requesting console log") + return + if console_log.lower() == "no console output": + # This test retries because we might not have the full console log + # on the first fetch. However, if we have no console output + # at all, we don't want to keep retrying as that would trigger + # another 5 minute wait on the pycloudlib side, which could + # leave us waiting for a couple hours + pytest.fail("no console output") + return + msg = "no authorized SSH keys fingerprints found for user barfoo." + assert 1 == console_log.count(msg) diff --git a/tests/integration_tests/modules/test_lxd_bridge.py b/tests/integration_tests/modules/test_lxd_bridge.py new file mode 100644 index 00000000..3292a833 --- /dev/null +++ b/tests/integration_tests/modules/test_lxd_bridge.py @@ -0,0 +1,46 @@ +"""Integration tests for LXD bridge creation. + +(This is ported from +``tests/cloud_tests/testcases/modules/lxd_bridge.yaml``.) +""" +import pytest +import yaml + +from tests.integration_tests.util import verify_clean_log + +USER_DATA = """\ +#cloud-config +lxd: + init: + storage_backend: dir + bridge: + mode: new + name: lxdbr0 + ipv4_address: 10.100.100.1 + ipv4_netmask: 24 + ipv4_dhcp_first: 10.100.100.100 + ipv4_dhcp_last: 10.100.100.200 + ipv4_nat: true + domain: lxd +""" + + +@pytest.mark.no_container +@pytest.mark.user_data(USER_DATA) +class TestLxdBridge: + @pytest.mark.parametrize("binary_name", ["lxc", "lxd"]) + def test_binaries_installed(self, class_client, binary_name): + """Check that the expected LXD binaries are installed""" + assert class_client.execute(["which", binary_name]).ok + + def test_bridge(self, class_client): + """Check that the given bridge is configured""" + cloud_init_log = class_client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(cloud_init_log) + + # The bridge should exist + assert class_client.execute("ip addr show lxdbr0") + + raw_network_config = class_client.execute("lxc network show lxdbr0") + network_config = yaml.safe_load(raw_network_config) + assert "10.100.100.1/24" == network_config["config"]["ipv4.address"] diff --git a/tests/integration_tests/modules/test_ntp_servers.py b/tests/integration_tests/modules/test_ntp_servers.py index e72389c1..fc62e63b 100644 --- a/tests/integration_tests/modules/test_ntp_servers.py +++ b/tests/integration_tests/modules/test_ntp_servers.py @@ -1,14 +1,18 @@ -"""Integration test for the ntp module's ``servers`` functionality with ntp. +"""Integration test for the ntp module's ntp functionality. This test specifies the use of the `ntp` NTP client, and ensures that the given NTP servers are configured as expected. -(This is ported from ``tests/cloud_tests/testcases/modules/ntp_servers.yaml``.) +(This is ported from ``tests/cloud_tests/testcases/modules/ntp_servers.yaml``, +``tests/cloud_tests/testcases/modules/ntp_pools.yaml``, +and ``tests/cloud_tests/testcases/modules/ntp_chrony.yaml``) """ import re -import yaml import pytest +import yaml + +from tests.integration_tests.instances import IntegrationInstance USER_DATA = """\ #cloud-config @@ -17,21 +21,25 @@ ntp: servers: - 172.16.15.14 - 172.16.17.18 + pools: + - 0.cloud-init.mypool + - 1.cloud-init.mypool + - 172.16.15.15 """ EXPECTED_SERVERS = yaml.safe_load(USER_DATA)["ntp"]["servers"] +EXPECTED_POOLS = yaml.safe_load(USER_DATA)["ntp"]["pools"] -@pytest.mark.ci @pytest.mark.user_data(USER_DATA) class TestNtpServers: - - def test_ntp_installed(self, class_client): + def test_ntp_installed(self, class_client: IntegrationInstance): """Test that `ntpd --version` succeeds, indicating installation.""" - result = class_client.execute("ntpd --version") - assert 0 == result.return_code + assert class_client.execute("ntpd --version").ok - def test_dist_config_file_is_empty(self, class_client): + def test_dist_config_file_is_empty( + self, class_client: IntegrationInstance + ): """Test that the distributed config file is empty. (This test is skipped on all currently supported Ubuntu releases, so @@ -42,17 +50,79 @@ class TestNtpServers: dist_file = class_client.read_from_file("/etc/ntp.conf.dist") assert 0 == len(dist_file.strip().splitlines()) - def test_ntp_entries(self, class_client): + def test_ntp_entries(self, class_client: IntegrationInstance): ntp_conf = class_client.read_from_file("/etc/ntp.conf") for expected_server in EXPECTED_SERVERS: assert re.search( r"^server {} iburst".format(expected_server), ntp_conf, - re.MULTILINE + re.MULTILINE, + ) + for expected_pool in EXPECTED_POOLS: + assert re.search( + r"^pool {} iburst".format(expected_pool), + ntp_conf, + re.MULTILINE, ) - def test_ntpq_servers(self, class_client): + def test_ntpq_servers(self, class_client: IntegrationInstance): result = class_client.execute("ntpq -p -w -n") assert result.ok - for expected_server in EXPECTED_SERVERS: - assert expected_server in result.stdout + for expected_server_or_pool in [*EXPECTED_SERVERS, *EXPECTED_POOLS]: + assert expected_server_or_pool in result.stdout + + +CHRONY_DATA = """\ +#cloud-config +ntp: + enabled: true + ntp_client: chrony + servers: + - 172.16.15.14 +""" + + +@pytest.mark.user_data(CHRONY_DATA) +def test_chrony(client: IntegrationInstance): + if client.execute("test -f /etc/chrony.conf").ok: + chrony_conf = "/etc/chrony.conf" + else: + chrony_conf = "/etc/chrony/chrony.conf" + contents = client.read_from_file(chrony_conf) + assert "server 172.16.15.14" in contents + + +TIMESYNCD_DATA = """\ +#cloud-config +ntp: + enabled: true + ntp_client: systemd-timesyncd + servers: + - 172.16.15.14 +""" + + +@pytest.mark.user_data(TIMESYNCD_DATA) +def test_timesyncd(client: IntegrationInstance): + contents = client.read_from_file( + "/etc/systemd/timesyncd.conf.d/cloud-init.conf" + ) + assert "NTP=172.16.15.14" in contents + + +EMPTY_NTP = """\ +#cloud-config +ntp: + ntp_client: ntp + pools: [] + servers: [] +""" + + +@pytest.mark.user_data(EMPTY_NTP) +def test_empty_ntp(client: IntegrationInstance): + assert client.execute("ntpd --version").ok + assert client.execute("test -f /etc/ntp.conf.dist").failed + assert "pool.ntp.org iburst" in client.execute( + 'grep -v "^#" /etc/ntp.conf' + ) diff --git a/tests/integration_tests/modules/test_package_update_upgrade_install.py b/tests/integration_tests/modules/test_package_update_upgrade_install.py index 8a38ad84..d668d81c 100644 --- a/tests/integration_tests/modules/test_package_update_upgrade_install.py +++ b/tests/integration_tests/modules/test_package_update_upgrade_install.py @@ -13,8 +13,8 @@ NOTE: the testcase for this looks for the command in history.log as """ import re -import pytest +import pytest USER_DATA = """\ #cloud-config @@ -26,9 +26,9 @@ package_upgrade: true """ +@pytest.mark.ubuntu @pytest.mark.user_data(USER_DATA) class TestPackageUpdateUpgradeInstall: - def assert_package_installed(self, pkg_out, name, version=None): """Check dpkg-query --show output for matching package name. @@ -37,7 +37,8 @@ class TestPackageUpdateUpgradeInstall: version. """ pkg_match = re.search( - "^%s\t(?P<version>.*)$" % name, pkg_out, re.MULTILINE) + "^%s\t(?P<version>.*)$" % name, pkg_out, re.MULTILINE + ) if pkg_match: installed_version = pkg_match.group("version") if not version: @@ -45,8 +46,10 @@ class TestPackageUpdateUpgradeInstall: if installed_version.startswith(version): return # Success raise AssertionError( - "Expected package version %s-%s not found. Found %s" % - name, version, installed_version) + "Expected package version %s-%s not found. Found %s" % name, + version, + installed_version, + ) raise AssertionError("Package not installed: %s" % name) def test_new_packages_are_installed(self, class_client): @@ -57,11 +60,13 @@ class TestPackageUpdateUpgradeInstall: def test_packages_were_updated(self, class_client): out = class_client.execute( - "grep ^Commandline: /var/log/apt/history.log") + "grep ^Commandline: /var/log/apt/history.log" + ) assert ( "Commandline: /usr/bin/apt-get --option=Dpkg::Options" "::=--force-confold --option=Dpkg::options::=--force-unsafe-io " - "--assume-yes --quiet install sl tree") in out + "--assume-yes --quiet install sl tree" in out + ) def test_packages_were_upgraded(self, class_client): """Test cloud-init-output for install & upgrade stuff.""" diff --git a/tests/integration_tests/modules/test_persistence.py b/tests/integration_tests/modules/test_persistence.py new file mode 100644 index 00000000..33527e1e --- /dev/null +++ b/tests/integration_tests/modules/test_persistence.py @@ -0,0 +1,32 @@ +# This file is part of cloud-init. See LICENSE file for license information. +"""Test the behavior of loading/discarding pickle data""" +from pathlib import Path + +import pytest + +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import ( + ASSETS_DIR, + verify_ordered_items_in_text, +) + +PICKLE_PATH = Path("/var/lib/cloud/instance/obj.pkl") +TEST_PICKLE = ASSETS_DIR / "trusty_with_mime.pkl" + + +@pytest.mark.lxd_container +def test_log_message_on_missing_version_file(client: IntegrationInstance): + client.push_file(TEST_PICKLE, PICKLE_PATH) + client.restart() + assert client.execute("cloud-init status --wait").ok + log = client.read_from_file("/var/log/cloud-init.log") + verify_ordered_items_in_text( + [ + "Unable to unpickle datasource: 'MIMEMultipart' object has no " + "attribute 'policy'. Ignoring current cache.", + "no cache found", + "Searching for local data source", + "SUCCESS: found local data from DataSourceNoCloud", + ], + log, + ) diff --git a/tests/integration_tests/modules/test_power_state_change.py b/tests/integration_tests/modules/test_power_state_change.py new file mode 100644 index 00000000..5cd19764 --- /dev/null +++ b/tests/integration_tests/modules/test_power_state_change.py @@ -0,0 +1,97 @@ +"""Integration test of the cc_power_state_change module. + +Test that the power state config options work as expected. +""" + +import time + +import pytest + +from tests.integration_tests.clouds import IntegrationCloud +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import verify_ordered_items_in_text + +USER_DATA = """\ +#cloud-config +power_state: + delay: {delay} + mode: {mode} + message: msg + timeout: {timeout} + condition: {condition} +""" + + +def _detect_reboot(instance: IntegrationInstance): + # We'll wait for instance up here, but we don't know if we're + # detecting the first boot or second boot, so we also check + # the logs to ensure we've booted twice. If the logs show we've + # only booted once, wait until we've booted twice + instance.instance.wait() + for _ in range(600): + try: + log = instance.read_from_file("/var/log/cloud-init.log") + boot_count = log.count("running 'init-local'") + if boot_count == 1: + instance.instance.wait() + elif boot_count > 1: + break + except Exception: + pass + time.sleep(1) + else: + raise Exception("Could not detect reboot") + + +def _can_connect(instance): + return instance.execute("true").ok + + +# This test is marked unstable because even though it should be able to +# run anywhere, I can only get it to run in an lxd container, and even then +# occasionally some timing issues will crop up. +@pytest.mark.unstable +@pytest.mark.ubuntu +@pytest.mark.lxd_container +class TestPowerChange: + @pytest.mark.parametrize( + "mode,delay,timeout,expected", + [ + ("poweroff", "now", "10", "will execute: shutdown -P now msg"), + ("reboot", "now", "0", "will execute: shutdown -r now msg"), + ("halt", "+1", "0", "will execute: shutdown -H +1 msg"), + ], + ) + def test_poweroff( + self, session_cloud: IntegrationCloud, mode, delay, timeout, expected + ): + with session_cloud.launch( + user_data=USER_DATA.format( + delay=delay, mode=mode, timeout=timeout, condition="true" + ), + launch_kwargs={"wait": False}, + ) as instance: + if mode == "reboot": + _detect_reboot(instance) + else: + instance.instance.wait_for_stop() + instance.instance.start(wait=True) + log = instance.read_from_file("/var/log/cloud-init.log") + assert _can_connect(instance) + lines_to_check = [ + "Running module power-state-change", + expected, + "running 'init-local'", + "config-power-state-change already ran", + ] + verify_ordered_items_in_text(lines_to_check, log) + + @pytest.mark.user_data( + USER_DATA.format( + delay="0", mode="poweroff", timeout="0", condition="false" + ) + ) + def test_poweroff_false_condition(self, client: IntegrationInstance): + log = client.read_from_file("/var/log/cloud-init.log") + assert _can_connect(client) + assert "Condition was false. Will not perform state change" in log diff --git a/tests/integration_tests/modules/test_puppet.py b/tests/integration_tests/modules/test_puppet.py new file mode 100644 index 00000000..1bd9cee4 --- /dev/null +++ b/tests/integration_tests/modules/test_puppet.py @@ -0,0 +1,39 @@ +"""Test installation configuration of puppet module.""" +import pytest + +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import verify_clean_log + +SERVICE_DATA = """\ +#cloud-config +puppet: + install: true + install_type: packages +""" + + +@pytest.mark.user_data(SERVICE_DATA) +def test_puppet_service(client: IntegrationInstance): + """Basic test that puppet gets installed and runs.""" + log = client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log) + assert client.execute("systemctl is-active puppet").ok + assert "Running command ['puppet', 'agent'" not in log + + +EXEC_DATA = """\ +#cloud-config +puppet: + install: true + install_type: packages + exec: true + exec_args: ['--noop'] +""" + + +@pytest.mark.user_data +@pytest.mark.user_data(EXEC_DATA) +def test_pupet_exec(client: IntegrationInstance): + """Basic test that puppet gets installed and runs.""" + log = client.read_from_file("/var/log/cloud-init.log") + assert "Running command ['puppet', 'agent', '--noop']" in log diff --git a/tests/integration_tests/modules/test_runcmd.py b/tests/integration_tests/modules/test_runcmd.py deleted file mode 100644 index 50d1851e..00000000 --- a/tests/integration_tests/modules/test_runcmd.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Integration test for the runcmd module. - -This test specifies a command to be executed by the ``runcmd`` module -and then checks if that command was executed during boot. - -(This is ported from -``tests/cloud_tests/testcases/modules/runcmd.yaml``.)""" - -import pytest - - -USER_DATA = """\ -#cloud-config -runcmd: - - echo cloud-init run cmd test > /var/tmp/run_cmd -""" - - -@pytest.mark.ci -class TestRuncmd: - - @pytest.mark.user_data(USER_DATA) - def test_runcmd(self, client): - runcmd_output = client.read_from_file("/var/tmp/run_cmd") - assert runcmd_output.strip() == "cloud-init run cmd test" diff --git a/tests/integration_tests/modules/test_seed_random_data.py b/tests/integration_tests/modules/test_seed_random_data.py deleted file mode 100644 index b365fa98..00000000 --- a/tests/integration_tests/modules/test_seed_random_data.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Integration test for the random seed module. - -This test specifies a command to be executed by the ``seed_random`` module, by -providing a different data to be used as seed data. We will then check -if that seed data was actually used. - -(This is ported from -``tests/cloud_tests/testcases/modules/seed_random_data.yaml``.)""" - -import pytest - - -USER_DATA = """\ -#cloud-config -random_seed: - data: 'MYUb34023nD:LFDK10913jk;dfnk:Df' - encoding: raw - file: /root/seed -""" - - -@pytest.mark.ci -class TestSeedRandomData: - - @pytest.mark.user_data(USER_DATA) - def test_seed_random_data(self, client): - seed_output = client.read_from_file("/root/seed") - assert seed_output.strip() == "MYUb34023nD:LFDK10913jk;dfnk:Df" diff --git a/tests/integration_tests/modules/test_set_hostname.py b/tests/integration_tests/modules/test_set_hostname.py index 2bfa403d..ae0aeae9 100644 --- a/tests/integration_tests/modules/test_set_hostname.py +++ b/tests/integration_tests/modules/test_set_hostname.py @@ -11,7 +11,6 @@ after the system is boot. import pytest - USER_DATA_HOSTNAME = """\ #cloud-config hostname: cloudinit2 @@ -24,15 +23,31 @@ hostname: cloudinit1 fqdn: cloudinit2.i9n.cloud-init.io """ +USER_DATA_PREFER_FQDN = """\ +#cloud-config +prefer_fqdn_over_hostname: {} +hostname: cloudinit1 +fqdn: cloudinit2.test.io +""" + @pytest.mark.ci class TestHostname: - @pytest.mark.user_data(USER_DATA_HOSTNAME) def test_hostname(self, client): hostname_output = client.execute("hostname") assert "cloudinit2" in hostname_output.strip() + @pytest.mark.user_data(USER_DATA_PREFER_FQDN.format(True)) + def test_prefer_fqdn(self, client): + hostname_output = client.execute("hostname") + assert "cloudinit2.test.io" in hostname_output.strip() + + @pytest.mark.user_data(USER_DATA_PREFER_FQDN.format(False)) + def test_prefer_short_hostname(self, client): + hostname_output = client.execute("hostname") + assert "cloudinit1" in hostname_output.strip() + @pytest.mark.user_data(USER_DATA_FQDN) def test_hostname_and_fqdn(self, client): hostname_output = client.execute("hostname") @@ -42,6 +57,8 @@ class TestHostname: assert "cloudinit2.i9n.cloud-init.io" in fqdn_output.strip() host_output = client.execute("grep ^127 /etc/hosts") - assert '127.0.1.1 {} {}'.format( - fqdn_output, hostname_output) in host_output - assert '127.0.0.1 localhost' in host_output + assert ( + "127.0.1.1 {} {}".format(fqdn_output, hostname_output) + in host_output + ) + assert "127.0.0.1 localhost" in host_output diff --git a/tests/integration_tests/modules/test_set_password.py b/tests/integration_tests/modules/test_set_password.py index b13f76fb..0e35cd26 100644 --- a/tests/integration_tests/modules/test_set_password.py +++ b/tests/integration_tests/modules/test_set_password.py @@ -8,11 +8,10 @@ other tests chpasswd's list being a string. Both expect the same results, so they use a mixin to share their test definitions, because we can (of course) only specify one user-data per instance. """ -import crypt - import pytest import yaml +from tests.integration_tests.util import retry COMMON_USER_DATA = """\ #cloud-config @@ -40,7 +39,9 @@ Uh69tP4GSrGW5XKHxMLiKowJgm/" lock_passwd: false """ -LIST_USER_DATA = COMMON_USER_DATA + """ +LIST_USER_DATA = ( + COMMON_USER_DATA + + """ chpasswd: list: - tom:mypassword123! @@ -48,8 +49,11 @@ chpasswd: - harry:RANDOM - mikey:$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89 """ +) -STRING_USER_DATA = COMMON_USER_DATA + """ +STRING_USER_DATA = ( + COMMON_USER_DATA + + """ chpasswd: list: | tom:mypassword123! @@ -57,6 +61,7 @@ chpasswd: harry:RANDOM mikey:$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89 """ +) USERS_DICTS = yaml.safe_load(COMMON_USER_DATA)["users"] USERS_PASSWD_VALUES = { @@ -116,14 +121,52 @@ class Mixin: # Which are not the same assert shadow_users["harry"] != shadow_users["dick"] + def test_random_passwords_not_stored_in_cloud_init_output_log( + self, class_client + ): + """We should not emit passwords to the in-instance log file. + + LP: #1918303 + """ + cloud_init_output = class_client.read_from_file( + "/var/log/cloud-init-output.log" + ) + assert "dick:" not in cloud_init_output + assert "harry:" not in cloud_init_output + + @retry(tries=30, delay=1) + def test_random_passwords_emitted_to_serial_console(self, class_client): + """We should emit passwords to the serial console. (LP: #1918303)""" + try: + console_log = class_client.instance.console_log() + except NotImplementedError: + # Assume that an exception here means that we can't use the console + # log + pytest.skip("NotImplementedError when requesting console log") + return + if console_log.lower() == "no console output": + # This test retries because we might not have the full console log + # on the first fetch. However, if we have no console output + # at all, we don't want to keep retrying as that would trigger + # another 5 minute wait on the pycloudlib side, which could + # leave us waiting for a couple hours + pytest.fail("no console output") + return + assert "dick:" in console_log + assert "harry:" in console_log + def test_explicit_password_set_correctly(self, class_client): """Test that an explicitly-specified password is set correctly.""" shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client) fmt_and_salt = shadow_users["tom"].rsplit("$", 1)[0] - expected_value = crypt.crypt("mypassword123!", fmt_and_salt) - - assert expected_value == shadow_users["tom"] + GEN_CRYPT_CONTENT = ( + "import crypt\n" + f"print(crypt.crypt('mypassword123!', '{fmt_and_salt}'))\n" + ) + class_client.write_to_file("/gen_crypt.py", GEN_CRYPT_CONTENT) + result = class_client.execute("python3 /gen_crypt.py") + assert result.stdout == shadow_users["tom"] def test_shadow_expected_users(self, class_client): """Test that the right set of users is in /etc/shadow.""" diff --git a/tests/integration_tests/modules/test_snap.py b/tests/integration_tests/modules/test_snap.py deleted file mode 100644 index b626f6b0..00000000 --- a/tests/integration_tests/modules/test_snap.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Integration test for the snap module. - -This test specifies a command to be executed by the ``snap`` module -and then checks that if that command was executed during boot. - -(This is ported from -``tests/cloud_tests/testcases/modules/runcmd.yaml``.)""" - -import pytest - - -USER_DATA = """\ -#cloud-config -package_update: true -snap: - squashfuse_in_container: true - commands: - - snap install hello-world -""" - - -@pytest.mark.ci -class TestSnap: - - @pytest.mark.user_data(USER_DATA) - def test_snap(self, client): - snap_output = client.execute("snap list") - assert "core " in snap_output - assert "hello-world " in snap_output diff --git a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py index b9b0d85e..89b49576 100644 --- a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py +++ b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py @@ -12,13 +12,14 @@ import re import pytest +from tests.integration_tests.util import retry USER_DATA_SSH_AUTHKEY_DISABLE = """\ #cloud-config no_ssh_fingerprints: true """ -USER_DATA_SSH_AUTHKEY_ENABLE="""\ +USER_DATA_SSH_AUTHKEY_ENABLE = """\ #cloud-config ssh_genkeytypes: - ecdsa @@ -30,19 +31,22 @@ ssh_authorized_keys: @pytest.mark.ci class TestSshAuthkeyFingerprints: - @pytest.mark.user_data(USER_DATA_SSH_AUTHKEY_DISABLE) def test_ssh_authkey_fingerprints_disable(self, client): cloudinit_output = client.read_from_file("/var/log/cloud-init.log") assert ( "Skipping module named ssh-authkey-fingerprints, " - "logging of SSH fingerprints disabled") in cloudinit_output + "logging of SSH fingerprints disabled" in cloudinit_output + ) + # retry decorator here because it can take some time to be reflected + # in syslog + @retry(tries=30, delay=1) @pytest.mark.user_data(USER_DATA_SSH_AUTHKEY_ENABLE) def test_ssh_authkey_fingerprints_enable(self, client): syslog_output = client.read_from_file("/var/log/syslog") - assert re.search(r'256 SHA256:.*(ECDSA)', syslog_output) is not None - assert re.search(r'256 SHA256:.*(ED25519)', syslog_output) is not None - assert re.search(r'1024 SHA256:.*(DSA)', syslog_output) is None - assert re.search(r'2048 SHA256:.*(RSA)', syslog_output) is None + assert re.search(r"256 SHA256:.*(ECDSA)", syslog_output) is not None + assert re.search(r"256 SHA256:.*(ED25519)", syslog_output) is not None + assert re.search(r"1024 SHA256:.*(DSA)", syslog_output) is None + assert re.search(r"2048 SHA256:.*(RSA)", syslog_output) is None diff --git a/tests/integration_tests/modules/test_ssh_generate.py b/tests/integration_tests/modules/test_ssh_generate.py index 60c36982..1dd0adf1 100644 --- a/tests/integration_tests/modules/test_ssh_generate.py +++ b/tests/integration_tests/modules/test_ssh_generate.py @@ -10,7 +10,6 @@ keys were created. import pytest - USER_DATA = """\ #cloud-config ssh_genkeytypes: @@ -23,28 +22,27 @@ authkey_hash: sha512 @pytest.mark.ci @pytest.mark.user_data(USER_DATA) class TestSshKeysGenerate: - @pytest.mark.parametrize( - "ssh_key_path", ( + "ssh_key_path", + ( "/etc/ssh/ssh_host_dsa_key.pub", "/etc/ssh/ssh_host_dsa_key", "/etc/ssh/ssh_host_rsa_key.pub", "/etc/ssh/ssh_host_rsa_key", - ) + ), ) def test_ssh_keys_not_generated(self, ssh_key_path, class_client): - out = class_client.execute( - "test -e {}".format(ssh_key_path) - ) + out = class_client.execute("test -e {}".format(ssh_key_path)) assert out.failed @pytest.mark.parametrize( - "ssh_key_path", ( + "ssh_key_path", + ( "/etc/ssh/ssh_host_ecdsa_key.pub", "/etc/ssh/ssh_host_ecdsa_key", "/etc/ssh/ssh_host_ed25519_key.pub", "/etc/ssh/ssh_host_ed25519_key", - ) + ), ) def test_ssh_keys_generated(self, ssh_key_path, class_client): out = class_client.read_from_file(ssh_key_path) diff --git a/tests/integration_tests/modules/test_ssh_import_id.py b/tests/integration_tests/modules/test_ssh_import_id.py deleted file mode 100644 index 45d37d6c..00000000 --- a/tests/integration_tests/modules/test_ssh_import_id.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Integration test for the ssh_import_id module. - -This test specifies ssh keys to be imported by the ``ssh_import_id`` module -and then checks that if the ssh keys were successfully imported. - -(This is ported from -``tests/cloud_tests/testcases/modules/ssh_import_id.yaml``.)""" - -import pytest - - -USER_DATA = """\ -#cloud-config -ssh_import_id: - - gh:powersj - - lp:smoser -""" - - -@pytest.mark.ci -class TestSshImportId: - - @pytest.mark.user_data(USER_DATA) - def test_ssh_import_id(self, client): - ssh_output = client.read_from_file( - "/home/ubuntu/.ssh/authorized_keys") - - assert '# ssh-import-id gh:powersj' in ssh_output - assert '# ssh-import-id lp:smoser' in ssh_output diff --git a/tests/integration_tests/modules/test_ssh_keys_provided.py b/tests/integration_tests/modules/test_ssh_keys_provided.py index 27d193c1..b79f18eb 100644 --- a/tests/integration_tests/modules/test_ssh_keys_provided.py +++ b/tests/integration_tests/modules/test_ssh_keys_provided.py @@ -9,7 +9,6 @@ system. import pytest - USER_DATA = """\ #cloud-config disable_root: false @@ -82,67 +81,60 @@ ssh_keys: @pytest.mark.ci @pytest.mark.user_data(USER_DATA) class TestSshKeysProvided: - - def test_ssh_dsa_keys_provided(self, class_client): - """Test dsa public key was imported.""" - out = class_client.read_from_file("/etc/ssh/ssh_host_dsa_key.pub") - assert ( - "AAAAB3NzaC1kc3MAAACBAPkWy1zbchVIN7qTgM0/yyY8q4R" - "ZS8cNM4ZpeuE5UB/Nnr6OSU/nmbO8LuM") in out - - """Test dsa private key was imported.""" - out = class_client.read_from_file("/etc/ssh/ssh_host_dsa_key") - assert ( - "MIIBuwIBAAKBgQD5Fstc23IVSDe6k4DNP8smPKuEWUvHDTOGaXr" - "hOVAfzZ6+jklP") in out - - def test_ssh_rsa_keys_provided(self, class_client): - """Test rsa public key was imported.""" - out = class_client.read_from_file("/etc/ssh/ssh_host_rsa_key.pub") - assert ( - "AAAAB3NzaC1yc2EAAAADAQABAAABAQC0/Ho+o3eJISydO2JvIgT" - "LnZOtrxPl+fSvJfKDjoOLY0HB2eOjy2s2/2N6d9X9SGZ4") in out - - """Test rsa private key was imported.""" - out = class_client.read_from_file("/etc/ssh/ssh_host_rsa_key") - assert ( - "4DOkqNiUGl80Zp1RgZNohHUXlJMtAbrIlAVEk+mTmg7vjfyp2un" - "RQvLZpMRdywBm") in out - - def test_ssh_rsa_certificate_provided(self, class_client): - """Test rsa certificate was imported.""" - out = class_client.read_from_file("/etc/ssh/ssh_host_rsa_key-cert.pub") - assert ( - "AAAAHHNzaC1yc2EtY2VydC12MDFAb3BlbnNzaC5jb20AAAAgMpg" - "BP4Phn3L8I7Vqh7lmHKcOfIokEvSEbHDw83Y3JloAAAAD") in out - - def test_ssh_certificate_updated_sshd_config(self, class_client): - """Test ssh certificate was added to /etc/ssh/sshd_config.""" - out = class_client.read_from_file("/etc/ssh/sshd_config").strip() - assert "HostCertificate /etc/ssh/ssh_host_rsa_key-cert.pub" in out - - def test_ssh_ecdsa_keys_provided(self, class_client): - """Test ecdsa public key was imported.""" - out = class_client.read_from_file("/etc/ssh/ssh_host_ecdsa_key.pub") - assert ( - "AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAAB" - "BBFsS5Tvky/IC/dXhE/afxxU") in out - - """Test ecdsa private key generated.""" - out = class_client.read_from_file("/etc/ssh/ssh_host_ecdsa_key") - assert ( - "AwEHoUQDQgAEWxLlO+TL8gL91eET9p/HFQbqR1A691AkJgZk3jY" - "5mpZqxgX4vcgb") in out - - def test_ssh_ed25519_keys_provided(self, class_client): - """Test ed25519 public key was imported.""" - out = class_client.read_from_file("/etc/ssh/ssh_host_ed25519_key.pub") - assert ( - "AAAAC3NzaC1lZDI1NTE5AAAAINudAZSu4vjZpVWzId5pXmZg1M6" - "G15dqjQ2XkNVOEnb5") in out - - """Test ed25519 private key was imported.""" - out = class_client.read_from_file("/etc/ssh/ssh_host_ed25519_key") - assert ( - "XAAAAAtzc2gtZWQyNTUxOQAAACDbnQGUruL42aVVsyHeaV5mYNT" - "OhteXao0Nl5DVThJ2+Q") in out + @pytest.mark.parametrize( + "config_path,expected_out", + ( + ( + "/etc/ssh/ssh_host_dsa_key.pub", + "AAAAB3NzaC1kc3MAAACBAPkWy1zbchVIN7qTgM0/yyY8q4R" + "ZS8cNM4ZpeuE5UB/Nnr6OSU/nmbO8LuM", + ), + ( + "/etc/ssh/ssh_host_dsa_key", + "MIIBuwIBAAKBgQD5Fstc23IVSDe6k4DNP8smPKuEWUvHDTOGaXr" + "hOVAfzZ6+jklP", + ), + ( + "/etc/ssh/ssh_host_rsa_key.pub", + "AAAAB3NzaC1yc2EAAAADAQABAAABAQC0/Ho+o3eJISydO2JvIgT" + "LnZOtrxPl+fSvJfKDjoOLY0HB2eOjy2s2/2N6d9X9SGZ4", + ), + ( + "/etc/ssh/ssh_host_rsa_key", + "4DOkqNiUGl80Zp1RgZNohHUXlJMtAbrIlAVEk+mTmg7vjfyp2un" + "RQvLZpMRdywBm", + ), + ( + "/etc/ssh/ssh_host_rsa_key-cert.pub", + "AAAAHHNzaC1yc2EtY2VydC12MDFAb3BlbnNzaC5jb20AAAAgMpg" + "BP4Phn3L8I7Vqh7lmHKcOfIokEvSEbHDw83Y3JloAAAAD", + ), + ( + "/etc/ssh/sshd_config", + "HostCertificate /etc/ssh/ssh_host_rsa_key-cert.pub", + ), + ( + "/etc/ssh/ssh_host_ecdsa_key.pub", + "AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAAB" + "BBFsS5Tvky/IC/dXhE/afxxU", + ), + ( + "/etc/ssh/ssh_host_ecdsa_key", + "AwEHoUQDQgAEWxLlO+TL8gL91eET9p/HFQbqR1A691AkJgZk3jY" + "5mpZqxgX4vcgb", + ), + ( + "/etc/ssh/ssh_host_ed25519_key.pub", + "AAAAC3NzaC1lZDI1NTE5AAAAINudAZSu4vjZpVWzId5pXmZg1M6" + "G15dqjQ2XkNVOEnb5", + ), + ( + "/etc/ssh/ssh_host_ed25519_key", + "XAAAAAtzc2gtZWQyNTUxOQAAACDbnQGUruL42aVVsyHeaV5mYNT" + "OhteXao0Nl5DVThJ2+Q", + ), + ), + ) + def test_ssh_provided_keys(self, config_path, expected_out, class_client): + out = class_client.read_from_file(config_path).strip() + assert expected_out in out diff --git a/tests/integration_tests/modules/test_ssh_keysfile.py b/tests/integration_tests/modules/test_ssh_keysfile.py new file mode 100644 index 00000000..8330a1ce --- /dev/null +++ b/tests/integration_tests/modules/test_ssh_keysfile.py @@ -0,0 +1,224 @@ +from io import StringIO + +import paramiko +import pytest +from paramiko.ssh_exception import SSHException + +from tests.integration_tests.clouds import ImageSpecification +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import get_test_rsa_keypair + +TEST_USER1_KEYS = get_test_rsa_keypair("test1") +TEST_USER2_KEYS = get_test_rsa_keypair("test2") +TEST_DEFAULT_KEYS = get_test_rsa_keypair("test3") + +_USERDATA = """\ +#cloud-config +bootcmd: + - {bootcmd} +ssh_authorized_keys: + - {default} +users: +- default +- name: test_user1 + ssh_authorized_keys: + - {user1} +- name: test_user2 + ssh_authorized_keys: + - {user2} +""".format( + bootcmd="{bootcmd}", + default=TEST_DEFAULT_KEYS.public_key, + user1=TEST_USER1_KEYS.public_key, + user2=TEST_USER2_KEYS.public_key, +) + + +def common_verify(client, expected_keys): + for user, filename, keys in expected_keys: + # Ensure key is in the key file + contents = client.read_from_file(filename) + if user in ["ubuntu", "root"]: + lines = contents.split("\n") + if user == "root": + # Our personal public key gets added by pycloudlib in + # addition to the default `ssh_authorized_keys` + assert len(lines) == 2 + else: + # Clouds will insert the keys we've added to our accounts + # or for our launches + assert len(lines) >= 2 + assert keys.public_key.strip() in contents + else: + assert contents.strip() == keys.public_key.strip() + + # Ensure we can actually connect + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + paramiko_key = paramiko.RSAKey.from_private_key( + StringIO(keys.private_key) + ) + + # Will fail with AuthenticationException if + # we cannot connect + ssh.connect( + client.instance.ip, + username=user, + pkey=paramiko_key, + look_for_keys=False, + allow_agent=False, + ) + + # Ensure other uses can't connect using our key + other_users = [u[0] for u in expected_keys if u[2] != keys] + for other_user in other_users: + with pytest.raises(SSHException): + print( + "trying to connect as {} with key from {}".format( + other_user, user + ) + ) + ssh.connect( + client.instance.ip, + username=other_user, + pkey=paramiko_key, + look_for_keys=False, + allow_agent=False, + ) + + # Ensure we haven't messed with any /home permissions + # See LP: #1940233 + home_dir = "/home/{}".format(user) + # Home permissions aren't consistent between releases. On ubuntu + # this can change to 750 once focal is unsupported. + if ImageSpecification.from_os_image().release in ("bionic", "focal"): + home_perms = "755" + else: + home_perms = "750" + if user == "root": + home_dir = "/root" + home_perms = "700" + assert "{} {}".format(user, home_perms) == client.execute( + 'stat -c "%U %a" {}'.format(home_dir) + ) + if client.execute("test -d {}/.ssh".format(home_dir)).ok: + assert "{} 700".format(user) == client.execute( + 'stat -c "%U %a" {}/.ssh'.format(home_dir) + ) + assert "{} 600".format(user) == client.execute( + 'stat -c "%U %a" {}'.format(filename) + ) + + # Also ensure ssh-keygen works as expected + client.execute("mkdir {}/.ssh".format(home_dir)) + assert client.execute( + "ssh-keygen -b 2048 -t rsa -f {}/.ssh/id_rsa -q -N ''".format( + home_dir + ) + ).ok + assert client.execute("test -f {}/.ssh/id_rsa".format(home_dir)) + assert client.execute("test -f {}/.ssh/id_rsa.pub".format(home_dir)) + + assert "root 755" == client.execute('stat -c "%U %a" /home') + + +DEFAULT_KEYS_USERDATA = _USERDATA.format(bootcmd='""') + + +@pytest.mark.ubuntu +@pytest.mark.user_data(DEFAULT_KEYS_USERDATA) +def test_authorized_keys_default(client: IntegrationInstance): + expected_keys = [ + ( + "test_user1", + "/home/test_user1/.ssh/authorized_keys", + TEST_USER1_KEYS, + ), + ( + "test_user2", + "/home/test_user2/.ssh/authorized_keys", + TEST_USER2_KEYS, + ), + ("ubuntu", "/home/ubuntu/.ssh/authorized_keys", TEST_DEFAULT_KEYS), + ("root", "/root/.ssh/authorized_keys", TEST_DEFAULT_KEYS), + ] + common_verify(client, expected_keys) + + +AUTHORIZED_KEYS2_USERDATA = _USERDATA.format( + bootcmd=( + "sed -i 's;#AuthorizedKeysFile.*;AuthorizedKeysFile " + "/etc/ssh/authorized_keys %h/.ssh/authorized_keys2;' " + "/etc/ssh/sshd_config" + ) +) + + +@pytest.mark.ubuntu +@pytest.mark.user_data(AUTHORIZED_KEYS2_USERDATA) +def test_authorized_keys2(client: IntegrationInstance): + expected_keys = [ + ( + "test_user1", + "/home/test_user1/.ssh/authorized_keys2", + TEST_USER1_KEYS, + ), + ( + "test_user2", + "/home/test_user2/.ssh/authorized_keys2", + TEST_USER2_KEYS, + ), + ("ubuntu", "/home/ubuntu/.ssh/authorized_keys2", TEST_DEFAULT_KEYS), + ("root", "/root/.ssh/authorized_keys2", TEST_DEFAULT_KEYS), + ] + common_verify(client, expected_keys) + + +NESTED_KEYS_USERDATA = _USERDATA.format( + bootcmd=( + "sed -i 's;#AuthorizedKeysFile.*;AuthorizedKeysFile " + "/etc/ssh/authorized_keys %h/foo/bar/ssh/keys;' " + "/etc/ssh/sshd_config" + ) +) + + +@pytest.mark.ubuntu +@pytest.mark.user_data(NESTED_KEYS_USERDATA) +def test_nested_keys(client: IntegrationInstance): + expected_keys = [ + ("test_user1", "/home/test_user1/foo/bar/ssh/keys", TEST_USER1_KEYS), + ("test_user2", "/home/test_user2/foo/bar/ssh/keys", TEST_USER2_KEYS), + ("ubuntu", "/home/ubuntu/foo/bar/ssh/keys", TEST_DEFAULT_KEYS), + ("root", "/root/foo/bar/ssh/keys", TEST_DEFAULT_KEYS), + ] + common_verify(client, expected_keys) + + +EXTERNAL_KEYS_USERDATA = _USERDATA.format( + bootcmd=( + "sed -i 's;#AuthorizedKeysFile.*;AuthorizedKeysFile " + "/etc/ssh/authorized_keys /etc/ssh/authorized_keys/%u/keys;' " + "/etc/ssh/sshd_config" + ) +) + + +@pytest.mark.ubuntu +@pytest.mark.user_data(EXTERNAL_KEYS_USERDATA) +def test_external_keys(client: IntegrationInstance): + expected_keys = [ + ( + "test_user1", + "/etc/ssh/authorized_keys/test_user1/keys", + TEST_USER1_KEYS, + ), + ( + "test_user2", + "/etc/ssh/authorized_keys/test_user2/keys", + TEST_USER2_KEYS, + ), + ("ubuntu", "/etc/ssh/authorized_keys/ubuntu/keys", TEST_DEFAULT_KEYS), + ("root", "/etc/ssh/authorized_keys/root/keys", TEST_DEFAULT_KEYS), + ] + common_verify(client, expected_keys) diff --git a/tests/integration_tests/modules/test_timezone.py b/tests/integration_tests/modules/test_timezone.py deleted file mode 100644 index 111d53f7..00000000 --- a/tests/integration_tests/modules/test_timezone.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Integration test for the timezone module. - -This test specifies a timezone to be used by the ``timezone`` module -and then checks that if that timezone was respected during boot. - -(This is ported from -``tests/cloud_tests/testcases/modules/timezone.yaml``.)""" - -import pytest - - -USER_DATA = """\ -#cloud-config -timezone: US/Aleutian -""" - - -@pytest.mark.ci -class TestTimezone: - - @pytest.mark.user_data(USER_DATA) - def test_timezone(self, client): - timezone_output = client.execute( - 'date "+%Z" --date="Thu, 03 Nov 2016 00:47:00 -0400"') - assert timezone_output.strip() == "HDT" diff --git a/tests/integration_tests/modules/test_user_events.py b/tests/integration_tests/modules/test_user_events.py new file mode 100644 index 00000000..e4a4241f --- /dev/null +++ b/tests/integration_tests/modules/test_user_events.py @@ -0,0 +1,110 @@ +"""Test user-overridable events. + +This is currently limited to applying network config on BOOT events. +""" + +import re + +import pytest +import yaml + +from tests.integration_tests.instances import IntegrationInstance + + +def _add_dummy_bridge_to_netplan(client: IntegrationInstance): + # Update netplan configuration to ensure it doesn't change on reboot + netplan = yaml.safe_load( + client.execute("cat /etc/netplan/50-cloud-init.yaml") + ) + # Just a dummy bridge to do nothing + try: + netplan["network"]["bridges"]["dummy0"] = {"dhcp4": False} + except KeyError: + netplan["network"]["bridges"] = {"dummy0": {"dhcp4": False}} + + dumped_netplan = yaml.dump(netplan) + client.write_to_file("/etc/netplan/50-cloud-init.yaml", dumped_netplan) + + +@pytest.mark.lxd_container +@pytest.mark.lxd_vm +@pytest.mark.ec2 +@pytest.mark.gce +@pytest.mark.oci +@pytest.mark.openstack +def test_boot_event_disabled_by_default(client: IntegrationInstance): + log = client.read_from_file("/var/log/cloud-init.log") + if "network config is disabled" in log: + pytest.skip("network config disabled. Test doesn't apply") + assert "Applying network configuration" in log + assert "dummy0" not in client.execute("ls /sys/class/net") + + _add_dummy_bridge_to_netplan(client) + client.execute("rm /var/log/cloud-init.log") + + client.restart() + log2 = client.read_from_file("/var/log/cloud-init.log") + + if "cache invalid in datasource" in log2: + # Invalid cache will get cleared, meaning we'll create a new + # "instance" and apply networking config, so events aren't + # really relevant here + pytest.skip("Test only valid for existing instances") + + # We attempt to apply network config twice on every boot. + # Ensure neither time works. + assert 2 == len( + re.findall( + r"Event Denied: scopes=\['network'\] EventType=boot[^-]", log2 + ) + ) + assert 2 == log2.count( + "Event Denied: scopes=['network'] EventType=boot-legacy" + ) + assert 2 == log2.count( + "No network config applied. Neither a new instance" + " nor datasource network update allowed" + ) + + assert "dummy0" in client.execute("ls /sys/class/net") + + +def _test_network_config_applied_on_reboot(client: IntegrationInstance): + log = client.read_from_file("/var/log/cloud-init.log") + if "network config is disabled" in log: + pytest.skip("network config disabled. Test doesn't apply") + assert "Applying network configuration" in log + assert "dummy0" not in client.execute("ls /sys/class/net") + + _add_dummy_bridge_to_netplan(client) + client.execute('echo "" > /var/log/cloud-init.log') + client.restart() + + log = client.read_from_file("/var/log/cloud-init.log") + if "cache invalid in datasource" in log: + # Invalid cache will get cleared, meaning we'll create a new + # "instance" and apply networking config, so events aren't + # really relevant here + pytest.skip("Test only valid for existing instances") + + assert "Event Allowed: scope=network EventType=boot" in log + assert "Applying network configuration" in log + assert "dummy0" not in client.execute("ls /sys/class/net") + + +@pytest.mark.azure +def test_boot_event_enabled_by_default(client: IntegrationInstance): + _test_network_config_applied_on_reboot(client) + + +USER_DATA = """\ +#cloud-config +updates: + network: + when: [boot] +""" + + +@pytest.mark.user_data(USER_DATA) +def test_boot_event_enabled(client: IntegrationInstance): + _test_network_config_applied_on_reboot(client) diff --git a/tests/integration_tests/modules/test_users_groups.py b/tests/integration_tests/modules/test_users_groups.py index 6a51f5a6..fddff681 100644 --- a/tests/integration_tests/modules/test_users_groups.py +++ b/tests/integration_tests/modules/test_users_groups.py @@ -1,12 +1,15 @@ -"""Integration test for the user_groups module. +"""Integration tests for the user_groups module. -This test specifies a number of users and groups via user-data, and confirms -that they have been configured correctly in the system under test. +TODO: +* This module assumes that the "ubuntu" user will be created when "default" is + specified; this will need modification to run on other OSes. """ import re import pytest +from tests.integration_tests.clouds import ImageSpecification +from tests.integration_tests.instances import IntegrationInstance USER_DATA = """\ #cloud-config @@ -41,6 +44,13 @@ AHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ @pytest.mark.ci @pytest.mark.user_data(USER_DATA) class TestUsersGroups: + """Test users and groups. + + This test specifies a number of users and groups via user-data, and + confirms that they have been configured correctly in the system under test. + """ + + @pytest.mark.ubuntu @pytest.mark.parametrize( "getent_args,regex", [ @@ -73,7 +83,9 @@ class TestUsersGroups: assert re.search(regex, result.stdout) is not None, ( "'getent {}' resulted in '{}', " "but expected to match regex {}".format( - ' '.join(getent_args), result.stdout, regex)) + " ".join(getent_args), result.stdout, regex + ) + ) def test_user_root_in_secret(self, class_client): """Test root user is in 'secret' group.""" @@ -81,3 +93,33 @@ class TestUsersGroups: _, groups_str = output.split(":", maxsplit=1) groups = groups_str.split() assert "secret" in groups + + +@pytest.mark.user_data(USER_DATA) +def test_sudoers_includedir(client: IntegrationInstance): + """Ensure we don't add additional #includedir to sudoers. + + Newer versions of /etc/sudoers will use @includedir rather than + #includedir. Ensure we handle that properly and don't include an + additional #includedir when one isn't warranted. + + https://github.com/canonical/cloud-init/pull/783 + """ + if ImageSpecification.from_os_image().release in [ + "bionic", + "focal", + ]: + raise pytest.skip( + "Test requires version of sudo installed on groovy and later" + ) + client.execute("sed -i 's/#include/@include/g' /etc/sudoers") + + sudoers = client.read_from_file("/etc/sudoers") + if "@includedir /etc/sudoers.d" not in sudoers: + client.execute("echo '@includedir /etc/sudoers.d' >> /etc/sudoers") + client.instance.clean() + client.restart() + sudoers = client.read_from_file("/etc/sudoers") + + assert "#includedir" not in sudoers + assert sudoers.count("includedir /etc/sudoers.d") == 1 diff --git a/tests/integration_tests/modules/test_version_change.py b/tests/integration_tests/modules/test_version_change.py new file mode 100644 index 00000000..3168cd60 --- /dev/null +++ b/tests/integration_tests/modules/test_version_change.py @@ -0,0 +1,76 @@ +from pathlib import Path + +import pytest + +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import ASSETS_DIR, verify_clean_log + +PICKLE_PATH = Path("/var/lib/cloud/instance/obj.pkl") +TEST_PICKLE = ASSETS_DIR / "test_version_change.pkl" + + +def _assert_no_pickle_problems(log): + assert "Failed loading pickled blob" not in log + verify_clean_log(log) + + +def test_reboot_without_version_change(client: IntegrationInstance): + log = client.read_from_file("/var/log/cloud-init.log") + assert "Python version change detected" not in log + assert "Cache compatibility status is currently unknown." not in log + _assert_no_pickle_problems(log) + + client.restart() + log = client.read_from_file("/var/log/cloud-init.log") + assert "Python version change detected" not in log + assert "Could not determine Python version used to write cache" not in log + _assert_no_pickle_problems(log) + + # Now ensure that loading a bad pickle gives us problems + client.push_file(TEST_PICKLE, PICKLE_PATH) + client.restart() + log = client.read_from_file("/var/log/cloud-init.log") + + # no cache found is an "expected" upgrade error, and + # "Failed" means we're unable to load the pickle + assert any( + [ + "Failed loading pickled blob from {}".format(PICKLE_PATH) in log, + "no cache found" in log, + ] + ) + + +@pytest.mark.ec2 +@pytest.mark.gce +@pytest.mark.oci +@pytest.mark.openstack +@pytest.mark.lxd_container +@pytest.mark.lxd_vm +# No Azure because the cache gets purged every reboot, so we'll never +# get to the point where we need to purge cache due to version change +def test_cache_purged_on_version_change(client: IntegrationInstance): + # Start by pushing the invalid pickle so we'll hit an error if the + # cache didn't actually get purged + client.push_file(TEST_PICKLE, PICKLE_PATH) + client.execute("echo '1.0' > /var/lib/cloud/data/python-version") + client.restart() + log = client.read_from_file("/var/log/cloud-init.log") + assert "Python version change detected. Purging cache" in log + _assert_no_pickle_problems(log) + + +def test_log_message_on_missing_version_file(client: IntegrationInstance): + # Start by pushing a pickle so we can see the log message + client.push_file(TEST_PICKLE, PICKLE_PATH) + client.execute("rm /var/lib/cloud/data/python-version") + client.execute("rm /var/log/cloud-init.log") + client.restart() + log = client.read_from_file("/var/log/cloud-init.log") + if "no cache found" not in log: + # We don't expect the python version file to exist if we have no + # pre-existing cache + assert ( + "Writing python-version file. " + "Cache compatibility status is currently unknown." in log + ) diff --git a/tests/integration_tests/modules/test_write_files.py b/tests/integration_tests/modules/test_write_files.py index 15832ae3..1eb7e945 100644 --- a/tests/integration_tests/modules/test_write_files.py +++ b/tests/integration_tests/modules/test_write_files.py @@ -7,8 +7,8 @@ and then checks if those files were created during boot. ``tests/cloud_tests/testcases/modules/write_files.yaml``.)""" import base64 -import pytest +import pytest ASCII_TEXT = "ASCII text" B64_CONTENT = base64.b64encode(ASCII_TEXT.encode("utf-8")) @@ -21,6 +21,9 @@ B64_CONTENT = base64.b64encode(ASCII_TEXT.encode("utf-8")) # USER_DATA = """\ #cloud-config +users: +- default +- name: myuser write_files: - encoding: b64 content: {} @@ -41,26 +44,50 @@ write_files: H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA= path: /root/file_gzip permissions: '0755' -""".format(B64_CONTENT.decode("ascii")) +- path: '/home/testuser/my-file' + content: | + echo 'hello world!' + defer: true + owner: 'myuser' + permissions: '0644' +""".format( + B64_CONTENT.decode("ascii") +) @pytest.mark.ci @pytest.mark.user_data(USER_DATA) class TestWriteFiles: - @pytest.mark.parametrize( - "cmd,expected_out", ( + "cmd,expected_out", + ( ("file /root/file_b64", ASCII_TEXT), ("md5sum </root/file_binary", "3801184b97bb8c6e63fa0e1eae2920d7"), - ("sha256sum </root/file_binary", ( + ( + "sha256sum </root/file_binary", "2c791c4037ea5bd7e928d6a87380f8ba" - "7a803cd83d5e4f269e28f5090f0f2c9a" - )), - ("file /root/file_gzip", - "POSIX shell script, ASCII text executable"), + "7a803cd83d5e4f269e28f5090f0f2c9a", + ), + ( + "file /root/file_gzip", + "POSIX shell script, ASCII text executable", + ), ("file /root/file_text", ASCII_TEXT), - ) + ), ) def test_write_files(self, cmd, expected_out, class_client): out = class_client.execute(cmd) assert expected_out in out + + def test_write_files_deferred(self, class_client): + """Test that write files deferred works as expected. + + Users get created after write_files module runs, so ensure that + with `defer: true`, the file gets written with correct ownership. + """ + out = class_client.read_from_file("/home/testuser/my-file") + assert "echo 'hello world!'" == out + assert ( + class_client.execute('stat -c "%U %a" /home/testuser/my-file') + == "myuser 644" + ) diff --git a/tests/integration_tests/network/test_net_config_load.py b/tests/integration_tests/network/test_net_config_load.py new file mode 100644 index 00000000..a6863b63 --- /dev/null +++ b/tests/integration_tests/network/test_net_config_load.py @@ -0,0 +1,27 @@ +"""Test loading the network config""" +import pytest + +from tests.integration_tests.instances import IntegrationInstance + + +def _customize_envionment(client: IntegrationInstance): + # Insert our "disable_network_config" file here + client.write_to_file( + "/etc/cloud/cloud.cfg.d/99-disable-network-config.cfg", + "network: {config: disabled}\n", + ) + client.execute("cloud-init clean --logs") + client.restart() + + +def test_network_disabled_via_etc_cloud(client: IntegrationInstance): + """Test that network can be disabled via config file in /etc/cloud""" + if client.settings.CLOUD_INIT_SOURCE == "IN_PLACE": + pytest.skip( + "IN_PLACE not supported as we mount /etc/cloud contents into the " + "container" + ) + _customize_envionment(client) + + log = client.read_from_file("/var/log/cloud-init.log") + assert "network config is disabled by system_cfg" in log diff --git a/tests/integration_tests/test_logging.py b/tests/integration_tests/test_logging.py new file mode 100644 index 00000000..b31a0434 --- /dev/null +++ b/tests/integration_tests/test_logging.py @@ -0,0 +1,22 @@ +"""Integration tests relating to cloud-init's logging.""" + + +class TestVarLogCloudInitOutput: + """Integration tests relating to /var/log/cloud-init-output.log.""" + + def test_var_log_cloud_init_output_not_world_readable(self, client): + """ + The log can contain sensitive data, it shouldn't be world-readable. + + LP: #1918303 + """ + # Check the file exists + assert client.execute("test -f /var/log/cloud-init-output.log").ok + + # Check its permissions are as we expect + perms, user, group = client.execute( + "stat -c %a:%U:%G /var/log/cloud-init-output.log" + ).split(":") + assert "640" == perms + assert "root" == user + assert "adm" == group diff --git a/tests/integration_tests/test_shell_script_by_frequency.py b/tests/integration_tests/test_shell_script_by_frequency.py new file mode 100644 index 00000000..25157722 --- /dev/null +++ b/tests/integration_tests/test_shell_script_by_frequency.py @@ -0,0 +1,48 @@ +"""Integration tests for various handlers.""" + +from io import StringIO + +import pytest + +from cloudinit.cmd.devel.make_mime import create_mime_message +from tests.integration_tests.instances import IntegrationInstance + +PER_FREQ_TEMPLATE = """\ +#!/bin/bash +touch /tmp/test_per_freq_{} +""" + +PER_ALWAYS_FILE = StringIO(PER_FREQ_TEMPLATE.format("always")) +PER_INSTANCE_FILE = StringIO(PER_FREQ_TEMPLATE.format("instance")) +PER_ONCE_FILE = StringIO(PER_FREQ_TEMPLATE.format("once")) + +FILES = [ + (PER_ALWAYS_FILE, "always.sh", "x-shellscript-per-boot"), + (PER_INSTANCE_FILE, "instance.sh", "x-shellscript-per-instance"), + (PER_ONCE_FILE, "once.sh", "x-shellscript-per-once"), +] + +USER_DATA, errors = create_mime_message(FILES) + + +@pytest.mark.ci +@pytest.mark.user_data(USER_DATA) +def test_per_freq(client: IntegrationInstance): + # Sanity test for scripts folder + cmd = "test -d /var/lib/cloud/scripts" + assert client.execute(cmd).ok + # Test per-boot + cmd = "test -f /var/lib/cloud/scripts/per-boot/always.sh" + assert client.execute(cmd).ok + cmd = "test -f /tmp/test_per_freq_always" + assert client.execute(cmd).ok + # Test per-instance + cmd = "test -f /var/lib/cloud/scripts/per-instance/instance.sh" + assert client.execute(cmd).ok + cmd = "test -f /tmp/test_per_freq_instance" + assert client.execute(cmd).ok + # Test per-once + cmd = "test -f /var/lib/cloud/scripts/per-once/once.sh" + assert client.execute(cmd).ok + cmd = "test -f /tmp/test_per_freq_once" + assert client.execute(cmd).ok diff --git a/tests/integration_tests/test_upgrade.py b/tests/integration_tests/test_upgrade.py new file mode 100644 index 00000000..b13d4703 --- /dev/null +++ b/tests/integration_tests/test_upgrade.py @@ -0,0 +1,188 @@ +import json +import logging +import os + +import pytest + +from tests.integration_tests.clouds import ImageSpecification, IntegrationCloud +from tests.integration_tests.conftest import get_validated_source +from tests.integration_tests.util import verify_clean_log + +LOG = logging.getLogger("integration_testing.test_upgrade") + +LOG_TEMPLATE = """\n\ +=== `systemd-analyze` before: +{pre_systemd_analyze} +=== `systemd-analyze` after: +{post_systemd_analyze} + +=== `systemd-analyze blame` before (first 10 lines): +{pre_systemd_blame} +=== `systemd-analyze blame` after (first 10 lines): +{post_systemd_blame} + +=== `cloud-init analyze show` before:') +{pre_analyze_totals} +=== `cloud-init analyze show` after:') +{post_analyze_totals} + +=== `cloud-init analyze blame` before (first 10 lines): ') +{pre_cloud_blame} +=== `cloud-init analyze blame` after (first 10 lines): ') +{post_cloud_blame} +""" + +UNSUPPORTED_INSTALL_METHOD_MSG = ( + "Install method '{}' not supported for this test" +) +USER_DATA = """\ +#cloud-config +hostname: SRU-worked +""" + + +def test_clean_boot_of_upgraded_package(session_cloud: IntegrationCloud): + source = get_validated_source(session_cloud) + if not source.installs_new_version(): + pytest.skip(UNSUPPORTED_INSTALL_METHOD_MSG.format(source)) + return # type checking doesn't understand that skip raises + if ( + ImageSpecification.from_os_image().release == "bionic" + and session_cloud.settings.PLATFORM == "lxd_vm" + ): + # The issues that we see on Bionic VMs don't appear anywhere + # else, including when calling KVM directly. It likely has to + # do with the extra lxd-agent setup happening on bionic. + # Given that we still have Bionic covered on all other platforms, + # the risk of skipping bionic here seems low enough. + pytest.skip("Upgrade test doesn't run on LXD VMs and bionic") + return + + launch_kwargs = { + "image_id": session_cloud.initial_image_id, + } + + with session_cloud.launch( + launch_kwargs=launch_kwargs, + user_data=USER_DATA, + ) as instance: + # get pre values + pre_hostname = instance.execute("hostname") + pre_cloud_id = instance.execute("cloud-id") + pre_result = instance.execute("cat /run/cloud-init/result.json") + pre_network = instance.execute("cat /etc/netplan/50-cloud-init.yaml") + pre_systemd_analyze = instance.execute("systemd-analyze") + pre_systemd_blame = instance.execute("systemd-analyze blame") + pre_cloud_analyze = instance.execute("cloud-init analyze show") + pre_cloud_blame = instance.execute("cloud-init analyze blame") + + # Ensure no issues pre-upgrade + log = instance.read_from_file("/var/log/cloud-init.log") + assert not json.loads(pre_result)["v1"]["errors"] + + try: + verify_clean_log(log) + except AssertionError: + LOG.warning( + "There were errors/warnings/tracebacks pre-upgrade. " + "Any failures may be due to pre-upgrade problem" + ) + + # Upgrade + instance.install_new_cloud_init(source, take_snapshot=False) + + # 'cloud-init init' helps us understand if our pickling upgrade paths + # have broken across re-constitution of a cached datasource. Some + # platforms invalidate their datasource cache on reboot, so we run + # it here to ensure we get a dirty run. + assert instance.execute("cloud-init init").ok + + # Reboot + instance.execute("hostname something-else") + instance.restart() + assert instance.execute("cloud-init status --wait --long").ok + + # get post values + post_hostname = instance.execute("hostname") + post_cloud_id = instance.execute("cloud-id") + post_result = instance.execute("cat /run/cloud-init/result.json") + post_network = instance.execute("cat /etc/netplan/50-cloud-init.yaml") + post_systemd_analyze = instance.execute("systemd-analyze") + post_systemd_blame = instance.execute("systemd-analyze blame") + post_cloud_analyze = instance.execute("cloud-init analyze show") + post_cloud_blame = instance.execute("cloud-init analyze blame") + + # Ensure no issues post-upgrade + assert not json.loads(pre_result)["v1"]["errors"] + + log = instance.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log) + + # Ensure important things stayed the same + assert pre_hostname == post_hostname + assert pre_cloud_id == post_cloud_id + try: + assert pre_result == post_result + except AssertionError: + if instance.settings.PLATFORM == "azure": + pre_json = json.loads(pre_result) + post_json = json.loads(post_result) + assert pre_json["v1"]["datasource"].startswith( + "DataSourceAzure" + ) + assert post_json["v1"]["datasource"].startswith( + "DataSourceAzure" + ) + assert pre_network == post_network + + # Calculate and log all the boot numbers + pre_analyze_totals = [ + x + for x in pre_cloud_analyze.splitlines() + if x.startswith("Finished stage") or x.startswith("Total Time") + ] + post_analyze_totals = [ + x + for x in post_cloud_analyze.splitlines() + if x.startswith("Finished stage") or x.startswith("Total Time") + ] + + # pylint: disable=logging-format-interpolation + LOG.info( + LOG_TEMPLATE.format( + pre_systemd_analyze=pre_systemd_analyze, + post_systemd_analyze=post_systemd_analyze, + pre_systemd_blame="\n".join( + pre_systemd_blame.splitlines()[:10] + ), + post_systemd_blame="\n".join( + post_systemd_blame.splitlines()[:10] + ), + pre_analyze_totals="\n".join(pre_analyze_totals), + post_analyze_totals="\n".join(post_analyze_totals), + pre_cloud_blame="\n".join(pre_cloud_blame.splitlines()[:10]), + post_cloud_blame="\n".join(post_cloud_blame.splitlines()[:10]), + ) + ) + + +@pytest.mark.ci +@pytest.mark.ubuntu +def test_subsequent_boot_of_upgraded_package(session_cloud: IntegrationCloud): + source = get_validated_source(session_cloud) + if not source.installs_new_version(): + if os.environ.get("TRAVIS"): + # If this isn't running on CI, we should know + pytest.fail(UNSUPPORTED_INSTALL_METHOD_MSG.format(source)) + else: + pytest.skip(UNSUPPORTED_INSTALL_METHOD_MSG.format(source)) + return # type checking doesn't understand that skip raises + + launch_kwargs = {"image_id": session_cloud.initial_image_id} + + with session_cloud.launch(launch_kwargs=launch_kwargs) as instance: + instance.install_new_cloud_init( + source, take_snapshot=False, clean=False + ) + instance.restart() + assert instance.execute("cloud-init status --wait --long").ok diff --git a/tests/integration_tests/util.py b/tests/integration_tests/util.py new file mode 100644 index 00000000..31fe69c0 --- /dev/null +++ b/tests/integration_tests/util.py @@ -0,0 +1,142 @@ +import functools +import logging +import multiprocessing +import os +import time +from collections import namedtuple +from contextlib import contextmanager +from pathlib import Path + +log = logging.getLogger("integration_testing") +key_pair = namedtuple("key_pair", "public_key private_key") + +ASSETS_DIR = Path("tests/integration_tests/assets") +KEY_PATH = ASSETS_DIR / "keys" + + +def verify_ordered_items_in_text(to_verify: list, text: str): + """Assert all items in list appear in order in text. + + Examples: + verify_ordered_items_in_text(['a', '1'], 'ab1') # passes + verify_ordered_items_in_text(['1', 'a'], 'ab1') # raises AssertionError + """ + index = 0 + for item in to_verify: + index = text[index:].find(item) + assert index > -1, "Expected item not found: '{}'".format(item) + + +def verify_clean_log(log): + """Assert no unexpected tracebacks or warnings in logs""" + warning_count = log.count("WARN") + expected_warnings = 0 + traceback_count = log.count("Traceback") + expected_tracebacks = 0 + + warning_texts = [ + # Consistently on all Azure launches: + # azure.py[WARNING]: No lease found; using default endpoint + "No lease found; using default endpoint" + ] + traceback_texts = [] + if "oracle" in log: + # LP: #1842752 + lease_exists_text = "Stderr: RTNETLINK answers: File exists" + warning_texts.append(lease_exists_text) + traceback_texts.append(lease_exists_text) + # LP: #1833446 + fetch_error_text = ( + "UrlError: 404 Client Error: Not Found for url: " + "http://169.254.169.254/latest/meta-data/" + ) + warning_texts.append(fetch_error_text) + traceback_texts.append(fetch_error_text) + # Oracle has a file in /etc/cloud/cloud.cfg.d that contains + # users: + # - default + # - name: opc + # ssh_redirect_user: true + # This can trigger a warning about opc having no public key + warning_texts.append( + "Unable to disable SSH logins for opc given ssh_redirect_user" + ) + + for warning_text in warning_texts: + expected_warnings += log.count(warning_text) + for traceback_text in traceback_texts: + expected_tracebacks += log.count(traceback_text) + + assert warning_count == expected_warnings + assert traceback_count == expected_tracebacks + + +@contextmanager +def emit_dots_on_travis(): + """emit a dot every 60 seconds if running on Travis. + + Travis will kill jobs that don't emit output for a certain amount of time. + This context manager spins up a background process which will emit a dot to + stdout every 60 seconds to avoid being killed. + + It should be wrapped selectively around operations that are known to take a + long time. + """ + if os.environ.get("TRAVIS") != "true": + # If we aren't on Travis, don't do anything. + yield + return + + def emit_dots(): + while True: + log.info(".") + time.sleep(60) + + dot_process = multiprocessing.Process(target=emit_dots) + dot_process.start() + try: + yield + finally: + dot_process.terminate() + + +def get_test_rsa_keypair(key_name: str = "test1") -> key_pair: + private_key_path = KEY_PATH / "id_rsa.{}".format(key_name) + public_key_path = KEY_PATH / "id_rsa.{}.pub".format(key_name) + with public_key_path.open() as public_file: + public_key = public_file.read() + with private_key_path.open() as private_file: + private_key = private_file.read() + return key_pair(public_key, private_key) + + +def retry(*, tries: int = 30, delay: int = 1): + """Decorator for retries. + + Retry a function until code no longer raises an exception or + max tries is reached. + + Example: + @retry(tries=5, delay=1) + def try_something_that_may_not_be_ready(): + ... + """ + + def _retry(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + last_error = None + for _ in range(tries): + try: + func(*args, **kwargs) + break + except Exception as e: + last_error = e + time.sleep(delay) + else: + if last_error: + raise last_error + + return wrapper + + return _retry |