QEMU image build improvements

* Added missed package dependencies (for Debian 10 based builders)
* Added additional options, which can be passed to the playbook:
  * iso_local - path to local ISO file
  * disk_size - target disk image size
  * cloud_init - enable or disable Cloud-init integration to an image
  * cloud_init_ds - set custom list of data sources for Cloud-init
* Added a version number and Cloud-init mark to a QCOW2 image name
* Cloud-init installation procedure tuned to use packages from a proper one VyOS repository, according to VyOS and Debian version
* Added workaround for /etc/network/interfaces to allow Cloud-init initialize network in a native way
* Replaced default config file to Jinja2 template
* Fixed GRUB installation on UEFI builders - added target i386-pc
* Replaced GRUB configuration:
  * enable both KVM and Serial console for all images (Serial by default)
  * added password recovery option for all images
  * added 5 seconds of a timeout to allow select proper menu entry
  * fixed booting for 1.3 VyOS
* Added loop device detach after image build - allows to build images multiple times without exhausting loop device limit
* Added fstrim applying for image - theoretically, this may save some space
* Enabled compression for QCOW2 image - this reduces image size significantly
* Added RAW image deletion after conversion to more accurate cleanup

25 files changed, 196 insertions, 29 deletions

diff --git a/README.md b/README.md
index 39560e7..677cf8a 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,22 @@
 
 [Ansible](https://www.ansible.com/) playbooks to build VyOS VM images.
 
+## Requirements
+
+You need a machine with at least 20 GB free space with Debian 10 (bare-metal, virtual, Docker container with --privileged flag). Also, you need to install ansible and python packages:
+
+```
+sudo apt update
+sudo apt install -y ansible python
+```
+
+All other requirements will be installed by ansible-playbook.
+
+
+## Prepare
+
+You need to copy the ISO image with VyOS to /tmp/vyos.iso before running ansible-playbook. Resulting images also will be located inside /tmp/ directory.
+
 ## Supported Platforms
 
 - QEMU
@@ -28,3 +44,46 @@
     ```
     ansible-playbook vagrant-libvirt.yml
     ```
+
+## Additional (optional) parameters
+
+- Path to local ISO image (default: /tmp/vyos.iso):
+
+    ```
+    -e iso_local=path
+    ```
+
+    Example:
+
+    ```
+    -e iso_local=/tmp/vyos/custom_image.iso
+    ```
+
+- Disk size (default: 10GB):
+
+    ```
+    -e disk_size=size
+    ```
+
+    Example for 2 GB:
+
+    ```
+    -e disk_size=2
+    ```
+
+- Enable Cloud-init (default: according to platform):
+
+    ```
+    -e cloud_init=true
+    ```
+
+- Configure custom Cloud-init datasources (default: according to platform):
+
+    ```
+    -e cloud_init_ds=datasources
+    ```
+
+    Example:
+    ```
+    -e cloud_init_ds=NoCloud,ConfigDrive,None
+    ```
diff --git a/group_vars/all.yml b/group_vars/all.yml
index 2e1c25b..f718626 100644
--- a/group_vars/all.yml
+++ b/group_vars/all.yml
@@ -1,7 +1,7 @@
 ansible_host_key_checking: False
 
 vyos_iso_url: ""
-vyos_iso_local: /tmp/vyos.iso
+vyos_iso_local: "{{ iso_local | default('/tmp/vyos.iso') }}"
 vyos_key_url: http://packages.vyos.net/vyos-release.gpg
 vyos_key_local: /tmp/vyos-release.gpg
 
@@ -12,7 +12,7 @@ vyos_write_root: /mnt/wroot
 vyos_read_root: /mnt/squashfs
 vyos_install_root: /mnt/inst_root
 
-vyos_disk_size: 10
+vyos_disk_size: "{{ disk_size | default(10) }}"
 vyos_root_fstype: ext4
 
 vyos_target_drive: ""
diff --git a/hyperv.yml b/hyperv.yml
index 365d839..75871fb 100644
--- a/hyperv.yml
+++ b/hyperv.yml
@@ -8,6 +8,9 @@
     vyos_format: vhd
     vyos_hyperv_img: /tmp/vyos_hyperv_image.vhd
     vyos_output_img: "{{ vyos_hyperv_img }}"
+    cloud_init: "true"
+    cloud_init_ds_string: "{{ cloud_init_ds | default('NoCloud, ConfigDrive, None') }}"
+    cloud_init_ds_list: "{{ cloud_init_ds_string.split(',') }}"
   roles:
     - install-packages
     - load-modules
@@ -22,7 +25,7 @@
     - install-config
     - install-grub
     - install-persistence-conf
-    - install-cloud-init
+    - install-cloud-init-wrapper
     - unmount-all
     - hyperv-vhd
     - release
diff --git a/qemu.yml b/qemu.yml
index e51e5e6..f5f79b5 100644
--- a/qemu.yml
+++ b/qemu.yml
@@ -5,8 +5,11 @@
   vars:
     vyos_platform: QEMU
     vyos_format: qcow2
-    vyos_qemu_img: /tmp/vyos_qemu_image.qcow2
+    vyos_qemu_img: "/tmp/vyos-{{ vyos_version }}{{ ci_tag  | default() }}-qemu.qcow2"
     vyos_output_img: "{{ vyos_qemu_img }}"
+    cloud_init: "false"
+    cloud_init_ds_string: "{{ cloud_init_ds | default('NoCloud,ConfigDrive,None') }}"
+    cloud_init_ds_list: "{{ cloud_init_ds_string.split(',') }}"
   roles:
     - install-packages
     - load-modules
@@ -21,7 +24,9 @@
     - install-config
     - install-grub
     - install-persistence-conf
-    - install-cloud-init
+    - install-cloud-init-wrapper
+    - fstrim
     - unmount-all
     - qemu-qcow2
+    - cleanup-ending
     - release
diff --git a/roles/cleanup-ending/tasks/main.yml b/roles/cleanup-ending/tasks/main.yml
new file mode 100644
index 0000000..1b2715b
--- /dev/null
+++ b/roles/cleanup-ending/tasks/main.yml
@@ -0,0 +1,4 @@
+- name: Delete RAW image
+  file:
+    path: "{{ vyos_raw_img }}"
+    state: absent
diff --git a/roles/fstrim/tasks/main.yml b/roles/fstrim/tasks/main.yml
new file mode 100644
index 0000000..e7230a7
--- /dev/null
+++ b/roles/fstrim/tasks/main.yml
@@ -0,0 +1,8 @@
+# Trim filesystems to minimize resulted image
+- name: Trim {{ vyos_install_root }}/boot
+  become: true
+  command: fstrim {{ vyos_install_root }}/boot
+
+- name: Trim {{ vyos_write_root }}
+  become: true
+  command: fstrim {{ vyos_write_root }}
diff --git a/roles/install-cloud-init-wrapper/tasks/main.yml b/roles/install-cloud-init-wrapper/tasks/main.yml
new file mode 100644
index 0000000..7ab9705
--- /dev/null
+++ b/roles/install-cloud-init-wrapper/tasks/main.yml
@@ -0,0 +1,12 @@
+- name: Check if we need to install Cloud-Init
+  include_role:
+    name: install-cloud-init
+  when: cloud_init == "true"
+- name: Set Cloud-Init tag for image file name
+  set_fact:
+    ci_tag: "-cloud-init"
+  when: cloud_init == "true"
+# - name: Set empty Cloud-Init tag for image file name
+#   set_fact:
+#     ci_tag: ""
+#   when: cloud_init == "false"
diff --git a/roles/install-cloud-init-wrapper/tests/inventory b/roles/install-cloud-init-wrapper/tests/inventory
new file mode 100644
index 0000000..878877b
--- /dev/null
+++ b/roles/install-cloud-init-wrapper/tests/inventory
@@ -0,0 +1,2 @@
+localhost
+
diff --git a/roles/install-cloud-init-wrapper/tests/test.yml b/roles/install-cloud-init-wrapper/tests/test.yml
new file mode 100644
index 0000000..6002afa
--- /dev/null
+++ b/roles/install-cloud-init-wrapper/tests/test.yml
@@ -0,0 +1,4 @@
+---
+- hosts: localhost
+  roles:
+    - install-cloud-init-wrapper
diff --git a/roles/install-cloud-init/files/debian.list.buster b/roles/install-cloud-init/files/debian.list.buster
new file mode 100644
index 0000000..fd5a770
--- /dev/null
+++ b/roles/install-cloud-init/files/debian.list.buster
@@ -0,0 +1,7 @@
+deb http://deb.debian.org/debian buster main contrib non-free
+deb-src http://deb.debian.org/debian buster main contrib non-free
+deb http://security.debian.org/debian-security/ buster/updates main contrib non-free
+deb-src http://security.debian.org/debian-security/ buster/updates main contrib non-free
+deb http://deb.debian.org/debian buster-updates main contrib non-free
+deb-src http://deb.debian.org/debian buster-updates main contrib non-free
+deb http://dev.packages.vyos.net/repositories/current current main
diff --git a/roles/install-cloud-init/files/debian.list b/roles/install-cloud-init/files/debian.list.jessie
index f657759..0750699 100644
--- a/roles/install-cloud-init/files/debian.list
+++ b/roles/install-cloud-init/files/debian.list.jessie
@@ -4,4 +4,5 @@ deb http://security.debian.org/debian-security/ jessie/updates main contrib non-
 deb-src http://security.debian.org/debian-security/ jessie/updates main contrib non-free
 deb http://deb.debian.org/debian jessie-updates main contrib non-free
 deb-src http://deb.debian.org/debian jessie-updates main contrib non-free
-deb http://deb.debian.org/debian jessie-backports main
+deb http://dev.packages.vyos.net/repositories/crux/vyos crux main
+deb http://dev.packages.vyos.net/repositories/crux/debian crux main
diff --git a/roles/install-cloud-init/files/hyper-v.cfg b/roles/install-cloud-init/files/hyper-v.cfg
deleted file mode 100644
index f66d7c0..0000000
--- a/roles/install-cloud-init/files/hyper-v.cfg
+++ /dev/null
@@ -1 +0,0 @@
-datasource_list: [ NoCloud, ConfigDrive, None ]
diff --git a/roles/install-cloud-init/files/qemu.cfg b/roles/install-cloud-init/files/qemu.cfg
deleted file mode 100644
index f66d7c0..0000000
--- a/roles/install-cloud-init/files/qemu.cfg
+++ /dev/null
@@ -1 +0,0 @@
-datasource_list: [ NoCloud, ConfigDrive, None ]
diff --git a/roles/install-cloud-init/files/resolv.conf b/roles/install-cloud-init/files/resolv.conf
new file mode 100644
index 0000000..81027f8
--- /dev/null
+++ b/roles/install-cloud-init/files/resolv.conf
@@ -0,0 +1 @@
+nameserver 1.1.1.1
\ No newline at end of file
diff --git a/roles/install-cloud-init/files/vmware.cfg b/roles/install-cloud-init/files/vmware.cfg
deleted file mode 100644
index 76a5efd..0000000
--- a/roles/install-cloud-init/files/vmware.cfg
+++ /dev/null
@@ -1 +0,0 @@
-datasource_list: [ NoCloud, ConfigDrive, OVF, None ]
diff --git a/roles/install-cloud-init/tasks/main.yml b/roles/install-cloud-init/tasks/main.yml
index c9d5d07..d894750 100644
--- a/roles/install-cloud-init/tasks/main.yml
+++ b/roles/install-cloud-init/tasks/main.yml
@@ -1,17 +1,36 @@
+- name: Get Debian version
+  become: true
+  command: chroot {{ vyos_install_root }} awk 'match($0, /VERSION=.*\((\w+)\)/, version) { print version[1] }' /etc/os-release
+  register: debian_version
+- name: Set VyOS branch name crux
+  set_fact:
+    vyos_branch: "crux"
+  when: vyos_version is regex("^1\.2.*$")
 - name: Put debian.list
   become: true
   copy:
-    src: files/debian.list
+    src: "files/debian.list.{{ debian_version.stdout }}"
     dest: "{{ vyos_install_root }}/etc/apt/sources.list.d/debian.list"
+- name: backup resolv.conf
+  become: true
+  command: mv {{ vyos_install_root }}/etc/resolv.conf /tmp/resolv.conf
+- name: add nameserver settings to chroot
+  become: true
+  copy:
+    src: "files/resolv.conf"
+    dest: "{{ vyos_install_root }}/etc/resolv.conf"
 - name: apt-get update
   become: true
   command: chroot {{ vyos_install_root }} apt-get update
 - name: Install cloud-init
   become: true
-  command: chroot {{ vyos_install_root }} apt-get install -y cloud-init cloud-utils
+  command: chroot {{ vyos_install_root }} apt-get -t {{ vyos_branch | default('current') }} install -y cloud-init cloud-utils
 - name: apt-get clean
   become: true
   command: chroot {{ vyos_install_root }} apt-get clean
+- name: delete apt lists from cache
+  become: true
+  command: chroot {{ vyos_install_root }} rm -rf /var/lib/apt/lists/*
 - name: Delete debian.list
   become: true
   file:
@@ -25,9 +44,15 @@
     mode: 0755
 - name: Put datasource_list.cfg
   become: true
-  copy:
-    src: "files/{{ vyos_platform | lower }}.cfg"
+  template:
+    src: 90_dpkg.cfg.j2
     dest: "{{ vyos_install_root }}/etc/cloud/cloud.cfg.d/90_dpkg.cfg"
 - name: run dpkg-reconfigure cloud-init
   become: true
   command: chroot {{ vyos_install_root }} dpkg-reconfigure -f noninteractive cloud-init
+- name: restore original resolv.conf
+  become: true
+  command: mv /tmp/resolv.conf {{ vyos_install_root }}/etc/resolv.conf
+- name: change /etc/network/interfaces to include config from Cloud-Init
+  become: true
+  command: chroot {{ vyos_install_root }} sed -i 's/source-directory \/etc\/network\/interfaces.d/source \/etc\/network\/interfaces.d\/*/g' /etc/network/interfaces
diff --git a/roles/install-cloud-init/templates/90_dpkg.cfg.j2 b/roles/install-cloud-init/templates/90_dpkg.cfg.j2
new file mode 100644
index 0000000..69d2461
--- /dev/null
+++ b/roles/install-cloud-init/templates/90_dpkg.cfg.j2
@@ -0,0 +1 @@
+datasource_list: [ {{ cloud_init_ds_list|join(', ') }} ]
diff --git a/roles/install-config/tasks/main.yml b/roles/install-config/tasks/main.yml
index 4447f87..0715fa1 100644
--- a/roles/install-config/tasks/main.yml
+++ b/roles/install-config/tasks/main.yml
@@ -4,9 +4,9 @@
     path: "{{ vyos_install_root }}/opt/vyatta/etc/config/.vyatta_config"
     state: touch
 
-- name: Copy the default config for QEMU to the installed image
+- name: Copy config to the installed image
   become: true
-  copy:
-    src: files/config.boot
+  template:
+    src: config.boot.j2
     dest: "{{ vyos_install_root }}/opt/vyatta/etc/config/config.boot"
     mode: 0755
diff --git a/roles/install-config/files/config.boot b/roles/install-config/templates/config.boot.j2
index f1ec4f5..9b3e8ec 100644
--- a/roles/install-config/files/config.boot
+++ b/roles/install-config/templates/config.boot.j2
@@ -1,14 +1,13 @@
-service {
-    ssh {
-        port 22
-    }
-}
 system {
     host-name vyos
     login {
         user vyos {
             authentication {
+{% if cloud_init == "true" %}
+                encrypted-password "*"
+{% else %}
                 encrypted-password "$6$MjV2YvKQ56q$QbL562qhRoyUu8OaqrXagicvcsNpF1HssCY06ZxxghDJkBCfSfTE/4FlFB41xZcd/HqYyVBuRt8Zyq3ozJ0dc."
+{% endif %}
                 plaintext-password ""
             }
             level admin
@@ -34,8 +33,18 @@ system {
     }
 }
 interfaces {
+{% if cloud_init == "true" %}
     ethernet eth0 {
         address dhcp
     }
-    loopback lo
+{% endif %}
+    loopback lo {
+    }
+}
+{% if cloud_init == "true" %}
+service {
+    ssh {
+        port 22
+    }
 }
+{% endif %}
diff --git a/roles/install-grub/tasks/main.yml b/roles/install-grub/tasks/main.yml
index 2a20271..33a1049 100644
--- a/roles/install-grub/tasks/main.yml
+++ b/roles/install-grub/tasks/main.yml
@@ -20,7 +20,7 @@
 
 - name: Install GRUB in the boot sector of {{ vyos_target_drive }}
   become: true
-  command: chroot {{ vyos_install_root }} grub-install --no-floppy --root-directory=/boot {{ vyos_target_drive }} --force
+  command: chroot {{ vyos_install_root }} grub-install --no-floppy --root-directory=/boot {{ vyos_target_drive }} --force --target=i386-pc
   args:
     creates: "{{ vyos_install_root }}/boot/grub/grubenv"
 
diff --git a/roles/install-grub/templates/boot/grub/grub.cfg.j2 b/roles/install-grub/templates/boot/grub/grub.cfg.j2
index 45f48ab..0440442 100644
--- a/roles/install-grub/templates/boot/grub/grub.cfg.j2
+++ b/roles/install-grub/templates/boot/grub/grub.cfg.j2
@@ -1,7 +1,25 @@
-set default=0
-set timeout=0
+set default=1
+set timeout=5
+serial --unit=0
+terminal_output --append serial
+terminal_input serial console
 
-menuentry "VyOS {{ vyos_platform }} Image {{ vyos_version }}" {
-  linux /boot/{{ vyos_version }}/vmlinuz boot=live selinux=0 vyos-union=/boot/{{ vyos_version }} console=tty1
+menuentry "VyOS {{ vyos_version }} for {{ vyos_platform }} (KVM console)" {
+  linux /boot/{{ vyos_version }}/vmlinuz boot=live rootdelay=5 noautologin net.ifnames=0 biosdevname=0 vyos-union=/boot/{{ vyos_version }} console=ttyS0 console=tty0
+  initrd /boot/{{ vyos_version }}/initrd.img
+}
+
+menuentry "VyOS {{ vyos_version }} for {{ vyos_platform }} (Serial console)" {
+  linux /boot/{{ vyos_version }}/vmlinuz boot=live rootdelay=5 noautologin net.ifnames=0 biosdevname=0 vyos-union=/boot/{{ vyos_version }} console=tty0 console=ttyS0
+  initrd /boot/{{ vyos_version }}/initrd.img
+}
+
+menuentry "VyOS {{ vyos_version }} for {{ vyos_platform }} - password reset (KVM console)" {
+  linux /boot/{{ vyos_version }}/vmlinuz boot=live rootdelay=5 noautologin net.ifnames=0 biosdevname=0 vyos-union=/boot/{{ vyos_version }} console=ttyS0 console=tty0 init=/opt/vyatta/sbin/standalone_root_pw_reset
+  initrd /boot/{{ vyos_version }}/initrd.img
+}
+
+menuentry "VyOS {{ vyos_version }} for {{ vyos_platform }} - password reset (Serial console)" {
+  linux /boot/{{ vyos_version }}/vmlinuz boot=live rootdelay=5 noautologin net.ifnames=0 biosdevname=0 vyos-union=/boot/{{ vyos_version }} console=tty0 console=ttyS0 init=/opt/vyatta/sbin/standalone_root_pw_reset
   initrd /boot/{{ vyos_version }}/initrd.img
 }
diff --git a/roles/install-packages/tasks/main.yml b/roles/install-packages/tasks/main.yml
index ffb0bbf..9a143ef 100644
--- a/roles/install-packages/tasks/main.yml
+++ b/roles/install-packages/tasks/main.yml
@@ -9,4 +9,8 @@
       - qemu-utils
       - python-lxml
       - aufs-tools
+      - grub2
+      - python-requests
+      - unzip
+      - zlib1g-dev
     state: present
diff --git a/roles/qemu-qcow2/tasks/main.yml b/roles/qemu-qcow2/tasks/main.yml
index c91c75b..c3893ef 100644
--- a/roles/qemu-qcow2/tasks/main.yml
+++ b/roles/qemu-qcow2/tasks/main.yml
@@ -1,2 +1,2 @@
 - name: Convert raw to qcow2
-  command: qemu-img convert -f raw "{{ vyos_raw_img }}" -O qcow2 "{{ vyos_qemu_img }}"
+  command: qemu-img convert -f raw "{{ vyos_raw_img }}" -O qcow2 -c "{{ vyos_qemu_img }}"
diff --git a/roles/unmount-all/tasks/main.yml b/roles/unmount-all/tasks/main.yml
index 1ff6daa..e097b2b 100644
--- a/roles/unmount-all/tasks/main.yml
+++ b/roles/unmount-all/tasks/main.yml
@@ -58,6 +58,10 @@
     fstype: "{{ vyos_root_fstype }}"
     state: absent
 
+- name: Detach {{ vyos_target_drive }}
+  become: true
+  command: "losetup -d {{ vyos_target_drive }}"
+
 - name: Unmount {{ vyos_cd_squash_root }}
   become: true
   mount:
diff --git a/vmware.yml b/vmware.yml
index d953ca7..6c6b8b9 100644
--- a/vmware.yml
+++ b/vmware.yml
@@ -8,6 +8,9 @@
     vyos_vmdk_size: 10
     vyos_vmware_ova: /tmp/vyos_vmware_image.ova
     vyos_output_img: "{{ vyos_vmware_ova }}"
+    cloud_init: "true"
+    cloud_init_ds_string: "{{ cloud_init_ds | default('OVF,None') }}"
+    cloud_init_ds_list: "{{ cloud_init_ds_string.split(',') }}"
   roles:
     - install-packages
     - load-modules
@@ -22,7 +25,7 @@
     - install-config
     - install-grub
     - install-persistence-conf
-    - install-cloud-init
+    - install-cloud-init-wrapper
     - unmount-all
     - install-open-vmdk
     - vmware-ova