Improve CI (#528)

* Update CI scripts to be more close to the ones in ansible-core.

* Extend CI matrix.

* Make sure that docker daemon is running (when not in a container).

* Make sure that connection plugin tests do not uninstall Docker daemon.

* Check some conditions.

* Fix error ignores.

* Skip SSH test on Alpine VMs.

* Take care of more errors.

* Adjust for more errors.

* Improve conditions.

* Remove new entries from CI matrix; make CI matrix nicer.

.azure-pipelines/azure-pipelines.yml

@@ -187,7 +187,7 @@ stages:
               test: debian-bullseye/3.9
             - name: ArchLinux
               test: archlinux/3.10
-            - name: CentOS Stream 8
+            - name: CentOS Stream 8 w/ Python 3.9
               test: centos-stream8/3.9
           groups:
             - 4
@@ -200,11 +200,12 @@ stages:
     jobs:
       - template: templates/matrix.yml
         parameters:
-          nameFormat: RHEL {0}
+          testFormat: devel/{0}
-          testFormat: devel/rhel/{0}
           targets:
-            - test: '7.9'
+            - name: RHEL 7.9
-            - test: '9.0-pypi-latest'
+              test: rhel/7.9
+            - name: RHEL 9.0 w/ latest Docker SDK from PyPi
+              test: rhel/9.0-pypi-latest
           groups:
             - 1
             - 2
@@ -217,10 +218,10 @@ stages:
     jobs:
       - template: templates/matrix.yml
         parameters:
-          nameFormat: RHEL {0}
+          testFormat: 2.14/{0}
-          testFormat: 2.14/rhel/{0}
           targets:
-            - test: '9.0'
+            - name: RHEL 9.0
+              test: rhel/9.0
           groups:
             - 1
             - 2
@@ -233,10 +234,10 @@ stages:
     jobs:
       - template: templates/matrix.yml
         parameters:
-          nameFormat: RHEL {0}
+          testFormat: 2.13/{0}
-          testFormat: 2.13/rhel/{0}
           targets:
-            - test: '8.5'
+            - name: RHEL 8.5
+              test: rhel/8.5
           groups:
             - 1
             - 2
@@ -249,10 +250,10 @@ stages:
     jobs:
       - template: templates/matrix.yml
         parameters:
-          nameFormat: RHEL {0}
+          testFormat: 2.12/{0}
-          testFormat: 2.12/rhel/{0}
           targets:
-            - test: '8.4'
+            - name: RHEL 8.4
+              test: rhel/8.4
           groups:
             - 1
             - 2

tests/integration/targets/connection_docker/setup.yml

@@ -6,7 +6,7 @@
 - hosts: localhost
   connection: local
   vars:
-    docker_skip_cleanup: yes
+    docker_skip_cleanup: true
 
   tasks:
     - name: Setup docker

tests/integration/targets/connection_docker/shutdown.yml

@@ -6,7 +6,7 @@
 - hosts: localhost
   connection: local
   vars:
-    docker_skip_cleanup: yes
+    docker_skip_cleanup: true
 
   tasks:
     - name: Remove docker packages
@@ -17,3 +17,4 @@
           - docker-ce
           - docker-ce-cli
         state: absent
+      when: not docker_skip_cleanup

tests/integration/targets/connection_docker_api/setup.yml

@@ -6,7 +6,7 @@
 - hosts: localhost
   connection: local
   vars:
-    docker_skip_cleanup: yes
+    docker_skip_cleanup: true
 
   tasks:
     - name: Setup docker

tests/integration/targets/connection_docker_api/shutdown.yml

@@ -6,7 +6,7 @@
 - hosts: localhost
   connection: local
   vars:
-    docker_skip_cleanup: yes
+    docker_skip_cleanup: true
 
   tasks:
     - name: Remove docker packages
@@ -17,3 +17,4 @@
           - docker-ce
           - docker-ce-cli
         state: absent
+      when: not docker_skip_cleanup

tests/integration/targets/connection_nsenter/setup.yml

@@ -6,7 +6,7 @@
 - hosts: localhost
   connection: local
   vars:
-    docker_skip_cleanup: yes
+    docker_skip_cleanup: true
 
   tasks:
     - name: Setup docker

tests/integration/targets/connection_nsenter/shutdown.yml

@@ -6,7 +6,7 @@
 - hosts: localhost
   connection: local
   vars:
-    docker_skip_cleanup: yes
+    docker_skip_cleanup: true
 
   tasks:
     - name: Remove docker packages
@@ -17,3 +17,4 @@
           - docker-ce
           - docker-ce-cli
         state: absent
+      when: not docker_skip_cleanup

tests/integration/targets/docker_container/tasks/tests/options.yml

@@ -53,6 +53,7 @@
     state: started
     blkio_weight: 123
   register: blkio_weight_1
+  ignore_errors: true
 
 - name: blkio_weight (idempotency)
   docker_container:
@@ -62,6 +63,7 @@
     state: started
     blkio_weight: 123
   register: blkio_weight_2
+  ignore_errors: true
 
 - name: blkio_weight (change)
   docker_container:
@@ -72,6 +74,7 @@
     blkio_weight: 234
     force_kill: yes
   register: blkio_weight_3
+  ignore_errors: true
 
 - name: cleanup
   docker_container:
@@ -80,10 +83,18 @@
     force_kill: yes
   diff: no
 
-- assert:
+- when: blkio_weight_1 is failed
+  assert:
+    that:
+    - "'setting cgroup config for procHooks process caused: failed to write' in blkio_weight_1.msg"
+
+- when: blkio_weight_1 is not failed
+  assert:
     that:
     - blkio_weight_1 is changed
+    - blkio_weight_2 is not failed
     - "blkio_weight_2 is not changed or 'Docker warning: Your kernel does not support Block I/O weight or the cgroup is not mounted. Weight discarded.' in (blkio_weight_2.warnings | default([]))"
+    - blkio_weight_3 is not failed
     - blkio_weight_3 is changed
 
 ####################################################################
@@ -197,8 +208,16 @@
     that:
     - cgroupns_mode_1 is changed
     - cgroupns_mode_2 is not changed and cgroupns_mode_2 is not failed
-    - "cgroupns_mode_3 is changed or 'Docker warning: Your kernel does not support cgroup namespaces.  Cgroup namespace setting discarded.' in (cgroupns_mode_3.warnings | default([]))"
+    - >-
-  when: docker_api_version is version('1.41', '>=')
+      cgroupns_mode_3 is changed or
+      'Docker warning: Your kernel does not support cgroup namespaces.  Cgroup namespace setting discarded.' in (cgroupns_mode_3.warnings | default([])) or
+      (cgroupns_mode_3 is failed and 'error mounting "cgroup" to rootfs at "/sys/fs/cgroup"' in cgroupns_mode_3.msg)
+  when: docker_api_version is version('1.41', '>=') and cgroupns_mode_1 is not failed
+- assert:
+    that:
+    - >-
+      'error mounting "cgroup" to rootfs at "/sys/fs/cgroup"' in cgroupns_mode_1.msg
+  when: docker_api_version is version('1.41', '>=') and cgroupns_mode_1 is failed
 - assert:
     that:
     - cgroupns_mode_1 is failed
@@ -861,6 +880,7 @@
       - path: /dev/urandom
         rate: 10K
   register: device_read_bps_1
+  ignore_errors: true
 
 - name: device_read_bps (idempotency)
   docker_container:
@@ -874,6 +894,7 @@
       - path: /dev/random
         rate: 20M
   register: device_read_bps_2
+  ignore_errors: true
 
 - name: device_read_bps (lesser entries)
   docker_container:
@@ -885,6 +906,7 @@
       - path: /dev/random
         rate: 20M
   register: device_read_bps_3
+  ignore_errors: true
 
 - name: device_read_bps (changed)
   docker_container:
@@ -899,6 +921,7 @@
         rate: 5K
     force_kill: yes
   register: device_read_bps_4
+  ignore_errors: true
 
 - name: cleanup
   docker_container:
@@ -907,13 +930,23 @@
     force_kill: yes
   diff: no
 
-- assert:
+- when: device_read_bps_1 is not failed
+  assert:
     that:
+    - device_read_bps_1 is not failed
     - device_read_bps_1 is changed
+    - device_read_bps_2 is not failed
     - device_read_bps_2 is not changed
+    - device_read_bps_3 is not failed
     - device_read_bps_3 is not changed
+    - device_read_bps_4 is not failed
     - device_read_bps_4 is changed
 
+- when: device_read_bps_1 is failed
+  assert:
+    that:
+    - "'error setting cgroup config for procHooks process' in device_read_bps_1.msg and 'blkio.throttle.read_bps_device: no such device' in device_read_bps_1.msg"
+
 ####################################################################
 ## device_read_iops ################################################
 ####################################################################
@@ -930,6 +963,7 @@
       - path: /dev/urandom
         rate: 20
   register: device_read_iops_1
+  ignore_errors: true
 
 - name: device_read_iops (idempotency)
   docker_container:
@@ -943,6 +977,7 @@
       - path: /dev/random
         rate: 10
   register: device_read_iops_2
+  ignore_errors: true
 
 - name: device_read_iops (less)
   docker_container:
@@ -954,6 +989,7 @@
       - path: /dev/random
         rate: 10
   register: device_read_iops_3
+  ignore_errors: true
 
 - name: device_read_iops (changed)
   docker_container:
@@ -968,6 +1004,7 @@
         rate: 50
     force_kill: yes
   register: device_read_iops_4
+  ignore_errors: true
 
 - name: cleanup
   docker_container:
@@ -976,13 +1013,23 @@
     force_kill: yes
   diff: no
 
-- assert:
+- when: device_read_iops_1 is not failed
+  assert:
     that:
+    - device_read_iops_1 is not failed
     - device_read_iops_1 is changed
+    - device_read_iops_2 is not failed
     - device_read_iops_2 is not changed
+    - device_read_iops_3 is not failed
     - device_read_iops_3 is not changed
+    - device_read_iops_4 is not failed
     - device_read_iops_4 is changed
 
+- when: device_read_iops_1 is failed
+  assert:
+    that:
+    - "'error setting cgroup config for procHooks process' in device_read_iops_1.msg and 'blkio.throttle.read_iops_device: no such device' in device_read_iops_1.msg"
+
 ####################################################################
 ## device_write_bps and device_write_iops ##########################
 ####################################################################
@@ -1000,6 +1047,7 @@
       - path: /dev/urandom
         rate: 30
   register: device_write_limit_1
+  ignore_errors: true
 
 - name: device_write_bps and device_write_iops (idempotency)
   docker_container:
@@ -1014,6 +1062,7 @@
       - path: /dev/urandom
         rate: 30
   register: device_write_limit_2
+  ignore_errors: true
 
 - name: device_write_bps device_write_iops (changed)
   docker_container:
@@ -1029,6 +1078,7 @@
         rate: 100
     force_kill: yes
   register: device_write_limit_3
+  ignore_errors: true
 
 - name: cleanup
   docker_container:
@@ -1037,12 +1087,19 @@
     force_kill: yes
   diff: no
 
-- assert:
+- when: device_write_limit_1 is not failed
+  assert:
     that:
+    - device_write_limit_1 is not failed and device_write_limit_2 is not failed and device_write_limit_3 is not failed
     - device_write_limit_1 is changed
     - device_write_limit_2 is not changed
     - device_write_limit_3 is changed
 
+- when: device_write_limit_1 is failed
+  assert:
+    that:
+    - "'error setting cgroup config for procHooks process' in device_write_limit_1.msg and 'blkio.throttle.write_bps_device: no such device' in device_write_limit_1.msg"
+
 ####################################################################
 ## device_requests #################################################
 ####################################################################

tests/integration/targets/docker_container/tasks/tests/update.yml

@@ -13,13 +13,16 @@
 # We do not test cpuset_cpus and cpuset_mems since changing it fails if the system does
 # not have 'enough' CPUs. We do not test kernel_memory since it is deprecated and fails.
 
+- set_fact:
+    has_blkio_weight: true
+
 - name: Create container
   docker_container:
     image: "{{ docker_test_image_alpine }}"
     command: '/bin/sh -c "sleep 10m"'
     name: "{{ cname }}"
     state: started
-    blkio_weight: 123
+    blkio_weight: "{{ 123 if has_blkio_weight else omit }}"
     cpu_period: 90000
     cpu_quota: 150000
     cpu_shares: 900
@@ -29,6 +32,39 @@
     restart_policy: on-failure
     restart_retries: 5
   register: create
+  ignore_errors: true
+
+- when: create is failed
+  block:
+    - name: Make sure container is not there
+      docker_container:
+        name: "{{ cname }}"
+        state: absent
+
+    - when: "'setting cgroup config for procHooks process caused: failed to write' in create.msg and 'io.bfq.weight' in create.msg"
+      set_fact:
+        has_blkio_weight: false
+
+    - name: Create container again
+      docker_container:
+        image: "{{ docker_test_image_alpine }}"
+        command: '/bin/sh -c "sleep 10m"'
+        name: "{{ cname }}"
+        state: started
+        blkio_weight: "{{ 123 if has_blkio_weight else omit }}"
+        cpu_period: 90000
+        cpu_quota: 150000
+        cpu_shares: 900
+        memory: 64M
+        memory_reservation: 64M
+        memory_swap: 64M
+        restart_policy: on-failure
+        restart_retries: 5
+      register: create_2
+
+    - when: "'setting cgroup config for procHooks process caused: failed to write' in create.msg and 'io.bfq.weight' in create.msg"
+      set_fact:
+        create: "{{ create_2 }}"
 
 - name: Update values
   docker_container:
@@ -36,7 +72,7 @@
     command: '/bin/sh -c "sleep 10m"'
     name: "{{ cname }}"
     state: started
-    blkio_weight: 234
+    blkio_weight: "{{ 234 if has_blkio_weight else omit }}"
     cpu_period: 50000
     cpu_quota: 50000
     cpu_shares: 1100
@@ -54,7 +90,7 @@
     command: '/bin/sh -c "sleep 10m"'
     name: "{{ cname }}"
     state: started
-    blkio_weight: 135
+    blkio_weight: "{{ 135 if has_blkio_weight else omit }}"
     cpu_period: 30000
     cpu_quota: 40000
     cpu_shares: 1000
@@ -72,7 +108,7 @@
     command: '/bin/sh -c "sleep 20m"'  # this will force re-creation
     name: "{{ cname }}"
     state: started
-    blkio_weight: 234
+    blkio_weight: "{{ 234 if has_blkio_weight else omit }}"
     cpu_period: 50000
     cpu_quota: 50000
     cpu_shares: 1100
@@ -111,8 +147,8 @@
   assert:
     that:
     # blkio_weight sometimes cannot be set, then we end up with 0 instead of the value we had
-    - update.diff.before.blkio_weight == 123 or 'Docker warning: Your kernel does not support Block I/O weight or the cgroup is not mounted. Weight discarded.' in (create.warnings | default([]))
+    - not has_blkio_weight or update.diff.before.blkio_weight == 123 or 'Docker warning: Your kernel does not support Block I/O weight or the cgroup is not mounted. Weight discarded.' in (create.warnings | default([]))
-    - update.diff.after.blkio_weight == 234
+    - not has_blkio_weight or update.diff.after.blkio_weight == 234
     - update.diff.before.cpu_period == 90000
     - update.diff.after.cpu_period == 50000
     - update.diff.before.cpu_quota == 150000
@@ -132,8 +168,8 @@
 - name: Check diff for second update
   assert:
     that:
-    - update2.diff.before.blkio_weight == 234 or 'Docker warning: Your kernel does not support Block I/O weight or the cgroup is not mounted. Weight discarded.' in (create.warnings | default([]))
+    - not has_blkio_weight or update2.diff.before.blkio_weight == 234 or 'Docker warning: Your kernel does not support Block I/O weight or the cgroup is not mounted. Weight discarded.' in (create.warnings | default([]))
-    - update2.diff.after.blkio_weight == 135
+    - not has_blkio_weight or update2.diff.after.blkio_weight == 135
     - update2.diff.before.cpu_period == 50000
     - update2.diff.after.cpu_period == 30000
     - update2.diff.before.cpu_quota == 50000
@@ -154,8 +190,8 @@
 - name: Check diff for recreation
   assert:
     that:
-    - recreate.diff.before.blkio_weight == 135 or 'Docker warning: Your kernel does not support Block I/O weight or the cgroup is not mounted. Weight discarded.' in (create.warnings | default([]))
+    - not has_blkio_weight or recreate.diff.before.blkio_weight == 135 or 'Docker warning: Your kernel does not support Block I/O weight or the cgroup is not mounted. Weight discarded.' in (create.warnings | default([]))
-    - recreate.diff.after.blkio_weight == 234
+    - not has_blkio_weight or recreate.diff.after.blkio_weight == 234
     - recreate.diff.before.cpu_period == 30000
     - recreate.diff.after.cpu_period == 50000
     - recreate.diff.before.cpu_quota == 40000

tests/integration/targets/docker_plugin/tasks/main.yml

@@ -11,6 +11,11 @@
 - debug:
     msg: "Using name prefix {{ name_prefix }}"
 
+- name: Check whether /dev/fuse exists
+  stat:
+    path: /dev/fuse
+  register: dev_fuse_stat
+
 - block:
   - include_tasks: run-test.yml
     with_fileglob:
@@ -23,7 +28,7 @@
       state: absent
     with_items: "{{ plugin_names }}"
 
-  when: docker_api_version is version('1.25', '>=')
+  when: docker_api_version is version('1.25', '>=') and dev_fuse_stat.stat.exists
 
 - fail: msg="Too old docker / docker-py version to run docker_plugin tests!"
   when: not(docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6)

tests/integration/targets/generic_ssh_connection/aliases

@@ -6,3 +6,4 @@ azp/4
 destructive
 needs/root
 skip/docker  # we need a VM, and not a container
+skip/alpine  # for some reason, SSH has problems with Alpine VMs

tests/integration/targets/setup_docker/tasks/Debian.yml

@@ -41,11 +41,11 @@
   - name: Restore /usr/sbin/policy-rc.d (if needed)
     command: mv {{ policy_rc_d.backup_file }} /usr/sbin/policy-rc.d
     when:
-    - '''backup_file'' in policy_rc_d'
+    - '"backup_file" in policy_rc_d'
 
   - name: Remove /usr/sbin/policy-rc.d (if needed)
     file:
       path: /usr/sbin/policy-rc.d
       state: absent
     when:
-    - '''backup_file'' not in policy_rc_d'
+    - '"backup_file" not in policy_rc_d'

tests/integration/targets/setup_docker/tasks/main.yml

@@ -43,6 +43,12 @@
           paths:
             - "{{ role_path }}/tasks"
 
+    - name: Make sure that docker is running
+      service:
+        name: docker
+        state: started
+      when: not ansible_module_running_in_container
+
     # Detect docker API version
     - name: Check Docker API version
       command: "docker version -f {% raw %}'{{(index .Server.Components 0).Details.ApiVersion}}'{% endraw %}"

tests/utils/shippable/alpine.sh

@@ -0,0 +1 @@
+remote.sh

tests/utils/shippable/fedora.sh

@@ -0,0 +1 @@
+remote.sh

tests/utils/shippable/remote.sh

@@ -10,7 +10,16 @@ IFS='/:' read -ra args <<< "$1"
 
 platform="${args[0]}"
 version="${args[1]}"
-target="shippable/posix/"
+pyver=default
+
+# check for explicit python version like 8.3@3.8
+declare -a splitversion
+IFS='@' read -ra splitversion <<< "$version"
+
+if [ "${#splitversion[@]}" -gt 1 ]; then
+    version="${splitversion[0]}"
+    pyver="${splitversion[1]}"
+fi
 
 if [ "${#args[@]}" -gt 2 ]; then
     target="azp/${args[2]}/"
@@ -22,10 +31,6 @@ force_python=""
 if [[ "${version}" =~ -pypi-latest$ ]]; then
     version="${version/-pypi-latest}"
     echo 'force_docker_sdk_for_python_pypi: true' >> tests/integration/interation_config.yml
-    if [ "${platform}" == "rhel" ] && [[ "${version}" =~ ^8\. ]]; then
-        # Use Python 3.8 on RHEL 8.x - TODO: this might be no longer necessary for high enough minor version! Check!
-        force_python="--python 3.8"
-    fi
 fi
 
 stage="${S:-prod}"
@@ -37,4 +42,4 @@ fi
 
 # shellcheck disable=SC2086
 ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
-    --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}" ${force_python}
+    --python "${pyver}" --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}" ${force_python}

tests/utils/shippable/ubuntu.sh

@@ -0,0 +1 @@
+remote.sh