CI: Run some tests with the latest development versions of Docker SDK for Python, requests, and urllib3 (#902)

* Run some tests with the latest development versions of Docker SDK for Python, requests, and urllib3.

* Use LooseVersion instead of StrictVersion to parse urllib3 versions.

.azure-pipelines/azure-pipelines.yml

@@ -205,8 +205,8 @@ stages:
         parameters:
           testFormat: devel/{0}
           targets:
-            - name: RHEL 9.4 with latest Docker SDK from PyPi
+            - name: RHEL 9.4 with Docker SDK, urllib3, requests from sources
-              test: rhel/9.4-pypi-latest
+              test: rhel/9.4-dev-latest
           groups:
             - 1
             - 2
@@ -222,8 +222,8 @@ stages:
         parameters:
           testFormat: 2.17/{0}
           targets:
-            - name: RHEL 9.3 with latest Docker SDK from PyPi
+            - name: RHEL 9.3
-              test: rhel/9.3-pypi-latest
+              test: rhel/9.3
           groups:
             - 1
             - 2

changelogs/fragments/902-loose-version.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - "vendored Docker SDK for Python - use ``LooseVersion`` instead of ``StrictVersion`` to compare urllib3 versions. This is needed for development versions (https://github.com/ansible-collections/community.docker/pull/902)."

plugins/module_utils/_api/transport/ssladapter.py

@@ -15,7 +15,7 @@ __metaclass__ = type
       https://github.com/kennethreitz/requests/pull/799
 """
 
-from ansible_collections.community.docker.plugins.module_utils.version import StrictVersion
+from ansible_collections.community.docker.plugins.module_utils.version import LooseVersion
 
 from .._import_helper import HTTPAdapter, urllib3
 from .basehttpadapter import BaseHTTPAdapter
@@ -66,4 +66,4 @@ class SSLHTTPAdapter(BaseHTTPAdapter):
             return False
         if urllib_ver == 'dev':
             return True
-        return StrictVersion(urllib_ver) > StrictVersion('1.5')
+        return LooseVersion(urllib_ver) > LooseVersion('1.5')

tests/integration/targets/setup_docker_python_deps/tasks/main.yml

@@ -13,3 +13,20 @@
     name: "{{ docker_pip_api_packages + (docker_pip_api_packages_python2 if ansible_facts.python.version.major == 2 else []) }}"
     extra_args: "-c {{ remote_constraints }}"
     state: present
+  when: not (force_docker_sdk_for_python_dev | default(false))
+
+- name: Make sure git is installed
+  package:
+    name:
+      - git
+    state: present
+  when: force_docker_sdk_for_python_dev | default(false)
+
+- name: Install/upgrade Python requirements from source repositories
+  pip:
+    name:
+      - git+https://github.com/psf/requests
+      - git+https://github.com/urllib3/urllib3
+    extra_args: "-c {{ remote_constraints }}"
+    state: latest
+  when: force_docker_sdk_for_python_dev | default(false)

tests/integration/targets/setup_docker_sdk_for_python/defaults/main.yml

@@ -8,3 +8,8 @@ docker_py_version: '0.0'
 docker_pip_extra_packages: []
 docker_pip_package: docker
 docker_pip_package_limit: ''
+
+docker_pip_git_packages:
+  - git+https://github.com/psf/requests
+  - git+https://github.com/urllib3/urllib3
+  - git+https://github.com/docker/docker-py

tests/integration/targets/setup_docker_sdk_for_python/tasks/main.yml

@@ -29,11 +29,28 @@
         docker_pip_package_limit: '<4.3.0'
       when: docker_api_version is version('1.39', '<')
 
+    - name: Make sure git is installed
+      package:
+        name:
+          - git
+      when: force_docker_sdk_for_python_dev | default(false)
+
     - name: Install/upgrade Python requirements
       pip:
-        name: "{{ [docker_pip_package ~ docker_pip_package_limit] + docker_pip_extra_packages }}"
+        name: >-
+          {{
+            docker_pip_git_packages
+            if (force_docker_sdk_for_python_dev | default(false)) else
+            ([docker_pip_package ~ docker_pip_package_limit] + docker_pip_extra_packages)
+          }}
         extra_args: "-c {{ remote_constraints }}"
-        state: "{{ 'latest' if force_docker_sdk_for_python_pypi | default(false) else 'present' }}"
+        state: >-
+          {{
+            'latest' if (
+              (force_docker_sdk_for_python_pypi | default(false)) or
+              (force_docker_sdk_for_python_dev | default(false))
+            ) else 'present'
+          }}
 
     - name: Check docker-py version
       command: "{{ ansible_python.executable }} -c 'import docker; print(docker.__version__)'"

tests/utils/shippable/linux-community.sh

@@ -17,6 +17,15 @@ else
     target="azp/"
 fi
 
+if [[ "${python}" =~ -pypi-latest$ ]]; then
+    python="${python/-pypi-latest}"
+    echo 'force_docker_sdk_for_python_pypi: true' >> tests/integration/integration_config.yml
+fi
+if [[ "${python}" =~ -dev-latest$ ]]; then
+    python="${python/-dev-latest}"
+    echo 'force_docker_sdk_for_python_dev: true' >> tests/integration/integration_config.yml
+fi
+
 # shellcheck disable=SC2086
 ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
     --docker "quay.io/ansible-community/test-image:${image}" --python "${python}"

tests/utils/shippable/linux.sh

@@ -16,6 +16,15 @@ else
     target="azp/"
 fi
 
+if [[ "${image}" =~ -pypi-latest$ ]]; then
+    image="${image/-pypi-latest}"
+    echo 'force_docker_sdk_for_python_pypi: true' >> tests/integration/integration_config.yml
+fi
+if [[ "${image}" =~ -dev-latest$ ]]; then
+    image="${image/-dev-latest}"
+    echo 'force_docker_sdk_for_python_dev: true' >> tests/integration/integration_config.yml
+fi
+
 # shellcheck disable=SC2086
 ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
     --docker "${image}"

tests/utils/shippable/remote.sh

@@ -27,10 +27,13 @@ else
     target="azp/"
 fi
 
-force_python=""
 if [[ "${version}" =~ -pypi-latest$ ]]; then
     version="${version/-pypi-latest}"
-    echo 'force_docker_sdk_for_python_pypi: true' >> tests/integration/interation_config.yml
+    echo 'force_docker_sdk_for_python_pypi: true' >> tests/integration/integration_config.yml
+fi
+if [[ "${version}" =~ -dev-latest$ ]]; then
+    version="${version/-dev-latest}"
+    echo 'force_docker_sdk_for_python_dev: true' >> tests/integration/integration_config.yml
 fi
 
 stage="${S:-prod}"
@@ -42,4 +45,4 @@ fi
 
 # shellcheck disable=SC2086
 ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
-    --python "${pyver}" --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}" ${force_python}
+    --python "${pyver}" --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}"