Run extra sanity tests with nox. (#1068)

.azure-pipelines/azure-pipelines.yml

@@ -60,8 +60,6 @@ stages:
           targets:
             - name: Sanity
               test: 'devel/sanity/1'
-            - name: Sanity Extra # Only on devel
-              test: 'devel/sanity/extra'
             - name: Units
               test: 'devel/units/1'
   - stage: Ansible_2_18

.github/workflows/import-galaxy.yml

@@ -1,20 +0,0 @@
----
-# Copyright (c) Ansible Project
-# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-name: import-galaxy
-'on':
-  # Run CI against all pushes (direct commits, also merged PRs) to main, and all Pull Requests
-  push:
-    branches:
-      - main
-      - stable-*
-  pull_request:
-
-jobs:
-  import-galaxy:
-    permissions:
-      contents: read
-    name: Test to import built collection artifact with Galaxy importer
-    uses: ansible-community/github-action-test-galaxy-import/.github/workflows/test-galaxy-import.yml@main

.github/workflows/nox.yml

@@ -3,31 +3,26 @@
 # GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-name: Verify REUSE
-
-on:
+name: nox
+'on':
   push:
     branches:
       - main
       - stable-*
   pull_request:
-    branches:
-      - main
-      - stable-*
   # Run CI once per day (at 04:30 UTC)
   schedule:
     - cron: '30 4 * * *'
+  workflow_dispatch:
 
 jobs:
-  check:
-    permissions:
-      contents: read
+  nox:
     runs-on: ubuntu-latest
-
+    name: "Run extra sanity tests"
     steps:
-      - uses: actions/checkout@v4
+      - name: Check out collection
+        uses: actions/checkout@v4
         with:
           persist-credentials: false
-
-      - name: REUSE Compliance Check
-        uses: fsfe/reuse-action@v5
+      - name: Run nox
+        uses: ansible-community/antsibull-nox@main

README.md

@@ -9,6 +9,7 @@ SPDX-License-Identifier: GPL-3.0-or-later
 [![Documentation](https://img.shields.io/badge/docs-brightgreen.svg)](https://docs.ansible.com/ansible/devel/collections/community/docker/)
 [![Build Status](https://dev.azure.com/ansible/community.docker/_apis/build/status/CI?branchName=main)](https://dev.azure.com/ansible/community.docker/_build?definitionId=25)
 [![EOL CI](https://github.com/ansible-collections/community.docker/actions/workflows/ansible-test.yml/badge.svg?branch=main)](https://github.com/ansible-collections/community.docker/actions)
+[![Nox CI](https://github.com/ansible-collections/community.docker/actions/workflows/nox.yml/badge.svg?branch=main)](https://github.com/ansible-collections/community.docker/actions)
 [![Codecov](https://img.shields.io/codecov/c/github/ansible-collections/community.docker)](https://codecov.io/gh/ansible-collections/community.docker)
 [![REUSE status](https://api.reuse.software/badge/github.com/ansible-collections/community.docker)](https://api.reuse.software/info/github.com/ansible-collections/community.docker)
 

antsibull-nox.toml

@@ -0,0 +1,32 @@
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2025 Felix Fontein <felix@fontein.de>
+
+[collection_sources]
+"community.internal_test_tools" = "git+https://github.com/ansible-collections/community.internal_test_tools.git,main"
+"community.library_inventory_filtering_v1" = "git+https://github.com/ansible-collections/community.library_inventory_filtering.git,stable-1"
+
+[sessions]
+
+[sessions.docs_check]
+validate_collection_refs="all"
+
+[sessions.license_check]
+
+[sessions.extra_checks]
+run_no_unwanted_files = true
+no_unwanted_files_module_extensions = [".py"]
+no_unwanted_files_yaml_extensions = [".yml"]
+run_action_groups = true
+
+[[sessions.extra_checks.action_groups_config]]
+name = "docker"
+pattern = "^.*$"
+exclusions = [
+    "current_container_facts",
+    "docker_context_info",
+]
+doc_fragment = "community.docker.attributes.actiongroup_docker"
+
+[sessions.build_import_check]
+run_galaxy_importer = true

noxfile.py

@@ -0,0 +1,27 @@
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2025 Felix Fontein <felix@fontein.de>
+
+# /// script
+# dependencies = ["nox>=2025.02.09", "antsibull-nox"]
+# ///
+
+import sys
+
+import nox
+
+
+try:
+    import antsibull_nox
+except ImportError:
+    print("You need to install antsibull-nox in the same Python environment as nox.")
+    sys.exit(1)
+
+
+antsibull_nox.load_antsibull_nox_toml()
+
+
+# Allow to run the noxfile with `python noxfile.py`, `pipx run noxfile.py`, or similar.
+# Requires nox >= 2025.02.09
+if __name__ == "__main__":
+    nox.main()

tests/sanity/extra/action-group.json

@@ -1,12 +0,0 @@
-{
-    "include_symlinks": true,
-    "prefixes": [
-        "meta/runtime.yml",
-        "plugins/modules/",
-        "tests/sanity/extra/action-group."
-    ],
-    "output": "path-message",
-    "requirements": [
-        "pyyaml"
-    ]
-}

tests/sanity/extra/action-group.json.license

@@ -1,3 +0,0 @@
-GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
-SPDX-License-Identifier: GPL-3.0-or-later
-SPDX-FileCopyrightText: Ansible Project

tests/sanity/extra/action-group.py

@@ -1,122 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2024, Felix Fontein <felix@fontein.de>
-# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
-# SPDX-License-Identifier: GPL-3.0-or-later
-"""Make sure all modules that should show up in the action group."""
-
-from __future__ import annotations
-
-import os
-import re
-import yaml
-
-
-ACTION_GROUPS = {
-    # The format is as follows:
-    # * 'pattern': a regular expression matching all module names potentially belonging to the action group;
-    # * 'exclusions': a list of modules that are not part of the action group; all other modules matching 'pattern' must be part of it;
-    # * 'doc_fragment': the docs fragment that documents membership of the action group.
-    'docker': {
-        'pattern': re.compile('^.*$'),
-        'exclusions': [
-            'current_container_facts',
-            'docker_context_info',
-        ],
-        'doc_fragment': 'community.docker.attributes.actiongroup_docker',
-    },
-}
-
-
-def main():
-    """Main entry point."""
-
-    # Load redirects
-    meta_runtime = 'meta/runtime.yml'
-    self_path = 'tests/sanity/extra/action-group.py'
-    try:
-        with open(meta_runtime, 'rb') as f:
-            data = yaml.safe_load(f)
-        action_groups = data['action_groups']
-    except Exception as exc:
-        print(f'{meta_runtime}: cannot load action groups: {exc}')
-        return
-
-    for action_group in action_groups:
-        if action_group not in ACTION_GROUPS:
-            print(f'{meta_runtime}: found unknown action group {action_group!r}; likely {self_path} needs updating')
-    for action_group, action_group_data in list(ACTION_GROUPS.items()):
-        if action_group not in action_groups:
-            print(f'{meta_runtime}: cannot find action group {action_group!r}; likely {self_path} needs updating')
-
-    modules_directory = 'plugins/modules/'
-    modules_suffix = '.py'
-
-    for file in os.listdir(modules_directory):
-        if not file.endswith(modules_suffix):
-            continue
-        module_name = file[:-len(modules_suffix)]
-
-        for action_group, action_group_data in ACTION_GROUPS.items():
-            action_group_content = action_groups.get(action_group) or []
-            path = os.path.join(modules_directory, file)
-
-            if not action_group_data['pattern'].match(module_name):
-                if module_name in action_group_content:
-                    print(f'{path}: module is in action group {action_group!r} despite not matching its pattern as defined in {self_path}')
-                continue
-
-            should_be_in_action_group = module_name not in action_group_data['exclusions']
-
-            if should_be_in_action_group:
-                if module_name not in action_group_content:
-                    print(f'{meta_runtime}: module {module_name!r} is not part of {action_group!r} action group')
-                else:
-                    action_group_content.remove(module_name)
-
-            documentation = []
-            in_docs = False
-            with open(path, 'r', encoding='utf-8') as f:
-                for line in f:
-                    if line.startswith('DOCUMENTATION ='):
-                        in_docs = True
-                    elif line.startswith(("'''", '"""')) and in_docs:
-                        in_docs = False
-                    elif in_docs:
-                        documentation.append(line)
-            if in_docs:
-                print(f'{path}: cannot find DOCUMENTATION end')
-            if not documentation:
-                print(f'{path}: cannot find DOCUMENTATION')
-                continue
-
-            try:
-                docs = yaml.safe_load('\n'.join(documentation))
-                if not isinstance(docs, dict):
-                    raise Exception('is not a top-level dictionary')
-            except Exception as exc:
-                print(f'{path}: cannot load DOCUMENTATION as YAML: {exc}')
-                continue
-
-            docs_fragments = docs.get('extends_documentation_fragment') or []
-            is_in_action_group = action_group_data['doc_fragment'] in docs_fragments
-
-            if should_be_in_action_group != is_in_action_group:
-                if should_be_in_action_group:
-                    print(
-                        f'{path}: module does not document itself as part of action group {action_group!r}, but it should;'
-                        f' you need to add {action_group_data["doc_fragment"]} to "extends_documentation_fragment" in DOCUMENTATION'
-                    )
-                else:
-                    print(f'{path}: module documents itself as part of action group {action_group!r}, but it should not be')
-
-    for action_group, action_group_data in ACTION_GROUPS.items():
-        action_group_content = action_groups.get(action_group) or []
-        for module_name in action_group_content:
-            print(
-                f'{meta_runtime}: module {module_name} mentioned in {action_group!r} action group'
-                f' does not exist or does not match pattern defined in {self_path}'
-            )
-
-
-if __name__ == '__main__':
-    main()

tests/sanity/extra/extra-docs.json

@@ -1,13 +0,0 @@
-{
-    "include_symlinks": false,
-    "prefixes": [
-        "docs/docsite/",
-        "plugins/",
-        "roles/"
-    ],
-    "output": "path-line-column-message",
-    "requirements": [
-        "ansible-core",
-        "antsibull-docs"
-    ]
-}

tests/sanity/extra/extra-docs.json.license

@@ -1,3 +0,0 @@
-GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
-SPDX-License-Identifier: GPL-3.0-or-later
-SPDX-FileCopyrightText: Ansible Project

tests/sanity/extra/extra-docs.py

@@ -1,29 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) Ansible Project
-# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
-# SPDX-License-Identifier: GPL-3.0-or-later
-"""Check extra collection docs with antsibull-docs."""
-from __future__ import (absolute_import, division, print_function)
-__metaclass__ = type
-
-import os
-import sys
-import subprocess
-
-
-def main():
-    """Main entry point."""
-    env = os.environ.copy()
-    suffix = ':{env}'.format(env=env["ANSIBLE_COLLECTIONS_PATH"]) if 'ANSIBLE_COLLECTIONS_PATH' in env else ''
-    env['ANSIBLE_COLLECTIONS_PATH'] = '{root}{suffix}'.format(root=os.path.dirname(os.path.dirname(os.path.dirname(os.getcwd()))), suffix=suffix)
-    p = subprocess.run(
-        ['antsibull-docs', 'lint-collection-docs', '--plugin-docs', '--skip-rstcheck', '.'],
-        env=env,
-        check=False,
-    )
-    if p.returncode not in (0, 3):
-        print('{0}:0:0: unexpected return code {1}'.format(sys.argv[0], p.returncode))
-
-
-if __name__ == '__main__':
-    main()

tests/sanity/extra/licenses.json

@@ -1,4 +0,0 @@
-{
-    "include_symlinks": false,
-    "output": "path-message"
-}

tests/sanity/extra/licenses.json.license

@@ -1,3 +0,0 @@
-GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
-SPDX-License-Identifier: GPL-3.0-or-later
-SPDX-FileCopyrightText: Ansible Project

tests/sanity/extra/licenses.py

@@ -1,110 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2022, Felix Fontein <felix@fontein.de>
-# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
-# SPDX-License-Identifier: GPL-3.0-or-later
-"""Prevent files without a correct license identifier from being added to the source tree."""
-from __future__ import (absolute_import, division, print_function)
-__metaclass__ = type
-
-import os
-import glob
-import sys
-
-
-def format_license_list(licenses):
-    if not licenses:
-        return '(empty)'
-    return ', '.join(['"%s"' % license for license in licenses])
-
-
-def find_licenses(filename, relax=False):
-    spdx_license_identifiers = []
-    other_license_identifiers = []
-    has_copyright = False
-    try:
-        with open(filename, 'r', encoding='utf-8') as f:
-            for line in f:
-                line = line.rstrip()
-                if 'Copyright ' in line:
-                    has_copyright = True
-                if 'Copyright: ' in line:
-                    print('%s: found copyright line with "Copyright:". Please remove the colon.' % (filename, ))
-                if 'SPDX-FileCopyrightText: ' in line:
-                    has_copyright = True
-                idx = line.find('SPDX-License-Identifier: ')
-                if idx >= 0:
-                    lic_id = line[idx + len('SPDX-License-Identifier: '):]
-                    spdx_license_identifiers.extend(lic_id.split(' OR '))
-                if 'GNU General Public License' in line:
-                    if 'v3.0+' in line:
-                        other_license_identifiers.append('GPL-3.0-or-later')
-                    if 'version 3 or later' in line:
-                        other_license_identifiers.append('GPL-3.0-or-later')
-                if 'Simplified BSD License' in line:
-                    other_license_identifiers.append('BSD-2-Clause')
-                if 'Apache License 2.0' in line:
-                    other_license_identifiers.append('Apache-2.0')
-                if 'PSF License' in line or 'Python-2.0' in line:
-                    other_license_identifiers.append('PSF-2.0')
-                if 'MIT License' in line:
-                    other_license_identifiers.append('MIT')
-    except Exception as exc:
-        print('%s: error while processing file: %s' % (filename, exc))
-    if len(set(spdx_license_identifiers)) < len(spdx_license_identifiers):
-        print('%s: found identical SPDX-License-Identifier values' % (filename, ))
-    if other_license_identifiers and set(other_license_identifiers) != set(spdx_license_identifiers):
-        print('%s: SPDX-License-Identifier yielded the license list %s, while manual guessing yielded the license list %s' % (
-            filename, format_license_list(spdx_license_identifiers), format_license_list(other_license_identifiers)))
-    if not has_copyright and not relax:
-        print('%s: found no copyright notice' % (filename, ))
-    return sorted(spdx_license_identifiers)
-
-
-def main():
-    """Main entry point."""
-    paths = sys.argv[1:] or sys.stdin.read().splitlines()
-
-    # The following paths are allowed to have no license identifier
-    no_comments_allowed = [
-        'changelogs/fragments/*.yml',
-        'changelogs/fragments/*.yaml',
-    ]
-
-    # These files are completely ignored
-    ignore_paths = [
-        '.ansible-test-timeout.json',
-        '.reuse/dep5',
-        'LICENSES/*.txt',
-        'COPYING',
-    ]
-
-    no_comments_allowed = [fn for pattern in no_comments_allowed for fn in glob.glob(pattern)]
-    ignore_paths = [fn for pattern in ignore_paths for fn in glob.glob(pattern)]
-
-    valid_licenses = [license_file[len('LICENSES/'):-len('.txt')] for license_file in glob.glob('LICENSES/*.txt')]
-
-    for path in paths:
-        if path.startswith('./'):
-            path = path[2:]
-        if path in ignore_paths or path.startswith('tests/output/'):
-            continue
-        if os.stat(path).st_size == 0:
-            continue
-        if not path.endswith('.license') and os.path.exists(path + '.license'):
-            path = path + '.license'
-        valid_licenses_for_path = valid_licenses
-        if path.startswith('plugins/') and not path.startswith(('plugins/modules/', 'plugins/module_utils/')):
-            valid_licenses_for_path = [license for license in valid_licenses if license == 'GPL-3.0-or-later']
-        licenses = find_licenses(path, relax=path in no_comments_allowed)
-        if not licenses:
-            if path not in no_comments_allowed:
-                print('%s: must have at least one license' % (path, ))
-        else:
-            for license in licenses:
-                if license not in valid_licenses_for_path:
-                    print('%s: found not allowed license "%s", must be one of %s' % (
-                        path, license, format_license_list(valid_licenses_for_path)))
-
-
-if __name__ == '__main__':
-    main()

tests/sanity/extra/licenses.py.license

@@ -1,3 +0,0 @@
-GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
-SPDX-License-Identifier: GPL-3.0-or-later
-SPDX-FileCopyrightText: 2022, Felix Fontein <felix@fontein.de>

tests/sanity/extra/no-unwanted-files.json

@@ -1,7 +0,0 @@
-{
-    "include_symlinks": true,
-    "prefixes": [
-        "plugins/"
-    ],
-    "output": "path-message"
-}

tests/sanity/extra/no-unwanted-files.json.license

@@ -1,3 +0,0 @@
-GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
-SPDX-License-Identifier: GPL-3.0-or-later
-SPDX-FileCopyrightText: Ansible Project

tests/sanity/extra/no-unwanted-files.py

@@ -1,44 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) Ansible Project
-# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
-# SPDX-License-Identifier: GPL-3.0-or-later
-"""Prevent unwanted files from being added to the source tree."""
-from __future__ import (absolute_import, division, print_function)
-__metaclass__ = type
-
-import os
-import sys
-
-
-def main():
-    """Main entry point."""
-    paths = sys.argv[1:] or sys.stdin.read().splitlines()
-
-    allowed_extensions = (
-        '.cs',
-        '.ps1',
-        '.psm1',
-        '.py',
-    )
-
-    skip_paths = set([
-    ])
-
-    skip_directories = (
-    )
-
-    for path in paths:
-        if path in skip_paths:
-            continue
-
-        if any(path.startswith(skip_directory) for skip_directory in skip_directories):
-            continue
-
-        ext = os.path.splitext(path)[1]
-
-        if ext not in allowed_extensions:
-            print('%s: extension must be one of: %s' % (path, ', '.join(allowed_extensions)))
-
-
-if __name__ == '__main__':
-    main()

tests/utils/shippable/sanity.sh

@@ -5,22 +5,12 @@
 
 set -o pipefail -eux
 
-declare -a args
-IFS='/:' read -ra args <<< "$1"
-
-group="${args[1]}"
-
 if [ "${BASE_BRANCH:-}" ]; then
     base_branch="origin/${BASE_BRANCH}"
 else
     base_branch=""
 fi
 
-if [ "${group}" == "extra" ]; then
-    ../internal_test_tools/tools/run.py --color --bot --junit
-    exit
-fi
-
 # shellcheck disable=SC2086
 ansible-test sanity --color -v --junit ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
     --docker --base-branch "${base_branch}" \

tests/utils/shippable/shippable.sh

@@ -67,17 +67,13 @@ fi
 
 export ANSIBLE_COLLECTIONS_PATHS="${PWD}/../../../"
 
-if [ "${test}" == "sanity/extra" ]; then
-    retry pip install junit-xml --disable-pip-version-check
-fi
-
 # START: HACK
 
 retry git clone --depth=1 --single-branch --branch stable-1 https://github.com/ansible-collections/community.library_inventory_filtering.git "${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/library_inventory_filtering_v1"
 # NOTE: we're installing with git to work around Galaxy being a huge PITA (https://github.com/ansible/galaxy/issues/2429)
 # retry ansible-galaxy -vvv collection install community.library_inventory_filtering_v1
 
-if [ "${test}" == "sanity/extra" ] || [ "${test}" == "units/1" ]; then
+if [ "${test}" == "units/1" ]; then
     # Nothing further should be added to this list.
     # This is to prevent modules or plugins in this collection having a runtime dependency on other collections.
     retry git clone --depth=1 --single-branch https://github.com/ansible-collections/community.internal_test_tools.git "${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/internal_test_tools"
@@ -85,7 +81,7 @@ if [ "${test}" == "sanity/extra" ] || [ "${test}" == "units/1" ]; then
     # retry ansible-galaxy -vvv collection install community.internal_test_tools
 fi
 
-if [ "${script}" != "sanity/1" ] && [ "${script}" != "units/1" ] && [ "${test}" != "sanity/extra" ]; then
+if [ "${script}" != "sanity/1" ] && [ "${script}" != "units/1" ]; then
     # To prevent Python dependencies on other collections only install other collections for integration tests
     retry git clone --depth=1 --single-branch https://github.com/ansible-collections/ansible.posix.git "${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/ansible/posix"
     retry git clone --depth=1 --single-branch https://github.com/ansible-collections/community.crypto.git "${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/crypto"