Add docker_image_export module (#774)

* Add docker_image_export module.

* Add basic tests.

* Add more seealsos.

README.md

@@ -63,6 +63,7 @@ If you use the Ansible package and do not update collections independently, use
     - community.docker.docker_host_info: retrieve information on the Docker daemon
     - community.docker.docker_image: manage Docker images
     - community.docker.docker_image_build: build Docker images using Docker buildx
+    - community.docker.docker_image_export: export (archive) Docker images
     - community.docker.docker_image_info: retrieve information on Docker images
     - community.docker.docker_image_load: load Docker images from archives
     - community.docker.docker_image_pull: pull Docker images from registries

docs/docsite/rst/scenario_guide.rst

@@ -203,6 +203,9 @@ For working with a plain Docker daemon, that is without Swarm, there are connect
     docker_image_build
         The :ansplugin:`community.docker.docker_image_build module <community.docker.docker_image_build#module>` allows you to build a Docker image using Docker buildx.
 
+    docker_image_export module
+        The :ansplugin:`community.docker.docker_image_export module <community.docker.docker_image_export#module>` allows you to export (archive) images.
+
     docker_image_info module
         The :ansplugin:`community.docker.docker_image_info module <community.docker.docker_image_info#module>` allows you to list and inspect images.
 

meta/runtime.yml

@@ -17,6 +17,7 @@ action_groups:
     - docker_host_info
     - docker_image
     - docker_image_build
+    - docker_image_export
     - docker_image_info
     - docker_image_load
     - docker_image_pull

plugins/module_utils/image_archive.py

@@ -23,7 +23,7 @@ class ImageArchiveManifestSummary(object):
         :param image_id:  File name portion of Config entry, e.g. abcde12345 from abcde12345.json
         :type image_id: str
         :param repo_tags  Docker image names, e.g. ["hello-world:latest"]
-        :type repo_tags: list
+        :type repo_tags: list[str]
         '''
 
         self.image_id = image_id
@@ -60,13 +60,13 @@ def api_image_id(archive_image_id):
     return 'sha256:%s' % archive_image_id
 
 
-def archived_image_manifest(archive_path):
+def load_archived_image_manifest(archive_path):
     '''
-    Attempts to get Image.Id and image name from metadata stored in the image
+    Attempts to get image IDs and image names from metadata stored in the image
     archive tar file.
 
-    The tar should contain a file "manifest.json" with an array with a single entry,
+    The tar should contain a file "manifest.json" with an array with one or more entries,
-    and the entry should have a Config field with the image ID in its file name, as
+    and every entry should have a Config field with the image ID in its file name, as
     well as a RepoTags list, which typically has only one entry.
 
     :raises:
@@ -75,7 +75,7 @@ def archived_image_manifest(archive_path):
     :param archive_path: Tar file to read
     :type archive_path: str
 
-    :return: None, if no file at archive_path, or the extracted image ID, which will not have a sha256: prefix.
+    :return: None, if no file at archive_path, or a list of ImageArchiveManifestSummary objects.
     :rtype: ImageArchiveManifestSummary
     '''
 
@@ -100,50 +100,51 @@ def archived_image_manifest(archive_path):
                     # In Python 2.6, this does not have __exit__
                     ef.close()
 
-                if len(manifest) != 1:
+                if len(manifest) == 0:
                     raise ImageArchiveInvalidException(
-                        "Expected to have one entry in manifest.json but found %s" % len(manifest),
+                        "Expected to have at least one entry in manifest.json but found none",
                         None
                     )
 
-                m0 = manifest[0]
+                result = []
+                for index, meta in enumerate(manifest):
+                    try:
+                        config_file = meta['Config']
+                    except KeyError as exc:
+                        raise ImageArchiveInvalidException(
+                            "Failed to get Config entry from {0}th manifest in manifest.json: {1}".format(index + 1, to_native(exc)),
+                            exc
+                        )
 
-                try:
+                    # Extracts hash without 'sha256:' prefix
-                    config_file = m0['Config']
+                    try:
-                except KeyError as exc:
+                        # Strip off .json filename extension, leaving just the hash.
-                    raise ImageArchiveInvalidException(
+                        image_id = os.path.splitext(config_file)[0]
-                        "Failed to get Config entry from manifest.json: %s" % to_native(exc),
+                    except Exception as exc:
-                        exc
+                        raise ImageArchiveInvalidException(
-                    )
+                            "Failed to extract image id from config file name %s: %s" % (config_file, to_native(exc)),
+                            exc
+                        )
 
-                # Extracts hash without 'sha256:' prefix
+                    for prefix in (
-                try:
+                        'blobs/sha256/',  # Moby 25.0.0, Docker API 1.44
-                    # Strip off .json filename extension, leaving just the hash.
+                    ):
-                    image_id = os.path.splitext(config_file)[0]
+                        if image_id.startswith(prefix):
-                except Exception as exc:
+                            image_id = image_id[len(prefix):]
-                    raise ImageArchiveInvalidException(
-                        "Failed to extract image id from config file name %s: %s" % (config_file, to_native(exc)),
-                        exc
-                    )
 
-                for prefix in (
+                    try:
-                    'blobs/sha256/',  # Moby 25.0.0, Docker API 1.44
+                        repo_tags = meta['RepoTags']
-                ):
+                    except KeyError as exc:
-                    if image_id.startswith(prefix):
+                        raise ImageArchiveInvalidException(
-                        image_id = image_id[len(prefix):]
+                            "Failed to get RepoTags entry from {0}th manifest in manifest.json: {1}".format(index + 1, to_native(exc)),
+                            exc
+                        )
 
-                try:
+                    result.append(ImageArchiveManifestSummary(
-                    repo_tags = m0['RepoTags']
+                        image_id=image_id,
-                except KeyError as exc:
+                        repo_tags=repo_tags
-                    raise ImageArchiveInvalidException(
+                    ))
-                        "Failed to get RepoTags entry from manifest.json: %s" % to_native(exc),
+                return result
-                        exc
-                    )
-
-                return ImageArchiveManifestSummary(
-                    image_id=image_id,
-                    repo_tags=repo_tags
-                )
 
             except ImageArchiveInvalidException:
                 raise
@@ -161,3 +162,33 @@ def archived_image_manifest(archive_path):
         raise
     except Exception as exc:
         raise ImageArchiveInvalidException("Failed to open tar file %s: %s" % (archive_path, to_native(exc)), exc)
+
+
+def archived_image_manifest(archive_path):
+    '''
+    Attempts to get Image.Id and image name from metadata stored in the image
+    archive tar file.
+
+    The tar should contain a file "manifest.json" with an array with a single entry,
+    and the entry should have a Config field with the image ID in its file name, as
+    well as a RepoTags list, which typically has only one entry.
+
+    :raises:
+        ImageArchiveInvalidException: A file already exists at archive_path, but could not extract an image ID from it.
+
+    :param archive_path: Tar file to read
+    :type archive_path: str
+
+    :return: None, if no file at archive_path, or the extracted image ID, which will not have a sha256: prefix.
+    :rtype: ImageArchiveManifestSummary
+    '''
+
+    results = load_archived_image_manifest(archive_path)
+    if results is None:
+        return None
+    if len(results) == 1:
+        return results[0]
+    raise ImageArchiveInvalidException(
+        "Expected to have one entry in manifest.json but found %s" % len(results),
+        None
+    )

plugins/modules/docker_image.py

@@ -253,6 +253,7 @@ author:
 
 seealso:
   - module: community.docker.docker_image_build
+  - module: community.docker.docker_image_export
   - module: community.docker.docker_image_info
   - module: community.docker.docker_image_load
   - module: community.docker.docker_image_pull

plugins/modules/docker_image_export.py

@@ -0,0 +1,283 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2023, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+DOCUMENTATION = '''
+---
+module: docker_image_export
+
+short_description: Export (archive) Docker images
+
+version_added: 3.7.0
+
+description:
+  - Creates an archive (tarball) from one or more Docker images.
+  - This can be copied to another machine and loaded with M(community.docker.docker_image_load).
+
+extends_documentation_fragment:
+  - community.docker.docker.api_documentation
+  - community.docker.attributes
+  - community.docker.attributes.actiongroup_docker
+
+attributes:
+  check_mode:
+    support: full
+  diff_mode:
+    support: none
+
+options:
+  names:
+    description:
+      - "One or more image names. Name format will be one of: C(name), C(repository/name), C(registry_server:port/name).
+        When pushing or pulling an image the name can optionally include the tag by appending C(:tag_name)."
+      - Note that image IDs (hashes) can also be used.
+    type: list
+    elements: str
+    required: true
+    aliases:
+      - name
+  tag:
+    description:
+      - Tag for the image name O(name) that is to be tagged.
+      - If O(name)'s format is C(name:tag), then the tag value from O(name) will take precedence.
+    type: str
+    default: latest
+  path:
+    description:
+      - The C(.tar) file the image should be exported to.
+    type: path
+  force:
+    description:
+      - Export the image even if the C(.tar) file already exists and seems to contain the right image.
+    type: bool
+    default: false
+
+requirements:
+  - "Docker API >= 1.25"
+
+author:
+  - Felix Fontein (@felixfontein)
+
+seealso:
+  - module: community.docker.docker_image
+  - module: community.docker.docker_image_info
+  - module: community.docker.docker_image_load
+'''
+
+EXAMPLES = '''
+- name: Export an image
+  community.docker.docker_image_export:
+    name: pacur/centos-7
+    path: /tmp/centos-7.tar
+
+- name: Export multiple images
+  community.docker.docker_image_export:
+    names:
+      - hello-world:latest
+      - pacur/centos-7:latest
+    path: /tmp/various.tar
+'''
+
+RETURN = '''
+images:
+    description: Image inspection results for the affected images.
+    returned: success
+    type: list
+    elements: dict
+    sample: []
+'''
+
+import traceback
+
+from ansible.module_utils.common.text.converters import to_native
+
+from ansible_collections.community.docker.plugins.module_utils.common_api import (
+    AnsibleDockerClient,
+    RequestException,
+)
+
+from ansible_collections.community.docker.plugins.module_utils.image_archive import (
+    load_archived_image_manifest,
+    api_image_id,
+    ImageArchiveInvalidException,
+)
+
+from ansible_collections.community.docker.plugins.module_utils.util import (
+    DockerBaseClass,
+    is_image_name_id,
+    is_valid_tag,
+)
+from ansible_collections.community.docker.plugins.module_utils._api.constants import (
+    DEFAULT_DATA_CHUNK_SIZE,
+)
+from ansible_collections.community.docker.plugins.module_utils._api.errors import DockerException
+from ansible_collections.community.docker.plugins.module_utils._api.utils.utils import (
+    parse_repository_tag,
+)
+
+
+class ImageExportManager(DockerBaseClass):
+    def __init__(self, client):
+        super(ImageExportManager, self).__init__()
+
+        self.client = client
+        parameters = self.client.module.params
+        self.check_mode = self.client.check_mode
+
+        self.path = parameters['path']
+        self.force = parameters['force']
+        self.tag = parameters['tag']
+
+        if not is_valid_tag(self.tag, allow_empty=True):
+            self.fail('"{0}" is not a valid docker tag'.format(self.tag))
+
+        # If name contains a tag, it takes precedence over tag parameter.
+        self.names = []
+        for name in parameters['names']:
+            if is_image_name_id(name):
+                self.names.append({'id': name, 'joined': name})
+            else:
+                repo, repo_tag = parse_repository_tag(name)
+                if not repo_tag:
+                    repo_tag = self.tag
+                self.names.append({'name': repo, 'tag': repo_tag, 'joined': '%s:%s' % (repo, repo_tag)})
+
+        if not self.names:
+            self.fail('At least one image name must be specified')
+
+    def fail(self, msg):
+        self.client.fail(msg)
+
+    def get_export_reason(self):
+        if self.force:
+            return 'Exporting since force=true'
+
+        try:
+            archived_images = load_archived_image_manifest(self.path)
+            if archived_images is None:
+                return 'Overwriting since no image is present in archive'
+        except ImageArchiveInvalidException as exc:
+            self.log('Unable to extract manifest summary from archive: %s' % to_native(exc))
+            return 'Overwriting an unreadable archive file'
+
+        left_names = list(self.names)
+        for archived_image in archived_images:
+            found = False
+            for i, name in enumerate(left_names):
+                if name['id'] == api_image_id(archived_image.image_id) and [name['joined']] == archived_image.repo_tags:
+                    del left_names[i]
+                    found = True
+                    break
+            if not found:
+                return 'Overwriting archive since it contains unexpected image %s named %s' % (
+                    archived_image.image_id, ', '.join(archived_image.repo_tags)
+                )
+        if left_names:
+            return 'Overwriting archive since it is missing image(s) %s' % (', '.join([name['joined'] for name in left_names]))
+
+        return None
+
+    def write_chunks(self, chunks):
+        try:
+            with open(self.path, 'wb') as fd:
+                for chunk in chunks:
+                    fd.write(chunk)
+        except Exception as exc:
+            self.fail("Error writing image archive %s - %s" % (self.path, to_native(exc)))
+
+    def export_images(self):
+        image_names = [name['joined'] for name in self.names]
+        image_names_str = ', '.join(image_names)
+        if len(image_names) == 1:
+            self.log("Getting archive of image %s" % image_names[0])
+            try:
+                chunks = self.client._stream_raw_result(
+                    self.client._get(self.client._url('/images/{0}/get', image_names[0]), stream=True),
+                    DEFAULT_DATA_CHUNK_SIZE,
+                    False,
+                )
+            except Exception as exc:
+                self.fail("Error getting image %s - %s" % (image_names[0], to_native(exc)))
+        else:
+            self.log("Getting archive of images %s" % image_names_str)
+            try:
+                chunks = self.client._stream_raw_result(
+                    self.client._get(
+                        self.client._url('/images/get'),
+                        stream=True,
+                        params={'names': image_names},
+                    ),
+                    DEFAULT_DATA_CHUNK_SIZE,
+                    False,
+                )
+            except Exception as exc:
+                self.fail("Error getting images %s - %s" % (image_names_str, to_native(exc)))
+
+        self.write_chunks(chunks)
+
+    def run(self):
+        tag = self.tag
+        if not tag:
+            tag = "latest"
+
+        images = []
+        for name in self.names:
+            if 'id' in name:
+                image = self.client.find_image_by_id(name['id'], accept_missing_image=True)
+            else:
+                image = self.client.find_image(name=name['name'], tag=name['tag'])
+            if not image:
+                self.fail("Image %s not found" % name['joined'])
+            images.append(image)
+
+            # Will have a 'sha256:' prefix
+            name['id'] = image['Id']
+
+        results = {
+            'changed': False,
+            'images': images,
+        }
+
+        reason = self.get_export_reason()
+        if reason is not None:
+            results['msg'] = reason
+            results['changed'] = True
+
+            if not self.check_mode:
+                self.export_images()
+
+        return results
+
+
+def main():
+    argument_spec = dict(
+        path=dict(type='path'),
+        force=dict(type='bool', default=False),
+        names=dict(type='list', elements='str', required=True, aliases=['name']),
+        tag=dict(type='str', default='latest'),
+    )
+
+    client = AnsibleDockerClient(
+        argument_spec=argument_spec,
+        supports_check_mode=True,
+    )
+
+    try:
+        results = ImageExportManager(client).run()
+        client.module.exit_json(**results)
+    except DockerException as e:
+        client.fail('An unexpected Docker error occurred: {0}'.format(to_native(e)), exception=traceback.format_exc())
+    except RequestException as e:
+        client.fail(
+            'An unexpected requests error occurred when trying to talk to the Docker daemon: {0}'.format(to_native(e)),
+            exception=traceback.format_exc())
+
+
+if __name__ == '__main__':
+    main()

plugins/modules/docker_image_load.py

@@ -49,6 +49,7 @@ author:
   - Felix Fontein (@felixfontein)
 
 seealso:
+  - module: community.docker.docker_image_export
   - module: community.docker.docker_image_push
   - module: community.docker.docker_image_remove
   - module: community.docker.docker_image_tag

tests/integration/targets/docker_image_export/aliases

@@ -0,0 +1,6 @@
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+azp/4
+destructive

tests/integration/targets/docker_image_export/meta/main.yml

@@ -0,0 +1,9 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+dependencies:
+  - setup_docker
+  - setup_docker_python_deps
+  - setup_remote_tmp_dir

tests/integration/targets/docker_image_export/tasks/main.yml

@@ -0,0 +1,13 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+####################################################################
+# WARNING: These are designed specifically for Ansible tests       #
+# and should not be used as examples of how to write Ansible roles #
+####################################################################
+
+- when: ansible_facts.distribution ~ ansible_facts.distribution_major_version not in ['CentOS6', 'RedHat6']
+  include_tasks:
+    file: test.yml

tests/integration/targets/docker_image_export/tasks/run-test.yml

@@ -0,0 +1,7 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: "Loading tasks from {{ test_name }}"
+  include_tasks: "{{ test_name }}"

tests/integration/targets/docker_image_export/tasks/test.yml

@@ -0,0 +1,39 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Create random name prefix
+  set_fact:
+    name_prefix: "{{ 'ansible-docker-test-%0x' % ((2**32) | random) }}"
+- name: Create image and container list
+  set_fact:
+    inames: []
+    cnames: []
+
+- debug:
+    msg: "Using name prefix {{ name_prefix }}"
+
+- block:
+    - include_tasks: run-test.yml
+      with_fileglob:
+        - "tests/*.yml"
+      loop_control:
+        loop_var: test_name
+
+  always:
+    - name: "Make sure all images are removed"
+      docker_image_remove:
+        name: "{{ item }}"
+      with_items: "{{ inames }}"
+    - name: "Make sure all containers are removed"
+      docker_container:
+        name: "{{ item }}"
+        state: absent
+        force_kill: true
+      with_items: "{{ cnames }}"
+
+  when: docker_api_version is version('1.25', '>=')
+
+- fail: msg="Too old docker / docker-py version to run docker_image tests!"
+  when: not(docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6)

tests/integration/targets/docker_image_export/tasks/tests/basic.yml

@@ -0,0 +1,69 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- set_fact:
+    image_names:
+      - "{{ docker_test_image_hello_world }}"
+      - "{{ docker_test_image_alpine_different }}"
+      - "{{ docker_test_image_alpine }}"
+
+- name: Make sure images are there
+  docker_image_pull:
+    name: "{{ item }}"
+  register: images
+  loop: "{{ image_names }}"
+
+- vars:
+    image_ids: "{{ images.results | map(attribute='image') | map(attribute='Id') | list }}"
+    all_images: "{{ image_names + (images.results | map(attribute='image') | map(attribute='Id') | list) }}"
+    image_tasks:
+      - file: archive-1.tar
+        images: "{{ image_names }}"
+      - file: archive-2.tar
+        images: "{{ image_ids }}"
+      - file: archive-3.tar
+        images:
+          - "{{ image_names[0] }}"
+          - "{{ image_ids[1] }}"
+      - file: archive-4.tar
+        images:
+          - "{{ image_ids[0] }}"
+          - "{{ image_names[0] }}"
+      - file: archive-5.tar
+        images:
+          - "{{ image_ids[0] }}"
+
+  block:
+    - name: Create archives
+      docker_image_export:
+        names: "{{ item.images }}"
+        path: "{{ remote_tmp_dir }}/{{ item.file }}"
+      loop: "{{ image_tasks }}"
+      loop_control:
+        label: "{{ item.file }}"
+      register: result
+
+    - name: Extract manifest.json files
+      command: tar xvf "{{ remote_tmp_dir }}/{{ item.file }}" manifest.json --to-stdout
+      loop: "{{ image_tasks }}"
+      loop_control:
+        label: "{{ item.file }}"
+      register: manifests
+
+    - name: Do basic tests
+      assert:
+        that:
+          - item.0.images | length == item.1 | length
+          - item.1 | unique | length == item.2 | length
+          - manifest_json_images == export_image_ids
+      loop: "{{ image_tasks | zip(export_images, manifests_json) }}"
+      loop_control:
+        label: "{{ item.0.file }}"
+      vars:
+        filenames: "{{ image_tasks | map(attribute='file') }}"
+        export_images: "{{ result.results | map(attribute='images') | map('map', attribute='Id') }}"
+        manifests_json: "{{ manifests.results | map(attribute='stdout') | map('from_json') }}"
+        manifest_json_images: "{{ item.2 | map(attribute='Config') | map('regex_replace', '.json$', '') | map('regex_replace', '^blobs/sha256/', '') | sort }}"
+        export_image_ids: "{{ item.1 | map('regex_replace', '^sha256:', '') | unique | sort }}"

tests/integration/targets/docker_image_load/tasks/tests/basic.yml

@@ -21,7 +21,9 @@
     all_images: "{{ image_names + (images.results | map(attribute='image') | map(attribute='Id') | list) }}"
 
 - name: Create archives
-  command: docker save {{ item.images | join(' ') }} -o {{ remote_tmp_dir }}/{{ item.file }}
+  docker_image_export:
+    names: "{{ item.images }}"
+    path: "{{ remote_tmp_dir }}/{{ item.file }}"
   loop:
     - file: archive-1.tar
       images: "{{ image_names }}"