From f142f8c86d39694755553c1407359ff82070a03d Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Fri, 5 Mar 2021 09:06:21 +0100 Subject: [PATCH] Add docker_image_load module (#90) * Add docker_image_load module. * Polish module. * Fix bug and add tests. * Apply suggestions from code review Co-authored-by: Amin Vakil * Make sure that containers that still exist are also cleared. * Always return stdout. * Try to work around removal problems. * Accept that the Docker daemon sometimes only reports the named image. * More debug output. * Also prune containers, in the hope that these cause the problems. * Let's see whether pruning containers (but not images) is enough. * Apply suggestions from code review Co-authored-by: Andrew Klychkov * Update plugins/modules/docker_image_load.py Co-authored-by: Andrew Klychkov Co-authored-by: Amin Vakil Co-authored-by: Andrew Klychkov --- plugins/modules/docker_image_load.py | 187 +++++++++++++++ .../targets/docker_image_load/aliases | 2 + .../targets/docker_image_load/meta/main.yml | 3 + .../targets/docker_image_load/tasks/main.yml | 8 + .../docker_image_load/tasks/run-test.yml | 3 + .../targets/docker_image_load/tasks/test.yml | 34 +++ .../docker_image_load/tasks/tests/basic.yml | 213 ++++++++++++++++++ 7 files changed, 450 insertions(+) create mode 100644 plugins/modules/docker_image_load.py create mode 100644 tests/integration/targets/docker_image_load/aliases create mode 100644 tests/integration/targets/docker_image_load/meta/main.yml create mode 100644 tests/integration/targets/docker_image_load/tasks/main.yml create mode 100644 tests/integration/targets/docker_image_load/tasks/run-test.yml create mode 100644 tests/integration/targets/docker_image_load/tasks/test.yml create mode 100644 tests/integration/targets/docker_image_load/tasks/tests/basic.yml diff --git a/plugins/modules/docker_image_load.py b/plugins/modules/docker_image_load.py new file mode 100644 index 00000000..920520b1 --- /dev/null +++ b/plugins/modules/docker_image_load.py @@ -0,0 +1,187 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# Copyright 2016 Red Hat | Ansible +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +DOCUMENTATION = ''' +--- +module: docker_image_load + +short_description: Load docker image(s) from archives + +version_added: 1.3.0 + +description: + - Load one or multiple Docker images from a C(.tar) archive, and return information on + the loaded image(s). + +options: + path: + description: + - The path to the C(.tar) archive to load Docker image(s) from. + type: path + required: true + +extends_documentation_fragment: +- community.docker.docker +- community.docker.docker.docker_py_2_documentation + +notes: + - Does not support C(check_mode). + +requirements: + - "L(Docker SDK for Python,https://docker-py.readthedocs.io/en/stable/) >= 2.5.0" + - "Docker API >= 1.23" + +author: + - Felix Fontein (@felixfontein) +''' + +EXAMPLES = ''' +- name: Load all image(s) from the given tar file + community.docker.docker_image_load: + path: /path/to/images.tar + register: result + +- name: Print the loaded image names + ansible.builtin.debug: + msg: "Loaded the following images: {{ result.image_names | join(', ') }}" +''' + +RETURN = ''' +image_names: + description: List of image names and IDs loaded from the archive. + returned: success + type: list + elements: str + sample: + - 'hello-world:latest' + - 'sha256:e004c2cc521c95383aebb1fb5893719aa7a8eae2e7a71f316a4410784edb00a9' +images: + description: Image inspection results for the loaded images. + returned: success + type: list + elements: dict + sample: [] +''' + +import errno +import traceback + +from ansible_collections.community.docker.plugins.module_utils.common import ( + AnsibleDockerClient, + DockerBaseClass, + is_image_name_id, + RequestException, +) + +try: + from docker.errors import DockerException +except ImportError: + # missing Docker SDK for Python handled in module_utils.docker.common + pass + + +class ImageManager(DockerBaseClass): + def __init__(self, client, results): + super(ImageManager, self).__init__() + + self.client = client + self.results = results + parameters = self.client.module.params + self.check_mode = self.client.check_mode + + self.path = parameters['path'] + + self.load_images() + + @staticmethod + def _extract_output_line(line, output): + ''' + Extract text line from stream output and, if found, adds it to output. + ''' + if 'stream' in line or 'status' in line: + # Make sure we have a string (assuming that line['stream'] and + # line['status'] are either not defined, falsish, or a string) + text_line = line.get('stream') or line.get('status') or '' + output.extend(text_line.splitlines()) + + def load_images(self): + ''' + Load images from a .tar archive + ''' + # Load image(s) from file + load_output = [] + try: + self.log("Opening image {0}".format(self.path)) + with open(self.path, 'rb') as image_tar: + self.log("Loading images from {0}".format(self.path)) + for line in self.client.load_image(image_tar): + self.log(line, pretty_print=True) + self._extract_output_line(line, load_output) + except EnvironmentError as exc: + if exc.errno == errno.ENOENT: + self.client.fail("Error opening archive {0} - {1}".format(self.path, str(exc))) + self.client.fail("Error loading archive {0} - {1}".format(self.path, str(exc)), stdout='\n'.join(load_output)) + except Exception as exc: + self.client.fail("Error loading archive {0} - {1}".format(self.path, str(exc)), stdout='\n'.join(load_output)) + + # Collect loaded images + loaded_images = [] + for line in load_output: + if line.startswith('Loaded image:'): + loaded_images.append(line[len('Loaded image:'):].strip()) + if line.startswith('Loaded image ID:'): + loaded_images.append(line[len('Loaded image ID:'):].strip()) + + if not loaded_images: + self.client.fail("Detected no loaded images. Archive potentially corrupt?", stdout='\n'.join(load_output)) + + images = [] + for image_name in loaded_images: + if is_image_name_id(image_name): + images.append(self.client.find_image_by_id(image_name)) + elif ':' in image_name: + image_name, tag = image_name.rsplit(':', 1) + images.append(self.client.find_image(image_name, tag)) + else: + self.client.module.warn('Image name "{0}" is neither ID nor has a tag'.format(image_name)) + + self.results['image_names'] = loaded_images + self.results['images'] = images + self.results['changed'] = True + self.results['stdout'] = '\n'.join(load_output) + + +def main(): + client = AnsibleDockerClient( + argument_spec=dict( + path=dict(type='path', required=True), + ), + supports_check_mode=False, + min_docker_version='2.5.0', + min_docker_api_version='1.23', + ) + + try: + results = dict( + image_names=[], + images=[], + ) + + ImageManager(client, results) + client.module.exit_json(**results) + except DockerException as e: + client.fail('An unexpected docker error occurred: {0}'.format(e), exception=traceback.format_exc()) + except RequestException as e: + client.fail('An unexpected requests error occurred when docker-py tried to talk ' + 'to the docker daemon: {0}'.format(e), exception=traceback.format_exc()) + + +if __name__ == '__main__': + main() diff --git a/tests/integration/targets/docker_image_load/aliases b/tests/integration/targets/docker_image_load/aliases new file mode 100644 index 00000000..02b78723 --- /dev/null +++ b/tests/integration/targets/docker_image_load/aliases @@ -0,0 +1,2 @@ +shippable/posix/group4 +destructive diff --git a/tests/integration/targets/docker_image_load/meta/main.yml b/tests/integration/targets/docker_image_load/meta/main.yml new file mode 100644 index 00000000..07da8c6d --- /dev/null +++ b/tests/integration/targets/docker_image_load/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - setup_docker diff --git a/tests/integration/targets/docker_image_load/tasks/main.yml b/tests/integration/targets/docker_image_load/tasks/main.yml new file mode 100644 index 00000000..2be493eb --- /dev/null +++ b/tests/integration/targets/docker_image_load/tasks/main.yml @@ -0,0 +1,8 @@ +#################################################################### +# WARNING: These are designed specifically for Ansible tests # +# and should not be used as examples of how to write Ansible roles # +#################################################################### + +- when: ansible_facts.distribution ~ ansible_facts.distribution_major_version not in ['CentOS6', 'RedHat6'] + include_tasks: + file: test.yml diff --git a/tests/integration/targets/docker_image_load/tasks/run-test.yml b/tests/integration/targets/docker_image_load/tasks/run-test.yml new file mode 100644 index 00000000..a2999370 --- /dev/null +++ b/tests/integration/targets/docker_image_load/tasks/run-test.yml @@ -0,0 +1,3 @@ +--- +- name: "Loading tasks from {{ item }}" + include_tasks: "{{ item }}" diff --git a/tests/integration/targets/docker_image_load/tasks/test.yml b/tests/integration/targets/docker_image_load/tasks/test.yml new file mode 100644 index 00000000..28248351 --- /dev/null +++ b/tests/integration/targets/docker_image_load/tasks/test.yml @@ -0,0 +1,34 @@ +--- +- name: Create random name prefix + set_fact: + name_prefix: "{{ 'ansible-test-%0x' % ((2**32) | random) }}" +- name: Create image and container list + set_fact: + inames: [] + cnames: [] + +- debug: + msg: "Using name prefix {{ name_prefix }}" + +- block: + - include_tasks: run-test.yml + with_fileglob: + - "tests/*.yml" + + always: + - name: "Make sure all images are removed" + docker_image: + name: "{{ item }}" + state: absent + with_items: "{{ inames }}" + - name: "Make sure all containers are removed" + docker_container: + name: "{{ item }}" + state: absent + force_kill: yes + with_items: "{{ cnames }}" + + when: docker_py_version is version('2.5.0', '>=') and docker_api_version is version('1.23', '>=') + +- fail: msg="Too old docker / docker-py version to run docker_image tests!" + when: not(docker_py_version is version('2.5.0', '>=') and docker_api_version is version('1.23', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) diff --git a/tests/integration/targets/docker_image_load/tasks/tests/basic.yml b/tests/integration/targets/docker_image_load/tasks/tests/basic.yml new file mode 100644 index 00000000..0748edf7 --- /dev/null +++ b/tests/integration/targets/docker_image_load/tasks/tests/basic.yml @@ -0,0 +1,213 @@ +--- +- set_fact: + image_names: + - "{{ docker_test_image_hello_world }}" + - "{{ docker_test_image_busybox }}" + - "{{ docker_test_image_alpine }}" + +- name: Make sure images are there + docker_image: + name: "{{ item }}" + source: pull + register: images + loop: "{{ image_names }}" + +- name: Compile list of all image names and IDs + set_fact: + image_ids: "{{ images.results | map(attribute='image') | map(attribute='Id') | list }}" + all_images: "{{ image_names + (images.results | map(attribute='image') | map(attribute='Id') | list) }}" + +- name: Create archives + command: docker save {{ item.images | join(' ') }} -o {{ output_dir }}/{{ item.file }} + loop: + - file: archive-1.tar + images: "{{ image_names }}" + - file: archive-2.tar + images: "{{ image_ids }}" + - file: archive-3.tar + images: + - "{{ image_names[0] }}" + - "{{ image_ids[1] }}" + - file: archive-4.tar + images: + - "{{ image_ids[0] }}" + - "{{ image_names[0] }}" + - file: archive-5.tar + images: + - "{{ image_ids[0] }}" + +# All images by IDs + +- name: Remove all images + docker_image: + name: "{{ item }}" + state: absent + force_absent: true + loop: "{{ all_images }}" + ignore_errors: true + register: remove_all_images + +- name: Prune all containers (if removing failed) + docker_prune: + containers: true + when: remove_all_images is failed + +- name: Obtain all docker containers and images (if removing failed) + shell: docker ps -a ; docker images -a + when: remove_all_images is failed + register: docker_container_image_list + +- name: Show all docker containers and images (if removing failed) + debug: + var: docker_container_image_list.stdout_lines + when: remove_all_images is failed + +- name: Remove all images (after pruning) + docker_image: + name: "{{ item }}" + state: absent + force_absent: true + loop: "{{ all_images }}" + when: remove_all_images is failed + +- name: Load all images (IDs) + docker_image_load: + path: "{{ output_dir }}/archive-2.tar" + register: result + +- name: Print loaded image names + debug: + var: result.image_names + +- assert: + that: + - result is changed + - result.image_names | sort == image_ids | sort + - result.image_names | length == result.images | length + +- name: Load all images (IDs, should be same result) + docker_image_load: + path: "{{ output_dir }}/archive-2.tar" + register: result_2 + +- name: Print loaded image names + debug: + var: result_2.image_names + +- assert: + that: + - result_2 is changed + - result_2.image_names | sort == image_ids | sort + - result_2.image_names | length == result_2.images | length + +# Mixed images and IDs + +- name: Remove all images + docker_image: + name: "{{ item }}" + state: absent + loop: "{{ all_images }}" + +- name: Load all images (mixed images and IDs) + docker_image_load: + path: "{{ output_dir }}/archive-3.tar" + register: result + +- name: Print loading log + debug: + var: result.stdout_lines + +- name: Print loaded image names + debug: + var: result.image_names + +- assert: + that: + - result is changed + # For some reason, *sometimes* only the named image is found; in fact, in that case, the log only mentions that image and nothing else + - "result.images | length == 3 or ('Loaded image: ' ~ docker_test_image_hello_world) == result.stdout" + - (result.image_names | sort) in [[image_names[0], image_ids[0], image_ids[1]] | sort, [image_names[0]]] + - result.images | length in [1, 3] + - (result.images | map(attribute='Id') | sort) in [[image_ids[0], image_ids[0], image_ids[1]] | sort, [image_ids[0]]] + +# Same image twice + +- name: Remove all images + docker_image: + name: "{{ item }}" + state: absent + loop: "{{ all_images }}" + +- name: Load all images (same image twice) + docker_image_load: + path: "{{ output_dir }}/archive-4.tar" + register: result + +- name: Print loaded image names + debug: + var: result.image_names + +- assert: + that: + - result is changed + - result.image_names | length == 1 + - result.image_names[0] == image_names[0] + - result.images | length == 1 + - result.images[0].Id == image_ids[0] + +# Single image by ID + +- name: Remove all images + docker_image: + name: "{{ item }}" + state: absent + loop: "{{ all_images }}" + +- name: Load all images (single image by ID) + docker_image_load: + path: "{{ output_dir }}/archive-5.tar" + register: result + +- name: Print loaded image names + debug: + var: result.image_names + +- assert: + that: + - result is changed + - result.image_names | length == 1 + - result.image_names[0] == image_ids[0] + - result.images | length == 1 + - result.images[0].Id == image_ids[0] + +- name: Try to get image info by name + docker_image_info: + name: "{{ image_names[0] }}" + register: result + +- name: Make sure that image does not exist by name + assert: + that: + - result.images | length == 0 + +# All images by names + +- name: Remove all images + docker_image: + name: "{{ item }}" + state: absent + loop: "{{ all_images }}" + +- name: Load all images (names) + docker_image_load: + path: "{{ output_dir }}/archive-1.tar" + register: result + +- name: Print loaded image names + debug: + var: result.image_names + +- assert: + that: + - result.image_names | sort == image_names | sort + - result.image_names | length == result.images | length