From 428dccdf15c119c32847db0374ff71152157c8a9 Mon Sep 17 00:00:00 2001 From: Danny Smit Date: Fri, 23 Apr 2021 17:58:21 +0200 Subject: [PATCH 1/8] fix(clean): fix syntax error with use_upstream repo With use_upstream: repo, several errors occur: - An invalid state id is used to require the repo state - An incorrect path to include the repo state is used - A dictionary with conifguration is incorrectly passed to the onlyif statement, which is already checked by the if-statement around it. --- docker/software/package/clean.sls | 4 ++-- docker/software/package/repo/install.sls | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docker/software/package/clean.sls b/docker/software/package/clean.sls index f4829d20..d9c3d543 100644 --- a/docker/software/package/clean.sls +++ b/docker/software/package/clean.sls @@ -9,7 +9,7 @@ {%- if grains.kernel|lower in ('linux',) %} {%- if d.pkg.docker.use_upstream == 'repo' %} include: - - .package.repo.clean + - .repo.clean {%- endif %} {{ formula }}-software-package-clean-pkg: @@ -18,7 +18,7 @@ include: - reload_modules: {{ d.misc.reload|default(true, true) }} {%- if d.pkg.docker.use_upstream == 'repo' %} - require: - - pkgrepo: {{ formula }}-package-repo-absent + - pkgrepo: {{ formula }}-software-package-repo-absent {%- endif %} {%- elif grains.os_family == 'MacOS' %} diff --git a/docker/software/package/repo/install.sls b/docker/software/package/repo/install.sls index c792721e..422b274e 100644 --- a/docker/software/package/repo/install.sls +++ b/docker/software/package/repo/install.sls @@ -13,7 +13,5 @@ {{- format_kwargs(d.pkg.docker.repo) }} - humanname: {{ grains["os"] }} {{ grains["oscodename"]|capitalize }} Docker Package Repository - refresh: {{ d.misc.refresh }} - - onlyif: - - {{ d.pkg.docker.repo }} {%- endif %} From 4730a725c773430674199be99fb8fffe65febfdf Mon Sep 17 00:00:00 2001 From: Danny Smit Date: Mon, 26 Apr 2021 09:47:35 +0200 Subject: [PATCH 2/8] fix(clean): fix clean of repo for the RedHat family The clean failed due to a mismatch of the name of the repo that was installed. --- docker/osfamilymap.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/osfamilymap.yaml b/docker/osfamilymap.yaml index aa970854..eedb872c 100644 --- a/docker/osfamilymap.yaml +++ b/docker/osfamilymap.yaml @@ -59,6 +59,7 @@ RedHat: docker: # environ_file: /etc/sysconfig/docker repo: + name: docker-ce baseurl: 'https://download.docker.com/linux/{{ grains.os|lower }}/$releasever/$basearch/stable' file: '/etc/yum.repos.d/docker-ce.repo' gpgkey: 'https://download.docker.com/linux/{{ grains.os|lower }}/gpg' From 5a62de91f8afeed3656d939951739c6da3907b9d Mon Sep 17 00:00:00 2001 From: Danny Smit Date: Mon, 26 Apr 2021 09:49:31 +0200 Subject: [PATCH 3/8] fix(test): add missing tests to verify installation from repo --- kitchen.yml | 19 ++++- test/salt/pillar/repo.sls | 166 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 test/salt/pillar/repo.sls diff --git a/kitchen.yml b/kitchen.yml index 7929640b..21217215 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -632,6 +632,23 @@ suites: verifier: inspec_tests: - path: test/integration/package + - name: repo + provisioner: + state_top: + base: + '*': + - docker._mapdata + - docker + pillars: + top.sls: + base: + '*': + - docker + pillars_from_files: + docker.sls: test/salt/pillar/repo.sls + verifier: + inspec_tests: + - path: test/integration/package - name: clean provisioner: state_top: @@ -645,7 +662,7 @@ suites: '*': - docker pillars_from_files: - docker.sls: test/salt/pillar/archive.sls + docker.sls: test/salt/pillar/repo.sls verifier: inspec_tests: - path: test/integration/clean diff --git a/test/salt/pillar/repo.sls b/test/salt/pillar/repo.sls new file mode 100644 index 00000000..dd721dd3 --- /dev/null +++ b/test/salt/pillar/repo.sls @@ -0,0 +1,166 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +--- +# example docker registry container +# if you want to your own docker registry, use this +docker: + wanted: + - docker + - compose + + pkg: + docker: + use_upstream: repo + config: + # yamllint disable-line rule:line-length + - OPTIONS='-s devicemapper --storage-opt dm.fs=xfs --exec-opt native.cgroupdriver=cgroupfs --selinux-enabled' + # yamllint disable-line rule:line-length + - DOCKER_OPTS="-s btrfs --dns 8.8.8.8" + - export http_proxy="http://172.17.42.1:3128" + daemon_config: + insecure-registries: [] + + containers: + running: + - nginx + - prometheus + + nginx: + image: "nginx:latest" + + prometheus: + image: "prom/prometheus:v1.7.1" + env: + - a=b + - ping=pong + - ding=dong + command: + - ls + - ls -l + auto_remove: true + blkio_weight: 1000 + cap_add: ["SYS_ADMIN", "MKNOD"] + dns: + - 8.8.8.8 + - 8.8.4.4 + dns_search: + - EXAMPLE.COM + domainname: + - EXAMPLE.COM + entrypoint: + - ls + - ls -l + - ls -last + - sleep 100 + init: false + labels: + - label1 + - label2 + - label3 + mem_limit: 1g + mem_swappiness: 50 + name: prometheus + network_disabled: false + network_mode: host # bridge or none or container:netcontainer or host + oom_kill_disable: true + oom_score_adj: 100 + pid_mode: host + pids_limit: -1 + privileged: false + publish_all_ports: true + read_only: false + stdin_open: false + tty: true + volume_driver: local + + registry: + image: "registry:latest" + env: + - REGISTRY_LOG_LEVEL=warn + - REGISTRY_STORAGE=s3 + - REGISTRY_STORAGE_S3_REGION=us-west-1 + - REGISTRY_STORAGE_S3_BUCKET=my-bucket + - REGISTRY_STORAGE_S3_ROOTDIRECTORY=/registry + command: + - "--log-driver=syslog" + - "-p 5000:5000" + - "--rm" + + compose: + ## salt dockercompose module ## + applications: + - composetest + composetest: + path: /srv/salt/docker/files/composetest/docker-compose.yml + + ## formerly compose-ng state ## + ng: + registry-datastore: + dvc: true + # image: ®istry_image 'docker.io/registry:latest' ## Fedora + image: ®istry_image 'registry:latest' + container_name: &dvc 'registry-datastore' + command: echo *dvc data volume container + volumes: + - &datapath '/registry' + registry-service: + image: *registry_image + container_name: 'registry-service' + volumes_from: + - *dvc + environment: + SETTINGS_FLAVOR: 'local' + STORAGE_PATH: *datapath + SEARCH_BACKEND: 'sqlalchemy' + REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY: '/registry' + ports: + - 127.0.0.1:5000:5000 + # restart: 'always' # compose v1.9 + deploy: # compose v3 + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 + window: 120s + nginx-latest: + # image: 'docker.io/nginx:latest' ##Fedora + image: 'nginx:latest' + container_name: 'nginx-latest' + links: + - 'registry-service:registry' + ports: + - '80:80' + - '443:443' + volumes: + - /srv/docker-registry/nginx/:/etc/nginx/conf.d + - /srv/docker-registry/auth/:/etc/nginx/conf.d/auth + - /srv/docker-registry/certs/:/etc/nginx/conf.d/certs + working_dir: '/var/www/html' + volume_driver: 'local' + userns_mode: 'host' + user: 'nginx' + # restart: 'always' # compose v1.9 + deploy: # compose v3 + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 + window: 120s + + swarm: + # Per https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.swarm.html + joinswarm: {} + leave_swarm: false + node_ls: {} + remove_node: {} + remove_service: {} + service_create: {} + swarm_init: {} + service_info: {} + swarm_tokens: true + update_node: {} + + misc: + skip_translate: ports + force_present: false + force_running: true From af16bb7781da5653c50375c8e197591a2b7d7c09 Mon Sep 17 00:00:00 2001 From: Danny Smit Date: Mon, 26 Apr 2021 19:20:53 +0200 Subject: [PATCH 4/8] fix(clean): make repo install and clean states consistent --- docker/software/package/clean.sls | 12 +++++++----- docker/software/package/install.sls | 7 ++++--- docker/software/package/repo/clean.sls | 4 ++++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/docker/software/package/clean.sls b/docker/software/package/clean.sls index d9c3d543..b82770e3 100644 --- a/docker/software/package/clean.sls +++ b/docker/software/package/clean.sls @@ -6,17 +6,19 @@ {%- set formula = d.formula %} {%- if d.pkg.docker.use_upstream in ('package', 'repo') %} - {%- if grains.kernel|lower in ('linux',) %} - {%- if d.pkg.docker.use_upstream == 'repo' %} + {%- set enable_repo = d.pkg.docker.use_upstream == 'repo' and grains.os_family in ('RedHat', 'Debian') %} + {%- if enable_repo %} + {%- set sls_repo_clean = tplroot ~ '.software.package.repo.clean' %} include: - - .repo.clean - {%- endif %} + - {{ sls_repo_clean }} + {%- endif %} + {%- if grains.kernel|lower in ('linux', 'darwin') %} {{ formula }}-software-package-clean-pkg: pkg.removed: - name: {{ d.pkg.docker.name }} - reload_modules: {{ d.misc.reload|default(true, true) }} - {%- if d.pkg.docker.use_upstream == 'repo' %} + {%- if enable_repo %} - require: - pkgrepo: {{ formula }}-software-package-repo-absent {%- endif %} diff --git a/docker/software/package/install.sls b/docker/software/package/install.sls index 6e232a7f..245003e4 100644 --- a/docker/software/package/install.sls +++ b/docker/software/package/install.sls @@ -6,13 +6,14 @@ {%- set formula = d.formula %} {%- if d.pkg.docker.use_upstream in ('package', 'repo') %} + {%- set enable_repo = d.pkg.docker.use_upstream == 'repo' and grains.os_family in ('RedHat', 'Debian') %} {%- set docker_pkg_version = d.version | default(d.pkg.version, true) %} - {%- if grains.os_family in ('RedHat', 'Debian') %} + {%- if enable_repo %} {%- set sls_repo_install = tplroot ~ '.software.package.repo.install' %} - include: - {{ sls_repo_install }} {%- endif %} + {%- if grains.kernel|lower in ('linux', 'darwin') %} {%- if 'deps' in d.pkg and d.pkg.deps %} @@ -43,7 +44,7 @@ include: {%- if grains.os|lower not in ('suse',) %} - hold: {{ d.misc.hold|default(false, true) }} {%- endif %} - {%- if grains.os_family in ('RedHat', 'Debian') %} + {%- if enable_repo %} - require: - pkgrepo: {{ formula }}-software-package-repo-managed {%- endif %} diff --git a/docker/software/package/repo/clean.sls b/docker/software/package/repo/clean.sls index 253d8293..af5877d5 100644 --- a/docker/software/package/repo/clean.sls +++ b/docker/software/package/repo/clean.sls @@ -5,8 +5,12 @@ {%- from tplroot ~ "/map.jinja" import data as d with context %} {%- set formula = d.formula %} + {%- if 'repo' in d.pkg.docker and d.pkg.docker.repo %} + {{ formula }}-software-package-repo-absent: pkgrepo.absent: - name: {{ d.pkg.docker.repo.name }} - onlyif: - {{ d.pkg.docker.repo }} + + {%- endif %} From 1f5190b20c77cfc5be0e9150cce09b46917f7fdd Mon Sep 17 00:00:00 2001 From: Danny Smit Date: Tue, 27 Apr 2021 11:07:31 +0200 Subject: [PATCH 5/8] fix(clean): update condition that enables the repo --- docker/software/package/clean.sls | 2 +- docker/software/package/install.sls | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/software/package/clean.sls b/docker/software/package/clean.sls index b82770e3..1006cbab 100644 --- a/docker/software/package/clean.sls +++ b/docker/software/package/clean.sls @@ -6,7 +6,7 @@ {%- set formula = d.formula %} {%- if d.pkg.docker.use_upstream in ('package', 'repo') %} - {%- set enable_repo = d.pkg.docker.use_upstream == 'repo' and grains.os_family in ('RedHat', 'Debian') %} + {%- set enable_repo = grains.os_family in ('RedHat', 'Debian') %} {%- if enable_repo %} {%- set sls_repo_clean = tplroot ~ '.software.package.repo.clean' %} include: diff --git a/docker/software/package/install.sls b/docker/software/package/install.sls index 245003e4..92649cb6 100644 --- a/docker/software/package/install.sls +++ b/docker/software/package/install.sls @@ -6,7 +6,7 @@ {%- set formula = d.formula %} {%- if d.pkg.docker.use_upstream in ('package', 'repo') %} - {%- set enable_repo = d.pkg.docker.use_upstream == 'repo' and grains.os_family in ('RedHat', 'Debian') %} + {%- set enable_repo = grains.os_family in ('RedHat', 'Debian') %} {%- set docker_pkg_version = d.version | default(d.pkg.version, true) %} {%- if enable_repo %} {%- set sls_repo_install = tplroot ~ '.software.package.repo.install' %} From b3c56cb9e1784d217eb4cba71caf2a75b0c3a0d4 Mon Sep 17 00:00:00 2001 From: Danny Smit Date: Tue, 4 May 2021 12:25:02 +0200 Subject: [PATCH 6/8] fix(clean): remove invalid 'onlyif' statement from repo.clean A dictionary is passed to the salt 'onlyif' condition. This is invalid syntax and the condition is already checked by in the jinja around it. --- docker/software/package/repo/clean.sls | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/software/package/repo/clean.sls b/docker/software/package/repo/clean.sls index af5877d5..53c03245 100644 --- a/docker/software/package/repo/clean.sls +++ b/docker/software/package/repo/clean.sls @@ -10,7 +10,5 @@ {{ formula }}-software-package-repo-absent: pkgrepo.absent: - name: {{ d.pkg.docker.repo.name }} - - onlyif: - - {{ d.pkg.docker.repo }} {%- endif %} From 53db7ad8ef775f725b5c339402473d15075174bc Mon Sep 17 00:00:00 2001 From: Danny Smit Date: Tue, 4 May 2021 12:41:51 +0200 Subject: [PATCH 7/8] fix(clean): fix syntax error in networks.clean state file --- docker/networks/clean.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/networks/clean.sls b/docker/networks/clean.sls index e507f5fc..09015605 100644 --- a/docker/networks/clean.sls +++ b/docker/networks/clean.sls @@ -20,3 +20,4 @@ include: - name: {{ name }} {%- endfor %} + {%- endif %} From a7f382f3f511b7006f0980efae3db7164ae2a95d Mon Sep 17 00:00:00 2001 From: Danny Smit Date: Tue, 4 May 2021 17:15:20 +0200 Subject: [PATCH 8/8] fix(clean): update enable_repo condition to be more robust --- docker/software/package/clean.sls | 2 +- docker/software/package/install.sls | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/software/package/clean.sls b/docker/software/package/clean.sls index 1006cbab..3183637b 100644 --- a/docker/software/package/clean.sls +++ b/docker/software/package/clean.sls @@ -6,7 +6,7 @@ {%- set formula = d.formula %} {%- if d.pkg.docker.use_upstream in ('package', 'repo') %} - {%- set enable_repo = grains.os_family in ('RedHat', 'Debian') %} + {%- set enable_repo = grains.os_family in ('RedHat', 'Debian') and d.pkg.docker.get('repo') %} {%- if enable_repo %} {%- set sls_repo_clean = tplroot ~ '.software.package.repo.clean' %} include: diff --git a/docker/software/package/install.sls b/docker/software/package/install.sls index 92649cb6..cf43eb91 100644 --- a/docker/software/package/install.sls +++ b/docker/software/package/install.sls @@ -6,7 +6,7 @@ {%- set formula = d.formula %} {%- if d.pkg.docker.use_upstream in ('package', 'repo') %} - {%- set enable_repo = grains.os_family in ('RedHat', 'Debian') %} + {%- set enable_repo = grains.os_family in ('RedHat', 'Debian') and d.pkg.docker.get('repo') %} {%- set docker_pkg_version = d.version | default(d.pkg.version, true) %} {%- if enable_repo %} {%- set sls_repo_install = tplroot ~ '.software.package.repo.install' %}