Merge pull request #2130 from ceph/2128-bkp

[skip ci] 2128-bkp

roles/ceph-client/tasks/create_users_keys.yml

@@ -6,7 +6,7 @@
 
 - name: set docker_exec_client_cmd for containers
   set_fact:
-    docker_exec_client_cmd: docker run -v /etc/ceph:/etc/ceph --entrypoint /usr/bin/{{ docker_exec_client_cmd_binary }} {{ ceph_docker_registry}}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
+    docker_exec_client_cmd: docker run --rm -v /etc/ceph:/etc/ceph --entrypoint /usr/bin/{{ docker_exec_client_cmd_binary }} {{ ceph_docker_registry}}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
   when: containerized_deployment
 
 - name: set docker_exec_client_cmd for non-containers

roles/ceph-config/templates/ceph.conf.j2

@@ -50,11 +50,19 @@ mon host = {% if nb_mon > 0 %}
       {%- elif ip_version == 'ipv6' -%}
         [{{ hostvars[host]['monitor_address'] }}]
       {%- endif %}
+    {%- elif hostvars[host]['monitor_interface'] is defined -%}
+      {% set interface = 'ansible_' + hostvars[host]['monitor_interface'] %}
+      {% if ip_version == 'ipv4' -%}
+        {{ hostvars[host][interface][ip_version]['address'] }}
+      {%- elif ip_version == 'ipv6' -%}
+        [{{ hostvars[host][interface][ip_version][0]['address'] }}]
+      {%- endif %}
     {%- else -%}
+      {% set interface = 'ansible_' + monitor_interface %}
       {% if ip_version == 'ipv4' -%}
-       {{ hostvars[host]['ansible_' + hostvars[host]['monitor_interface']][ip_version]['address'] }}
+        {{ hostvars[host][interface][ip_version]['address'] }}
       {%- elif ip_version == 'ipv6' -%}
-       [{{ hostvars[host]['ansible_' + hostvars[host]['monitor_interface']][ip_version][0]['address'] }}]
+        [{{ hostvars[host][interface][ip_version][0]['address'] }}]
       {%- endif %}
     {%- endif %}
    {% if not loop.last -%},{%- endif %}
@@ -80,13 +88,20 @@ mon host = {% if nb_mon > 0 %}
       {%- elif ip_version == 'ipv6' -%}
         [{{ hostvars[host]['monitor_address'] }}]
       {%- endif %}
-    {%- else -%}
+    {%- elif hostvars[host]['monitor_interface'] is defined -%}
       {% set interface = 'ansible_' + hostvars[host]['monitor_interface'] %}
       {% if ip_version == 'ipv4' -%}
         {{ hostvars[host][interface][ip_version]['address'] }}
       {%- elif ip_version == 'ipv6' -%}
         [{{ hostvars[host][interface][ip_version][0]['address'] }}]
       {%- endif %}
+    {%- else -%}
+      {% set interface = 'ansible_' + monitor_interface %}
+      {% if ip_version == 'ipv4' -%}
+        {{ [interface][ip_version]['address'] }}
+      {%- elif ip_version == 'ipv6' -%}
+        [{{ [interface][ip_version][0]['address'] }}]
+      {%- endif %}
     {%- endif %}
     {% if not loop.last -%},{%- endif %}
   {%- endfor %}

roles/ceph-docker-common/tasks/main.yml

@@ -90,7 +90,7 @@
   include: fetch_image.yml
 
 - name: get ceph version
-  command: docker run --entrypoint /usr/bin/ceph {{ ceph_docker_registry}}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} --version
+  command: docker run --rm --entrypoint /usr/bin/ceph {{ ceph_docker_registry}}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} --version
   changed_when: false
   always_run: yes
   register: ceph_version

roles/ceph-mon/templates/ceph-mon.service.j2

@@ -8,48 +8,54 @@ ExecStartPre=-/usr/bin/docker rm ceph-mon-%i
 ExecStartPre=$(command -v mkdir) -p /etc/ceph /var/lib/ceph/mon
 ExecStart=/usr/bin/docker run --rm --name ceph-mon-%i --net=host \
   --memory={{ ceph_mon_docker_memory_limit }} \
-  {% if docker_version.split('.')[0] | version_compare('13', '>=') -%}
-  --cpus={{ ceph_mon_docker_cpu_limit }} \
-  {% else -%}
+{% if docker_version.split('.')[0] | version_compare('13', '>=') -%}
+  --cpus={{ ceph_mon_docker_cpu_limit }} \
+{% else -%}
   --cpu-quota={{ ceph_mon_docker_cpu_limit * 100000 }} \
-  {% endif -%}
-  {% if not containerized_deployment_with_kv -%}
+{% endif -%}
+{% if not containerized_deployment_with_kv -%}
   -v /var/lib/ceph:/var/lib/ceph \
   -v /etc/ceph:/etc/ceph \
-  {% else -%}
+{% else -%}
   -e KV_TYPE={{kv_type}} \
   -e KV_IP={{kv_endpoint}}\
   -e KV_PORT={{kv_port}} \
-  {% endif -%}
+{% endif -%}
   -v /etc/localtime:/etc/localtime:ro \
-  {% if mon_docker_privileged -%}
+{% if mon_docker_privileged -%}
   --privileged \
-  {% endif -%}
-  {% if mon_docker_net_host -%}
+{% endif -%}
+{% if mon_docker_net_host -%}
   --net=host \
-  {% endif -%}
+{% endif -%}
   -e IP_VERSION={{ ip_version[-1:] }} \
-  {% if monitor_address_block != 'subnet' %}
+{% if monitor_address_block != 'subnet' %}
     {% if ip_version == 'ipv4' -%}
   -e MON_IP={{ hostvars[inventory_hostname]['ansible_all_' + ip_version + '_addresses'] | ipaddr(monitor_address_block) | first }} \
     {% elif ip_version == 'ipv6' -%}
   -e MON_IP=[{{ hostvars[inventory_hostname]['ansible_all_' + ip_version + '_addresses'] | ipaddr(monitor_address_block) | first }}] \
     {% endif -%}
-  {% elif hostvars[inventory_hostname]['monitor_address'] is defined and hostvars[inventory_hostname]['monitor_address'] != '0.0.0.0' -%}
+{% elif hostvars[inventory_hostname]['monitor_address'] is defined and hostvars[inventory_hostname]['monitor_address'] != '0.0.0.0' %}
     {% if ip_version == 'ipv4' -%}
   -e MON_IP={{ hostvars[inventory_hostname]['monitor_address'] }} \
     {% elif ip_version == 'ipv6' -%}
   -e MON_IP=[{{ hostvars[inventory_hostname]['monitor_address'] }}] \
     {% endif -%}
-  {% else -%}
+{% elif hostvars[inventory_hostname]['monitor_interface'] is defined %}
     {% set interface = 'ansible_' + hostvars[inventory_hostname]['monitor_interface'] %}
-      {% if ip_version == 'ipv6' -%}
-  -e MON_IP=[{{ hostvars[inventory_hostname][interface][ip_version][0]['address'] }}] \
-      {% elif ip_version == 'ipv4' -%}
+    {% if ip_version == 'ipv4' -%}
   -e MON_IP={{ hostvars[inventory_hostname][interface][ip_version]['address'] }} \
-      {% endif -%}
-  {% endif -%}
-
+    {% elif ip_version == 'ipv6' -%}
+  -e MON_IP=[{{ hostvars[inventory_hostname][interface][ip_version][0]['address'] }}] \
+    {% endif -%}
+{% else %}
+    {% set interface = 'ansible_' + monitor_interface %}
+    {% if ip_version == 'ipv4' -%}
+  -e MON_IP={{ [interface][ip_version]['address'] }} \
+    {% elif ip_version == 'ipv6' -%}
+  -e MON_IP=[{{ [interface][ip_version][0]['address'] }}] \
+    {% endif -%}
+{%- endif %}
   -e CLUSTER={{ cluster }} \
   -e FSID={{ fsid }} \
   -e CEPH_PUBLIC_NETWORK={{ public_network }} \

roles/ceph-osd/tasks/docker/start_docker_osd.yml

@@ -11,7 +11,7 @@
     - ceph_docker_on_openstack
 
 - name: test if the container image has the disk_list function
-  command: docker run --entrypoint=stat {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} disk_list.sh
+  command: docker run --rm --entrypoint=stat {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} disk_list.sh
   changed_when: false
   failed_when: false
   register: disk_list

roles/ceph-osd/tasks/scenarios/collocated.yml

@@ -5,7 +5,6 @@
 - name: prepare ceph containerized osd disk collocated
   shell: |
     docker run --net=host \
-    --rm \
     --pid=host \
     --privileged=true \
     --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \
@@ -32,7 +31,6 @@
 - name: automatic prepare ceph containerized osd disk collocated
   shell: |
     docker run --net=host \
-    --rm \
     --pid=host \
     --privileged=true \
     --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.key }} \

roles/ceph-osd/tasks/scenarios/non-collocated.yml

@@ -5,7 +5,6 @@
 - name: prepare ceph "{{ osd_objectstore }}" containerized osd disk(s) non-collocated
   shell: |
     docker run --net=host \
-    --rm \
     --pid=host \
     --privileged=true \
     --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \
@@ -34,7 +33,6 @@
 - name: prepare ceph "{{ osd_objectstore }}" containerized osd disk(s) non-collocated with a dedicated device for db and wal
   shell: |
     docker run --net=host \
-    --rm \
     --pid=host \
     --privileged=true \
     --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \

roles/ceph-osd/templates/ceph-osd-run.sh.j2

@@ -7,17 +7,30 @@
 #############
 {% if disk_list.get("rc", 1) == 0 -%}
 function expose_partitions () {
-DOCKER_ENV=$(docker run --name expose_partitions_${1} --privileged=true -v /dev/:/dev/ -v /etc/ceph:/etc/ceph -e CLUSTER={{ cluster }} -e OSD_DEVICE=/dev/${1} {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} disk_list)
+DOCKER_ENV=$(docker run --rm --name expose_partitions_${1} --privileged=true -v /dev/:/dev/ -v /etc/ceph:/etc/ceph -e CLUSTER={{ cluster }} -e OSD_DEVICE=/dev/${1} {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} disk_list)
   docker rm -f expose_partitions_${1}
 }
 {% else -%}
 # NOTE(leseb): maintains backwards compatibility with old ceph-docker Jewel images
 # Jewel images prior to https://github.com/ceph/ceph-docker/pull/797
 REGEX="[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"
 function expose_partitions {
-  local partition
-  if docker logs ceph-osd-prepare-{{ ansible_hostname }}-devdev${1} |& grep -Eo "$partition is GPT partition"; then
-    part=$(docker logs ceph-osd-prepare-{{ ansible_hostname }}-devdev${1} |& grep "$partition is GPT partition" | grep -Eo /dev/disk/by-partuuid/${REGEX} | uniq)
+  if docker ps -a | grep -sq ceph-osd-prepare-{{ ansible_hostname }}-devdev${1}; then
+    if [[ ! -f {{ ceph_osd_docker_run_script_path }}/ceph-osd-prepare-{{ ansible_hostname }}-devdev${1}.log ]]; then
+      docker logs ceph-osd-prepare-{{ ansible_hostname }}-devdev${1} &> {{ ceph_osd_docker_run_script_path }}/ceph-osd-prepare-{{ ansible_hostname }}-devdev${1}.log
+    fi
+  fi
+  if docker ps -a | grep -sq ceph-osd-prepare-{{ ansible_hostname }}-${1}; then
+    if [[ ! -f {{ ceph_osd_docker_run_script_path }}/ceph-osd-prepare-{{ ansible_hostname }}-${1}.log ]]; then
+      docker logs ceph-osd-prepare-{{ ansible_hostname }}-${1} &> {{ ceph_osd_docker_run_script_path }}/ceph-osd-prepare-{{ ansible_hostname }}-${1}.log
+    fi
+  fi
+  if [[ -f {{ ceph_osd_docker_run_script_path }}/ceph-osd-prepare-{{ ansible_hostname }}-devdev${1}.log ]]; then
+    part=$(grep "Journal is GPT partition" {{ ceph_osd_docker_run_script_path }}/ceph-osd-prepare-{{ ansible_hostname }}-devdev${1}.log | grep -Eo /dev/disk/by-partuuid/${REGEX} | uniq)
+    DOCKER_ENV="$DOCKER_ENV -e OSD_JOURNAL=$part"
+  fi
+  if [[ -f {{ ceph_osd_docker_run_script_path }}/ceph-osd-prepare-{{ ansible_hostname }}-${1}.log ]]; then
+    part=$(grep "Journal is GPT partition" {{ ceph_osd_docker_run_script_path }}/ceph-osd-prepare-{{ ansible_hostname }}-${1}.log | grep -Eo /dev/disk/by-partuuid/${REGEX} | uniq)
     DOCKER_ENV="$DOCKER_ENV -e OSD_JOURNAL=$part"
   fi
 }