rolling_update.yml

---
# Copyright 2016 PingCAP, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.

# The rolling update playbook of TiDB

- name: check config locally
  hosts: localhost
  any_errors_fatal: true
  tags:
    - always
  roles:
    - check_config_static

- name: gather all facts, and check dest
  hosts: all
  any_errors_fatal: true
  tags:
    - always
  roles:
    - check_config_dynamic


- name: rolling update PD cluster
  hosts: pd_servers
  any_errors_fatal: true
  serial: 1
  tags:
    - pd

  pre_tasks:
    - name: stop PD by supervise
      shell: cd {{ deploy_dir }}/scripts && ./stop_pd.sh
      when: process_supervision == 'supervise'

    - name: stop PD by systemd
      systemd: name=pd.service state=stopped
      become: true
      when: process_supervision == 'systemd' and ansible_user != "root"

    - name: stop PD by systemd
      systemd: name=pd.service state=stopped
      when: process_supervision == 'systemd' and ansible_user == "root"
      vars:
        - ansible_become: false
        - ansible_become_user: root

    - name: wait for PD down
      wait_for: host={{ ansible_host }} port={{ pd_client_port }} state=stopped

  roles:
    - pd

  post_tasks:
    - name: start PD by supervise
      shell: cd {{ deploy_dir }}/scripts && ./start_pd.sh
      when: process_supervision == 'supervise'

    - name: start PD by systemd
      systemd: name=pd.service state=started
      become: true
      when: process_supervision == 'systemd' and ansible_user != "root"

    - name: start PD by systemd
      systemd: name=pd.service state=started
      when: process_supervision == 'systemd' and ansible_user == "root"
      vars:
        - ansible_become: false
        - ansible_become_user: root

    - name: wait for PD up
      wait_for: |
        host={{ ansible_host }} port={{ pd_client_port }} state=present
        send='GET /pd/api/v1/members HTTP/1.0\r\n\r\n' search_regex='200 OK'
      when: not enable_tls|default(false)

    - name: wait for PD up when enable_tls|default(false)
      uri:
        url: "https://{{ ansible_host }}:{{ pd_client_port }}/pd/api/v1/members"
        validate_certs: no
        client_cert: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem"
        client_key: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem"
      register: result
      until: result.status is defined and result.status == 200
      retries: 10
      delay: 5
      when: enable_tls|default(false)


- name: rolling update TiKV cluster
  hosts: tikv_servers
  any_errors_fatal: true
  serial: 1
  tags:
    - tikv

  pre_tasks:
    - set_fact:
        pd_host: "{{ hostvars[groups.pd_servers[0]].ansible_host | default(hostvars[groups.pd_servers[0]].inventory_hostname) }}"
        pd_client_port: "{{ hostvars[groups.pd_servers[0]].pd_client_port }}"

    - set_fact:
        pd_addr: "{{ pd_host }}:{{ pd_client_port }}"
        tikv_addr: "{{ ansible_host }}:{{ tikv_port }}"

    - debug:
        var: pd_addr

    - debug:
        var: tikv_addr

    - include_tasks: get_store_id.yml
      when: not enable_tls|default(false)

    - include_tasks: get_store_id_tls.yml
      when: enable_tls|default(false)

    - name: remove evict-leader-scheduler
      uri:
        url: "http://{{ pd_addr }}/pd/api/v1/schedulers/evict-leader-scheduler-{{ store_id }}"
        method: DELETE
        status_code: 200,500
        return_content: yes
      register: scheduler_info
      until: "'scheduler not found' in scheduler_info.content"
      retries: 3
      delay: 5
      when: not enable_tls|default(false)

    - name: remove evict-leader-scheduler when enable_tls|default(false)
      uri:
        url: "https://{{ pd_addr }}/pd/api/v1/schedulers/evict-leader-scheduler-{{ store_id_tls }}"
        validate_certs: no
        client_cert: "{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}.pem"
        client_key: "{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}-key.pem"
        method: DELETE
        status_code: 200,500
        return_content: yes
      register: scheduler_info_tls
      until: "'scheduler not found' in scheduler_info_tls.content"
      retries: 3
      delay: 5
      when: enable_tls|default(false)

    - name: add evict-leader-scheduler
      uri:
        url: "http://{{ pd_addr }}/pd/api/v1/schedulers"
        method: POST
        status_code: 200
        body_format: json
        body:
          name: "evict-leader-scheduler"
          store_id: "{{ store_id }}"
      when: not enable_tls|default(false)

    - name: add evict-leader-scheduler when enable_tls|default(false)
      uri:
        url: "https://{{ pd_addr }}/pd/api/v1/schedulers"
        validate_certs: no
        client_cert: "{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}.pem"
        client_key: "{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}-key.pem"
        method: POST
        status_code: 200
        body_format: json
        body:
          name: "evict-leader-scheduler"
          store_id: "{{ store_id_tls }}"
      when: enable_tls|default(false)

    - name: check tikv's leader count
      uri:
        url: "http://{{ pd_addr }}/pd/api/v1/store/{{ store_id }}"
        method: GET
        return_content: yes
        body_format: json
        status_code: 200
      register: store_info
      until: (store_info.json.status.leader_count is defined and store_info.json.status.leader_count|int < 10) or store_info.json.status.leader_count is not defined
      retries: 12
      delay: 10
      failed_when: false
      when: not enable_tls|default(false)

    - name: check tikv's leader count when enable_tls|default(false)
      uri:
        url: "https://{{ pd_addr }}/pd/api/v1/store/{{ store_id_tls }}"
        validate_certs: no
        client_cert: "{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}.pem"
        client_key: "{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}-key.pem"
        method: GET
        return_content: yes
        body_format: json
        status_code: 200
      register: store_info_tls
      until: (store_info_tls.json.status.leader_count is defined and store_info_tls.json.status.leader_count|int < 10) or store_info_tls.json.status.leader_count is not defined
      retries: 12
      delay: 10
      failed_when: false
      when: enable_tls|default(false)

    - debug:
        var: store_info.json.status.leader_count
      when: not enable_tls|default(false)

    - debug:
        var: store_info_tls.json.status.leader_count
      when: enable_tls|default(false)

    - name: stop TiKV by supervise
      shell: cd {{ deploy_dir }}/scripts && ./stop_tikv.sh
      when: process_supervision == 'supervise'

    - name: stop TiKV by systemd
      systemd: name=tikv-{{ tikv_port }}.service state=stopped
      become: true
      when: process_supervision == 'systemd' and ansible_user != "root"

    - name: stop TiKV by systemd
      systemd: name=tikv-{{ tikv_port }}.service state=stopped
      when: process_supervision == 'systemd' and ansible_user == "root"
      vars:
        - ansible_become: false
        - ansible_become_user: root

    - name: wait for TiKV down (via Port)
      wait_for: host={{ ansible_host }} port={{ tikv_port }} state=stopped

    - name: wait for TiKV down (via PID)
      wait_for_pid: pid_file={{ deploy_dir }}/status/tikv.pid timeout=300 state=absent

    - command: cat {{ deploy_dir }}/status/tikv.pid
      register: old_tikv_pid
      ignore_errors: yes
      changed_when: false

    - name: display old tikv pid
      debug:
        msg: "tikv binary or docker pid: {{ old_tikv_pid.stdout }}"

  roles:
    - tikv

  post_tasks:
    - name: start TiKV by supervise
      shell: cd {{ deploy_dir }}/scripts && ./start_tikv.sh
      when: process_supervision == 'supervise'

    - name: start TiKV by systemd
      systemd: name=tikv-{{ tikv_port }}.service state=started
      become: true
      when: process_supervision == 'systemd' and ansible_user != "root"

    - name: start TiKV by systemd
      systemd: name=tikv-{{ tikv_port }}.service state=started
      when: process_supervision == 'systemd' and ansible_user == "root"
      vars:
        - ansible_become: false
        - ansible_become_user: root

    - name: wait for TiKV up
      wait_for_pid: |
        pid_file={{ deploy_dir }}/status/tikv.pid timeout=300
        thread_name_regex='endpoint' state=present
      when: deployment_method == 'binary'

    - name: wait for TiKV up
      wait_for: host={{ ansible_host }} port={{ tikv_port }} state=present
      when: deployment_method == 'docker'

    - command: cat {{ deploy_dir }}/status/tikv.pid
      register: new_tikv_pid
      ignore_errors: yes
      changed_when: false

    - name: display new tikv pid
      debug:
        msg: "tikv binary or docker pid: {{ new_tikv_pid.stdout }}"

    - name: remove evict-leader-scheduler
      uri:
        url: "http://{{ pd_addr }}/pd/api/v1/schedulers/evict-leader-scheduler-{{ store_id }}"
        method: DELETE
        status_code: 200
      when: not enable_tls|default(false)

    - name: remove evict-leader-scheduler when enable_tls|default(false)
      uri:
        url: "https://{{ pd_addr }}/pd/api/v1/schedulers/evict-leader-scheduler-{{ store_id_tls }}"
        validate_certs: no
        client_cert: "{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}.pem"
        client_key: "{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}-key.pem"
        method: DELETE
        status_code: 200
      when: enable_tls|default(false)


- name: rolling update TiDB cluster
  hosts: tidb_servers
  any_errors_fatal: true
  serial: 1
  tags:
    - tidb

  pre_tasks:
    - name: stop TiDB by supervise
      shell: cd {{ deploy_dir }}/scripts && ./stop_tidb.sh
      when: process_supervision == 'supervise'

    - name: stop TiDB by systemd
      systemd: name=tidb-{{ tidb_port }}.service state=stopped
      become: true
      when: process_supervision == 'systemd' and ansible_user != "root"

    - name: stop TiDB by systemd
      systemd: name=tidb-{{ tidb_port }}.service state=stopped
      when: process_supervision == 'systemd' and ansible_user == "root"
      vars:
        - ansible_become: false
        - ansible_become_user: root

    - name: wait for TiDB down
      wait_for: host={{ ansible_host }} port={{ tidb_port }} state=stopped

    - name: stop pump by supervise
      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh
      with_items:
        - pump
      when: enable_binlog and process_supervision == 'supervise'

    - name: stop pump by systemd
      systemd: name=pump.service state=stopped
      become: true
      when: enable_binlog and process_supervision == 'systemd' and ansible_user != "root"

    - name: stop pump by systemd
      systemd: name=pump.service state=stopped
      when: enable_binlog and process_supervision == 'systemd' and ansible_user == "root"
      vars:
        - ansible_become: false
        - ansible_become_user: root

    - name: wait for pump down
      wait_for: |
        host={{ ansible_host }} port={{ pump_port }} state=stopped
      when: enable_binlog

  roles:
    - { role: pump, when: enable_binlog }
    - { role: tidb }

  post_tasks:
    - name: start pump by supervise
      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh
      when: enable_binlog and process_supervision == 'supervise'
      with_items:
        - pump

    - name: start pump by systemd
      systemd: name=pump.service state=started
      become: true
      when: enable_binlog and process_supervision == 'systemd' and ansible_user != 'root'

    - name: start pump by systemd
      systemd: name=pump.service state=started
      when: enable_binlog and process_supervision == 'systemd' and ansible_user == 'root'
      vars:
        - ansible_become: false
        - ansible_become_user: root

    - name: wait for pump up
      wait_for: |
        host={{ ansible_host }} port={{ pump_port }} state=present
      when: enable_binlog

    - name: start TiDB by supervise
      shell: cd {{ deploy_dir }}/scripts && ./start_tidb.sh
      when: process_supervision == 'supervise'

    - name: start TiDB by systemd
      systemd: name=tidb-{{ tidb_port }}.service state=started
      become: true
      when: process_supervision == 'systemd' and ansible_user != "root"

    - name: start TiDB by systemd
      systemd: name=tidb-{{ tidb_port }}.service state=started
      when: process_supervision == 'systemd' and ansible_user == "root"
      vars:
        - ansible_become: false
        - ansible_become_user: root

    - name: wait for TiDB up
      wait_for: |
        host={{ ansible_host }} port={{ tidb_status_port }} state=present
        send='GET /status HTTP/1.0\r\n\r\n' search_regex='TiDB'
      when: not enable_tls|default(false)

    - name: wait for TiDB up
      uri:
        url: "https://{{ ansible_host }}:{{ tidb_status_port }}/status"
        validate_certs: no
        client_cert: "{{ tidb_cert_dir }}/tidb-server-{{ ansible_host }}.pem"
        client_key: "{{ tidb_cert_dir }}/tidb-server-{{ ansible_host }}-key.pem"
      register: result
      until: result.status is defined and result.status == 200
      retries: 10
      delay: 5
      when: enable_tls|default(false)