update: follow new recommandation to upgrade mds cluster

Refact the mds cluster upgrade code in order to follow the documented
recommandation.
See: https://github.com/ceph/ceph/blob/master/doc/cephfs/upgrading.rst

Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1569689

Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>

infrastructure-playbooks/rolling_update.yml

@@ -536,25 +536,91 @@
         - (ceph_versions.get('stdout', '{}')|from_json).get('osd', {}) | length == 1
         - ceph_versions_osd | string is search("ceph version 12")
 
-- name: upgrade ceph mdss cluster
-  vars:
-    upgrade_ceph_packages: True
-  hosts: "{{ mds_group_name|default('mdss') }}"
-  serial: 1
-  become: True
+
+- name: upgrade ceph mdss cluster, deactivate all rank > 0
+  hosts: "{{ groups[mon_group_name|default('mons')][0] }}"
+  become: true
   tasks:
-    - name: stop ceph mds
+    - import_role:
+        name: ceph-defaults
+    - import_role:
+        name: ceph-facts
+
+    - name: set max_mds 1 on ceph fs
+      command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs set {{ cephfs }} max_mds 1"
+      changed_when: false
+
+    - name: wait until only rank 0 is up
+      command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs get {{ cephfs }} -f json"
+      changed_when: false
+      register: wait_rank_zero
+      retries: 720
+      delay: 5
+      until: (wait_rank_zero.stdout | from_json).mdsmap.in | length == 1 and (wait_rank_zero.stdout | from_json).mdsmap.in[0]  == 0
+
+    - name: get name of remaining active mds
+      command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs dump -f json"
+      changed_when: false
+      register: _mds_active_name
+
+    - name: set_fact mds_active_name
+      set_fact:
+        mds_active_name: "{{ [(_mds_active_name.stdout | from_json)['filesystems'][0]['mdsmap']['info'][item.key]['name']] }}"
+      with_dict: "{{ (_mds_active_name.stdout | from_json).filesystems[0]['mdsmap']['info'] }}"
+
+    - name: create active_mdss group
+      add_host:
+        name: "{{ mds_active_name[0] }}"
+        groups: active_mdss
+        ansible_host: "{{ hostvars[mds_active_name]['ansible_host'] | default(omit) }}"
+        ansible_port: "{{ hostvars[mds_active_name]['ansible_port'] | default(omit) }}"
+
+    - name: create standby_mdss group
+      add_host:
+        name: "{{ standby_mdss }}"
+        groups: standby_mdss
+        ansible_host: "{{ hostvars[item]['ansible_host'] | default(omit) }}"
+        ansible_port: "{{ hostvars[item]['ansible_port'] | default(omit) }}"
+      with_items: "{{ groups[mds_group_name] | difference(mds_active_name) }}"
+
+    - name: stop standby ceph mds
       systemd:
-        name: ceph-mds@{{ ansible_hostname }}
+        name: ceph-mds@{{ hostvars[item]['ansible_hostname'] }}
         state: stopped
         enabled: no
         masked: yes
-      when: not containerized_deployment | bool
+      delegate_to: "{{ item }}"
+      with_items: "{{ groups['standby_mdss'] }}"
+      when: groups['standby_mdss'] | default([]) | length > 0
+
+    - name: wait until all standbys mds are stopped
+      command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs dump -f json"
+      changed_when: false
+      register: wait_standbys_down
+      retries: 300
+      delay: 5
+      until: (wait_standbys_down.stdout | from_json).standbys | length == 0
+
 
+- name: upgrade active mds
+  vars:
+    upgrade_ceph_packages: True
+  hosts: active_mdss
+  become: true
+  tasks:
     - import_role:
         name: ceph-defaults
+
     - import_role:
         name: ceph-facts
+
+    - name: prevent restart from the packaging
+      systemd:
+        name: ceph-mds@{{ ansible_hostname }}
+        enabled: no
+        masked: yes
+      when: not containerized_deployment | bool
+
     - import_role:
         name: ceph-handler
     - import_role:
@@ -568,14 +634,48 @@
     - import_role:
         name: ceph-mds
 
-    - name: start ceph mds
+    - name: restart ceph mds
       systemd:
         name: ceph-mds@{{ ansible_hostname }}
-        state: started
+        state: restarted
         enabled: yes
         masked: no
+        daemon_reload: yes
+
+
+- name: upgrade standbys ceph mdss cluster
+  vars:
+    upgrade_ceph_packages: True
+  hosts: standby_mdss | default([])
+  become: True
+
+  tasks:
+    - import_role:
+        name: ceph-defaults
+
+    - import_role:
+        name: ceph-facts
+
+    - name: prevent restarts from the packaging
+      systemd:
+        name: ceph-mds@{{ ansible_hostname }}
+        enabled: no
+        masked: yes
       when: not containerized_deployment | bool
 
+    - import_role:
+        name: ceph-handler
+    - import_role:
+        name: ceph-common
+      when: not containerized_deployment | bool
+    - import_role:
+        name: ceph-container-common
+      when: containerized_deployment | bool
+    - import_role:
+        name: ceph-config
+    - import_role:
+        name: ceph-mds
+
     - name: restart ceph mds
       systemd:
         name: ceph-mds@{{ ansible_hostname }}
@@ -583,7 +683,12 @@
         enabled: yes
         masked: no
         daemon_reload: yes
-      when: containerized_deployment | bool
+
+    - name: set max_mds
+      command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs set {{ cephfs }} max_mds {{ mds_max_mds }}"
+      changed_when: false
+      delegate_to: "{{ groups[mon_group_name][0] }}"
+      when: inventory_hostname == groups['standby_mdss'] | last
 
 
 - name: upgrade ceph rgws cluster

roles/ceph-mds/tasks/create_mds_filesystems.yml

@@ -74,4 +74,6 @@
   command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs set {{ cephfs }} max_mds {{ mds_max_mds }}"
   changed_when: false
   delegate_to: "{{ groups[mon_group_name][0] }}"
-  when: mds_max_mds > 1
+  when:
+    - mds_max_mds > 1
+    - not rolling_update