#!/bin/bash

declare -a SLOTS
declare -a OSDS_BY_NVME
declare -a IDS
declare -a LVS
declare -a LUKS
declare -A OSDS_UUID

# Obtenir disques probl      matiques
DEVS=$(lsblk | awk '{print $1}' | grep n2)

# OSDs Down
OSDS=$(ceph osd tree-from $(hostname -s) | grep down | awk '{print $1}')
for OSD in $OSDS; do
    ID=$(cat /var/lib/ceph/osd/ceph-${OSD}/lockbox.keyring | grep client | awk -F'.' '{print $3}'| tr -d "]")
    OSDS_UUID[$ID]=$OSD
    DEV=$(ceph osd metadata $OSD |jq .devices)
    IDX=$(echo ${DEV} | grep -o '[[:digit:]]*' |head -1)
    OSDS_BY_NVME[$IDX]="${OSDS_BY_NVME[$IDX]} $OSD"
done

echo Analyze...
for DEV in $DEVS; do
  IDX=$(echo ${DEV} | grep -o '[[:digit:]]*' |head -1)
  # Slot physique d'un NVME
  SLOTS[$IDX]=$(cat /sys/block/${DEV}/device/address | xargs -i{} lspci -v -s {} | grep "Physical Slot"| awk -F':' '{print $2}'|xargs)
  # VG
  VG=$(pvs --noheadings -o vg_name /dev/${DEV} 2>/dev/null|xargs))
  # Obtenir le path du LV et l'UUID du conteneur CrytSetup (=UUID du LV)
  OLDIFS=$IFS
  IFS=$'\n'
  for line in $(lvs --noheadings -o lv_path,lv_uuid,lv_name ${VG}); do
    NAME=$(echo $line| awk '{print $3}'|awk -F 'block-' '{print $2}')
    OSD=${OSDS_UUID[$NAME]}
    IDS[$OSD]=$NAME
    LUKS[$OSD]=$(echo $line| awk '{print $2}')
    LVS[$OSD]=$(echo $line| awk '{print $1}')
  done
  IFS=$OLDIFS
done

echo
echo NVMEs Resetted :
for nvme in ${!SLOTS[@]}; do
    echo Nvme${nvme}:
    echo -e "\tSLOT: ${SLOTS[$nvme]}"
    echo -e "\tOSDS:${OSDS_BY_NVME[$nvme]}"
    for OSD in ${OSDS_BY_NVME[$nvme]}; do
        echo osd.${OSD}
        echo -e "\tIDS: ${IDS[$OSD]}"
        echo -e "\tLUKS: ${LUKS[$OSD]}"
        echo -e "\tLVS: ${LVS[$OSD]}"
    done
done

echo
echo Repair...
# Pour chaque disque
for nvme in ${!SLOTS[@]}; do
    # arreter le disque
    echo Stopping nvme${nvme}
    (echo 0 > /sys/bus/pci/slots/${SLOTS[$nvme]}/power)
    sleep 5
    # Nettoyer la table des Device Mapper
    for OSD in ${OSDS_BY_NVME[$nvme]}; do
        echo Cleaning ${LUKS[$OSD]}
        dmsetup remove ${LUKS[$OSD]}
        echo Cleaning ${LVS[$OSD]}
        dmsetup remove ${LVS[$OSD]}
    done
    sleep 1

    # Redemarrer le disque 
    echo Restart nvme${nvme}
    (echo 1 > /sys/bus/pci/slots/${SLOTS[$nvme]}/power)
    sleep 5

    for OSD in ${OSDS_BY_NVME[$nvme]}; do
        echo Open LUKS ${LUKS[$OSD]}
        (ceph config-key get dm-crypt/osd/${IDS[$OSD]}/luks | /usr/sbin/cryptsetup --key-file - --allow-discards luksOpen ${LVS[$OSD]} ${LUKS[$OSD]})

        echo Enable osd.${OSD}
        chown -R ceph:ceph /var/lib/ceph/osd/ceph-${OSD}
        ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/mapper/${LUKS[$OSD]} --path /var/lib/ceph/osd/ceph-${OSD} --no-mon-config
        rm -f /var/lib/ceph/osd/ceph-${OSD}/block
        ln -snf /dev/mapper/${LUKS[$OSD]} /var/lib/ceph/osd/ceph-${OSD}/block
        DM=$(ls -l /dev/mapper/${LUKS[$OSD]} | awk -F'->' '{print $2}'| awk -F'/' '{print $2}')
        chown -R ceph:ceph /dev/${DM}
        chown -R ceph:ceph /var/lib/ceph/osd/ceph-${OSD}
        systemctl enable ceph-volume@lvm-${OSD}-${IDS[$OSD]}
        systemctl enable --runtime ceph-osd@${OSD}

        echo restart OSD service : ${OSD}
        systemctl start ceph-osd@${OSD}

    done
    if (lsblk /dev/nvme${nvme}n1 &>/dev/null| grep crypt); then
        OUTPUT="${OUTPUT}\nOSD Repair Failed for NVME ${nvme}"
        OUTPUT="${OUTPUT}\nYou can retry this script !"
     else
        OUTPUT="${OUTPUT}\nOSD Repair for NVME ${nvme} OK :"
        OUTPUT="${OUTPUT}\n $(lsblk /dev/nvme${nvme}n1 --noheading)'"
    fi
done

printf "$OUTPUT\n"