From a12ea0244e75804793d3b1e2bde714643caf1edf Mon Sep 17 00:00:00 2001 From: Steve Taylor Date: Mon, 27 Apr 2020 13:01:42 -0600 Subject: [PATCH] [ceph-osd] Use lvm commands instead of ceph-volume to get OSD properties This change removes "ceph-volume inventory" and "ceph-volume lvm list" commands from the ceph-volume OSD initialization script and Bluestore start script and replaces them with "pvdisplay" and "lvs" to retrieve lvm tags directly from lvm volumes instead. Ceph-volume makes repeated calls to blkid, which is very slow in some cases and deadlocks in others when there are RBDs mapped on the host. Change-Id: Ia999770d4a59729e38dbb494b34c30e5a1b36a8b --- .../bin/osd/ceph-volume/_bluestore.sh.tpl | 8 +-- .../bin/osd/ceph-volume/_common.sh.tpl | 71 +++++++++++++++++++ .../ceph-volume/_init-with-ceph-volume.sh.tpl | 14 ++-- 3 files changed, 82 insertions(+), 11 deletions(-) diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_bluestore.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_bluestore.sh.tpl index 3bc8e0d22..87d77aacf 100644 --- a/ceph-osd/templates/bin/osd/ceph-volume/_bluestore.sh.tpl +++ b/ceph-osd/templates/bin/osd/ceph-volume/_bluestore.sh.tpl @@ -39,7 +39,7 @@ CEPH_OSD_OPTIONS="" udev_settle -OSD_ID=$(ceph-volume inventory ${OSD_DEVICE} | grep "osd id" | awk '{print $3}') +OSD_ID=$(get_osd_id_from_device ${OSD_DEVICE}) simple_activate=0 if [[ -z ${OSD_ID} ]]; then echo "Looks like ceph-disk has been used earlier to activate the OSD." @@ -49,7 +49,7 @@ if [[ -z ${OSD_ID} ]]; then umount ${tmpmnt} simple_activate=1 fi -OSD_FSID=$(ceph-volume inventory ${OSD_DEVICE} | grep "osd fsid" | awk '{print $3}') +OSD_FSID=$(get_osd_fsid_from_device ${OSD_DEVICE}) if [[ -z ${OSD_FSID} ]]; then echo "Looks like ceph-disk has been used earlier to activate the OSD." tmpmnt=$(mktemp -d) @@ -73,7 +73,7 @@ else --auto-detect-objectstore \ --no-systemd ${OSD_ID} ${OSD_FSID} # Cross check the db and wal symlinks if missed - DB_DEV=$(ceph-volume lvm list ${OSD_DEVICE} | grep "db device" | awk '{print $3}') + DB_DEV=$(get_osd_db_device_from_device ${OSD_DEVICE}) if [[ ! -z ${DB_DEV} ]]; then if [[ ! -h /var/lib/ceph/osd/ceph-${OSD_ID}/block.db ]]; then ln -snf ${DB_DEV} /var/lib/ceph/osd/ceph-${OSD_ID}/block.db @@ -81,7 +81,7 @@ else chown -h ceph:ceph /var/lib/ceph/osd/ceph-${OSD_ID}/block.db fi fi - WAL_DEV=$(ceph-volume lvm list ${OSD_DEVICE} | grep "wal device" | awk '{print $3}') + WAL_DEV=$(get_osd_wal_device_from_device ${OSD_DEVICE}) if [[ ! -z ${WAL_DEV} ]]; then if [[ ! -h /var/lib/ceph/osd/ceph-${OSD_ID}/block.wal ]]; then ln -snf ${WAL_DEV} /var/lib/ceph/osd/ceph-${OSD_ID}/block.wal diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl index 0ff777e30..48194e0fe 100644 --- a/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl +++ b/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl @@ -251,3 +251,74 @@ function udev_settle { done } +# Helper function to get an lvm tag from a logical volume +function get_lvm_tag_from_volume { + logical_volume="$1" + tag="$2" + + if [[ -z "${logical_volume}" ]]; then + # Return an empty string if the logical volume doesn't exist + echo + else + # Get and return the specified tag from the logical volume + echo "$(lvs -o lv_tags ${logical_volume} | tr ',' '\n' | grep ${tag} | cut -d'=' -f2)" + fi +} + +function get_lvm_tag_from_device { + device="$1" + tag="$2" + # Attempt to get a logical volume for the physical device + logical_volume="$(pvdisplay -m ${device} | awk '/Logical volume/{print $3}')" + + # Use get_lvm_tag_from_volume to get the specified tag from the logical volume + echo "$(get_lvm_tag_from_volume ${logical_volume} ${tag})" +} + +# Helper function get a cluster FSID from a physical device +function get_cluster_fsid_from_device { + device="$1" + + # Use get_lvm_tag_from_device to get the cluster FSID from the device + echo "$(get_lvm_tag_from_device ${device} ceph.cluster_fsid)" +} + +# Helper function to get an OSD ID from a logical volume +function get_osd_id_from_volume { + logical_volume="$1" + + # Use get_lvm_tag_from_volume to get the OSD ID from the logical volume + echo "$(get_lvm_tag_from_volume ${logical_volume} ceph.osd_id)" +} + +# Helper function get an OSD ID from a physical device +function get_osd_id_from_device { + device="$1" + + # Use get_lvm_tag_from_device to get the OSD ID from the device + echo "$(get_lvm_tag_from_device ${device} ceph.osd_id)" +} + +# Helper function get an OSD FSID from a physical device +function get_osd_fsid_from_device { + device="$1" + + # Use get_lvm_tag_from_device to get the OSD FSID from the device + echo "$(get_lvm_tag_from_device ${device} ceph.osd_fsid)" +} + +# Helper function get an OSD DB device from a physical device +function get_osd_db_device_from_device { + device="$1" + + # Use get_lvm_tag_from_device to get the OSD DB device from the device + echo "$(get_lvm_tag_from_device ${device} ceph.db_device)" +} + +# Helper function get an OSD WAL device from a physical device +function get_osd_wal_device_from_device { + device="$1" + + # Use get_lvm_tag_from_device to get the OSD WAL device from the device + echo "$(get_lvm_tag_from_device ${device} ceph.wal_device)" +} diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl index 8ae4ac1ab..abe4cdf13 100644 --- a/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl +++ b/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl @@ -62,7 +62,7 @@ function osd_disk_prepare { CEPH_LVM_PREPARE=1 osd_dev_string=$(echo ${OSD_DEVICE} | awk -F "/" '{print $2}{print $3}' | paste -s -d'-') udev_settle - OSD_ID=$(ceph-volume inventory ${OSD_DEVICE} | grep "osd id" | awk '{print $3}') + OSD_ID=$(get_osd_id_from_device ${OSD_DEVICE}) if [ "${OSD_BLUESTORE:-0}" -ne 1 ]; then if [[ ! -z ${OSD_ID} ]]; then DM_NUM=$(dmsetup ls | grep $(lsblk -J ${OSD_DEVICE} | jq -r '.blockdevices[].children[].name') | awk '{print $2}' | cut -d':' -f2 | cut -d')' -f1) @@ -197,8 +197,8 @@ function osd_disk_prepare { if [[ ${block_db_string} == ${block_wal_string} ]]; then if [[ $(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_db_string}") ]]; then VG=$(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_db_string}") - WAL_OSD_ID=$(ceph-volume lvm list /dev/ceph-db-wal-${block_wal_string}/ceph-wal-${osd_dev_string} | grep "osd id" | awk '{print $3}') - DB_OSD_ID=$(ceph-volume lvm list /dev/ceph-db-wal-${block_db_string}/ceph-db-${osd_dev_string} | grep "osd id" | awk '{print $3}') + WAL_OSD_ID=$(get_osd_id_from_volume /dev/ceph-db-wal-${block_wal_string}/ceph-wal-${osd_dev_string}) + DB_OSD_ID=$(get_osd_id_from_volume /dev/ceph-db-wal-${block_db_string}/ceph-db-${osd_dev_string}) if [ ! -z ${OSD_ID} ] && ([ ${WAL_OSD_ID} != ${OSD_ID} ] || [ ${DB_OSD_ID} != ${OSD_ID} ]); then echo "Found VG, but corresponding DB || WAL are not, zapping the ${OSD_DEVICE}" disk_zap ${OSD_DEVICE} @@ -236,7 +236,7 @@ function osd_disk_prepare { else if [[ $(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_db_string}") ]]; then VG=$(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_db_string}") - DB_OSD_ID=$(ceph-volume lvm list /dev/ceph-db-wal-${block_db_string}/ceph-db-${block_db_string} | grep "osd id" | awk '{print $3}') + DB_OSD_ID=$(get_osd_id_from_volume /dev/ceph-db-wal-${block_db_string}/ceph-db-${block_db_string}) if [ ! -z ${OSD_ID} ] && [ ${DB_OSD_ID} != ${OSD_ID} ]; then echo "Found VG, but corresponding DB is not, zapping the ${OSD_DEVICE}" disk_zap ${OSD_DEVICE} @@ -262,7 +262,7 @@ function osd_disk_prepare { fi if [[ $(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_wal_string}") ]]; then VG=$(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_wal_string}") - WAL_OSD_ID=$(ceph-volume lvm list /dev/ceph-db-wal-${block_wal_string}/ceph-wal-${block_wal_string} | grep "osd id" | awk '{print $3}') + WAL_OSD_ID=$(get_osd_id_from_volume /dev/ceph-db-wal-${block_wal_string}/ceph-wal-${block_wal_string}) if [ ! -z ${OSD_ID} ] && [ ${WAL_OSD_ID} != ${OSD_ID} ]; then echo "Found VG, but corresponding WAL is not, zapping the ${OSD_DEVICE}" disk_zap ${OSD_DEVICE} @@ -298,7 +298,7 @@ function osd_disk_prepare { elif [[ -z ${BLOCK_DB} && ${BLOCK_WAL} ]]; then if [[ $(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_wal_string}") ]]; then VG=$(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_wal_string}") - WAL_OSD_ID=$(ceph-volume lvm list /dev/ceph-wal-${block_wal_string}/ceph-wal-${osd_dev_string} | grep "osd id" | awk '{print $3}') + WAL_OSD_ID=$(get_osd_id_from_volume /dev/ceph-wal-${block_wal_string}/ceph-wal-${osd_dev_string}) if [ ! -z ${OSD_ID} ] && [ ${WAL_OSD_ID} != ${OSD_ID} ]; then echo "Found VG, but corresponding WAL is not, zapping the ${OSD_DEVICE}" disk_zap ${OSD_DEVICE} @@ -329,7 +329,7 @@ function osd_disk_prepare { elif [[ ${BLOCK_DB} && -z ${BLOCK_WAL} ]]; then if [[ $(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_db_string}") ]]; then VG=$(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_db_string}") - DB_OSD_ID=$(ceph-volume lvm list /dev/ceph-db-${block_db_string}/ceph-db-${osd_dev_string} | grep "osd id" | awk '{print $3}') + DB_OSD_ID=$(get_osd_id_from_volume /dev/ceph-db-${block_db_string}/ceph-db-${osd_dev_string}) if [ ! -z ${OSD_ID} ] && [ ${DB_OSD_ID} != ${OSD_ID} ]; then echo "Found VG, but corresponding DB is not, zapping the ${OSD_DEVICE}" disk_zap ${OSD_DEVICE}