forked from yindun/ansible-devops
194 lines
6.5 KiB
Bash
194 lines
6.5 KiB
Bash
#!/bin/bash
|
||
# 集群健康检查脚本 - 生成node_exporter可采集的监控指标
|
||
# 指标规则：0=正常，1=异常
|
||
|
||
METRIC_FILE="/var/lib/node_exporter/cluster_metrics.prom"
|
||
TMP_FILE=$(mktemp)
|
||
|
||
# 确保输出文件目录存在
|
||
mkdir -p $(dirname $METRIC_FILE)
|
||
|
||
# 写入指标注释
|
||
cat << EOF > $TMP_FILE
|
||
# HELP node_cluster_check_gpu_count 检查GPU数量是否为8（0=正常，1=异常）
|
||
# HELP node_cluster_check_ib_status 检查IB网卡状态（0=正常，1=异常）
|
||
# HELP node_cluster_check_peermem 检查peermem模块加载状态（0=正常，1=异常）
|
||
# HELP node_cluster_check_fabric 检查nvidia-fabricmanager运行状态（0=正常，1=异常）
|
||
# HELP node_cluster_check_acs 检查ACS状态是否关闭（0=正常，1=异常）
|
||
# HELP node_cluster_check_ib_order 检查IB网卡排列顺序（0=正常，1=异常）
|
||
# HELP node_cluster_check_memory_brand 检查内存品牌是否一致（0=正常，1=异常）
|
||
# HELP node_cluster_check_gpu_ecc GPU ECC错误状态（0=正常，1=异常）
|
||
# HELP node_cluster_check_gpu_xid GPU XID错误状态（0=正常，1=异常）
|
||
# HELP node_cluster_check_fan_speed 检查风扇转速是否为最高（0=正常，1=异常）
|
||
# HELP node_cluster_check_ib_ber IB网卡Symbol BER值是否正常（0=正常，1=异常）
|
||
# HELP node_cluster_check_ib_power IB网卡光衰是否正常（0=正常，1=异常）
|
||
# HELP node_cluster_check_ib_bandwidth IB网卡带宽是否正常（0=正常，1=异常）
|
||
EOF
|
||
|
||
# 1. 检查计算卡数量（应为8）
|
||
gpu_count=$(nvidia-smi -L | grep -i nvidia | wc -l)
|
||
if [ "$gpu_count" -ne 8 ]; then
|
||
echo "node_cluster_check_gpu_count 1" >> $TMP_FILE
|
||
else
|
||
echo "node_cluster_check_gpu_count 0" >> $TMP_FILE
|
||
fi
|
||
|
||
# 2. 检查IB网卡状态
|
||
ib_down=$(ibdev2netdev | grep ib | grep 'own' | wc -l)
|
||
if [ "$ib_down" -ne 0 ]; then
|
||
echo "node_cluster_check_ib_status 1" >> $TMP_FILE
|
||
else
|
||
echo "node_cluster_check_ib_status 0" >> $TMP_FILE
|
||
fi
|
||
|
||
# 3. 检查peermem状态
|
||
peermem_status=$(lsmod | grep -cE 'nvidia_peermem|peer_mem|peer' 2>/dev/null)
|
||
if [ "$peermem_status" -ne 3 ]; then
|
||
modprobe nvidia_peermem >/dev/null 2>&1
|
||
echo "node_cluster_check_peermem 1" >> $TMP_FILE
|
||
else
|
||
echo "node_cluster_check_peermem 0" >> $TMP_FILE
|
||
fi
|
||
|
||
# 4. 检查fabric状态
|
||
fabric_status=$(systemctl is-active nvidia-fabricmanager.service 2>/dev/null)
|
||
if [ "$fabric_status" != "active" ]; then
|
||
echo "node_cluster_check_fabric 1" >> $TMP_FILE
|
||
else
|
||
echo "node_cluster_check_fabric 0" >> $TMP_FILE
|
||
fi
|
||
|
||
# 5. 检查acs状态（应关闭）
|
||
acs_status=$(lspci -vvv | grep 'PCI bridge' -A 80 | grep ACSCtl | grep -c SrcValid+ 2>/dev/null)
|
||
if [ "$acs_status" -ne 0 ]; then
|
||
echo "node_cluster_check_acs 1" >> $TMP_FILE
|
||
else
|
||
echo "node_cluster_check_acs 0" >> $TMP_FILE
|
||
fi
|
||
|
||
# 6. 检查IB网卡排列顺序
|
||
value3=$(cat << EOF
|
||
hca_id: mlx5_0
|
||
hca_id: mlx5_1
|
||
hca_id: mlx5_2
|
||
hca_id: mlx5_3
|
||
hca_id: mlx5_4
|
||
hca_id: mlx5_5
|
||
hca_id: mlx5_6
|
||
hca_id: mlx5_7
|
||
hca_id: mlx5_bond_0
|
||
hca_id: mlx5_bond_1
|
||
EOF
|
||
)
|
||
value4=$(ibv_devinfo | grep mlx)
|
||
if ! test "$value3" = "$value4"; then
|
||
echo "node_cluster_check_ib_order 1" >> $TMP_FILE
|
||
else
|
||
echo "node_cluster_check_ib_order 0" >> $TMP_FILE
|
||
fi
|
||
|
||
# 7. 检查硬盘数量
|
||
disk_error=0
|
||
if ! lsblk | grep nvme5n1 > /dev/null; then
|
||
disk_error=1
|
||
elif lsblk | grep nvme6n1 > /dev/null; then
|
||
disk_error=1
|
||
fi
|
||
echo "node_cluster_check_disk_count $disk_error" >> $TMP_FILE
|
||
|
||
# 8. 检查内存品牌一致性
|
||
memory2=$(sudo dmidecode -t memory | grep -i "Manufacturer:" | uniq | wc -l)
|
||
if [ $memory2 -ne 1 ]; then
|
||
echo "node_cluster_check_memory_brand 1" >> $TMP_FILE
|
||
else
|
||
echo "node_cluster_check_memory_brand 0" >> $TMP_FILE
|
||
fi
|
||
|
||
# 9. 检查GPU ECC状态
|
||
ECC=$(nvidia-smi -q | grep -i 'Remapping Failure Occurred' | grep -c ': Yes' 2>/dev/null)
|
||
if [ "$ECC" -ne 0 ]; then
|
||
echo "node_cluster_check_gpu_ecc 1" >> $TMP_FILE
|
||
else
|
||
echo "node_cluster_check_gpu_ecc 0" >> $TMP_FILE
|
||
fi
|
||
|
||
# 10. 检查GPU XID异常
|
||
XID=$(dmesg | grep -i 'NVRM: Xid' | awk -F': ' '{print $3}' | awk '{print $1}' | wc -l)
|
||
if [ "$XID" -ne 0 ]; then
|
||
echo "node_cluster_check_gpu_xid 1" >> $TMP_FILE
|
||
else
|
||
echo "node_cluster_check_gpu_xid 0" >> $TMP_FILE
|
||
fi
|
||
|
||
# 11. 检查风扇模式
|
||
fan_mode=$(racadm get system.thermalsettings.ThermalProfile 2>/dev/null | grep -c 'ThermalProfile=Maximum Performance')
|
||
if [ "$fan_mode" -ne 1 ]; then
|
||
echo "node_cluster_check_fan_mode 1" >> $TMP_FILE
|
||
else
|
||
echo "node_cluster_check_fan_mode 0" >> $TMP_FILE
|
||
fi
|
||
|
||
# 12. 检查风扇转速
|
||
fan_speed=$(racadm get system.thermalsettings.FanSpeedOffset 2>/dev/null | grep -c 'FanSpeedOffset=Max')
|
||
if [ "$fan_speed" -ne 1 ]; then
|
||
echo "node_cluster_check_fan_speed 1" >> $TMP_FILE
|
||
else
|
||
echo "node_cluster_check_fan_speed 0" >> $TMP_FILE
|
||
fi
|
||
|
||
# 13. 检查IB链路BER值
|
||
for i in {0..7}; do
|
||
dev="mlx5_$i"
|
||
ber_value=$(mlxlink -d $dev -c 2>/dev/null | grep 'Symbol BER' | awk -F- '{print $2}' | xargs)
|
||
if [[ -n "$ber_value" && "$ber_value" -lt 14 ]]; then
|
||
echo "node_cluster_check_ib_ber{device=\"$dev\"} 1" >> $TMP_FILE
|
||
else
|
||
echo "node_cluster_check_ib_ber{device=\"$dev\"} 0" >> $TMP_FILE
|
||
fi
|
||
done
|
||
|
||
# 14. 检查IB网卡光衰
|
||
for i in {0..7}; do
|
||
dev="mlx5_$i"
|
||
output=$(mlxlink -d "$dev" -m 2>/dev/null | grep -E 'Tx Power Current|Rx Power Current')
|
||
power_error=0
|
||
|
||
if [[ -n "$output" ]]; then
|
||
while IFS= read -r line; do
|
||
power_type=$(echo "$line" | awk '{print $1 " " $2}')
|
||
values=$(echo "$line" | awk -F: '{print $2}' | awk '{print $1}')
|
||
IFS=',' read -r -a value_array <<< "$values"
|
||
|
||
for val in "${value_array[@]}"; do
|
||
integer_part=$(echo "$val" | awk -F. '{print $1}')
|
||
if [[ "$integer_part" =~ ^-?[0-9]+$ && "$integer_part" -lt -2 ]]; then
|
||
power_error=1
|
||
break 2 # 跳出双层循环
|
||
fi
|
||
done
|
||
done <<< "$output"
|
||
fi
|
||
|
||
echo "node_cluster_check_ib_power{device=\"$dev\"} $power_error" >> $TMP_FILE
|
||
done
|
||
|
||
# 15. 检查IB网卡带宽
|
||
for i in {0..7}; do
|
||
dev="mlx5_$i"
|
||
bandwidth_error=0
|
||
output=$(run_perftest_loopback 0 1 ib_write_bw -d "$dev" --report_gbits 2>/dev/null | grep 65536 | awk '{print int($3)}')
|
||
|
||
if [[ -n "$output" ]]; then
|
||
while read -r bw; do
|
||
if (( bw < 360 )); then
|
||
bandwidth_error=1
|
||
break
|
||
fi
|
||
done <<< "$output"
|
||
fi
|
||
|
||
echo "node_cluster_check_ib_bandwidth{device=\"$dev\"} $bandwidth_error" >> $TMP_FILE
|
||
done
|
||
|
||
# 替换指标文件（原子操作避免读取不完整）
|
||
mv $TMP_FILE $METRIC_FILE |