contrib-munin/plugins/emc/emc_vnx_nfs_stats

#!/bin/bash
######################################################################################################################
#                 Plugin to monitor NFS statistics of EMC VNX 5300 Unified Storage system                            #
######################################################################################################################

# Author: Evgeny Beysembaev <megabotva@gmail.com>

#####################################
#         Description               #
#####################################

# The plugin monitors NFS v3 and v4 statistics of EMC Unified Storage system. Probably it can also be compatible with 
# other Isilon or Celerra systems. It uses SSH to connect to Control Stations, then remotely executes 
# /nas/sbin/server_stats and fetches and parses data from it. It supports gathering data both from active/active 
# and active/passive Datamover configurations, ignoring offline or standby Datamovers. If all Datamovers are 
# offline or absent, the plugin returns error.
# This plugin also automatically chooses Primary Control Station from the list by calling /nasmcd/sbin/getreason and 
# /nasmcd/sbin/t2slot.
# 
# At the moment data is gathered from the following statistics sources:
# nfs.v3.op - Tons of timings about NFS RPC calls
# nfs.v4.op - Tons of timings about NFS RPC calls
# nfs.client - Here new Client addresses are rescanned and added automatically.
# 
# It's quite easy to comment out unneeded data to make graphs less overloaded or to add new statistics sources.
 
#####################################
#        Configuration              #
#####################################

# The plugin uses SSH to connect to Control Stations. It's possible to use 'nasadmin' user, but it would be better
# if you create read-only global user by Unisphere Client. The user should have only Operator role.
# I created "operator" user but due to the fact that Control Stations already had one internal "operator" user,
# the new one was called "operator1". So be careful.
# 
# On munin-node side choose a user which will be used to connect through SSH. Generally user "munin" is ok. Then,
# execute "sudo su munin -s /bin/bash", "ssh-keygen" and "ssh-copy-id" to both Control Stations with newly created 
# user.
# 
# Make a link from /usr/share/munin/plugins/emc_vnx_nfs_stats to /etc/munin/plugins/emc_vnx_nfs_stats_<NAME>,
# where <NAME> is any arbitrary name of your storage system. The plugin will return <NAME> in its answer 
# as "host_name" field.
# Assume your storage system is called "VNX5300".
# 
# Make a configuration file at /etc/munin/plugin-conf.d/emc_vnx_nfs_stats_VNX5300
# 
# [emc_vnx_nfs_stats_VNX5300]
# user munin							# SSH Client local user
# env.username operator1					# Remote user with Operator role
# env.cs_addr 192.168.1.1 192.168.1.2				# Control Stations addresses
# env.nas_servers server_2 server_3				# This is the default value and can be omitted

#####################################
#           History                 #
#####################################

# 08.11.2016 - First Release
# 17.11.2016 - NFSv4 support

######################################################################################################################

export LANG=C

TARGET=$(echo "${0##*/}" | cut -d _ -f 5)
: ${nas_servers:="server_2 server_3"}
SSH_CHECK='ssh -q $username@$CS "/nasmcd/sbin/getreason | grep -w slot_\`/nasmcd/sbin/t2slot\` | cut -d- -f1"'

if [ "$1" = "autoconf" ]; then
	echo "yes"
	exit 0
fi

if [ -z "$username" ]; then
	echo "No username!"
	exit 1
fi

if [ -z "$cs_addr" ]; then
	echo "No control station addresses!"
	exit 1
fi

#Choosing Cotrol Station. Code have to be "10"
for CS in $cs_addr; do
	if [[ "10" -eq "$(eval $SSH_CHECK)" ]]; then
#		echo "$CS is Primary"
		PRIMARY_CS=$CS
		break
	fi
done

if [ -z "$PRIMARY_CS" ]; then
	echo "No alive primary Control Station from list \"$cs_addr\"";
	exit 1;
fi

SSH="ssh -q $username@$PRIMARY_CS . /home/operator1/.bash_profile; "

echo "host_name ${TARGET}"

if [ "$1" = "config" ] ; then
	for server in $nas_servers; do
		$SSH nas_server -i $server | grep -q 'type *= nas'
		if [ "$?" != 0 ]  ; then continue; fi
		nas_server_ok=TRUE
#nfs.v3.op data
		member_elements=$($SSH server_stats $server -info nfs.v3.op | grep member_elements | sed -ne 's/^.*= //p')
		IFS=',' read -ra graphs <<< $member_elements
		echo "multigraph vnx_emc_v3_calls_s
graph_title EMC VNX 5300 NFSv3 Calls per second
graph_vlabel Calls
graph_category nfs
graph_args --base 1000"
		for graph in "${graphs[@]}"; do
        	        field=$(echo "$graph" | cut -d '.' -f4 )
			echo "${server}_$field.label $server $field"
        	done

		echo -e "\nmultigraph vnx_emc_v3_usec_call
graph_title EMC VNX 5300 NFSv3 uSeconds per call
graph_vlabel uSec / call
graph_category nfs
graph_args --base 1000"
		for graph in "${graphs[@]}"; do
                	field=$(echo "$graph" | cut -d '.' -f4 )
			echo "${server}_$field.label $server $field"
        	done
		echo -e "\nmultigraph vnx_emc_v3_op_percent
graph_title EMC VNX 5300 NFSv3 Op %
graph_vlabel %
graph_scale no
graph_category nfs"
		for graph in "${graphs[@]}"; do
                	field=$(echo "$graph" | cut -d '.' -f4 )
			echo "${server}_$field.label $server $field"
			echo "${server}_$field.min 0"
        	done
		graphs=()
#nfs.v4.op data
		member_elements=$($SSH server_stats $server -info nfs.v4.op | grep member_elements | sed -ne 's/^.*= //p')
		IFS=',' read -ra graphs <<< $member_elements
		echo "multigraph vnx_emc_v4_calls_s
graph_title EMC VNX 5300 NFSv4 Calls per second
graph_vlabel Calls
graph_category nfs
graph_args --base 1000"
		for graph in "${graphs[@]}"; do
        	        field=$(echo "$graph" | cut -d '.' -f4 )
			echo "${server}_$field.label $server $field"
        	done

		echo -e "\nmultigraph vnx_emc_v4_usec_call
graph_title EMC VNX 5300 NFSv4 uSeconds per call
graph_vlabel uSec / call
graph_category nfs
graph_args --base 1000"
		for graph in "${graphs[@]}"; do
                	field=$(echo "$graph" | cut -d '.' -f4 )
			echo "${server}_$field.label $server $field"
        	done
		echo -e "\nmultigraph vnx_emc_v4_op_percent
graph_title EMC VNX 5300 NFSv4 Op %
graph_vlabel %
graph_scale no
graph_category nfs"
		for graph in "${graphs[@]}"; do
                	field=$(echo "$graph" | cut -d '.' -f4 )
			echo "${server}_$field.label $server $field"
			echo "${server}_$field.min 0"
        	done

#nfs.client data
#							 Total    Read     Write   Suspicious   Total    Read     Write      Avg   
#                                                      Ops/s    Ops/s    Ops/s    Ops diff    KiB/s    KiB/s    KiB/s   uSec/call
		member_elements=$($SSH server_stats server_2 -monitor nfs.client -count 1 -terminationsummary no -titles never | sed -ne 's/^.*id=//p' | cut -d' ' -f1)
		readarray graphs2 <<< $member_elements
		echo -e "\nmultigraph vnx_emc_nfs_client_ops_s
graph_title EMC VNX 5300 NFS Client Ops/s
graph_vlabel Ops/s
graph_category nfs"
		echo -n "graph_order "
		for graph in "${graphs2[@]}"; do
                        field=$(echo "$graph" | sed -ne 's/\./_/pg' )
			echo -n "${server}_${field}_r ${server}_${field}_w ${server}_${field}_t ${server}_${field}_s "
		done
		echo " "
		for graph in "${graphs2[@]}"; do
                        field=$(echo "$graph" | sed -ne 's/\./_/pg' )
                        echo "${server}_${field}_r.label $server $field Read Ops/s"
                        echo "${server}_${field}_w.label $server $field Write Ops/s"
                        echo "${server}_${field}_w.draw STACK"
                        echo "${server}_${field}_t.label $server $field Total Ops/s"
			echo "${server}_${field}_s.label $server $field Suspicious Ops diff"
                done

		echo -e "\nmultigraph vnx_emc_nfs_client_kib_s
graph_title EMC VNX 5300 NFS Client B/s
graph_vlabel B/s
graph_category nfs"
		echo -n "graph_order "
		for graph in "${graphs2[@]}"; do
                        field=$(echo "$graph" | sed -ne 's/\./_/pg' )
			echo -n "${server}_${field}_r ${server}_${field}_w ${server}_${field}_t "
		done
		echo " "
		for graph in "${graphs2[@]}"; do
                        field=$(echo "$graph" | sed -ne 's/\./_/pg' )
                        echo "${server}_${field}_r.label $server $field Read B/s"
                        echo "${server}_${field}_r.cdef ${server}_${field}_r,1024,*"
                        echo "${server}_${field}_w.label $server $field Write B/s"
                        echo "${server}_${field}_w.cdef ${server}_${field}_w,1024,*"
                        echo "${server}_${field}_w.draw STACK"
                        echo "${server}_${field}_t.label $server $field Total B/s"
                        echo "${server}_${field}_t.cdef ${server}_${field}_t,1024,*"
                done

		echo -e "\nmultigraph vnx_emc_nfs_client_avg_usec
graph_title EMC VNX 5300 NFS Client Avg uSec/call
graph_vlabel uSec/call
graph_category nfs"
		for graph in "${graphs2[@]}"; do
                        field=$(echo "$graph" | sed -ne 's/\./_/pg' )
                        echo "${server}_${field}.label $server $field Avg uSec/call"
                done
	done
	if [ -z $nas_server_ok ]; then
		echo "No active data movers!"
		exit 1
	fi
exit 0
fi

for server in $nas_servers; do
	$SSH nas_server -i $server | grep -q 'type *= nas'
	if [ "$?" != 0 ]  ; then continue; fi
	nas_server_ok=TRUE
#nfs.v3.op data
	member_elements=$($SSH server_stats $server -monitor nfs.v3.op -count 1 -terminationsummary no -titles never | sed -ne 's/^.*v3/v3/p')
	NUMCOL=5
	LINES=$(wc -l <<< $member_elements)
	while IFS=$'\n' read -ra graphs ; do
		element+=( $graphs )
	done <<< $member_elements

	echo "multigraph vnx_emc_v3_calls_s"
	for ((i=0; i<$((LINES)); i++ )); do
		echo "${server}_${element[i*$NUMCOL]}".value "${element[i*$NUMCOL+1]}"
	done

	echo -e "\nmultigraph vnx_emc_v3_usec_call"
	for ((i=0; i<$((LINES)); i++ )); do
		echo "${server}_${element[i*$NUMCOL]}".value "${element[i*$NUMCOL+3]}"
	done

	echo -e "\nmultigraph vnx_emc_v3_op_percent"
	for ((i=0; i<$((LINES)); i++ )); do
		echo "${server}_${element[i*$NUMCOL]}".value "${element[i*$NUMCOL+4]}"
	done

	element=()

#nfs.v4.op data
	member_elements=$($SSH server_stats $server -monitor nfs.v4.op -count 1 -terminationsummary no -titles never | sed -ne 's/^.*v4/v4/p')
	NUMCOL=5
	LINES=$(wc -l <<< $member_elements)
	while IFS=$'\n' read -ra graphs ; do
		element+=( $graphs )
	done <<< $member_elements

	echo -e "\nmultigraph vnx_emc_v4_calls_s"
	for ((i=0; i<$((LINES)); i++ )); do
		echo "${server}_${element[i*$NUMCOL]}".value "${element[i*$NUMCOL+1]}"
	done

	echo -e "\nmultigraph vnx_emc_v4_usec_call"
	for ((i=0; i<$((LINES)); i++ )); do
		echo "${server}_${element[i*$NUMCOL]}".value "${element[i*$NUMCOL+3]}"
	done

	echo -e "\nmultigraph vnx_emc_v4_op_percent"
	for ((i=0; i<$((LINES)); i++ )); do
		echo "${server}_${element[i*$NUMCOL]}".value "${element[i*$NUMCOL+4]}"
	done

	element=()

#nfs.client data
	echo -e "\nmultigraph vnx_emc_nfs_client_ops_s"
        member_elements=$($SSH server_stats server_2 -monitor nfs.client -count 1 -terminationsummary no -titles never | sed -ne 's/^.*id=//p')
	NUMCOL=9
        LINES=$(wc -l <<< $member_elements)
	while IFS=$'\n' read -ra graphs; do
		element+=($graphs)
	done  <<< $member_elements
	for (( i=0; i<$((LINES)); i++ )); do
		client=$( echo ${element[i*$NUMCOL]} | sed -ne 's/\./_/pg')
               echo "${server}_${client}_r".value "${element[$i*$NUMCOL+2]}"
               echo "${server}_${client}_w".value "${element[$i*$NUMCOL+3]}"
               echo "${server}_${client}_t".value "${element[$i*$NUMCOL+1]}"
               echo "${server}_${client}_s".value "${element[$i*$NUMCOL+4]}"
        done
	echo -e "\nmultigraph vnx_emc_nfs_client_kib_s"
	for (( i=0; i<$((LINES)); i++ )); do
		client=$( echo ${element[i*$NUMCOL]} | sed -ne 's/\./_/pg')
               echo "${server}_${client}_r".value "${element[$i*$NUMCOL+6]}"
               echo "${server}_${client}_w".value "${element[$i*$NUMCOL+7]}"
               echo "${server}_${client}_t".value "${element[$i*$NUMCOL+5]}"
        done
	echo -e "\nmultigraph vnx_emc_nfs_client_avg_usec"
	for (( i=0; i<$((LINES)); i++ )); do
		client=$( echo ${element[i*$NUMCOL]} | sed -ne 's/\./_/pg')
		echo "${server}_${client}".value "${element[$i*$NUMCOL+8]}"
	done
done
if [ -z $nas_server_ok ]; then
	echo "No active data movers!"
	exit 1
fi
exit 0