Northeastern IT LLC - Forums

Code: Select all

echo; echo "****************** START ******************"; echo; echo ">> Any Locking?:"; echo 483388 : How to resolve cluster Deadlocking issue. https://support.emc.com/kb/483388; echo; isi_for_array -X 'sysctl efs.lin.lock.initiator.oldest_waiter | grep -E "address|started"' | grep -v "exited with status 1" | sed 's/= .* (/= /' | sort -nk4; echo; echo ++++++++++++++++++++++++++++++++++++++++; echo; echo; date ; echo; echo ">> OneFS:" `uname -r`; echo; isi_for_array -sX isi_hw_status | grep -i prod; echo; echo ">> Group Statement:"; isi_group_info; echo; isi status -q ; echo; echo ++++++++++++++++++++++++++++++++++++++++; echo; echo ">> LWIO USAGE" ; echo "Node\t\tCPU\tMem"; isi_for_array -sXn 1-144 'ps auwx | egrep "lwio" | grep -v grep' | awk '$6 = $6/1024{print $1"\t"$4"\t"$6"MB"}' ; echo; echo ">> LSASS USAGE" ; isi_for_array -sXn 1-144 'ps auwx | egrep "lsass" | grep -v grep' | awk '$6 = $6/1024{print $1"\t"$4"\t"$6"MB"}'; sleep 3; echo; echo ++++++++++++++++++++++++++++++++++++++++; echo; echo ">> DISK UTILISATION"; echo; echo ++++++++++++++++++++++++++++++++++++++++; echo; echo ">> Any SATA Ops/In over 100?"; echo; isi statistics drive --nodes=all --sort=opsin --degraded | grep -vi ssd | head -10; echo; echo ">> Any SATA Ops/Out over 100?"; echo; isi statistics drive --nodes=all --sort=opsout --degraded | grep -vi ssd | head -10;echo; echo ">> Are the drives Busy"; echo; isi statistics drive --nodes=all --sort=busy --degraded | head -10 ; echo; echo ">> Are there any queued Drives"; echo; isi statistics drive --nodes=all --sort=queued --degraded | head -10 ; echo; echo ">> How is the TimeinQ?"; echo; isi statistics drive --nodes=all --sort=timeinq --degraded | head -10 ; echo; echo ">> How many open files per/node?" ; isi_for_array -sX sysctl kern.openfiles ; sleep 3 ; echo; echo ">> How are load averages?"; isi_for_array -sX uptime ; echo; echo ">> How is lsass latency?"; isi_classic statistics protocol --degraded --nodes=all --protocols=lsass_in,lsass_out | awk '0+$4 >= 300000' ;echo ">> Any latency? No output is good, more than 10 seconds is bad" ; isi_for_array -sX sysctl efs.lin.lock.initiator.oldest_waiter | grep -i seconds ; echo ">> Which files are causing latency?" ; isi_for_array -sX sysctl efs.lin.lock.initiator.oldest_waiter | grep -i lin | grep = | awk '{print $5}' | while read f; do isi get -L ${f} | grep -vi ifsvar ; done ; echo; echo ++++++++++++++++++++++++++++++++++++++++; echo; echo ">> How is Job progress?"; isi_classic job status -v | head -17 ;echo; echo ++++++++++++++++++++++++++++++++++++++++; echo; echo ">> Top 3 CPU consumers"; echo; isi_for_array -sX 'top -n |grep PID -A3' ; echo; echo ">> How is global CPU" ; isi_classic statistics system --nodes --degraded ; echo; echo ++++++++++++++++++++++++++++++++++++++++; echo; echo ">> Are any sync jobs running?" ; isi sync jobs list; echo; echo ">> Number of free vnodes per/node, less 3,000 is concerning" ; isi_for_array -X sysctl vfs.freevnodes ; echo; echo ">> How are journal sectors? Less than 1000 concerning" ; isi_for_array -sX sysctl efs.journal.free_sectors; echo; echo '>> Is NFS responding on all nodes?'; isi_for_array -sX "rpcinfo -T tcp localhost 100003 3" ; echo; echo ">> Connection counts" ; isi statistics query current --nodes=all --stats=node.clientstats.connected.nfs,node.clientstats.active.nfs ; isi statistics query current list --nodes=all --stats=node.clientstats.connected.smb,node.clientstats.active.smb1,node.clientstats.active.smb2 ; echo; echo ">> Top write latencies" ; isi statistics protocol --nodes=all --degraded --sort=timeavg --protocols=nfs3 --classes=namespace_write | head -n10 ; echo; echo ">> Average Latencies" ; isi statistics client --nodes=all --degraded --protocols nfs3 --totalby=class ; echo; echo ++++++++++++++++++++++++++++++++++++++++; echo; isi upgrade view ; echo; echo ++++++++++++++++++++++++++++++++++++++++; echo; echo ">> Installed Patches:"; isi upgrade patches list ; echo; echo ++++++++++++++++++++++++++++++++++++++++; echo; echo ">> Check for Network errors or dropped packets:"; isi_for_array -sX netstat -i | grep -vi lo0 | egrep -i "ierrs|link" | column -t; echo; echo ++++++++++++++++++++++++++++++++++++++++; echo; echo ">> Authentication Latencies, time in Microseconds:"; isi statistics protocol list --protocols=lsass_in,lsass_out --degraded --nodes=all --sort=time_avg; echo; echo ++++++++++++++++++++++++++++++++++++++++; echo ">> SMB Class Latencies, time in Microseconds:"; isi statistics protocol --nodes=all --degraded --protocols=smb1,smb2 --sort=TimeAvg --limit=30; echo; echo ++++++++++++++++++++++++++++++++++++++++; echo; echo ">> ANTIVIRUS settings:"; echo; isi antivirus settings view | egrep -i "open|close" | grep -vi fail; echo; echo ++++++++++++++++++++++++++++++++++++++++; echo; echo ">> Any Hangdumps from today:"; echo; isi_for_array -sX find /var/crash -maxdepth 1 -type f -mtime -1 | grep -i hang | grep -vi latest; echo; echo "Done==============================================Done"; echo;