# Keep "global" functions here, should reside in /usr/local/axcient/ax_ops_functions
# add this to your bashrc using this line "source /usr/local/axcient/ax_ops_functions"

. /usr/local/axcient/ax_ops_vars

appliance () {
	if [ $# -lt 2 ] ; then
		ssh -i /usr/local/axcient/keys/uptiva-root-key -U root@/home/$1/tunnels/ssh
	else
		ssh -i /usr/local/axcient/keys/uptiva-root-key -U root@/home/$1/tunnels/ssh $2
	fi
}

appliance_kill_all_offsite_jobs () {
	echo "Killing the following processes:"
	ssh -i /usr/local/axcient/keys/uptiva-root-key -U root@/home/$1/tunnels/ssh "ps ax|grep rsync|grep -v grep; ps ax|grep rsync_wrapper|grep -v grep|cut -c1-5|xargs kill; killall rsync"
	sleep 5
	ssh -i /usr/local/axcient/keys/uptiva-root-key -U root@/home/$1/tunnels/ssh "ps ax|grep rsync|grep -v grep; ps ax|grep rsync_wrapper|grep -v grep|cut -c1-5|xargs kill; killall rsync"
	sleep 2
	echo "Rsync processes remaining:"
	ssh -i /usr/local/axcient/keys/uptiva-root-key -U root@/home/$1/tunnels/ssh "ps ax|grep rsync|grep -v grep"
}

appliance_check_offsite_running () {
	count=`ssh -i /usr/local/axcient/keys/uptiva-root-key -U root@/home/$1/tunnels/ssh "ps ax|grep rsync_wrapper|grep -v grep|wc -l"`
	if [ $count -ne 0 ] 
	then 
		echo -n "offsite running on "
	else 
		echo -n "offsite not running on "
	fi
	echo $1
}

pause_offsites () {
if [ $# -gt 0 ]; then
        while (( "$#" )); do
		echo -n "[*] Pausing Offsites $1"; ssh -o ConnectTimeout=5 root@$1 "killall -SIGSTOP encfs" 2>/dev/null; if [ $? == 0 ]; then echo " [DONE]"; else echo -e " [${red}FAIL${endcolor}]";fi
		shift
	done
else
	echo "need at least one namespace defined"
fi
}

unpause_offsites () {
if [ $# -gt 0 ]; then
        while (( "$#" )); do
		echo -n "[*] Unpausing Offsites $1"; ssh -o ConnectTimeout=5 root@$1 "killall -SIGCONT encfs" 2>/dev/null; if [ $? == 0 ]; then echo " [DONE]"; else echo -e " [${red}FAIL${endcolor}]";fi
		shift
	done
else
	echo "need at least one namespace defined"
fi
}


check_for_ssh () {
 for i in {$SJC_NAMESPACE1,$SJC_NAMESPACE2,$SJC_NAMESPACE3,$SJC_NAMESPACE4};do echo -n "----====$i====---- "; (ssh $i "ps ax | grep -v "grep" | grep $@ | grep ssh" ) || echo "nothing"; done
}

check_nodes_locks () {
if [ $# -gt 0 ]; then
        while (( "$#" )); do
                echo -n $1;
                ssh $1 "axl -l s -s p4a5 ls"&> /dev/null && echo " success" || echo " fail"
                shift
        done
else
	for i in ${SJC_NAMESPACES};do 
#echo -n "$i ";(ssh $i "axl -l x -s p4a5 ls"&> /dev/null && echo "success") || echo " fail"; done
	echo -n "[*] Checking axl lock capabilities on $i "; ssh -o ConnectTimeout=10 root@$i "cat /dev/null" 2>/dev/null; if [ $? == 0 ]; then timeout 30 ssh -o ConnectTimeout=10 root@$i "axl -l s -s p4a5 ls &> /dev/null" 2>/dev/null; if [ $? == 0 ]; then echo "[GOOD]"; else echo -e "[${red}FAIL${endcolor}]";fi; else echo -e "[${red}DOWN${endcolor}]";fi;done
#echo -n "[*] Checking axl lock capabilities on $i "; timeout 30 ssh -o ConnectTimeout=10 root@$i "axl -l s -s p4a5 ls &> /dev/null" 2>/dev/null; if [ $? == 0 ]; then echo "[GOOD]"; else echo -e "[${red}FAIL${endcolor}]";fi; done
fi
}

check_nodes_locks_namespace5 () {
if [ $# -gt 0 ]; then
        while (( "$#" )); do
                echo -n $1;
                ssh $1 "axl -l s -s p4a5 ls"&> /dev/null && echo "success" || echo " fail"
                shift
        done
else
        for i in ${SJC_NAMESPACE5};do
#echo -n "$i ";(ssh $i "axl -l x -s p4a5 ls"&> /dev/null && echo "success") || echo " fail"; done
        echo -n "[*] Checking axl lock capabilities on $i "; timeout 10 ssh -o ConnectTimeout=5 root@$i "axl -l x -s p4a5 ls &> /dev/null" 2>/dev/null; if [ $? == 0 ]; then echo "[GOOD]"; else echo -e "[${red}FAIL${endcolor}]";fi; done
fi
}


check_nodes_mounts () {
 if [ $# -gt 0 ]; then
        while (( "$#" )); do
                echo -n $1;
                ssh -o ConnectTimeout=5 $1 "if [ -d '/store' ]; then ls /uptiva/mounts /uptiva/raw/ /store/moo; elif [ -d '/ifs01' ]; then ls /uptiva/mounts /uptiva/raw/ /ifs01/snaptest/; elif [ -d '/ifs04' ]; then ls /uptiva/mounts /uptiva/raw/ /ifs04/trash/; else ls /uptiva/mounts /uptiva/raw /ifs02/trash; fi" &> /dev/null && echo ' success' || echo " fail"
         shift
        done
 else
        for i in  ${SJC_NAMESPACES}; do echo -n "$i"; (ssh -o ConnectTimeout=5 $i "if [ -d '/store' ]; then ls /uptiva/mounts /uptiva/raw/ /store/moo; elif [ -d '/ifs01' ]; then ls /uptiva/mounts /uptiva/raw/ /ifs01/snaptest/; elif [ -d '/ifs04' ]; then ls /uptiva/mounts /uptiva/raw/ /ifs04/trash/; else ls /uptiva/mounts /uptiva/raw /ifs02/trash; fi" &> /dev/null && echo ' success') || echo " fail"; done
 fi
}

check_nodes_uptime () {
if [ $# -gt 0 ]; then
        while (( "$#" )); do
                echo -n $1;
                ssh -o ConnectTimeout=5 $1 "uptime"
                shift
        done
else
	 for i in ${SJC_NAMESPACES}; do echo -n "$i "; ssh -o ConnectTimeout=5 $i "uptime"; done
fi
}

sids_running_on_nod (){
if [ -z "$1" ]; then 
	echo "please provide node name as parameter1";
else
	ssh -o ConnectTimeout=5 root@$1 "ps -ef | grep rsync.rea[l] | cut -d \" \" -f1 | sort -u";
fi
}

reboot_bad_nodes () {
bad_sids=" "
for i in ${SJC_NAMESPACES};do
        echo -n "[*] Checking axl lock capabilities on $i "; 
	timeout 10 ssh -o ConnectTimeout=5 root@$i "axl -l x -s p4a5 ls &> /dev/null" 2>/dev/null; 
	if [ $? == 0 ]; 
		then echo "[GOOD]"; 
	else 
		echo -e "[${red}FAIL${endcolor}]"; 
		sids_running_on_nod $i >> $bad_sids; 
		echo -e "[${red}REBOOTED $i${endcolor}]"; timeout 5 ssh -o ConnectTimeout=5 root@$i "reboot -f -n &> /dev/null" 2>/dev/null;
	fi; 
done
echo "these sids were affected and will require a rescan:"
echo $bad_sids | sort | uniq
}

customer_ibdircheck () {
if [ $# -eq 0 ] ; then
    echo 'please provide an arguement such as "1pym/offsite/SVA/3/9/C/Intuit/Statement Writer/Corinthian Services of Iowa, LLC/test 2/"'
    exit 0
fi

dir="/store/custdata/volumes/$1"
echo "checking for presense of $dir"
ssh -o ConnectTimeout=5 root@fusion "if [ -d '$dir' ];
        then
                echo 'found on NS1'
                ls -al '$dir'
                ibdircheck -y -v -p=1 '$dir'
        else
                echo 'not found on NS1'
fi"

dir="/ifs01/custdata/volumes/$1"
echo "checking for presense of $dir"
ssh -o ConnectTimeout=5 root@fusion "if [ -d '$dir' ];
        then
                echo 'found on NS2'
                ls -al '$dir'
                ibdircheck -y -v -p=1 '$dir'
        else
                echo 'not found on NS2'
fi"

dir="/ifs02/custdata/volumes/$1"
echo "checking for presense of $dir"
ssh -o ConnectTimeout=5 root@fusion03 "if [ -d '$dir' ];
        then
                echo 'found on NS3'
                ls -al '$dir'
                ibdircheck -y -v -p=1 '$dir'
        else
                echo 'not found on NS3'
fi"

dir="/ifs04/custdata/volumes/$1"
echo "checking for presense of $dir"
ssh -o ConnectTimeout=5 root@fusion04 "if [ -d '$dir' ];
        then
                echo 'found on NS4'
                ls -al '$dir'
                ibdircheck -y -v -p=1 '$dir'
        else
                echo 'not found on NS4'
fi"
}

das_report () {
	/usr/local/axcient/das_report/das_report.sh
}

check_pin_status () {
if [ $# -gt 1 ]; then
	svcid="$1"
	shift
	random_host=($1)
	rhost=${random_host[$(( RANDOM % 9 ))]}
	pins=`ssh -q root@${rhost} "axl -s ${svcid} find links -name \"hotsite_*\""`
	echo $pins
fi
}
pin_action () {
if [ $# -ne 4 ]; then
	echo "pin_action requires (pin|unpin) svcid namespace links path"
else
	pact="$1"
	svcid="$2"
	namespace=($3)
	links="$4"
	directory="`dirname \`dirname ${links}\``"
	path="/uptiva/mounts/${svcid}/${directory}/current"
	pinname="`basename ${links}`"
	rhost=${namespace[$(( RANDOM % 9 ))]}
	echo "${rhost} ${SVCID} ${pact} ${path} ${pinname}"
	 ssh -q root@${rhost} "axl -s ${svcid} /usr/local/axcient/transaction_pin.sh ${pact} ${path} ${pinname}"
fi	
}

check_rsync_nodes () {
if [ $# -gt 1 ]; then
        svcid=`id -u $1`
        shift
        while (( "$#" )); do
                echo -n $1;
                value=`ssh -q -o ConnectTimeout=5 $1 "ps x -U ${svcid} | grep -v grep | grep uptiva | awk -F / '{print \\$(NF-3),\\$(NF-2),\\$(NF-1)}'| uniq"`
                if [ -n "${value}" ]; then
                        echo " ${value//[!0-9 ]/}"
                else
                        echo " nothing"
                fi
                shift
        done
else
 for i in ${SJC_NAMESPACES} ;do echo -n "$i "; value=`ssh -q -o ConnectTimeout=5 $i "ps x -U \`id -u $@\` | grep -v grep | grep uptiva | awk -F / '{print \\$(NF-3),\\$(NF-2),\\$(NF-1)}'| uniq"`; if [ -n "${value}" ]; then echo " ${value//[!0-9 ]/}"; else echo " nothing"; fi; done
fi
}

test_rsync_nodes () {
#adds job type (to output
if [ $# -gt 1 ]; then
        svcid=`id -u $1`
        shift
        while (( "$#" )); do
                echo -n $1;
                value=`ssh -q -o ConnectTimeout=5 $1 "ps -ocmd -U ${svcid} | grep -v grep | egrep 'real|Linux' | awk '{print \\$1,\\$NF}' | awk -F / '{print \\$1,\\$7,\\$8}'| sort | uniq"`
                if [ -n "${value}" ]; then
                        echo " ${value}"
                else
                        echo " nothing"
                fi
                shift
        done
else
 for i in ${SJC_NAMESPACES} ;do echo -n "$i "; value=`ssh -q -o ConnectTimeout=5 $i "ps -ocmd -U ${svcid} | grep -v grep | egrep 'real|Linux' | awk '{print \\$1,\\$NF}' | awk -F / '{print \\$1,\\$7,\\$8}'| sort | uniq"`; if [ -n "${value}" ]; then echo " ${value}"; else echo " nothing"; fi; done
fi
}


return_namespace_id () {
	if [ "$1" = "NS1" ]; then echo "23"; fi
	if [ "$1" = "NS2" ]; then echo "36"; fi
	if [ "$1" = "NS3" ]; then echo "37"; fi
	if [ "$1" = "NS4" ]; then echo "40"; fi
	if [ "$1" = "NS5" ]; then echo "41"; fi
	if [ "$1" = "NS6" ]; then echo "42"; fi
        if [ "$1" = "NS7" ]; then echo "43"; fi
        if [ "$1" = "NS8" ]; then echo "44"; fi
        if [ "$1" = "NS9" ]; then echo "45"; fi
        if [ "$1" = "NS10" ]; then echo "46"; fi
        if [ "$1" = "NS11" ]; then echo "47"; fi
        if [ "$1" = "NS12" ]; then echo "48"; fi
}

cycle_bad_nodes () {
	for i in ${SJC_NAMESPACES}; do 
		count=$(ping -c 4 $i | grep 'received' | awk -F',' '{ print $2 }' | awk '{ print $1 }')
		if [ $count -eq 0 ]; then
			echo "$i is down, power cycling...."
			/root/nodpower.sh $i cycle
		fi
	done
}

check_appliance_open_ports () { 
if [ $# -gt 0 ]; then
	echo "trying to connect to axcient datacenter from SID $1 on port $2 ..."
	ssh_result=$(timeout 10 /usr/local/bin/ssh -l root -i /usr/local/axcient/keys/uptiva-root-key -o "StrictHostKeyChecking no" -o "PasswordAuthentication no" -o "LogLevel quiet" -U /home/$1/tunnels/ssh "nc -n -v -z -w5 38.122.174.2 $2 2>&1")
#	${ssh_result}=echo "${ssh_result}" | cut -d " " -f5 
	echo "${ssh_result}"
#	grep -q 'open' "${ssh_result}" 
#	if [ "${ssh_result}" = "(UNKNOWN) [38.122.174.2] 4021 (?) open" ] ; then echo "outbound traffic from sid $1 to port $2 is open" ; else echo "outbound traffic from sid $1 to port $2 is closed" ; fi
	else
	echo "usage: check_appliance_open_ports <SID> <PORT>"
fi

}

configure_ilo () {
export IPMI_PASSWORD='ADMIN'
if [ $# -gt 0 ]; then
        while (( "$#" )); do
		host="$1"
                ipmitool -l lan -E -U ADMIN -H ilo-$host user set name 3 admin
		ipmitool -l lan -E -U ADMIN -H ilo-$host user enable 3
                ipmitool -l lan -E -U ADMIN -H ilo-$host user set password 3 '0axilo!!'
                ipmitool -l lan -E -U ADMIN -H ilo-$host channel setaccess 1 3 link=on ipmi=on callin=on privilege=4
		shift
        done
else
	echo "please supply at least one server to configure ilo on"
fi
}

kill_a_process_on_CC_workers () {
process_name=$@
if [ $# -eq 1 ]; then
        for i in ${SJC_CC_WORKERS1}; do
        echo -n "[*] Checking for presence of \"$process_name\" on $i ";
        ssh -o ConnectTimeout=10 root@$i "cat /dev/null" 2>/dev/null;
                if [ $? == 0 ];
                        then
                        ssh_result=$(timeout 10 ssh -o ConnectTimeout=10 root@$i "ps aux | grep "$process_name" | grep -v "grep"")
                        pid=$(echo $ssh_result | awk '{print $2}')
                                if [ -z "$pid" ];
                                        then echo -e "[FAIL, no process found]"
                                else
                                        echo "[killing $pid]"
                                        ssh root@$i "kill $pid"
                                fi
                else echo -e "[${red}DOWN${endcolor}]"
                fi
        done
else
        echo "please supply the name of the process you would like to kill, be careful"
fi
}

check_zfs_disks_oldMethod () {
for i in $ZX_HEADS; do
	echo "---------------------$i--------------------"
	ssh coraid@$i "/usr/sbin/zpool status | grep '\<c..t.*d.*\>'| grep -v ONLINE |  grep -v continue ; zpool status | grep scan; zpool status | grep done"
	echo " "
done
}

check_zfs_disks () {
for i in $ZX_HEADS; do
        echo "---------------------$i--------------------"
        ssh coraid@$i "/usr/sbin/zpool status | egrep 'UNAVAIL|DEGRAD|OFF|FAULT' ; zpool status | grep scan; zpool status | grep done"
        echo " "
done
}

appliance_tail_rsync_errors () {
        recent_job=$(ssh -i /usr/local/axcient/keys/uptiva-root-key -U root@/home/$1/tunnels/ssh "ls -atr /mnt/ax/scratch/rsync_wrapper_logs | tail -n 1")
        echo "FOLLOWING LOG FOR MOST RECENT JOB: $recent_job"
        ssh -i /usr/local/axcient/keys/uptiva-root-key -U root@/home/$1/tunnels/ssh "tail /mnt/ax/scratch/rsync_wrapper_logs/$recent_job/offsite_stderr"
}

appliance_tail_rsync_errors_follow () {
        recent_job=$(ssh -i /usr/local/axcient/keys/uptiva-root-key -U root@/home/$1/tunnels/ssh "ls -Atr /mnt/ax/scratch/rsync_wrapper_logs | tail -n 1")
        echo "FOLLOWING LOG FOR MOST RECENT JOB: $recent_job"
        ssh -i /usr/local/axcient/keys/uptiva-root-key -U root@/home/$1/tunnels/ssh "tail -f /mnt/ax/scratch/rsync_wrapper_logs/$recent_job/offsite_stderr"
}

coraidShelfFinder () {
        /usr/local/axcient/coraidShelfFinder.sh $1
}
findcoraidshelf() {
        /usr/local/axcient/coraidShelfFinder.sh $1
}



whoIsOnCall () {
        /usr/local/axcient/whoIsOnCall.py
}

whoisoncall () {
api_key="8df60f71-9d3c-4738-96dc-0751aa04be2a"
wget -O - "https://api.opsgenie.com/v1.1/json/schedule/whoIsOnCall?apiKey=${api_key}" 2>&1 | tr ',' '\012' | grep -i name
}

das_list_port() {
	echo -e "
Namespace	 Port	 Status
Gen2_BPN	 4014	 Active
Gen2_SJC1	 4015	 Active
Gen2_SJC5	 4019	 Active
Gen2_SJC6	 4020	 Active
Gen2_SJC7	 4021	 Active
Gen2_SJC8	 4022	 Active
Gen2_SJC9	 4023	 Active
Gen2_SJC10	 4024	 Active
Gen2_SJC11	 4025	 Active
Gen2_SJC12	 4015	 Active (Namespace 1)
Gen2_SJC13	 4027 	 Active
Gen2_SJC14	 4028	 Active
Gen2_SJC15	 4029 	 Active
Gen2_SJC16	 4030	 Active
Gen2_SJC17	 4016	 Active
Gen2_SJC18	 4017	 Active
Gen2_SJC19	 4018	 Active
Gen2_SJC20	 4026	 Active
"
}

coraidShelfFinder () {
DC=$1
SHELF=$2

if [[ -z $DC || -z $SHELF ]]; then
  echo "Usage: coraidShelfFinder [-s|-b] \$shelf"
  echo ""
  echo "-s              SJC"
  echo "-b              BPN"
  echo ""
  return 1
elif [[ $DC != "-s" && $DC != "-b" ]]; then
  echo "Please specify '-s' for SJC DC or '-b' for BPN DC"
  return 1
fi

if [ $DC == "-b" ]; then
  if [ $SHELF -lt 5 ]; then
    ESM=bpn-esm-p1.bpn.axdc.net
    ZX=bpn-zx-p1.bpn.axdc.net
    ZPOOL=zpool1
    echo "Shelf $SHELF can be accessed via $ESM"
    echo "The shelf is a member of $ZPOOL on $ZX"
  else
    echo "Shelf $SHELF isn't a known BPN shelf."
    return 2
  fi
else
  if [ $SHELF -lt 5 ]; then
    ESM=esm-p5
    ZX=zx-p5
    ZPOOL=zpool1
    echo "Shelf $SHELF can be accessed via $ESM"
    echo "The shelf is a member of $ZPOOL on $ZX"
  fi
  if [ $SHELF -gt 4 -a $SHELF -lt 10 ]; then
    ESM=esm-p5
    ZX=zx-p6
    ZPOOL=zpool2
    echo "Shelf $SHELF can be accessed via $ESM"
    echo "The shelf is a member of $ZPOOL on $ZX"
  fi
  if [ $SHELF -gt 9 -a $SHELF -lt 15 ]; then
    ESM=esm-p7
    ZX=zx-p7
    ZPOOL=zpool3
    echo "Shelf $SHELF can be accessed via $ESM"
    echo "The shelf is a member of $ZPOOL on $ZX"
  fi
  if [ $SHELF -gt 14 -a $SHELF -lt 20 ]; then
    ESM=esm-p7
    ZX=zx-p8
    ZPOOL=zpool4
    echo "Shelf $SHELF can be accessed via $ESM"
    echo "The shelf is a member of $ZPOOL on $ZX"
  fi
  if [ $SHELF -gt 19 -a $SHELF -lt 25 ]; then
    ESM=esm-p9
    ZX=zx-p9
    ZPOOL=zpool9
    echo "Shelf $SHELF can be accessed via $ESM"
    echo "The shelf is a member of $ZPOOL on $ZX"
  fi
  if [ $SHELF -gt 24 -a $SHELF -lt 30 ]; then
    ESM=esm-p9
    ZX=zx-p10
    ZPOOL=zpool10
    echo "Shelf $SHELF can be accessed via $ESM"
    echo "The shelf is a member of $ZPOOL on $ZX"
  fi
  if [ $SHELF -gt 29 -a $SHELF -lt 40 ]; then
    ESM=esm-p11
    ZX=(zx-p11 zx-p12)
    ZPOOL=(zpool11 zpool12)
    echo "Shelf $SHELF can be accessed via $ESM"
    echo "The shelf is a member of ${ZPOOL[0]} for disks 0-17 on ${ZX[0]} and ${ZPOOL[1]} for disks 18-35 on ${ZX[1]}"
  fi
  if [ $SHELF -gt 39 -a $SHELF -lt 50 ]; then
    ESM=esm-p13
    ZX=(zx-p13 zx-p14)
    ZPOOL=(zpool13 zpool14)
    echo "Shelf $SHELF can be accessed via $ESM"
    echo "The shelf is a member of ${ZPOOL[0]} for disks 0-17 on ${ZX[0]} and ${ZPOOL[1]} for disks 18-35 on ${ZX[1]}"
  fi
  if [ $SHELF -gt 49 -a $SHELF -lt 60 ]; then
    ESM=esm-d2c-p1
    ZX=zx-d2c-p1
    ZPOOL=d2c-zpool1
    echo "Shelf $SHELF can be accessed via $ESM"
    echo "The shelf is a member of $ZPOOL on $ZX"
  fi
  if [ $SHELF -gt 59 -a $SHELF -lt 70 ]; then
    ESM=esm-p15
    ZX=(zx-p15 zx-p16)
    ZPOOL=(zpool15 zpool16)
    echo "Shelf $SHELF can be accessed via $ESM"
    echo "The shelf is a member of ${ZPOOL[0]} for disks 0-17 on ${ZX[0]} and ${ZPOOL[1]} for disks 18-35 on ${ZX[1]}"
  fi
  if [ $SHELF -gt 69 -a $SHELF -lt 80 ]; then
    ESM=esm-p17
    ZX=(zx-p17 zx-p18)
    ZPOOL=(zpool17 zpool18)
    echo "Shelf $SHELF can be accessed via $ESM"
    echo "The shelf is a member of ${ZPOOL[0]} for disks 0-17 on ${ZX[0]} and ${ZPOOL[1]} for disks 18-35 on ${ZX[1]}"
  fi
  if [ $SHELF -ge 80 ]; then
    echo "Shelf $SHELF is not known to be a member of any zpool on any current production ZX"
    return 3
  fi
fi
}
