#/usr/bin/env bash
#SAP HANA checkmk agent plugin
#Nov. 2021
#Forvia, Michael FRANK <michael.frank@forvia.com>
#April 2022, made it checkmk 2.0 compatible
#May 2022, fix timezone issue in backup query
#
#

EMPTY_SECTIONS="
<<<sap_hana_status:sep(59)>>>
\${TENANT}
<<<sap_hana_backup:sep(59)>>>
\${TENANT}
<<<sap_hana_diskusage:sep(59)>>>
\${TENANT}
<<<sap_hana_data_volume:sep(59)>>>
\${TENANT}
<<<sap_hana_license:sep(59)>>>
\${TENANT}
<<<sap_hana_ess:sep(59)>>>
\${TENANT}
<<<sap_hana_memrate:sep(59)>>>
[[${sid} ${sysid}]]
<<<sap_hana_events:sep(59)>>>
\${TENANT}
<<<sap_hana_proc:sep(59)>>>
\${TENANT}
<<<fileinfo:sep(59)>>>
$(date +%s)
<<<logwatch>>>
[[[SAPHANA DB \${sid} ${sysid} ${dbname}]]]
"

#preserve error return code of piped commands
set -o pipefail
set -o errtrace
set -o noglob

# This function creates the nice and colorful debug messages
function __msg_debug() {
	color_debug="\\x1b[35m"
	color_reset="\\x1b[0m"
	[[ "${DEBUG}" == "1" ]] && echo -e "[DEBUG]: ${color_debug}$*${color_reset}"
}
export -f __msg_debug

# This function connects to the database and run SQL queries
# Currently it works only with a KEY
function mk_hdbsql() {
    local dir_exe="${1}"
	local sysnr="${2}"
	local key="${3}"
	local query="${4}"
	__msg_debug "EXE DIR: ${dir_exe} sysid: ${sysnr} key: ${key} Running query: ${query}";
	if OUTPUT=$(${dir_exe}/hdbsql -C -F\; -x -a -n localhost -i ${sysnr} -U ${key} "${query}")
    then 
        __msg_debug "OUTPUT: ${OUTPUT}"
		sed -e "s/^;//" -e "s/;$//" <<< "${OUTPUT}"
		return 0;
    else
	    rc=${?};
        __msg_debug "SQL query failed with output: ${OUTPUT}";
		return ${rc};
    fi
	unset OUTPUT
}	
export -f mk_hdbsql

# handle command line options for debugging.
while getopts dv opt 2> /dev/null; do
  case $opt in
    d) export DEBUG=1;exec 3>&2; exec 2> >(sed -u 's/^\(.*\)$/'$'\e''[31m\1'$'\e''[m/' >&3);;
	v) export VERBOSE=1;set -o xtrace; exec 3>&2; exec 2> >(sed -u 's/^\(.*\)$/'$'\e''[31m\1'$'\e''[m/' >&3);;
	?) echo "(${0}): USAGE: -d for debug, -v for verbose";exit 1;;
  esac
done

# Lets close stderr to not bother checkmk with error messages
{ [[ -n "${DEBUG}" ]] || [[ -n "${VERBOSE}" ]]; } || exec 2>&-

#Check if SAPHostAgent is running and reacheable 
#/usr/sap/hostctrl/exe/saphostctrl -function Ping

# Get the config
# The config file is a BASH array
# dbs = (sysid,key,db sysid,key,db)
# Each DB has a unique KEY!
# Fills array $dbs
#. ${MK_CONFDIR}/sap_hana_tenant.cfg
if [ -r /etc/check_mk/test.cfg ]; then
   . /etc/check_mk/test.cfg
elif [ -r ${MK_CONFDIR}/sap_hana_tenant.cfg ]; then
   . ${MK_CONFDIR}/sap_hana_tenant.cfg
else
   __msg_debug "Configuration file not found! set MK_CONFDIR environment variable"
   exit
fi

# Allow a local DB config file
if [ -r ${MK_CONFDIR}/sap_hana_tenant_local.cfg ]; then
   dbs_=(${dbs[@]})
   . ${MK_CONFDIR}/check_mk/sap_hana_tenant_local.cfg
   dbs=(${dbs_[@]} ${dbs[@]})
fi

# Build a list of Sysids
#read the instances from configuration file
declare -a sysids
for db in ${dbs[@]}; do
  sysids+=($(cut -d ',' -f 1 <<< ${db} | tr -d ' '))
done
unset db

# Lets sort out all duplicates
# sysids is then a list if instances
IFS=$'\n' sysids=($(sort -u <<<"${sysids[*]}"))

# List running instances
#/usr/sap/hostctrl/exe/saphostctrl -function ListInstances -running
#-format [flat|tree|cimobject] where supported.
#sidS=$(/usr/sap/hostctrl/exe/saphostctrl -function ListInstances | cut -d "-" -f 2 | tr -d ' ')
#/usr/sap/hostctrl/exe/sapcontrol -nr 00 -function GetSystemInstanceList

#List DBs
#/usr/sap/hostctrl/exe/saphostctrl -function ListDatabases

for sysid in ${sysids[*]}; do
  env=$(/usr/sap/hostctrl/exe/sapcontrol -nr ${sysid} -function GetEnvironment)
  __msg_debug "${env}"
  user=$(grep USER= <<< "${env}" | cut -d '=' -f 2)
  #TODO: Check if later built of sid in DB loop is still necessary
  sid=$(grep SAPSYSTEMNAME <<< "${env}" | cut -d '=' -f 2)
  host=$(grep HOST= <<< "${env}" | cut -d '=' -f 2)
  if dir_exe=$(grep DIR_EXECUTABLE <<< "${env}" | cut -d '=' -f 2); then
    __msg_debug "SAP User:${user} EXE Path: ${dir_exe} HOST: ${host}"
    else
	__msg_debug "SAP dir_exe path not set"
	continue
  fi

  __msg_debug "Checking instance ${sysid}"
  echo -e "<<<sap_hana_instance_status:sep(59)>>>"
  echo -e "[[${sid} ${sysid}]]"
  OUTPUT=$(/usr/sap/hostctrl/exe/sapcontrol -nr ${sysid} -function GetProcessList )
  rc=${?}
  echo -e "instanceStatus: ${rc}"
  #EXITCODES
  # 0  Last webmethod call successful
  # 1  Last webmethod call failed, invalid parameter
  # 2  StartWait, StopWait, WaitforStarted, WaitforStopped, RestartServiceWait, timed out
  # 3  GetProcessList succeeded, all processes running correctly
  # 4  GetProcessList succeeded, all processes stopped
  # Exit code 0 in case a process is not GREEN
  if (( ${rc} == 3)) || ((${rc} == 0 )); then
    sed -n '4,$p' <<< "${OUTPUT}" | sed  's/, /;/g'
	__msg_debug "Instance ${sysid} running. exit code ${rc}" 
  else
    __msg_debug "Instance ${sysid} not running. exit code ${rc}\n ${OUTPUT}" 
    echo -e "instanceStatus: ${rc}\n"
    echo "${OUTPUT}"
    continue
  fi
  
  #################### Check Replication Status ####################################
  #The script provides the following return codes:
  #10: No System Replication
  #11: Error
  #12: Unknown
  #13: Initializing
  #14: Syncing
  #15: Active
  # On Backup System we receive rc=12
  
  output="$(su "${user}" -c "/bin/env python ${dir_exe}/python_support/systemReplicationStatus.py")"
  rc=$?
  mode=$(grep mode: <<<"${output}" | sed 's/mode: //g')
  if [ ${mode} != PRIMARY ]
  then
      __msg_debug "Replication mode: ${mode} DB Tests run only on PRIMARY node"
      exit
  fi
  echo "<<<sap_hana_replication_status>>>"
  echo "[[${sid} ${sysid}]]"
  echo "systemReplicationStatus: ${rc}"
  echo "$output"
  unset output
  
  ######################## Run DB checks ###########################################  
  for db in ${dbs[@]}; do
    sysid_=$(cut -d ',' -f 1 <<< ${db} | tr -d ' ')
    key=$(cut -d ',' -f 2 <<< ${db} | tr -d ' ' )
	dbname=$(cut -d ',' -f 3 <<< ${db} | tr -d ' ' )
	## Check if the db is valid for current instance
	if (( ${sysid} != ${sysid_} )); then
      __msg_debug "Sys ID: ${sysid} doesnt fit to ${sysid_}"
      continue
    fi
    __msg_debug "Running DB checks for DB:${dbname} Key:${key} SID: ${sid} Sys ID.:${sysid} "
    
##################### Run as instance user ############################################   
    
    su "${user}" -c /bin/bash << EOF
    set -o pipefail
	set -o errtrace
    # Set BASH to verbose
    [[ "${VERBOSE}" == "1" ]] && set -o xtrace
        
    ################## Build the TENANT output ###################################
    TENANT="[[${sid} ${sysid} ${dbname}]]"

	################### Get database information and status ###################################
	echo -e "<<<sap_hana_db_status:sep(59)>>>"
	echo \$TENANT
    DBHOST=${host}
    DBNAME=${dbname}
    QUERY="SELECT Name, Status, Value FROM M_SYSTEM_OVERVIEW Where NAME='All Started'"
    if OUTPUT=\$(${dir_exe}/hdbsql -a -x -n localhost -i ${sysid} -U ${key} "\${QUERY}" 2>&1 ); then
        echo -e "OK\nConnected to \${DBNAME}\n\${OUTPUT}"
    else
        echo -e "Cannot Connect to DB \${OUTPUT}"
        echo -e "${EMPTY_SECTIONS}"
        __msg_debug "Cannot connect to DB ${dbname} exit code \${?} and output:\n \${OUTPUT}";
        exit
    fi
	unset OUTPUT
	
	# Lets close stderr to not send errors to checkmk agent
    ([[ -n "${DEBUG}" ]] || [[ -n "${VERBOSE}" ]]) || exec 2>&-
	
	################### Backup ###################################
	echo "<<<sap_hana_backup:sep(59)>>>"
	echo "\${TENANT}"
	QUERY="Select TOP 1 entry_type_name, utc_end_time, state_name, comment,\
	message from M_BACKUP_CATALOG where entry_type_name='data snapshot' AND \
	state_name <> 'running' order by sys_start_time desc"
	mk_hdbsql "${dir_exe}" "${sysid}" ${key} "\${QUERY}"
	QUERY="Select TOP 1 entry_type_name, utc_end_time, state_name, comment,\
	message from M_BACKUP_CATALOG where entry_type_name = 'complete data backup'\
	AND state_name <> 'running' order by sys_start_time desc"
	mk_hdbsql "${dir_exe}" "${sysid}" ${key} "\${QUERY}"
	QUERY="Select TOP 1 entry_type_name, utc_end_time, state_name, comment, message\
	from M_BACKUP_CATALOG where entry_type_name = 'log backup' AND state_name <> 'running'\
	order by sys_start_time desc"
	mk_hdbsql "${dir_exe}" "${sysid}" ${key} "\${QUERY}"
   
   ################### License ################################################
   echo "<<<sap_hana_license:sep(59)>>>"
   echo "\${TENANT}"
   QUERY="SELECT ENFORCED,PERMANENT,LOCKED_DOWN,PRODUCT_USAGE,PRODUCT_LIMIT,\
   VALID,EXPIRATION_DATE FROM M_LICENSE"
   mk_hdbsql "${dir_exe}" "${sysid}" ${key} "\${QUERY}"
   
   ######################### ESS #################################################
   echo "<<<sap_hana_ess:sep(59)>>>"
   echo "\${TENANT}"
   QUERY="SELECT 'started', count(*) FROM M_SERVICE_THREADS where\
   thread_type='WorkerThread (StatisticsServer)' and HOST = '\${DBHOST}'"
   mk_hdbsql "${dir_exe}" "${sysid}" ${key} "\${QUERY}"
   QUERY="select 'active', MAP(IFNULL(SYSTEM_VALUE, IFNULL(HOST_VALUE,DEFAULT_VALUE)),\
   'true', 'yes', 'false', 'no', 'unknown') FROM (SELECT  MAX(MAP(LAYER_NAME,\
   'DEFAULT', VALUE)) DEFAULT_VALUE, MAX(MAP(LAYER_NAME, 'HOST',VALUE))\
   HOST_VALUE, MAX(MAP(LAYER_NAME, 'SYSTEM',  VALUE, 'DATABASE', VALUE))\
   SYSTEM_VALUE FROM  M_INIFILE_CONTENTS WHERE  FILE_NAME IN ('indexserver.ini',\
   'nameserver.ini') AND SECTION = 'statisticsserver' AND  key = 'active')"
   mk_hdbsql "${dir_exe}" "${sysid}" ${key} "\${QUERY}"
   
   ######################### MEMRATE #################################################
   if [ \${DBNAME} = SYSTEMDB ] ; then
       echo "<<<sap_hana_memrate:sep(59)>>>"
       echo "[[${sid} ${sysid}]]"
       QUERY="SELECT 'mem_rate', INSTANCE_TOTAL_MEMORY_USED_SIZE, ALLOCATION_LIMIT FROM \
         M_HOST_RESOURCE_UTILIZATION WHERE HOST = '\${DBHOST}'"
       mk_hdbsql "${dir_exe}" "${sysid}" ${key} "\${QUERY}"
   fi
   
   ######################### EVENTS #################################################
   echo "<<<sap_hana_events:sep(59)>>>"
   echo "\${TENANT}"
   QUERY="select 'open_events', count(*) from m_events where acknowledged='FALSE'"
   mk_hdbsql "${dir_exe}" "${sysid}" ${key} "\${QUERY}"
   QUERY="select 'disabled_alerts', count(*) from _sys_statistics.STATISTICS_SCHEDULE \
   where status='Disabled'"
   mk_hdbsql "${dir_exe}" "${sysid}" ${key} "\${QUERY}"
   QUERY="select 'high_alerts', count(*) from _sys_statistics.statistics_current_alerts \
   where  alert_rating >=4"
   mk_hdbsql "${dir_exe}" "${sysid}" ${key} "\${QUERY}"

EOF
  done
done