5 Sep 2012

Working with log files in the /var/log/ directory Part 3 ssh

Continuing from part 1

Part 3 will deal with using these commands in scripts and over ssh to get some information on remote servers. 

As most of what I wanted to say is covered in an other blog I am provding a link to that blog with some of the important parts cut out.

 http://backreference.org/2011/08/10/running-local-script-remotely-with-arguments/ 

Well, ssh runs a shell on the remote host, so why not do
 $ ssh user@remote < local.sh  

And if your script has arguments
$ ssh user@remote 'cat | bash /dev/stdin arg1 arg2 arg3' < local.sh 
Argument is __arg1__  
Argument is __arg2__  
Argument is __arg3__

Now what would you put inside a script. In my case I have a script that I use to do some basic checks. I will cut out our work specific stuff but you should get a really good idea about what you can do and how to do it. This script is a work in progress please fell free to make changes. 

How I run the script. Use the multiple arguments method

ssh root@192.168.0.51 'cat | bash /dev/stdin -a < ./check.sh


What my script does



 /tmp/check.sh -h
Usage: check.sh [-hvapdbs] [-o arg] args
Options avalible in the script -h Help
  -v version
  -s disable sleep time beween checks

Chose which checks you want to perfom 
  -a (all) checks 
  -p not implemented
  -d defualt linux health checks 
  -w webserver checks
  -l list all the check functions

How to use

ssh root@192.168.0.51 'cat | bash /dev/stdin -a -s' < `basename $0` | tee /somedir/checks_output

The script 



#!/bin/bash

# simple example of how to use this script

# simple script that will run some of the basic checks that need
# to be done on the bob servers. 

# simple boolean to enable or disapable sleep/pauses in script out put 
# set waits=$NULL or waits=  to ensure the test evalues false. 
# how to test
# waits=
# [ $waits ] && echo true
# waits=$NULL
# [ $waits ] && echo true
#
# setting $waits to anything other then null evaluates true
# waits=1
# [ $waits ] && echo true
# true
waits=TRUE
pause=2

#Nicely formated title on checks
title() 
{
    Sstart="\n\e[04;31m="
    Send="=\e[00m\n"
    Sfill="================================================================="
    Stitl="$1"
    Sbuild="$Sstart $HOSTNAME -- $Stitl"
    fill=${#Sbuild}
    #echo $fill
    #echo ${Sfill:fill}
    #echo -e "\e[04;31m= $HOSTNAME ==== $1 =============\e[00m\n"

    Sbuild="$Sstart $HOSTNAME -- $Stitl ${Sfill:fill}$Send"
    echo -e "$Sbuild"
}

# check disk usage on server
disk_check()
{
    title "Checking Disk Space"
    #echo -e "\e[04;31m=$HOSTNAME=====Checking Disk Space===================\e[00m"
    df -h
    [ $waits ] && sleep 3

    # check inodes
    title "Checking Available inodes"
    #echo -e "\e[04;31m=$HOSTNAME=======Checking Available inodes===========\e[00m"
    df -i
    [ $waits ] && sleep 3
}

dmesg_check() 
{
    #dmesg
    title "Checking DMESG"
    #echo -e "\e[04;31m=$HOSTNAME======Checking DMESG========================\e[00m"
    [ $waits ] && sleep 2
    echo "dmesg tail -n 60"
    dmesg | tail -n 60
    [ $waits ] && sleep 8
    echo "summary of dmesg"
    dmesg | sed -e "s/\[[^][]*\]//" -e "s/[0-9]\+//g" | sort | uniq -dc | sort -n
    [ $waits ] && sleep 8
}

mesg_check() 
{
title "Messages Log"
    #cut -d" " -f 4- /var/log/messages | sort | uniq -dc
    #cut -d" " -f 5- /var/log/messages |\
    # sort |\
    # sed -e "s/pid=[0-9]* //"\
    # -e "s/EXIT\: rsync status\=0.*/EXIT: rsync status=0/"\
    # -e "s/last message repeated.*/last message repeated/" |\
    # uniq -c |\
    # sort -n

    sed -e "s/\(.*\)\($HOSTNAME\)/\2/" -e "s/\[[^][]*\]//" -e "s/[0-9]\+//g" /var/log/messages | sort |  uniq -c | sort -n
    [ $waits ] && sleep 5
}

DT_check()
{
    # date
    title "Checking Date & Time"
    [ $waits ] && sleep 3
    date
    [ $waits ] && sleep 3
}

network_check()
{
    # check interfaces for droped packets and errors
    title "Checking Network Interfaces"
    [ $waits ] && sleep 3
    ifconfig
    [ $waits ] && sleep 5
    # check firewall
    echo "Showing ip tables"
    iptables -L
    [ $waits ] && sleep 5
}

crontab_check()
{
    # check the jobs schedule 
    title "Checking Crontab Entries"
    #echo -e "\e[04;31m=$HOSTNAME======Checking Crontab Entries==============\e[00m"
    [ $waits ] && sleep 3
    #find /var/spool/cron/* -printf  \\n%p\\n%t\\n -exec cat {} \;
    find /var/spool/cron/* -printf "\\n\\n%p\\nLast Modified : %t\\nLast Accessed : %a\\n\\n" -exec cat {} \;
    [ $waits ] && sleep 3
    #[ $waits ] && sleep 3
    #[ $waits ] && sleep 3
}

keys_check()
{
# check ssh configuration to make sure it has not changed
    title "Checking Known Hosts"
    [ $waits ] && sleep 2
    cat /root/.ssh/known_hosts 
    echo -e "\n"
    echo -e "\nList of authorized IPs"
    cut -d" " -f1 /root/.ssh/known_hosts | sort -n
    [ $waits ] && sleep 5
    title "Checking Authorized Keys"
    [ $waits ] && sleep 2
    cat /etc/ssh/authorized_keys 
    echo -e "\nList of authorized IDs\n"
    grep -v "#" /etc/ssh/authorized_keys | cut -d" " -f3
    [ $waits ] && sleep 5
}

mail_check()
{
# chech maillog for status not sent should not return any output
    title "Checking Mail Logs"
    [ $waits ] && sleep 3
    echo -e "\nChecking mail logs\n"
    grep -P "(stat|status)=(?"\!"[Ss]ent)" /var/log/maillog
    grep -P "(stat|status)=(?"\!"[Ss]ent)" /var/log/maillog | wc
    #grep -P "stat=(?"\!"Sent)" /var/log/maillog
    #grep -P "stat=(?"\!"Sent)" /var/log/maillog | wc 
    #grep -P "status=(?"\!"sent)" /var/log/maillog
    #grep -P "status=(?"\!"sent)" /var/log/maillog | wc
    [ $waits ] && sleep 3
}


apache_check()
{
    # Apache Error Logs
    title "Checking Apache"
    if  [ -e /var/log/httpd/error.log ]
    then
        [ $waits ] && sleep 2
        #sed -e "s/\[[^][]*\]//g" -e "s/[0-9]\+//g" /var/log/httpd/error.log | sort | uniq -c | sort -n
        title "Sumary of httpd error"
        #echo -e "\n Sumary of httpd error messages"
        sed -e "s/\[[^][]*\]//g" -e "s/ referer:.*//" -e "s/[0-9]\+//g" /var/log/httpd/error.log | sort | uniq -dc | sort -n
    #sed -e "s/\[.*\]\([^:]*\)\(.*\)/\1/" /var/log/httpd/error.log | sort | uniq -dc | sort -nr
    #sed -e "s/\[.*\]\([^:]*\)\(.*\)/\1/" /var/log/httpd/error.log | sort | uniq -u | sort -nr
        #grep "phpmyadmin" /var/log/httpd/error.log
        [ $waits ] && sleep 5
    else 
        echo -e "no apache error logs at /var/log/httpd/error.log\n"
    fi
}



all_checks()
{
disk_check
dmesg_check 
mesg_check 
DT_check
network_check
crontab_check
keys_check
mail_check
apache_check

}

default_checks() 
{
disk_check
dmesg_check
mesg_check
DT_check
network_check
crontab_check
keys_check
mail_check

}


checks()
{
#g/()/

checklist=$(cat <
title() \n
disk_check()\n
dmesg_check() \n
mesg_check() \n
DT_check()\n
network_check()\n
crontab_check()\n
keys_check()\n
mail_check()\n
apache_check()\n
checks()\n

EOF
)
echo -e $checklist

}


USAGE="Usage: `basename $0` [-hvapdbs] [-o arg] args"
USAGED=$(cat <
Options avalible in the script
\t-h  Help\n 
\t-v  version\n
\t-s  disable sleep time beween checks\n

\nChose which checks you want to perfom \n
\t-a  (all) checks \n
\t-p  not implemented\n
\t-d  defualt linux health checks\n
\t-w  webserver checks\n
\t-l  list all the check functions\n

\nHow to use\n
ssh root@192.168.0.51 'cat | bash /dev/stdin -a < ./check.sh | tee /somedir/server_report\n
EOF
)


# Parse command line options.
while getopts hvsabdwtlo: OPT; do
    case "$OPT" in
    h)
        echo $USAGE
        echo -e $USAGED
        exit 0
        ;;
    v)
        echo "`basename $0` version 0.4"
        exit 0
        ;;
    s)
        waits=
        ;;
    o)
        OUTPUT_FILE=$OPTARG
        ;;
    a)
        echo "doing all checks"
        all_checks
        exit 0
        ;;

    b)
        echo "doing bob checks"
        bob_checks
        exit 0
        ;;

    p)
        echo "doing all checks"
        all_checks
        exit 0
        ;;
    d)
        echo "doing default checks"

        default_checks
        exit 0
        ;;
    w)
        echo "doing apache checks"
        apache_check

        exit 0
        ;;
    l)
        echo "showing all checks"
        checks
        exit 0
        ;;

   \?)
        # getopts issues an error message
        echo $USAGE >&2
        exit 1
        ;;
    esac
done

# Remove the switches we parsed above.
shift `expr $OPTIND - 1`

# We want at least one non-option argument.
# Remove this block if you don't need it.
if [ $# -eq 0 ]; then
    echo $USAGE >&2
    exit 1
fi

# Access additional arguments as usual through
# variables $@, $*, $1, $2, etc. or using this loop:
for PARAM; do
    echo $PARAM
done

# EOF