bzip2 compress log files

Control File

[user@log01 scripts]$ cat dir-to-compress.txt
/var/log/app/app01
/var/log/app/app02

Scripts

[works@log01 scripts]$ more compresslogfiles.sh
#!/bin/bash                   
                                                                                                                                                               
#Written Andrew S 28 September 2012 to compress log files on srv-log01                     
                                                                                                                                                               
#turn on debugging, -x starts, +x stops
set +x

DIRTOCOMPRESS=/home/scripts/dir-to-compress.txt

#set to 3 for testing, this forces sending mail, 1 & 2 are used for errors, normally should be 0
SENDEMAIL=3
#EMAIL=ww@zzz.com
EMAIL=andrew.xx@zzz.com


TMPLOG=/tmp/compresslog-$$.txt
DEBUGLOG=/tmp/compressdebuglog-$$.txt


DATE=`date +%b" "%d" "%Y`
DAY=`date +%a`

#Check user is works
if [ "${USER}" = "logworks" ]; then
   echo "Running as user logworks"  >> ${TMPLOG}
   echo " "  >> ${TMPLOG}
   else
   echo "Sorry, you need to be the logworks user to run this script."
   exit 1
fi

DATENUM=`date +%d`
#Test to force 1st of the month
#DATENUM=01
if [ ${DATENUM} == 01 ]
then
        MONTHTEST=" - 1st of the month test email."
        SENDEMAIL=2
fi

#Write out header to email tmp file.
echo "Log file compressor on srv-log01" > ${TMPLOG}
echo "================================" >> ${TMPLOG}
echo "" >> ${TMPLOG}
echo "From /home/logworks/scripts/compresslogfiles.sh on srv-log01" >> ${TMPLOG}
echo "" >> ${TMPLOG};
echo "" >> ${TMPLOG};


#Main program section
#====================

#from df
SPACEATSTART=`df -h | grep /var/log/GRY | cut -d' ' -f28`


#find /var/log/app/srv-app01/ -maxdepth 1 -type f -exec bzip2 {} \;

#set up some counters
FILECOMPRESS=0
FILEZIP=0
FILEGZ=0
FILEBZ2=0
COMPRESSTOTAL=0
ZEROSIZE=0

for DIR in `cat ${DIRTOCOMPRESS}` ;
do
echo "Looking in ${DIR}."  >> ${TMPLOG}
echo "=========================================="  >> ${TMPLOG}
echo " "  >> ${TMPLOG}

echo "Looking in ${DIR}."  >> ${DEBUGLOG}
echo "=========================================="  >> ${DEBUGLOG}
echo " "  >> ${DEBUGLOG}

        for FILE in `find ${DIR} -mtime +1 -maxdepth 1 -type f` ;
        do

        #test if file is greater than zero bytes long, else skip it.
        if [ -s "${FILE}" ] ; then

        if [[ ${FILE} =~ \.bz2$ ]] ; then
                #do nothing except move file. Don't add bzip2 to mv as it already ends bzip2
                echo "$FILE is already bzip2'ed"  >> ${DEBUGLOG}
                echo "-> mv ${FILE} to  ${DIR}/archive."  >> ${DEBUGLOG}
                echo " "  >> ${DEBUGLOG}
                mv ${FILE} ${DIR}/archive
                ((FILEBZ2++))

        elif [[ ${FILE} =~ \.zip$ ]] ; then
                #do nothing except move, zip file may be a compressed directory structure which bzip won't handle ok
                echo "${FILE} is zip archive"  >> ${DEBUGLOG}
                echo "-> mv ${FILE} to  ${DIR}/archive."  >> ${DEBUGLOG}
                echo " "  >> ${DEBUGLOG}
                mv ${FILE} ${DIR}/archive
                ((FILEZIP++))

        elif [[ ${FILE} =~ \.gz$ ]] ; then
                echo "-> ${FILE} is gzip'ed, so decompress it"  >> ${DEBUGLOG}
                #use -f to force overwriting to prevent script stalling waiting for user input "y"
                gunzip -f -v ${FILE}

                #FILE is now wrong as it still contains gz, so remove it.
                FILE=${FILE%.gz}

                #bzip2 -v form.log.2012-07-16.16Jul 2>&1 | cut -d',' -f3 | cut -d'%' -f1 - #getcompress stats
                echo -n "--> bzipping ${FILE}, "  >> ${DEBUGLOG}
                RATIO=`bzip2 -v ${FILE} 2>&1 | cut -d',' -f3 | cut -d'%' -f1`
                echo "ratio is >$RATIO<"  >> ${DEBUGLOG}

                COMPRESSTOTAL=`echo "${COMPRESSTOTAL}+${RATIO}" | bc -l`

                echo "---> mv ${FILE}.bz2 to ${DIR}/archive."  >> ${DEBUGLOG}
                echo " "  >> ${DEBUGLOG}
                mv ${FILE}.bz2 ${DIR}/archive
                ((FILEGZ++))
                ((FILECOMPRESS++))


        else 
                #assume file is not compressed
                echo -n "-> ${FILE} is not gz or bzip2, so compress it, "  >> ${DEBUGLOG}
                RATIO=`bzip2 -v ${FILE} 2>&1 | cut -d',' -f3 | cut -d'%' -f1`
                echo "ratio is >$RATIO<"  >> ${DEBUGLOG}

                COMPRESSTOTAL=`echo "${COMPRESSTOTAL}+${RATIO}" | bc -l`

                #bzip2 adds extension, so we add it to the mv below
                echo "--> mv ${FILE}.bz2 to ${DIR}/archive."  >> ${DEBUGLOG}
                echo " "  >> ${DEBUGLOG}
                mv ${FILE}.bz2 ${DIR}/archive
                ((FILECOMPRESS++))
        fi

        #end file zero test
        else
        echo "${FILE} is zero length, skipping."  >> ${DEBUGLOG}
        ((ZEROSIZE++))
        fi

        #End of file test "for"
        done

#end of dirs to test "for"
done

RESULT=$?

#don't attempt to calculate mean if no files were processed
if [[ ${FILECOMPRESS} -ne 0 ]] ;
then
        MEANCOMPRESS=`echo "scale=2; ${COMPRESSTOTAL}/${FILECOMPRESS}" | bc -l`
else
        MEANCOMPRESS="undefined, cannot divide by zero"
fi

echo "Average (Mean) compression was ${MEANCOMPRESS} %."  >> ${TMPLOG}


#from df
SPACEATEND=`df -h | grep /var/log | cut -d' ' -f28`

echo "Returncode is >${RESULT}<" >> ${TMPLOG}


echo " " >> ${TMPLOG}
echo "Free space at start ${SPACEATSTART}, free space at end ${SPACEATEND}." >> ${TMPLOG}

echo " " >> ${TMPLOG}
echo "files compressed with bzip2:- ${FILECOMPRESS}" >> ${TMPLOG}
echo "zip files encountered:- ${FILEZIP}" >> ${TMPLOG}
echo "gzip files converted:- ${FILEGZ}" >> ${TMPLOG}
echo "bz2 files found:- ${FILEBZ2}" >> ${TMPLOG}
echo "zero sized files discovered:- ${ZEROSIZE}"  >> ${TMPLOG}

#Uncomment for testing....
#echo "++++++++++++++++++++++++++++++++"  >> ${TMPLOG}
#cat  ${DEBUGLOG}  >> ${TMPLOG}
#echo "++++++++++++++++++++++++++++++++"  >> ${TMPLOG}


if [ ${RESULT} != 0 ]
        then
        echo "Gzip of files on srv-log01 may have errors, return code was not 0, ( >${RESULT}< ). " >> ${TMPLOG}
        echo "Command returned \"${OUTPUT}\" " >> ${TMPLOG}
        SENDEMAIL=1
else
        echo "bzip2 run completed."  >> ${TMPLOG}
fi



echo ""
echo "------------------------------------------------------------------------" >> ${TMPLOG}

#Send mail
#=========
if [ ${SENDEMAIL} -eq 1 ]
        then
        /bin/mailx -s "Log file compressor had problems." ${EMAIL} < ${TMPLOG}
elif [ ${SENDEMAIL} -eq 2 ]
        then
        /bin/mailx -s "Log file compressor had problems ${MONTHTEST}." ${EMAIL} < ${TMPLOG}
elif [ ${SENDEMAIL} -eq 3 ]
        then
        /bin/mailx -s "Log file compressor is set to testing." ${EMAIL} < ${TMPLOG}
else
        echo "No errors" > /dev/null
fi


rm ${TMPLOG}

exit 0

rb/compresslogfiles.txt · Last modified: 05/09/2019 10:34 by andrew