Python: скрипт LogParcer

Автор: | 09/11/2014
 

PythonЗадача скрипта — выполнять постоянную проверку лога приложения и, в случае обнаружения сообщений об ошибке подключения к базе данных, останавливать Tomcat.

Т.к. после трёх неудачных попыток подключения аккаунт на Oracle-сервере блокируется — необходимо было жестко останавливать приложение до того, как аккаунт будет заблокирован.

Для PROD-сервера — конечно, «убивать» приложение нельзя, поэтому — там скрипт только отправляет письмо.

Для чтения лога — используется модуль python-tail.

#!/usr/bin/env python

import subprocess
import os
import sys
import tail
import smtplib
import signal
import datetime
import logging
import socket

try:
        env = os.environ['ENV']
except KeyError as e:
        print('ERROR - no such variable: %s' % e)
        sys.exit(6)

host = os.environ['HOSTNAME']

if env == 'DEV':
        app_basedir = '/home/user1/APP'
elif env == 'SIT':
        app_basedir = '/home/user2/APP'
elif env == 'UAT':
        app_basedir = '/home/user3/APP'
elif env == 'REG':
        app_basedir = '/home/user4/APP'
elif env == 'UDEP':
        app_basedir = '/home/user5/APP'
elif env == 'PROD':
        app_basedir = '/usr/local/app_install/APP'
else:
        print('nERROR! Unknown ENV. Exit.')
        sys.exit(1)

catalina_home = app_basedir + '/application-server'

# APP lo-file
applicationlog = app_basedir + '/logs/application-app.log'
# Tomcat log
catalog = app_basedir + '/logs/catalina.out'
# APPmanager log
manalog = app_basedir + '/logs/application-manager.log'
# LogParcer lof
parcerlog = app_basedir + '/logs/logparcer.log'

# list to check logfile
logs = (applicationlog, catalog, manalog)
# list to print names of log-files
names = ('APP', 'Tomcat', 'APPmanager')

# LogParcer PID to kill it when APPmanager -stop
ppid = os.getpid()
# file to save LogParcer PID
ppidf = (app_basedir + '/logparser.pid')
# Tomcat's PID file
tpidf = (catalina_home + '/conf/catalina.pid')


# email sender function
sender = 'ApplicationMonitor@list.domain.com'
receivers = ['user1@domain.com', 'user2@domain.com', 'user3@list.domain.com']

# message to be mailed
message = """From: Logparser <ApplicationMonitor@list.domain.com>
To: APP team <user@domain.com>
Subject:  Logparser on %s ALERT

ALARM on %s!n
""" % (env, host)

# list of errors to parce
errors = ('ORA-01017', 'ORA-28000')

# logging properties
logging.basicConfig(format = '%(filename)s[LINE:%(lineno)d] - %(levelname)-3s [%(asctime)s] %(message)s ', filename=parcerlog, level=logging.DEBUG)

logging.info('LogParcer started')
logging.info('Working on %s and using %s as $APP_BASEDIR, and %s as APP logfile.' % (env, app_basedir, applicationlog))

# save own PID to file
def savepid(ppidf, ppid):

        if os.path.isfile(ppidf):
                logging.info('Found my own old PID file...')
                os.remove(ppidf)
                logging.info('Removed.')

        pidfile = open(ppidf, 'w')
        pidfile.write(ppid)
        pidfile.close()
        logging.info('Saved my PID %s in file %s.' % (ppid, ppidf))

# get Tomcat PID to kill
def catapid(tpidf):
        try:
                catalina = open(tpidf, 'r')
                pid = catalina.readline()
                return pid
        except IOError as e:
                logging.critical("ERROR! Can't find catalina.pid: %sn" % e)
                sys.exit(3)

# check if catalina.pid exist - otherwise LogParcer will can't kill it
def catacheck(tpidf):
        if not os.path.isfile(tpidf):
                logging.critical("ERROR! Can't find catalina.pid: %sn" % tpid)
                sys.exit(4)

# check log-files is exists
def checkfile(log, name):
        if os.path.isfile(log):
                logging.info('%s log is: %s' % (name, log))
        else:
                logging.critical('nERROR! Can't find log file! Exit.n')
                sys.exit(2)

# sending notifications
def sendmail(event):
        try:
                # get time
                time = datetime.datetime.now()
                timemes = 'nnSysevent at %s' % time

                # connect to local SMTP
                smtpconnect = smtplib.SMTP('localhost:25')
                # debug to sys.stdout currently is $OUTPUT_DIR/logparcer_mail.log
                smtpconnect.set_debuglevel(1)
                # sending email
                smtpconnect.sendmail(sender, receivers, message + event + timemes)
                # close conenction
                smtpconnect.quit()
                logging.info('Email sent from %s to %s.' % (sender, receivers))
        # if anything wrong on local SMTP
        except socket.gaierror as s:
                logging.exception('Local SMTP error: %s' % s)
        except smtplib.SMTPException as e:
                logging.exception('ERROR during email sent: %s' % e)

# APPmanager stops LogParcer with SIGINT (Ctrl+C same)
def signal_handler(signal, frame):

        logging.info('Exited by APPmanager SIGINT.')
        sys.exit(0)

# main error parces function
def err(data):
        # check both errors
        if any(ora in data for ora in errors):

                # on PROD we don't need kill Tomcat
                if not env == 'PROD':
                        # get Tomcat's PID
                        pid = catapid(tpidf)
                        # kill Tomcat with kill -9
                        try:
                                # killing Tomcat with kill -9
                                os.kill(int(pid), signal.SIGKILL)
                                # save message for email and log
                                event = "Tomcat killed with PID %s.nnError data: nn%s" % (pid, data)
                                logging.error(event)
                                # sending email
                                sendmail(event)
                                logging.info('Email sent from %s to %s.n' % (sender, receivers))
                                # as Tomcat killed - LogParcer doesn't need anymore
                                # will be started again when APPmanager -start
                                sys.exit(0)
                        # if can't find catalina.pid
                        except OSError as e:
                                event = "Got ALARM, but can't kill Tomcat: %s with PID %s" % (e, pid)
                                sendmail(event)
                                logging.critical(event)
                # if PROD - just send email, no kill
                else:
                        event = "Tomcat wasn't killed. Please - check logs manually.n"
                        sendmail(event)
                        logging.error(event)

# capture exit to write log message
signal.signal(signal.SIGINT, signal_handler)

# check if Tomcat was started and catalina.pid created
# and it's PID file present
catacheck(tpidf)

# need PID of logparceer own process to kill it
# when APPmanger -stop executed
savepid(ppidf, str(ppid))

logging.info('Check log-files:')

# running log-files checks
for file, name in zip(logs, names):
        checkfile(file, name)

logging.info('Files checked, OK.')

# running main tail
tail = tail.Tail(applicationlog)
# when new line appears - sent it to 'err' function
tail.register_callback(err)
tail.follow()

ToDo: Добавить парсинг одновременно лога Tomcat и приложения.