Задача скрипта – выполнять постоянную проверку лога приложения и, в случае обнаружения сообщений об ошибке подключения к базе данных, останавливать Tomcat.
Т.к. после трёх неудачных попыток подключения аккаунт на Oracle-сервере блокируется – необходимо было жестко останавливать приложение до того, как аккаунт будет заблокирован.
Для PROD-сервера – конечно, “убивать” приложение нельзя, поэтому – там скрипт только отправляет письмо.
Для чтения лога – используется модуль python-tail.
#!/usr/bin/env python
import subprocess
import os
import sys
import tail
import smtplib
import signal
import datetime
import logging
import socket
try:
env = os.environ['ENV']
except KeyError as e:
print('ERROR - no such variable: %s' % e)
sys.exit(6)
host = os.environ['HOSTNAME']
if env == 'DEV':
app_basedir = '/home/user1/APP'
elif env == 'SIT':
app_basedir = '/home/user2/APP'
elif env == 'UAT':
app_basedir = '/home/user3/APP'
elif env == 'REG':
app_basedir = '/home/user4/APP'
elif env == 'UDEP':
app_basedir = '/home/user5/APP'
elif env == 'PROD':
app_basedir = '/usr/local/app_install/APP'
else:
print('nERROR! Unknown ENV. Exit.')
sys.exit(1)
catalina_home = app_basedir + '/application-server'
# APP lo-file
applicationlog = app_basedir + '/logs/application-app.log'
# Tomcat log
catalog = app_basedir + '/logs/catalina.out'
# APPmanager log
manalog = app_basedir + '/logs/application-manager.log'
# LogParcer lof
parcerlog = app_basedir + '/logs/logparcer.log'
# list to check logfile
logs = (applicationlog, catalog, manalog)
# list to print names of log-files
names = ('APP', 'Tomcat', 'APPmanager')
# LogParcer PID to kill it when APPmanager -stop
ppid = os.getpid()
# file to save LogParcer PID
ppidf = (app_basedir + '/logparser.pid')
# Tomcat's PID file
tpidf = (catalina_home + '/conf/catalina.pid')
# email sender function
sender = '[email protected]'
receivers = ['[email protected]', '[email protected]', '[email protected]']
# message to be mailed
message = """From: Logparser <[email protected]>
To: APP team <[email protected]>
Subject: Logparser on %s ALERT
ALARM on %s!n
""" % (env, host)
# list of errors to parce
errors = ('ORA-01017', 'ORA-28000')
# logging properties
logging.basicConfig(format = '%(filename)s[LINE:%(lineno)d] - %(levelname)-3s [%(asctime)s] %(message)s ', filename=parcerlog, level=logging.DEBUG)
logging.info('LogParcer started')
logging.info('Working on %s and using %s as $APP_BASEDIR, and %s as APP logfile.' % (env, app_basedir, applicationlog))
# save own PID to file
def savepid(ppidf, ppid):
if os.path.isfile(ppidf):
logging.info('Found my own old PID file...')
os.remove(ppidf)
logging.info('Removed.')
pidfile = open(ppidf, 'w')
pidfile.write(ppid)
pidfile.close()
logging.info('Saved my PID %s in file %s.' % (ppid, ppidf))
# get Tomcat PID to kill
def catapid(tpidf):
try:
catalina = open(tpidf, 'r')
pid = catalina.readline()
return pid
except IOError as e:
logging.critical("ERROR! Can't find catalina.pid: %sn" % e)
sys.exit(3)
# check if catalina.pid exist - otherwise LogParcer will can't kill it
def catacheck(tpidf):
if not os.path.isfile(tpidf):
logging.critical("ERROR! Can't find catalina.pid: %sn" % tpid)
sys.exit(4)
# check log-files is exists
def checkfile(log, name):
if os.path.isfile(log):
logging.info('%s log is: %s' % (name, log))
else:
logging.critical('nERROR! Can't find log file! Exit.n')
sys.exit(2)
# sending notifications
def sendmail(event):
try:
# get time
time = datetime.datetime.now()
timemes = 'nnSysevent at %s' % time
# connect to local SMTP
smtpconnect = smtplib.SMTP('localhost:25')
# debug to sys.stdout currently is $OUTPUT_DIR/logparcer_mail.log
smtpconnect.set_debuglevel(1)
# sending email
smtpconnect.sendmail(sender, receivers, message + event + timemes)
# close conenction
smtpconnect.quit()
logging.info('Email sent from %s to %s.' % (sender, receivers))
# if anything wrong on local SMTP
except socket.gaierror as s:
logging.exception('Local SMTP error: %s' % s)
except smtplib.SMTPException as e:
logging.exception('ERROR during email sent: %s' % e)
# APPmanager stops LogParcer with SIGINT (Ctrl+C same)
def signal_handler(signal, frame):
logging.info('Exited by APPmanager SIGINT.')
sys.exit(0)
# main error parces function
def err(data):
# check both errors
if any(ora in data for ora in errors):
# on PROD we don't need kill Tomcat
if not env == 'PROD':
# get Tomcat's PID
pid = catapid(tpidf)
# kill Tomcat with kill -9
try:
# killing Tomcat with kill -9
os.kill(int(pid), signal.SIGKILL)
# save message for email and log
event = "Tomcat killed with PID %s.nnError data: nn%s" % (pid, data)
logging.error(event)
# sending email
sendmail(event)
logging.info('Email sent from %s to %s.n' % (sender, receivers))
# as Tomcat killed - LogParcer doesn't need anymore
# will be started again when APPmanager -start
sys.exit(0)
# if can't find catalina.pid
except OSError as e:
event = "Got ALARM, but can't kill Tomcat: %s with PID %s" % (e, pid)
sendmail(event)
logging.critical(event)
# if PROD - just send email, no kill
else:
event = "Tomcat wasn't killed. Please - check logs manually.n"
sendmail(event)
logging.error(event)
# capture exit to write log message
signal.signal(signal.SIGINT, signal_handler)
# check if Tomcat was started and catalina.pid created
# and it's PID file present
catacheck(tpidf)
# need PID of logparceer own process to kill it
# when APPmanger -stop executed
savepid(ppidf, str(ppid))
logging.info('Check log-files:')
# running log-files checks
for file, name in zip(logs, names):
checkfile(file, name)
logging.info('Files checked, OK.')
# running main tail
tail = tail.Tail(applicationlog)
# when new line appears - sent it to 'err' function
tail.register_callback(err)
tail.follow()
ToDo: Добавить парсинг одновременно лога Tomcat и приложения.
