+- Python Forum (https://python-forum.io)
+-- Forum: Python Coding (https://python-forum.io/forum-7.html)
+--- Forum: General Coding Help (https://python-forum.io/forum-8.html)
+--- Thread: python realtime parsing logs (/thread-28058.html)
python realtime parsing logs - anna - Jul-03-2020
I am reading and parsing real time logs from remote server. I am getting huge logs and wanted to parse only New Alarm and Clear alarm. I have tried 'New Alarm' or 'Clear to Alarm' in line, but not worked for me.I have tried with regex (sorry for bad regex attempt).
Intermediately regex not capturing details... please support.
From Clear log alert, I want to capture below strings
**Clear Alarm log:**
[1833:22 30/06/20 18:13:36 InternalExternalAlarmList.cc:55] > Clear to Alarm was reported on : EMS~~LSN/EMS_BG-40_95@@ManagedElement~~2911@@PTP~~/shelf=1/slot=6/ebtype=SAM/port=1@@CTP~~/sts3c_au4-j=1/vt2_tu12-k=3-l=5-m=1 CAM ID: 3179232(A2:SAM_1 oPort 1-VC4#1VC12#55), P.cause 39, native P.C: LP-AIS, rate : 11 and Qualifier : 2911@@5@@-1@@-1@@-1@@-1@@-1@@7@@0@@3@@54@@3@@0@@317 EMS time: 20200630180625.0
1. LSN/EMS_BG-40_95 --- nodename
2. 2911 ---- Node ID
3. /shelf=1/slot=6/ebtype=SAM/port=1@@CTP~~/sts3c_au4-j=1/vt2_tu12-k=3-l=5-m=1
4. A2:SAM_1 oPort 1-VC4#1VC12#55 from () -- Port
5. 3179232 --- CAM ID
6. LP-AIS --- Native P.C.
7. 20200630180625.0 -- EMS time
**New Alarm log**
[1833:22 01/07/20 14:08:10 InternalExternalAlarmList.cc:287] > New Alarm was reported on : EMS~~LSN/EMS_BG-40_200@@ManagedElement~~10009@@PTP~~/shelf=1/slot=9/ebtype=MPS_4F/feport=1 ( MPS_4F FE-ETY Port 1(Electrical)) with CAM Id : 3835346, P.cause 18, Native P.C: PortLinkDown, rate : 97 and Qualifier : 10009@@8@@-1@@-1@@-1@@-1@@-1@@-1@@-1@@25@@0@@4@@0@@42 EMS time: 20200701140050.0
wants to capture as below
1. LSN/EMS_BG-40_200 --- node name
2. 10009 ---Node ID
3. /shelf=1/slot=9/ebtype=MPS_4F/feport=1 --self details
4. MPS_4F FE-ETY Port 1(Electrical) --port
5. 3835346 --CAM ID
6. PortLinkDown ---Native P.C
7. 20200701140050.0 -- EMS time
My code try, apologize for regex
import paramiko
import select
import time
import re
import datetime as dt
host = '172.23.88.23'
nmsHostname = 'NMS'
port = 22
user = 'nms'
password = 'Nms'
#####
def follow(thefile):
thefile.seek(0,2)
while True:
line = thefile.readline()
if not line:
time.sleep(0.1)
continue
yield line
#####
def string2time(systime):
emd_time = systime.strip()
t = dt.datetime.strptime(str(systime), '%Y%m%d%H%M%S')
return t
#########
def sysDateTime(systime):
t = dt.datetime.strptime(str(systime).split('.')[0], '%Y%m%d%H%M%S')
return t
#########
client = paramiko.SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(host, port=port, username=user, password=password)
transport = client.get_transport()
channel = transport.open_session()
sftp_client = client.open_sftp()
remote_file = sftp_client.open("/opt/NMS/server/logs/NMSAlarms.log")
try:
loglines = follow(remote_file)
for line in loglines:
#Check at least Clear to Alarm in line
Clear_alarm = re.findall(r'(\s*[C]\w*\s\w+\s\w+\w.\w+\s\w+\s\w.)', line)
#Check at least New Alarm was in line
New_alarm = re.findall(r'(\s*[N]\w.\s\w+\s\w+\s\w+\s\w+)', line)
# Found, check for other strings
if Clear_alarm or New_alarm:
if '@@SubnetworkConnection' in line:
pass
if 'ERROR' in line:
pass
if 'EMS~~LSN/EMS' not in line:
pass
if 'EMS time:' not in line:
pass
else:
#print(line)
status = re.findall(r'\s*[C|N]\w.', line)[0].strip() #Clear or New
systime = re.findall(r'\d{14}\.[0]', line)[0] #EMS Time
node = re.findall(r'\bEMS\D+.*\~',line)[0] #node
port = re.findall('\((.*?)\)',line)[0] #port
node_name = re.split('~~|@@',node)[1] #node_name
if 'Clear to Alarm' in line:
#in Clear alarm, CAM ID is Capital
camID = re.findall(r'\s[A-Z]{3}\s[A-Z]{2}\D\s\d{7}',line)[0].strip().split(':')[1]
#Native P.C. ... for this facing issue intermediately
pc = re.findall(r'(\s[N|n][a-z]{5}\s\w{1}\.\w{1}\:?)(\s\w.+,)', line)[0]
pcause = pc[1]
else:
#in New alarm, CAM Id (not all capital)
camID = re.findall(r'([C].{3})([I].{1}\s:)(\s\d{7})', line)[0]
camID = camID[2]
pc = re.findall(r'\s[N]a.+,',line)[0] # find Native PC in new Alarm
pcause = pc.split(':')[1]
if '~~' or '@@' in node:
neID = (re.split('~~|@@',node)[3]) # node ID
else:
neID = 'NA'
finally:
remote_file.close()
RE: python realtime parsing logs - ibreeden - Jul-03-2020
(Jul-03-2020, 04:40 AM)anna Wrote: but not worked for meWhy did it not work for you? Did you get an error? Is the text not found?
I am not sure but I would not trust the construction in lines 47 and 48. In line 47 you gather all the lines in a list and in the next line you process the list. That is good for static files. But why is de function called "follow()"? Is it perhaps the intention to follow the lines in an ever growing logfile?
RE: python realtime parsing logs - anna - Jul-05-2020
Hi All,
@ibreeden - tried if 'string1' or 'string2' in line: still unwanted data was coming, as i want only 'New' and 'Clear' alarm, regex implemented.
I am parsing ever growing log file from remote server, I am able to parse till file gets truncate post that script not parsing data, get stuck.
How to start reading or parsing data if log file get truncate or erased and created new by remote system.
Currently using 'tailer' module.
import paramiko
#import select
import time
import re
#import datetime as dt
import tailer
#import logging
#import os
host = 'xx'
nmsHostname = 'yyzz'
port = 22
user = 'xya'
password = 'abc'
####
client = paramiko.SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(host, port=port, username=user, password=password)
sftp_client = client.open_sftp()
remote_file = sftp_client.open("/opt/NMS/server/logs/NMSAlarms.log")
try:
for line in tailer.follow(remote_file):
regex = re.compile(r'(\d+/\d+/\d+ \d+:\d+:\d+)\s\w+\S+\D\s(\>\s[C|N][e|l][w|e])') # match New or Clear
match = re.search(regex, line)
if match:
print(line)
finally:
remote_file.close()Code is optimized,