Context:
python 2.7.3
ubuntu
Symptoms:
after the code run, logs can be dumped in the RSS memory with GDB. BTW. the problem does not exist in Python 2.7.10+, but I can not find any related ChangeLog.
python 2.7.3
ubuntu
Symptoms:
after the code run, logs can be dumped in the RSS memory with GDB. BTW. the problem does not exist in Python 2.7.10+, but I can not find any related ChangeLog.
# coding=utf-8 import os import sys import time import Queue import logging import threading import traceback class Reader(threading.Thread): def __init__(self, q): super(Reader, self).__init__(name="Reader") self.queue = q def run(self): while True: log = "1542085197 2018-11-13<SP>12:59:57 0.524 NONE - 124.89.70.249 124.89.70.249 - www.test.com /beauty.jpg GET - - -<||>- 400 0 - Mozilla/4.0<SP>(compatible;<SP>MSIE<SP>6.0;<SP>Windows<SP>NT<SP>4.0;<SP>); zh-cn<||>- -<||>80<||>-<||>http - - - 23332 - 165 165 0 - -" counter = 1 arr = [] items = log.split() items[8] = '-' items[9] = '/' while True: counter += 1 ip = "1.1.%s.%s" % ((counter/256) % 255, counter%255) items[5] = ip items[6] = ip log1=" ".join(items) arr.append(log1) if len(arr) >= 3000: try: self.queue.put(arr, timeout=1) print("add a msg into queue!") except Queue.Full: print( 'reader: queue is full') arr = [] if counter > 2100000: time.sleep(10000) class Matcher(threading.Thread): def __init__(self, q): super(Matcher, self).__init__(name='Matcher') self.queue = q def run(self): counter = 0 while True: try: logs = self.queue.get(timeout=3) counter += len(logs) for log in logs: item = log.split() except Queue.Empty: print("queue empty! ") print ("rec %s" % counter) time.sleep(0.1) class XXer(object): def __init__(self): q = Queue.Queue(2048) self.reader = Reader(q) self.matcher = Matcher(q) self._worker_init(self.reader) self._worker_init(self.matcher) def _worker_init(self, w): w.setDaemon(True) w.start() return w def run(self): while True: time.sleep(1) if __name__ == "__main__": obj = XXer() obj.run()