Python Forum
Queue get memory leak when used in multithreading
Thread Rating:
  • 0 Vote(s) - 0 Average
  • 1
  • 2
  • 3
  • 4
  • 5
Queue get memory leak when used in multithreading
#1
Context:
python 2.7.3
ubuntu

Symptoms:
after the code run, logs can be dumped in the RSS memory with GDB. BTW. the problem does not exist in Python 2.7.10+, but I can not find any related ChangeLog.

# coding=utf-8
import os
import sys
import time
import Queue
import logging
import threading
import traceback


class Reader(threading.Thread):
    def __init__(self, q):
        super(Reader, self).__init__(name="Reader")
        self.queue = q

    def run(self):
        while True:
            log = "1542085197 2018-11-13<SP>12:59:57 0.524 NONE - 124.89.70.249 124.89.70.249 - www.test.com /beauty.jpg GET - - -<||>- 400 0 - Mozilla/4.0<SP>(compatible;<SP>MSIE<SP>6.0;<SP>Windows<SP>NT<SP>4.0;<SP>); zh-cn<||>- -<||>80<||>-<||>http - - - 23332 - 165 165 0 - -"
            counter = 1
            arr = []
            items = log.split()
            items[8] = '-'
            items[9] = '/'
            while True:
                counter += 1
                ip = "1.1.%s.%s" % ((counter/256) % 255, counter%255)
                items[5] = ip
                items[6] = ip
                log1=" ".join(items)
                arr.append(log1)
                if len(arr) >= 3000:
                    try:
                        self.queue.put(arr, timeout=1)
                        
                        print("add a msg into queue!")
                    except Queue.Full:
                        print( 'reader: queue is full')

                    arr = []

                if counter > 2100000:
                    time.sleep(10000)


class Matcher(threading.Thread):
    def __init__(self, q):
        super(Matcher, self).__init__(name='Matcher')
        self.queue = q

    def run(self):
        counter = 0
        while True:
            try:
                logs = self.queue.get(timeout=3)
                counter += len(logs)
                for log in logs:
                    item = log.split()

            except Queue.Empty:
                print("queue empty! ")
            print ("rec %s" % counter)
            time.sleep(0.1)


class XXer(object):
    def __init__(self):
        q = Queue.Queue(2048)

        self.reader = Reader(q)
        self.matcher = Matcher(q)
        self._worker_init(self.reader)
        self._worker_init(self.matcher)

    def _worker_init(self, w):
        w.setDaemon(True)
        w.start()
        return w

    def run(self):
        while True:
            time.sleep(1)


if __name__ == "__main__":
    obj = XXer()
    obj.run()
Reply
#2
context:
python 2.7.3
ubuntu

Symptoms:
when the code run finished, logs can be dumped in the RSS memory with GDB. The problem does not exist in Python 2.7.10+, but I can't find any related Changelog Angel .

# coding=utf-8
import os
import sys
import time
import Queue
import logging
import threading
import traceback


class Reader(threading.Thread):
    def __init__(self, q):
        super(Reader, self).__init__(name="Reader")
        self.queue = q

    def run(self):
        while True:
            log = "1542085197 2018-11-13<SP>12:59:57 0.524 NONE - 124.89.70.249 124.89.70.249 - www.test.com /beauty.jpg GET - - -<||>- 400 0 - Mozilla/4.0<SP>(compatible;<SP>MSIE<SP>6.0;<SP>Windows<SP>NT<SP>4.0;<SP>); zh-cn<||>- -<||>80<||>-<||>http - - - 23332 - 165 165 0 - -"
            counter = 1
            arr = []
            items = log.split()
            items[8] = '-'
            items[9] = '/'
            while True:
                counter += 1
                ip = "1.1.%s.%s" % ((counter/256) % 255, counter%255)
                items[5] = ip
                items[6] = ip
                log1=" ".join(items)
                arr.append(log1)
                if len(arr) >= 3000:
                    try:
                        self.queue.put(arr, timeout=1)
                        
                        print("add a msg into queue!")
                    except Queue.Full:
                        print( 'reader: queue is full')

                    arr = []

                if counter > 2100000:
                    time.sleep(10000)


class Matcher(threading.Thread):
    def __init__(self, q):
        super(Matcher, self).__init__(name='Matcher')
        self.queue = q

    def run(self):
        counter = 0
        while True:
            try:
                logs = self.queue.get(timeout=3)
                counter += len(logs)
                for log in logs:
                    item = log.split()

            except Queue.Empty:
                print("queue empty! ")
            print ("rec %s" % counter)
            time.sleep(0.1)


class XXer(object):
    def __init__(self):
        q = Queue.Queue(2048)

        self.reader = Reader(q)
        self.matcher = Matcher(q)
        self._worker_init(self.reader)
        self._worker_init(self.matcher)

    def _worker_init(self, w):
        w.setDaemon(True)
        w.start()
        return w

    def run(self):
        while True:
            time.sleep(1)


if __name__ == "__main__":
    obj = XXer()
    obj.run()
Reply


Possibly Related Threads…
Thread Author Replies Views Last Post
  multithreading Hanyx 4 1,283 Jul-29-2022, 07:28 AM
Last Post: Larz60+
Question Problems with variables in multithreading Wombaz 2 1,286 Mar-08-2022, 03:32 PM
Last Post: Wombaz
  Where is the memory leak in this Flask Code? Morkus 1 5,472 Jun-18-2021, 02:48 PM
Last Post: Oliver
  Multithreading question amadeok 0 1,748 Oct-17-2020, 12:54 PM
Last Post: amadeok
  How can i add multithreading in this example WoodyWoodpecker1 3 2,451 Aug-11-2020, 05:30 PM
Last Post: deanhystad
  memory leak on embedded python in c++ asdf3721 3 3,334 Jul-16-2020, 06:33 AM
Last Post: Gribouillis
  task queue Valon1981 8 3,517 Jul-07-2020, 07:41 AM
Last Post: freeman
  matplotlib multithreading catosp 0 2,909 Jul-03-2020, 09:33 AM
Last Post: catosp
  Queue in Pygame constantin01 1 3,640 Jan-07-2020, 04:02 PM
Last Post: metulburr
  Multithreading dynamically syncronism Rodrigo 0 1,504 Nov-08-2019, 02:33 AM
Last Post: Rodrigo

Forum Jump:

User Panel Messages

Announcements
Announcement #1 8/1/2020
Announcement #2 8/2/2020
Announcement #3 8/6/2020