import mysql.connector
    import mysql.connector.pooling as x
    import time
    from threading import Thread
    from queue import Queue

    dbconfig = {
      "database": "mydb",
      "user":     "root",
      "password": "rootpass",
      "host":"127.0.0.1"
    }

    cnxpool =x.MySQLConnectionPool(pool_name = "mypool",pool_size = 2,**dbconfig)
    q = Queue(maxsize=20)

    threads = []
    limit = 0
    fetch_size = 20
    flag = True

    def populate():
        cnx = cnxpool.get_connection()
        cursor = cnx.cursor()
        global flag,fetch_size,limit
        while flag:
            cursor.execute("SELECT mycol from mytab limit " + str(limit)+","+ str(fetch_size))
            rows = cursor.fetchall()
            for row in rows:
                print("Putting "+ str(row[0]))
                q.put(row[0])
            if cursor.rowcount < 20 :    
                for i in range(4):
                    q.put(None)
                flag = False
            limit = limit + 20
        cursor.close()
        cnx.close()

    def consume():
        while True:
            item = q.get()
            if item is None:
                q.task_done()
                break
            time.sleep(1)
            print(item)
            q.task_done()

    for i in range(4):
        t = Thread(target=consume)
        t.start()
        threads.append(t)

    y = Thread(target=populate)
    y.start()

    q.join()
    y.join()

    for t in threads:
        t.join()

    print("Completed")

        import mysql.connector
        import mysql.connector.pooling as x
        import time
        from threading import Thread
        from queue import Queue

        dbconfig = {
          "database": "mydb",
          "user":     "root",
          "password": "rootpass",
          "host":"127.0.0.1"
        }

        cnxpool =x.MySQLConnectionPool(pool_name = "mypool",pool_size = 2,**dbconfig)
        q = Queue(maxsize=20)

        threads = []
        limit = 0
        fetch_size = 20
        flag = True

        def populate():
            cnx = cnxpool.get_connection()
            cursor = cnx.cursor()
            global flag,fetch_size,limit
            while flag:
                cursor.execute("SELECT mycol from mytab limit " + str(limit)+","+ str(fetch_size))
                rows = cursor.fetchall()
                for row in rows:
                    print("Putting "+ str(row[0]))
                    q.put(row[0])
                if cursor.rowcount < 20 :    
                    for i in range(4):
                        q.put(None)
                    flag = False
                limit = limit + 20
            cursor.close()
            cnx.close()

        def consume():
            while True:
                item = q.get()
                if item is None:
                    q.task_done()
                    break
                time.sleep(1)
                print(item)
                q.task_done()

        for i in range(4):
            t = Thread(target=consume)
            t.start()
            threads.append(t)

        y = Thread(target=populate)
        y.start()

        q.join()
        y.join()

        for t in threads:
            t.join()

        print("Completed")

        del q

        q = Queue(maxsize=20)

        for i in range(4):
            t = Thread(target=consume)
            t.start()
            threads.append(t)

        y = Thread(target=populate)
        y.start()

        q.join()
        y.join()

        for t in threads:
            t.join()




        print("Completed")

• Add q to a global declaration. In this way both consume and populate will look for the q variable in the upper namespace... but using global feels always dirty and in many occasions the result is a painful bug 3 days later.
  
• Do not declare a new variable, empty the queue with something like:
  

  while not q.empty():
      try:
          q.get_nowait()
      except queue.Empty:
          break

• modify populate and consume so they get the queue to work with from the argument list and pass the queue object with the args= of Thread.
  

(May-21-2018, 12:05 PM)killerrex Wrote: It is difficult to test your code (missing the DB) but I think the problem is that the Queue q that both methods populate and consume are seeing is the one they find when declared (the one at line 15) Once declared, it does not matter if you delete and declare again the variable q, it is a "new" variable and populate and consume does not known about it.

The 3 things I think I would try are:

• Add q to a global declaration. In this way both consume and populate will look for the q variable in the upper namespace... but using global feels always dirty and in many occasions the result is a painful bug 3 days later.
  
• Do not declare a new variable, empty the queue with something like:
  

  while not q.empty():
      try:
          q.get_nowait()
      except queue.Empty:
          break

• modify populate and consume so they get the queue to work with from the argument list and pass the queue object with the args= of Thread.
  

I do not know which one works better -if any of them does-- but if I have to choose I prefer the 3rd one, I do not like the global variables...

I think the reason in jupyter works has to do with the fact that jupyter makes some dirty tricks with the global namespace to handle the different cells.