Tkinter Web Scraping w/Multithreading Question....

[AaronCatolico1]

I have a simple program in Tkinter that I'm using to scrape the age of various people from wikipedia just for webscraping practice.

I am able to scrape the age of each person one-by-one on one thread, but I'm trying to have one thread for each person to handle scraping their ages all at the same time so that the program will be much faster.

So, in other words, the program currently scrapes only 1 person at a time and can only return 1 row at a time in the Treeview, but I'd like to have it to where a thread works for each person at the same time (concurrently) so that the Treeview will return each person's age in one shot as well.

Here's the code that I've come up with so far:

from tkinter import Tk, Button, Listbox
from tkinter import ttk
import threading
import requests
import re #imports RegEx

    class MainWindow(Tk):

    def __init__(self):
        super().__init__()

        self.option_add("*Font", "poppins, 11 bold")

        self.lb1 = Listbox(self, width=22, cursor='hand2')
        self.lb1.pack(side='left', fill='y', padx=20, pady=20)

        #create list of names
        names = ['Adam Levine', 'Arnold Schwarzenegger', 'Artur Beterbiev', 'Chris Hemsworth',
                 'Dan Henderson', 'Dustin Poirier', 'Fedor Emelianenko', 'Gennady Golovkin', 'Igor Vovchanchyn', 'Ken Shamrock',
                 'Mirko Cro Cop', 'Oleksandr Usyk', 'Ronnie Coleman', 'Vasiliy Lomachenko']

        #populate listbox with names
        for name in names:
            self.lb1.insert('end', name)


        self.tv1 = ttk.Treeview(self, show='tree headings', cursor='hand2')

        columns = ('NAME', 'AGE')

        self.tv1.config(columns = columns)

        style = ttk.Style()
        style.configure("Treeview", highlightthickness=2, bd=0, rowheight=26,font=('Poppins', 11))  # Modify the font of the body
        style.configure("Treeview.Heading", font=('Poppins', 12, 'bold'))  # Modify the font of the headings

        #configure headers
        self.tv1.column('#0', width=0, stretch=0)
        self.tv1.column('NAME', width=190)
        self.tv1.column('AGE', width=80, stretch=0)

        #define headings
        self.tv1.heading('NAME', text='NAME', anchor='w')
        self.tv1.heading('AGE', text='AGE', anchor='w')

        self.tv1.pack(fill='both', expand=1, padx=(0, 20), pady=20)

        #create start button
        self.b1 = Button(self, text='START', bg='green', fg='white', cursor='hand2', command=self.start)
        self.b1.pack(pady=(0, 20))


    #scrape data from WikiPedia.org
    def start(self):
        for item in self.tv1.get_children():
            self.tv1.delete(item)

        t1 = threading.Thread(target=self.scrape_wiki, daemon=True)
        t1.start()


    def scrape_wiki(self):
        for i in range(self.lb1.size()):
            #select the name
            self.name = self.lb1.get(i).replace(' ', '_')

            # create a simple dictionary to hold the user agent inside of the headers
            headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0'}

            # the required first parameter of the 'get' method is the 'url':
            r = requests.get('https://en.wikipedia.org/wiki/' + self.name, headers=headers)

            # regex setup

            regex = re.search('(age \d+)', r.text)
            age = regex.group(0).replace('age ', '').replace(')', '')

            # Populate Treeview with row data
            self.name = self.name.replace('_', ' ')
            self.tv1.insert(parent='', index='end', iid=i, values=(self.name, age))



if __name__ == '__main__':
    app = MainWindow()
    #app.iconbitmap('imgs/logo-icon.ico')
    app.title('Main Window')
    app.configure(bg='#333333')

    #center the Main Window:
    w = 600  # Width
    h = 520  # Height

    screen_width = app.winfo_screenwidth()  # Width of the screen
    screen_height = app.winfo_screenheight()  # Height of the screen

    # Calculate Starting X and Y coordinates for Window
    x = (screen_width / 2) - (w / 2)
    y = (screen_height / 2) - (h / 2)

    app.mainloop()

So, how exactly do I create multiple threads to handle each person in the list and return them concurrently/simultaneously instead of row-by-row, one at a time?
I'd appreciate any support.

[deanhystad]

Take a look at ProcessPoolExecutor.

https://docs.python.org/3/library/concur...olExecutor

[AaronCatolico1]

Hi Dean, and thank you for your reply.

So, I've already tried the ThreadPoolExector from concurrent.futures class, but it was not compatible with updating Tkinter's GUI. Is ProcessPoolExecutor going to be any different?

I'm currently away from home, but I'll give it a shot once I return if you have tested this class & have concluded that it will indeed work with Tkinter's GUI..

Anyhow, thanks again in the meantime. 👍👍

Take a look at ProcessPoolExecutor.

https://docs.python.org/3/library/concur...olExecutor

[deanhystad]

After my post I gave ProcessPoolExecuter a better look. I thought you could launch all the processes and use futures to get the results when available. Unfortunately, it looks like the executor blocks until all processes are completed, or at least started.

Have you tried Asnyc IO? I think async io works great, but it is invasive. You only want to call one function asynchronously and before you know it you have async def sprinkled everywhere.

Here's an idea that uses threads, a list of queue objects, and the tkinter after() funciton.

import tkinter as tk
from threading import Thread
from queue import Queue
from time import sleep
from random import randint
import functools


class TkinterThreadManager:
    """A thread pool manager that maintains function call order
    and doesn't block tkinter
    """
    def __init__(self, window, period=100):
        self.queue = []
        self.window = window
        self.period = period

    def submit(self, func, args=None, callback=None):
        """Add thread to the pool
        func: Function executed in thread
        args: Optional arguments passed to function.
        callback:  Optional function that is called using return value
        """
        if callback is None:
            Thread(target=func, args=args).start()
        else:
            q = Queue(maxsize=1)
            q.callback = callback
            Thread(target=func, args=(q,)+args).start()
            self.queue.append(q)
            self.join()

    def join(self):
        """Periodically check for thread completion.  Execute callback."""
        while self.queue and self.queue[0].full():
            q = self.queue.pop(0)
            q.callback(q.get())
        if self.queue:
            self.window.after(self.period, self.join)


def thread_func(func):
    """TkinterThreadManager function wrapper.  Puts function return value
    in Queue to notify manager of thread completion.
    """
    @functools.wraps(func)
    def wrapper(q, *args, **kwargs):
        value = func(*args, **kwargs)
        print(value)  # For demonstration purposes
        q.put(value)
    return wrapper


@thread_func
def get_name(name, delay):
    """Dummy function to execute via thread"""
    sleep(randint(0, delay))
    return name


class MainWindow(tk.Tk):
    """Window to demonstrate TkinterThreadManager."""
    def __init__(self):
        super().__init__()
        self.threads = TkinterThreadManager(self)
        self.lb1 = tk.Listbox(self, width=22, height=30)
        self.lb1.pack(side='left', fill='y', padx=10, pady=10)
        button = tk.Button(self, text='Letters', command=self.letters)
        button.pack(padx=10, pady=10)
        button = tk.Button(self, text='Numbers', command=self.numbers)
        button.pack(padx=10, pady=10)

    def letters(self):
        """Add some letters to the listbox"""
        for delay, name in enumerate(('A', 'B', 'C')):
            self.threads.submit(
                func=get_name,
                args=(name, delay),
                callback=lambda x: self.lb1.insert(tk.END, x))

    def numbers(self):
        """Add some numbers to the listbox"""
        for delay, name in enumerate((1, 2, 3)):
            self.threads.submit(
                func=get_name,
                args=(name, delay),
                callback=lambda x: self.lb1.insert(tk.END, x))


if __name__ == '__main__':
    MainWindow().mainloop()