Python Forum
python multiprocessing help -- to extract 10 sql table into csv
Thread Rating:
  • 0 Vote(s) - 0 Average
  • 1
  • 2
  • 3
  • 4
  • 5
python multiprocessing help -- to extract 10 sql table into csv
#4
is this correct way of doing multi processing.


import pyodbc
import csv
import os
import time
import multiprocessing


def Extract_to_csv(tbl,cursor):
    qry = f"Select * from {tbl}"
    cursor.execute(qry)
    data = cursor.fetchall()
    print(len(data))
    folderPath = f"D:\\test_data\\output{tbl}"
    count = 0
    # for reply in cursor.execute(qry):**
    for x in data:
        count = count + 1
        print(count)
        fname = "Row"+str(count)+".csv"
        fullpath = os.path.join(folderPath,fname)
        print(fullpath)
        with open(fullpath,"w",newline="") as outfile:
            writer = csv.writer(outfile,delimiter="|",quoting=csv.QUOTE_NONNUMERIC)
            writer.writerow(col[0] for col in cursor.description)
            writer.writerow(x)
            print(f"I am row {count}",x)

def main():
    connection = pyodbc.connect(
        'DRIVER={ODBC Driver 17 for SQL Server};SERVER=DESKTOP-GQK64O6;DATABASE=Customer;Trusted_Connection=yes;')
    cursor = connection.cursor()

    p1 = multiprocessing.Process(target=Extract_to_csv, args=(table1,cursor,))
    p2 = multiprocessing.Process(target=Extract_to_csv, args=(table2,cursor,))
    p3 = multiprocessing.Process(target=Extract_to_csv, args=(table3,cursor,))
    p4 = multiprocessing.Process(target=Extract_to_csv, args=(table4,cursor,))
    p5 = multiprocessing.Process(target=Extract_to_csv, args=(table5,cursor,))
    p6 = multiprocessing.Process(target=Extract_to_csv, args=(table6,cursor,))
    p7 = multiprocessing.Process(target=Extract_to_csv, args=(table7,cursor,))
    p8 = multiprocessing.Process(target=Extract_to_csv, args=(table8,cursor,))
    p9 = multiprocessing.Process(target=Extract_to_csv, args=(table9,cursor,))
    p10 = multiprocessing.Process(target=Extract_to_csv, args=(table10,cursor,))

    p1.start()
    p2.start()
    p3.start()
    p4.start()
    p5.start()
    p6.start()
    p7.start()
    p8.start()
    p9.start()
    p10.start()

    p1.join()
    p2.join()
    p3.join()
    p4.join()
    p5.join()
    p6.join()
    p7.join()
    p8.join()
    p9.join()
    p10.join()
    
    print("End of main")


if __name__ == "__main__":
    start_time = time.process_time()
    main()
    end_time = time.process_time()
    print("Execution time is : ", end_time - start_time)
Reply


Messages In This Thread
RE: python multiprocessing help -- to extract 10 sql table into csv - by mg24 - Nov-20-2022, 11:50 PM

Possibly Related Threads…
Thread Author Replies Views Last Post
  Is this a multiprocessing bug in Python - or am I doing something wrong? haimat 1 1,264 Oct-18-2023, 06:07 AM
Last Post: absalom1
  extract table from multiple pages sshree43 8 5,490 Dec-12-2022, 10:34 AM
Last Post: arvin
  python extract mg24 1 995 Nov-02-2022, 06:30 PM
Last Post: Larz60+
  python multiprocessing to download sql table mg24 5 1,562 Oct-31-2022, 03:53 PM
Last Post: Larz60+
  PyRun_SimpleFile calling multiprocessing Python Class cause endless init loop Xeno 2 1,093 Sep-19-2022, 02:32 AM
Last Post: Xeno
  Python multiprocessing Pool apply async wait for process to complete sunny9495 6 6,650 Apr-02-2022, 06:31 AM
Last Post: sunny9495
  Need help on extract dynamic table data Dr_Strange 0 2,528 Apr-30-2021, 07:03 AM
Last Post: Dr_Strange
  python multiprocessing import Pool, cpu_count: causes forever loop | help to remove Hassibayub 0 1,901 Jun-18-2020, 05:27 PM
Last Post: Hassibayub
  How to extract digits in table of image using python SuSeegio 3 3,124 Dec-05-2018, 10:47 AM
Last Post: Larz60+
  Issue in my multiprocessing Python code? PrateekG 7 4,308 Jul-19-2018, 06:47 PM
Last Post: gontajones

Forum Jump:

User Panel Messages

Announcements
Announcement #1 8/1/2020
Announcement #2 8/2/2020
Announcement #3 8/6/2020