Process (pool,map) strange behaviour

[maverick76]

Still editing sorry

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101

import configparser,os,sys,time,mysql.connector,logging,io,locale,threading,queue from gen_rapport import * from multiprocessing import Pool,Lock, Process, Queue, current_process from threading import Thread import plotly.graph_objects as go   #Recuperation du fichier conf.ini Config = configparser.ConfigParser() Config.read(sys.argv[1]) num_process=sys.argv[2] # print('proc='+num_process) robotfolder=Config.get('dossier','robotfolder')   #Definition des logs logging.info('') et redirection de laffichage et des erreurs dans le fichier de log log_path=Config.get('dossier','log_path') log_file=Config.get('dossier','log_file') logging.basicConfig(filename=log_path+log_file, filemode='w',level=logging.INFO,format='%(levelname)s:%(message)s') f = open('data.csv','w') f.write('nomrapport,time\n') f.flush() # sys.stdout = f # sys.stderr = f     #Configuration de lutilisateur NNI=Config.get('credentials','Login_Gardian') Gardian=Config.get('credentials','MdP_Gardian')     #Configuration de lacces a la base de donnees mydb = mysql.connector.connect(     host=Config.get('credentials','SQL_IPaddress'),     port=Config.get('credentials','SQL_Port'),     user=Config.get('credentials','SQL_User'),     password=Config.get('credentials','SQL_Password'))   base=Config.get('credentials','SQL_Base') table=Config.get('credentials','SQL_Table') max_threads=Config.get('thread','maxthreads')   #Recuperation du nombre darguments pour savoir quelles fonctions on lance NmbArg = len(sys.argv) # print(NmbArg) #Affichage et formatage de la date et heure locale.setlocale(locale.LC_TIME,'') today = time.strftime("%d").lstrip("0").replace("0", "")   print('Commencement :'+str(time.asctime())) tomonth = time.strftime("%B") tomonth1 = tomonth[:1] tomonth2 = tomonth[-1:] toyear = time.strftime("%Y") today = tomonth1+tomonth2+" "+toyear   def work_log(work_data):           t = robot(NNI,Gardian,work_data[4],work_data[2],work_data[0],work_data[1],work_data[5],work_data[6],today)     print(" Process %s Finished." % work_data[0])     f.write(str(t[0])+','+str(t[1])+'\n')     f.flush()       def pool_handler():     p = Pool(int(num_process))     p.map(work_log, work)     f.close()   if NmbArg == 4:       if __name__ == '__main__':             start = time.time()         #Requete SQL pour recuperer les rapports a jouer         mycursor = mydb.cursor()         mycursor.execute("SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM "+base+"."+table+" where Type LIKE '%"+sys.argv[3]+"%' AND Executer='OUI'")         #mycursor.execute("SELECT NomRapport,xlsfile,ImportFolder2,Type,ID, MinSize, Invites FROM "+base+"."+table+" where Type LIKE '%BI%' AND Executer='OUI'")         print(("requete SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM "+base+"."+table+" where Type='"+sys.argv[3]+"' AND Executer='OUI'"))         myresult = mycursor.fetchall()           work = ()           for rapports in myresult:                                     NomRapport=rapports[0]             xlsfile=rapports[1]             ImportFolder=rapports[2]             ID=rapports[4]             MinSize=rapports[5]             Invites=rapports[6]                                         work = work + (rapports,)             pool_handler()         print("Fin:"+str(time.asctime()))   else:     print('Usage : createbi.exe conf.ini PCA; ou bien createbi.exe conf.ini ID,ImportFolder,xlsfile,(Invites)')

Output:
PS C:\Users\d74074\Documents\Devs\Python\BIV6> python .\createbi.py .\conf.ini 6 BI
Commencement :Mon Feb  3 13:39:40 2020
requete SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM tabatta.l_app_createbi where Type='BI' AND Executer='OUI'
Commencement :Mon Feb  3 13:39:45 2020
Fichier present:BI6_CauseFigeageTardif
C:\Users\d74074\Documents\Devs\Python\BIV6\Imports\figeagetardif\PAL-tabatta_tot_cause_figeage_tardif.csv
Commencement :Mon Feb  3 13:39:45 2020
Fichier present:BI6_GenCloture
C:\Users\d74074\Documents\Devs\Python\BIV6\Imports\BITOTTypeCloture\PAL-Tabatta_BI_-_TOT_-_TypeCloture.csv
Commencement :Mon Feb  3 13:39:45 2020
doesnt exist:BI6_GenCompl2340
Commencement :Mon Feb  3 13:39:45 2020
doesnt exist:BI6_Gencomplt340
Commencement :Mon Feb  3 13:39:45 2020
doesnt exist:BI6_GenDatesTOT
Commencement :Mon Feb  3 13:39:45 2020

[maverick76]

Hello,

I have learnt Python from myself because at work, i had to automate some action on a web app(i am using selenium).
For that, i have a list of url in a database.

As i am a beginner, there is something i don't understand.

Last week, i have been able to start some process (the number is a command line argument).
These processes are a function, called 'robot()'.

It works!

But i am wondering, why the "print('Commencement :'+str(time.asctime()))" line 48 appear more than 1 time in the console?

because it is written only one time(line 48) in the script, and it is not in the 'work_log' function.

If someone can explain, i would be very happy !

next step for me, process which write on the same file....

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101

import configparser,os,sys,time,mysql.connector,logging,io,locale,threading,queue from gen_rapport import * from multiprocessing import Pool,Lock, Process, Queue, current_process from threading import Thread import plotly.graph_objects as go   #Recuperation du fichier conf.ini Config = configparser.ConfigParser() Config.read(sys.argv[1]) num_process=sys.argv[2] # print('proc='+num_process) robotfolder=Config.get('dossier','robotfolder')   #Definition des logs logging.info('') et redirection de laffichage et des erreurs dans le fichier de log log_path=Config.get('dossier','log_path') log_file=Config.get('dossier','log_file') logging.basicConfig(filename=log_path+log_file, filemode='w',level=logging.INFO,format='%(levelname)s:%(message)s') f = open('data.csv','w') f.write('nomrapport,time\n') f.flush() # sys.stdout = f # sys.stderr = f     #Configuration de lutilisateur Login=Config.get('credentials','Login') MdP=Config.get('credentials','MdP')     #Configuration de lacces a la base de donnees mydb = mysql.connector.connect(     host=Config.get('credentials','SQL_IPaddress'),     port=Config.get('credentials','SQL_Port'),     user=Config.get('credentials','SQL_User'),     password=Config.get('credentials','SQL_Password'))   base=Config.get('credentials','SQL_Base') table=Config.get('credentials','SQL_Table') max_threads=Config.get('thread','maxthreads')   #Recuperation du nombre darguments pour savoir quelles fonctions on lance NmbArg = len(sys.argv) # print(NmbArg) #Affichage et formatage de la date et heure locale.setlocale(locale.LC_TIME,'') today = time.strftime("%d").lstrip("0").replace("0", "")   print('Commencement :'+str(time.asctime())) tomonth = time.strftime("%B") tomonth1 = tomonth[:1] tomonth2 = tomonth[-1:] toyear = time.strftime("%Y") today = tomonth1+tomonth2+" "+toyear   def work_log(work_data):           t = robot(Login,MdP,work_data[4],work_data[2],work_data[0],work_data[1],work_data[5],work_data[6],today)     print(" Process %s Finished." % work_data[0])     f.write(str(t[0])+','+str(t[1])+'\n')     f.flush()       def pool_handler():     p = Pool(int(num_process))     p.map(work_log, work)     f.close()   if NmbArg == 4:       if __name__ == '__main__':             start = time.time()         #Requete SQL pour recuperer les rapports a jouer         mycursor = mydb.cursor()         mycursor.execute("SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM "+base+"."+table+" where Type LIKE '%"+sys.argv[3]+"%' AND Executer='OUI'")         #mycursor.execute("SELECT NomRapport,xlsfile,ImportFolder2,Type,ID, MinSize, Invites FROM "+base+"."+table+" where Type LIKE '%BI%' AND Executer='OUI'")         print(("requete SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM "+base+"."+table+" where Type='"+sys.argv[3]+"' AND Executer='OUI'"))         myresult = mycursor.fetchall()           work = ()           for rapports in myresult:                                     NomRapport=rapports[0]             xlsfile=rapports[1]             ImportFolder=rapports[2]             ID=rapports[4]             MinSize=rapports[5]             Invites=rapports[6]                                         work = work + (rapports,)             pool_handler()         print("Fin:"+str(time.asctime()))   else:     print('Usage : createbi.exe conf.ini PCA; ou bien createbi.exe conf.ini ID,ImportFolder,xlsfile,(Invites)')

Output:
PS C:\Users\sssss\Documents\Devs\Python\ssss> python .\sssss.py .\conf.ini 6 BI
Commencement :Mon Feb  3 13:39:40 2020
requete SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM xxxxxxxxx where Type='BI' AND Executer='OUI'
Commencement :Mon Feb  3 13:39:45 2020
Fichier present:BI6_CauseFigeageTardif
C:\Users\sssss\Documents\Devs\Python\BIV6\Imports\figeagetardif\xxxxxx.csv
Commencement :Mon Feb  3 13:39:45 2020
Fichier present:BI6_GenCloture
C:\Users\sssss\Documents\Devs\Python\BIV6\Imports\BITOTTypeCloture\xcxcxcxc.csv
Commencement :Mon Feb  3 13:39:45 2020
doesnt exist:BI6_GenCompl2340
Commencement :Mon Feb  3 13:39:45 2020
doesnt exist:BI6_Gencomplt340
Commencement :Mon Feb  3 13:39:45 2020
doesnt exist:BI6_GenDatesTOT
Commencement :Mon Feb  3 13:39:45 2020