Python Forum
Process (pool,map) strange behaviour
Thread Rating:
  • 0 Vote(s) - 0 Average
  • 1
  • 2
  • 3
  • 4
  • 5
Process (pool,map) strange behaviour
#1
Still editing sorry


import configparser,os,sys,time,mysql.connector,logging,io,locale,threading,queue
from gen_rapport import *
from multiprocessing import Pool,Lock, Process, Queue, current_process
from threading import Thread
import plotly.graph_objects as go

#Recuperation du fichier conf.ini
Config = configparser.ConfigParser()
Config.read(sys.argv[1])
num_process=sys.argv[2]
# print('proc='+num_process)
robotfolder=Config.get('dossier','robotfolder')

#Definition des logs logging.info('') et redirection de laffichage et des erreurs dans le fichier de log
log_path=Config.get('dossier','log_path')
log_file=Config.get('dossier','log_file')
logging.basicConfig(filename=log_path+log_file, filemode='w',level=logging.INFO,format='%(levelname)s:%(message)s')
f = open('data.csv','w')
f.write('nomrapport,time\n')
f.flush()
# sys.stdout = f
# sys.stderr = f


#Configuration de lutilisateur
NNI=Config.get('credentials','Login_Gardian')
Gardian=Config.get('credentials','MdP_Gardian')


#Configuration de lacces a la base de donnees
mydb = mysql.connector.connect(
    host=Config.get('credentials','SQL_IPaddress'),
    port=Config.get('credentials','SQL_Port'),
    user=Config.get('credentials','SQL_User'),
    password=Config.get('credentials','SQL_Password'))

base=Config.get('credentials','SQL_Base')
table=Config.get('credentials','SQL_Table')
max_threads=Config.get('thread','maxthreads')

#Recuperation du nombre darguments pour savoir quelles fonctions on lance
NmbArg = len(sys.argv)
# print(NmbArg)
#Affichage et formatage de la date et heure
locale.setlocale(locale.LC_TIME,'')
today = time.strftime("%d").lstrip("0").replace("0", "")

print('Commencement :'+str(time.asctime()))
tomonth = time.strftime("%B")
tomonth1 = tomonth[:1]
tomonth2 = tomonth[-1:]
toyear = time.strftime("%Y")
today = tomonth1+tomonth2+" "+toyear

def work_log(work_data):
	
	t = robot(NNI,Gardian,work_data[4],work_data[2],work_data[0],work_data[1],work_data[5],work_data[6],today)
	print(" Process %s Finished." % work_data[0])
	f.write(str(t[0])+','+str(t[1])+'\n')
	f.flush()	


def pool_handler():
	p = Pool(int(num_process))
	p.map(work_log, work)
	f.close()

if NmbArg == 4:

	if __name__ == '__main__':


		start = time.time()
		#Requete SQL pour recuperer les rapports a jouer
		mycursor = mydb.cursor()
		mycursor.execute("SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM "+base+"."+table+" where Type LIKE '%"+sys.argv[3]+"%' AND Executer='OUI'")
		#mycursor.execute("SELECT NomRapport,xlsfile,ImportFolder2,Type,ID, MinSize, Invites FROM "+base+"."+table+" where Type LIKE '%BI%' AND Executer='OUI'")
		print(("requete SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM "+base+"."+table+" where Type='"+sys.argv[3]+"' AND Executer='OUI'"))
		myresult = mycursor.fetchall()

		work = ()

		for rapports in myresult:
		
			
			NomRapport=rapports[0]
			xlsfile=rapports[1]
			ImportFolder=rapports[2]
			ID=rapports[4]
			MinSize=rapports[5]
			Invites=rapports[6]
			
			
			work = work + (rapports,)


		pool_handler()
		print("Fin:"+str(time.asctime()))

else:
	print('Usage : createbi.exe conf.ini PCA; ou bien createbi.exe conf.ini ID,ImportFolder,xlsfile,(Invites)')
Output:
PS C:\Users\d74074\Documents\Devs\Python\BIV6> python .\createbi.py .\conf.ini 6 BI Commencement :Mon Feb 3 13:39:40 2020 requete SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM tabatta.l_app_createbi where Type='BI' AND Executer='OUI' Commencement :Mon Feb 3 13:39:45 2020 Fichier present:BI6_CauseFigeageTardif C:\Users\d74074\Documents\Devs\Python\BIV6\Imports\figeagetardif\PAL-tabatta_tot_cause_figeage_tardif.csv Commencement :Mon Feb 3 13:39:45 2020 Fichier present:BI6_GenCloture C:\Users\d74074\Documents\Devs\Python\BIV6\Imports\BITOTTypeCloture\PAL-Tabatta_BI_-_TOT_-_TypeCloture.csv Commencement :Mon Feb 3 13:39:45 2020 doesnt exist:BI6_GenCompl2340 Commencement :Mon Feb 3 13:39:45 2020 doesnt exist:BI6_Gencomplt340 Commencement :Mon Feb 3 13:39:45 2020 doesnt exist:BI6_GenDatesTOT Commencement :Mon Feb 3 13:39:45 2020
Reply
#2
Hello,

I have learnt Python from myself because at work, i had to automate some action on a web app(i am using selenium).
For that, i have a list of url in a database.

As i am a beginner, there is something i don't understand.

Last week, i have been able to start some process (the number is a command line argument).
These processes are a function, called 'robot()'.

It works!

But i am wondering, why the "print('Commencement :'+str(time.asctime()))" line 48 appear more than 1 time in the console?

because it is written only one time(line 48) in the script, and it is not in the 'work_log' function.

If someone can explain, i would be very happy !

next step for me, process which write on the same file....

import configparser,os,sys,time,mysql.connector,logging,io,locale,threading,queue
from gen_rapport import *
from multiprocessing import Pool,Lock, Process, Queue, current_process
from threading import Thread
import plotly.graph_objects as go

#Recuperation du fichier conf.ini
Config = configparser.ConfigParser()
Config.read(sys.argv[1])
num_process=sys.argv[2]
# print('proc='+num_process)
robotfolder=Config.get('dossier','robotfolder')

#Definition des logs logging.info('') et redirection de laffichage et des erreurs dans le fichier de log
log_path=Config.get('dossier','log_path')
log_file=Config.get('dossier','log_file')
logging.basicConfig(filename=log_path+log_file, filemode='w',level=logging.INFO,format='%(levelname)s:%(message)s')
f = open('data.csv','w')
f.write('nomrapport,time\n')
f.flush()
# sys.stdout = f
# sys.stderr = f


#Configuration de lutilisateur
Login=Config.get('credentials','Login')
MdP=Config.get('credentials','MdP')


#Configuration de lacces a la base de donnees
mydb = mysql.connector.connect(
    host=Config.get('credentials','SQL_IPaddress'),
    port=Config.get('credentials','SQL_Port'),
    user=Config.get('credentials','SQL_User'),
    password=Config.get('credentials','SQL_Password'))

base=Config.get('credentials','SQL_Base')
table=Config.get('credentials','SQL_Table')
max_threads=Config.get('thread','maxthreads')

#Recuperation du nombre darguments pour savoir quelles fonctions on lance
NmbArg = len(sys.argv)
# print(NmbArg)
#Affichage et formatage de la date et heure
locale.setlocale(locale.LC_TIME,'')
today = time.strftime("%d").lstrip("0").replace("0", "")

print('Commencement :'+str(time.asctime()))
tomonth = time.strftime("%B")
tomonth1 = tomonth[:1]
tomonth2 = tomonth[-1:]
toyear = time.strftime("%Y")
today = tomonth1+tomonth2+" "+toyear

def work_log(work_data):
	
	t = robot(Login,MdP,work_data[4],work_data[2],work_data[0],work_data[1],work_data[5],work_data[6],today)
	print(" Process %s Finished." % work_data[0])
	f.write(str(t[0])+','+str(t[1])+'\n')
	f.flush()	


def pool_handler():
	p = Pool(int(num_process))
	p.map(work_log, work)
	f.close()

if NmbArg == 4:

	if __name__ == '__main__':


		start = time.time()
		#Requete SQL pour recuperer les rapports a jouer
		mycursor = mydb.cursor()
		mycursor.execute("SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM "+base+"."+table+" where Type LIKE '%"+sys.argv[3]+"%' AND Executer='OUI'")
		#mycursor.execute("SELECT NomRapport,xlsfile,ImportFolder2,Type,ID, MinSize, Invites FROM "+base+"."+table+" where Type LIKE '%BI%' AND Executer='OUI'")
		print(("requete SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM "+base+"."+table+" where Type='"+sys.argv[3]+"' AND Executer='OUI'"))
		myresult = mycursor.fetchall()

		work = ()

		for rapports in myresult:
		
			
			NomRapport=rapports[0]
			xlsfile=rapports[1]
			ImportFolder=rapports[2]
			ID=rapports[4]
			MinSize=rapports[5]
			Invites=rapports[6]
			
			
			work = work + (rapports,)


		pool_handler()
		print("Fin:"+str(time.asctime()))

else:
	print('Usage : createbi.exe conf.ini PCA; ou bien createbi.exe conf.ini ID,ImportFolder,xlsfile,(Invites)')
Output:
PS C:\Users\sssss\Documents\Devs\Python\ssss> python .\sssss.py .\conf.ini 6 BI Commencement :Mon Feb 3 13:39:40 2020 requete SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM xxxxxxxxx where Type='BI' AND Executer='OUI' Commencement :Mon Feb 3 13:39:45 2020 Fichier present:BI6_CauseFigeageTardif C:\Users\sssss\Documents\Devs\Python\BIV6\Imports\figeagetardif\xxxxxx.csv Commencement :Mon Feb 3 13:39:45 2020 Fichier present:BI6_GenCloture C:\Users\sssss\Documents\Devs\Python\BIV6\Imports\BITOTTypeCloture\xcxcxcxc.csv Commencement :Mon Feb 3 13:39:45 2020 doesnt exist:BI6_GenCompl2340 Commencement :Mon Feb 3 13:39:45 2020 doesnt exist:BI6_Gencomplt340 Commencement :Mon Feb 3 13:39:45 2020 doesnt exist:BI6_GenDatesTOT Commencement :Mon Feb 3 13:39:45 2020
Reply


Possibly Related Threads…
Thread Author Replies Views Last Post
  logger behaviour setdetnet 1 853 Apr-15-2023, 05:20 AM
Last Post: Gribouillis
  can someone explain this __del__ behaviour? rjdegraff42 1 692 Apr-12-2023, 03:25 PM
Last Post: deanhystad
  Asyncio weird behaviour vugz 2 1,190 Apr-09-2023, 01:48 AM
Last Post: vugz
  Weird behaviour using if statement in python 3.10.8 mikepy 23 3,427 Jan-18-2023, 04:51 PM
Last Post: mikepy
  Multiprocessing Pool Multiple Instances How to Kill by Pool ID sunny9495 0 735 Nov-16-2022, 05:57 AM
Last Post: sunny9495
  Generator behaviour bla123bla 2 1,072 Jul-26-2022, 07:30 PM
Last Post: bla123bla
  Python multiprocessing Pool apply async wait for process to complete sunny9495 6 6,220 Apr-02-2022, 06:31 AM
Last Post: sunny9495
  Inconsistent behaviour in output - web scraping Steve 6 2,445 Sep-20-2021, 01:54 AM
Last Post: Larz60+
  Pool multiprocessing - know current status in loop? korenron 0 1,604 Jul-28-2021, 08:49 AM
Last Post: korenron
  pool mysql error - not catch by try\except? korenron 1 2,101 Jul-05-2021, 11:26 AM
Last Post: ibreeden

Forum Jump:

User Panel Messages

Announcements
Announcement #1 8/1/2020
Announcement #2 8/2/2020
Announcement #3 8/6/2020