Python Forum
How to run different processes in a pool of 5 - Printable Version

+- Python Forum (https://python-forum.io)
+-- Forum: Python Coding (https://python-forum.io/forum-7.html)
+--- Forum: General Coding Help (https://python-forum.io/forum-8.html)
+--- Thread: How to run different processes in a pool of 5 (/thread-10633.html)



How to run different processes in a pool of 5 - lravikumarvsp - May-29-2018

Hi,

I had a python script which executes different processes in parallel.
How can I modify this script so that it can run the parallel processes in a pool of 5 at a time?
Please help
from multiprocessing import Pool
from multiprocessing import Process
import glob
import os
import csv
from threading import Thread
from multiprocessing import Pool as ThreadPool
import time
import multiprocessing
patterns = ['ACM','ACX','AW','BC','XU0', 'DRM', 'DHD', 'CR', 'GSK', 'DMS', 'BLS']
	
def process(pattern=[]):
	
	base_path = '/ai2/data/dev/admin/inf/*{}*'
	print("ID of process: {}".format(os.getpid()))
	print "Name of the process is :" ,pattern
	
	

	search_path =  base_path.format(pattern)
	for f in glob.glob(search_path):
		print("-----------------------")
		print ("The directory path is:")
		print f
		print("List of files in the directory are:")
		os.chdir('/ai2/data/dev/admin/inf/')
		os.chdir(f)
		cwd = os.getcwd()
		for subdir, dirs, files in os.walk(cwd, topdown=True):
			for file23 in glob.glob('*.err'):
				print file23	
if __name__ == "__main__":  # confirms that the code is under main function
	patterns = ['ACM','ACX','AW','BC','XU0', 'DRM', 'DHD', 'CR', 'GSK', 'DMS', 'BLS']
	procs = []
	#proc = Process(target=process)  # instantiating without any argument
	#procs.append(proc)
	#proc.start()
	
    # instantiating process with arguments
	for pattern in patterns:
		#print(name)
		proc = Process(target=process,args=(pattern,))
		procs.append(proc)
		proc.start()
	
Output:
ID of process: 5789 Name of the process is : ACM ----------------------- The directory path is: /ai2/data/dev/admin/inf/inf_ACM_pvt List of files in the directory are: dsplit.err ID of process: 5793 Name of the process is : XU0 ----------------------- The directory path is: /ai2/data/dev/admin/inf/inf_XU0_pvt List of files in the directory are: t_itm.err ID of process: 5791 Name of the process is : AW ----------------------- The directory path is: /ai2/data/dev/admin/inf/inf_AW_pvt List of files in the directory are: aware.err ID of process: 5796 Name of the process is : CR ----------------------- The directory path is: /ai2/data/dev/admin/inf/inf_CR_pvt List of files in the directory are: issue.err ID of process: 5792 Name of the process is : BC ----------------------- The directory path is: /ai2/data/dev/admin/inf/inf_BC_pvt List of files in the directory are: run_ingest_BC_daily_1249.err ID of process: 5797 Name of the process is : GSK ----------------------- The directory path is: /ai2/data/dev/admin/inf/inf_GSK_pvt List of files in the directory are: gs_attribute.err ID of process: 5790 Name of the process is : ACX ----------------------- The directory path is: /ai2/data/dev/admin/inf/inf_ACX_pvt List of files in the directory are: accountshighfocus.err acm_access_log.err ID of process: 5795 Name of the process is : DHD ----------------------- The directory path is: /ai2/data/dev/admin/inf/inf_DHD_pvt List of files in the directory are: retpaid.err ID of process: 5794 Name of the process is : DRM ----------------------- The directory path is: /ai2/data/dev/admin/inf/inf_DRM_pvt List of files in the directory are: call_report_y9clookup.err ID of process: 5798 Name of the process is : DMS ----------------------- The directory path is: /ai2/data/dev/admin/inf/inf_DMS_pvt List of files in the directory are: run_ingest_DMS_daily_1235.err ID of process: 5799 Name of the process is : BLS ----------------------- The directory path is: /ai2/data/dev/admin/inf/inf_BLS_pvt List of files in the directory are: run_ingest_BLS_daily_1241.err