#from multiprocessing.pool import ThreadPool #from multiprocessing.dummy import Pool as ThreadPool from multiprocessing import Pool from multiprocessing.dummy import Pool as ThreadPool import tweepy import random import json import simplejson import re from tweepy import OAuthHandler from tweepy.auth import OAuthHandler import gc import sys import time from datetime import datetime import threading import psycopg2 # Consumer keys and access tokens, used for OAuth consumer_key = "xxxxxxxx" consumer_secret = "xxxxxxxx" access_token = "xxxxxxxxxx" access_token_secret = "xxxxxxx" # OAuth process, using the keys and tokens auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) # Creation of the actual interface, using authentication try: api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True, retry_count=10, retry_delay=60) #auth = api.VerifyCredentials() except Exception as e: print("Exception is occured",e) count =0 def twitterjson(string): results = tweepy.Cursor(api.user_timeline, screen_name=sm_handle, count=100).items() for tweet in results: with open(sm_handle+'.json', 'w') as outfile: outfile.write(json.dumps(tweet._json)) print (tweet) return 1; if __name__ == "__main__": while True: try: # Establishing a PSQL connection conn = psycopg2.connect("dbname='sample' user='postgres' host='localhost' port='5432'") print("opened database successfully") # create a psycopg2 cursor that can execute queries cursor = conn.cursor() # twitter_users_query = "select twitter_user_id, twitter_screen_name from twitter_users;" twitter_users_query = "select usernames from test;" cursor.execute(twitter_users_query) #pool = ThreadPool(10) rows = cursor.fetchall() names=[] for row in rows: sm_handle = row[0] names.append(sm_handle) print("sm_handles: "+sm_handle) threads = [] # for i in rows: for name in names: print ("name:"+name) except Exception as e: print("unable to connect to the database : ", e) try: pool = ThreadPool(100) gc.collect() #pool.terminate() results=pool.map(twitterjson, names) pool.terminate(100) print ("Completed"+results) pool.terminate() pool.close() pool.join() print("This is sleeping for 1 minute..............") #time.sleep(60) print('Done') #pool.terminate() #pool.close() #pool.join() #pool.terminate() except Exception as e: print("Exception is occured",e) pool.terminate();
How to reallocating memory space in multithreading program in python ?
Users browsing this thread: 1 Guest(s)