Python Forum

#from multiprocessing.pool import ThreadPool
#from multiprocessing.dummy import Pool as ThreadPool
from multiprocessing import Pool
from multiprocessing.dummy import Pool as ThreadPool 
import tweepy
import random
import json
import simplejson
import re
from tweepy import OAuthHandler
from tweepy.auth import OAuthHandler
import gc
import sys

import time
from datetime import datetime
import threading
import psycopg2

# Consumer keys and access tokens, used for OAuth

consumer_key = "xxxxxxxx"
consumer_secret = "xxxxxxxx"
access_token = "xxxxxxxxxx"
access_token_secret = "xxxxxxx"


# OAuth process, using the keys and tokens
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)


# Creation of the actual interface, using authentication
try:
    api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True, retry_count=10, retry_delay=60)
    #auth = api.VerifyCredentials()
except Exception as e:
    print("Exception is occured",e)
count =0 

def twitterjson(string):

            results = tweepy.Cursor(api.user_timeline, screen_name=sm_handle, count=100).items()
            for tweet in results:
                with open(sm_handle+'.json', 'w') as outfile:
                    outfile.write(json.dumps(tweet._json))
                    print (tweet)
     return 1;
            
if __name__ == "__main__":
  while True:
    try:
        # Establishing a PSQL connection
        conn = psycopg2.connect("dbname='sample' user='postgres' host='localhost' port='5432'")
        print("opened database successfully")

        # create a psycopg2 cursor that can execute queries
        cursor = conn.cursor()

        # twitter_users_query = "select twitter_user_id, twitter_screen_name from twitter_users;"
        twitter_users_query = "select usernames from test;"
        cursor.execute(twitter_users_query)
        #pool = ThreadPool(10) 

        rows = cursor.fetchall()
        names=[]
        for row in rows:
            sm_handle = row[0]
            names.append(sm_handle)
            print("sm_handles: "+sm_handle)
            threads = []

           # for i in rows:
        for name in names:
            print ("name:"+name)
     
    except Exception as e:
        print("unable to connect to the database : ", e)
    try:   
        pool = ThreadPool(100) 
        gc.collect()
        #pool.terminate()
        results=pool.map(twitterjson, names)
        pool.terminate(100)
        print ("Completed"+results)
        pool.terminate()
        pool.close() 
        pool.join()
        print("This is sleeping for 1 minute..............")
        #time.sleep(60)
        print('Done')
        
        #pool.terminate()
        #pool.close() 
        #pool.join()
        #pool.terminate()

    except Exception as e:
         print("Exception is occured",e)
         pool.terminate();