Nov-20-2020, 09:23 PM
Got this python code that works great sending parallel http requests. When i get a HTTP 429 (Too Many Requests) it is appended to a list. The question is how can i rerun these urls?
or is there a better way.
or is there a better way.
# Yield successive n-sized # chunks from l. def divide_chunks(l, n): # looping till length l for i in range(0, len(l), n): yield l[i:i + n] # yield l[i:i + n] # How many elements each # list should have n = 5 lst = [] dlist = [] try: with FuturesSession(max_workers=10) as session: urls = list(divide_chunks(my_list, n)) for items in urls: futures = [session.get(url[2]) for url in items] for future in futures: r = future.result() if r.status_code == 429: lst.append((r.url, divide_chunks(my_list, n ))) elif r.status_code == 200: dict = r.json() dlist.append(dict) print (dlist) except requests.ConnectionError as e: print (e)