Dec-20-2018, 12:54 AM
Like nilamo said, multithreading is the best bet for a major performance boost. For something more marginal, I made a few edits to remove superfluous lines and use some more performant functions/methods.
from bs4 import BeautifulSoup import requests import csv import urllib.request import os def get(url, values): res = requests.get(url, data=values) d = res.status_code if d!=200: print("Invalid Link") else: print("Valid link") data = res.text return data def readfile(): final_data = [] with open("./plans.csv", "r") as csvfile: reader = csv.reader(csvfile) next(reader) for row in reader: lists = row[0] ind = row[1]+".jpg" d = requests.get(lists) sublist = [] sublist.append(os.path.abspath(ind)) # Eliminate mj variable try: d = urllib.request.urlretrieve(lists, ind) sublist.append(lists) except Exception as e: sublist.append(e) final_data.append(sublist) return final_data def writefiles(alldata, filename): with open ("./"+ filename, "a+") as file: csvfile = csv.writer(file, delimiter=",") csvfile.writerow("") csvfile.writerows(alldata) # csvwriter.writerows() instead of loop def main(): writefiles(readfile(), "downloads.csv") main()