Oct-27-2017, 01:34 PM
hi, i got it solved this is the solution,
import requests from bs4 import BeautifulSoup url = "http://upreraportal.cloudapp.net/View_projects.aspx" response = requests.get(url).text soup = BeautifulSoup(response,"html.parser") VIEWSTATE = soup.select("#__VIEWSTATE")[0]['value'] EVENTVALIDATION = soup.select("#__EVENTVALIDATION")[0]['value'] for title in soup.select("#ContentPlaceHolder1_DdlprojectDistrict [value]")[:-1]: search_item = title.text # print(search_item) headers= {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 'Content-Type':'application/x-www-form-urlencoded', 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'} formfields = {'__VIEWSTATE':VIEWSTATE, #Put the value in this variable '__VIEWSTATEGENERATOR':'4F1A7E70', '__EVENTVALIDATION':EVENTVALIDATION, #Put the value in this variable 'ctl00$ContentPlaceHolder1$DdlprojectDistrict':search_item, 'ctl00$ContentPlaceHolder1$txtProject':'', 'ctl00$ContentPlaceHolder1$btnSearch':'Search'} #here in form details check agra , i am able to scrape one city only, # how to loop for all cities res = requests.post(url, data=formfields, headers=headers).text soup = BeautifulSoup(res, "html.parser") get_list = soup.find_all('option') #gets list of all <option> tag for element in get_list : cities = element["value"] #final.append(cities) #print(final) get_details = soup.find_all("table", attrs={"id":"ContentPlaceHolder1_GridView1"}) for details in get_details: text = details.find_all("tr")[1:] for tds in text: td = tds.find_all("td")[1] rera = td.find_all("span") rnumber = "" for num in rera: rnumber = num.text print(rnumber)By this code i can scrape this website completely.