Sep-29-2022, 09:42 AM
import json import time import requests from bs4 import BeautifulSoup from selenium import webdriver from time import sleep seleniumChromeDriver = webdriver.Chrome() seleniumChromeDriver.maximize_window() returnResponse = [] for pageNumber in range(5): url = "https://suchen.mobile.de/fahrzeuge/search.html?damageUnrepaired=ALSO_DAMAGE_UNREPAIRED&isSearchRequest=true&od=down&pageNumber=" + str( pageNumber + 1) seleniumChromeDriver.get(url) #allCookies = seleniumChromeDriver.get_cookies() #cookies = '' #for cookie in allCookies: # cookies += cookie['name'] + '=' + cookie['value'] # if allCookies.index(cookie) != (len(allCookies) - 1): # cookies += ';' # time.sleep(10) headers = { 'Accept': 'application/json', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36', 'Referer': 'https://suchen.mobile.de/fahrzeuge/search.html?dam=0&isSearchRequest=true&ref=quickSearch&sb=rel&vc=Car', 'Cookie': ' **doh** ' } response = requests.get(url, headers=headers, verify=False).content soup = BeautifulSoup(response, "html.parser") body = soup.find("body", {}) #bodyCaptchaChecking = body.find("iframe2", {}) time.sleep(10) carList = soup.find("div", {"class": "cBox--resultList"}).find_all("div", {"cBox-body--resultitem"}) for car in carList: carName = car.find("span", {"class": "h3 u-text-break-word"}).text carPrice = car.find("span", {"class": "h3 u-block"}).text carPrice = carPrice.replace("\xa0", " ") returnResponse.append({ 'carName': carName, 'carPrice': carPrice }) time.sleep(100) else: print('Sistem Captcha\'ya Takıldı!') jsonFormattedString = json.dumps(returnResponse, indent=2) fileManager = open('result.json', 'a') fileManager.write(jsonFormattedString) fileManager.close()