Python Forum
Thread Rating:
  • 0 Vote(s) - 0 Average
  • 1
  • 2
  • 3
  • 4
  • 5
Nested for loop not looping
#1
I'm trying to get this code to open a page, pull item and info data, add it to a dictionary then repeat on the next page. It works fine on the first page, but then when it continues it will go to as many pages as needed but does not run the for loops. I can see it looking at each page but the dictionary only contains info from the first page. Thanks in advance!

from lxml import html
import requests
import re


from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time


def proxi_scrape(auction_url):

	PATH = "/Users/jacobdonelson/Desktop/chromedriver"
	driver = webdriver.Chrome(PATH)
	driver.get(auction_url)
	print('Auction URL: ' + auction_url)
	page = requests.get(driver.current_url)
	tree = html.fromstring(page.content)
	number_of_pages = tree.xpath('//*[@id="Viewing"]/div[1]/text()')
	number_of_pages = number_of_pages[-1]
	number_of_pages = int(number_of_pages.replace(' of ', ''))
	desired_page = 1
	clean_prices = []
	total_items = []
	while desired_page <= 3:
		go_to_page = WebDriverWait(driver, 20).until(
		EC.presence_of_element_located((By.XPATH, '//*[@id="txtGoToPage"]')))
		
		go_to_page.clear()
		go_to_page.send_keys(desired_page)
		go_to_page.send_keys(Keys.RETURN)
		

		current_page = driver.current_url
		new_page = requests.get(current_page)
		new_tree = html.fromstring(new_page.content)
		

		new_items = new_tree.xpath('//a[@class="showVisited responsive-width "]/text()')
		new_prices = new_tree.xpath('//div[@class="smInstaBidButton"]/text()')
		
		
		for new_price in new_prices:
			clean_price = new_price.replace('BID NOW $', '')
			clean_price = clean_price.replace(',', '')
			clean_prices.append(clean_price)

		for new_item in new_items:
			total_items.append(new_item)

		desired_page += 1

		
		

	auction_items = dict(zip(total_items, clean_prices))

	return auction_items
Reply


Possibly Related Threads…
Thread Author Replies Views Last Post
  Big O runtime nested for loop and append yarinsh 4 1,331 Dec-31-2022, 11:50 PM
Last Post: stevendaprano
  Nested for loops - help with iterating a variable outside of the main loop dm222 4 1,532 Aug-17-2022, 10:17 PM
Last Post: deanhystad
  "while" loop is looping back too early mangurian 1 1,243 Jan-28-2022, 09:15 AM
Last Post: ibreeden
  Looping through nested elements and updating the original list Alex_James 3 2,069 Aug-19-2021, 12:05 PM
Last Post: Alex_James
  How do I add another loop to my nested loop greenpine 11 4,441 Jan-12-2021, 04:41 PM
Last Post: greenpine
  Error on nested loop : Invalid syntax dvazquezgu 3 3,177 Nov-25-2020, 10:04 AM
Last Post: palladium
  Nested loop indexing Morte 4 3,808 Aug-04-2020, 07:24 AM
Last Post: Morte
  While loop keeps looping mcoliver88 3 2,156 Jul-29-2020, 12:48 PM
Last Post: buran
  while loop will not stop looping TheTechRobo 5 3,646 Apr-20-2020, 01:47 PM
Last Post: TheTechRobo
  Help: for loop with dictionary and nested lists mart79 1 1,834 Apr-12-2020, 02:52 PM
Last Post: TomToad

Forum Jump:

User Panel Messages

Announcements
Announcement #1 8/1/2020
Announcement #2 8/2/2020
Announcement #3 8/6/2020