So I likely changed the code a bit, and not so sure what's wrong with it, tried searching for issues but nothing happening, would you mind looking at what I've done so far?
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import requests
import re
import urllib.request
from urllib.request import urlopen
from bs4 import BeautifulSoup
import unicodedata
import codecs
import os
import time
import pandas as pd
import xlwt
import math
from xlwt.Workbook import *
from pandas import ExcelWriter
import xlsxwriter
from itertools import groupby
import datetime
import xlsxwriter
url = 'https://www.tui.lt/?departureCityId=389090&arrivalCountryId=18498&arrivalRegionIds=&arrivalCityIds=&hotelIds=&minStartDate=2022-02-09&maxStartDate=2022-02-22&minNightsCount=7&maxNightsCount=14&adults=2&children=&searchLevel=&isGeoInfoRequired=false&type=country'
r = requests.get(url, headers={"User-Agent": "Chrome"})
soup = BeautifulSoup(r.text, "html.parser")
xurl = soup.find_all("a", {"class": "paginator__page paginator__page--active"})
workbook = xlsxwriter.Workbook('Hoteliai.xlsx')
workpro = workbook.add_worksheet('HotelName')
workimg = workbook.add_worksheet('location')
workspec = workbook.add_worksheet('Features')
workrew = workbook.add_worksheet('Price')
pro_list = ["product_id", "name(en-gb)", "location", "adults", "children", "rate", "date", "price"]
workpro.write_row(0, 0, pro_list)
img_list = ["product_id", "image", "sort_order"]
workimg.write_row(0, 0, img_list)
atr_list = ["product_id", "attribute_group", "attribute", "text(en-gb)"]
row = 1
imgrow = 1
ocd = 1
rows = 1
category = 88
proid = 200
atrribute_group = "Tui deals"
location = "Egypt"
adults = 2
children = 0
rate = 4
date = 2022/2/22
price = "1000.00"
url = "https://www.tui.lt" + xurl[0]["href"] + "&page=" + str(ocd)
for ocd in range(1, 48):
url = "https://www.tui.lt" + xurl[0]["href"] + "&page=" + str(ocd)
m = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
soup = BeautifulSoup(m.text, "html.parser")
link = soup.find_all("a", {"class": "catalog-taxons-product__image-anchor"})
nextlink = "https://www.tui.lt" + link[0]["href"]
print("\n" "--------change page---------")
print(url)
print("--------change page---------")
for number in range(48):
nextlink = "https://www.tui.lt" + link[number]["href"]
r = requests.get(nextlink, headers={"User-Agent": "Mozilla/5.0"})
soup = BeautifulSoup(r.text, "html.parser")
productcode = soup.findAll("div",{"class":"product-righter"})
sku = re.sub('\D', '', productcode[0].p.text.strip())
pavad = soup.find("h1").text.strip()
print("\n" "--------++++++---------")
print(nextlink)
print("--------++++++---------")
workpro.write(row, 0, proid)
workpro.write(row, 1, category)
workpro.write(row, 2, location)
workpro.write(row, 3, adults)
workpro.write(row, 4, children)
workpro.write(row, 5, rate)
workpro.write(row, 6, date)
price = soup.findAll("span", {'class': 'price'})
if price != []:
pricenr = price[0].text.strip().replace('€', '').replace('vnt.', '').replace('\n', '').replace('/ ',
'').replace(',',
'.')
else:
print("no price")
containers = soup.findAll("div", {"class": "site-block inner-content"})
table = soup.findAll("td")
find = soup.find_all("div", {'class': 'ck-info-tooltip-wrap'})
for div in find:
div.decompose()
syntaxes = ["Number of people", "date", "location", "price"]
nodata = ""
workbook.close()
print("ALL DONE !")