+- Python Forum (https://python-forum.io)
+-- Forum: Python Coding (https://python-forum.io/forum-7.html)
+--- Forum: Web Scraping & Web Development (https://python-forum.io/forum-13.html)
+--- Thread: Python - Scrapy Login form (/thread-13381.html)
Python - Scrapy Login form - Baggelhsk95 - Oct-12-2018
Hello guyes....i was messing with my script to login in this website but seems doesnt work....i'll like to login first to loggin_url and then request the rest urls to get data
# -*- coding: utf-8 -*-
import scrapy
class StrongbotSpider(scrapy.Spider):
name = 'StrongBot'
login_url = 'https://www.strongflex.eu/en/login?back=my-account'
start_urls = ['https://www.strongflex.eu/en/a3-96-03-8l-fwd/298-221077b-front-wishbone-rear-bush-5902553512151.html?search_query=221077&results=5']
def parse(self, response):
data = {
'email': '[email protected]',
'passwd': 'example',
}
yield scrapy.FormRequest(url=self.login_url, formdata=data, callback=self.after_login)
def after_login(self, response):
for item in response.css('div.pb-right-column.col-xs-12.col-sm-4.col-md-3'):
yield {
'Price' : item.css('#our_price_display::text').extract()
}
RE: Python - Scrapy Login form - stranac - Oct-12-2018
Your spider is doing a few things wrong:
- It starts by opening search url (from
start_urls) before login is even attempted
- Submits the form request with some missing data (
SubmitLoginis required)
- Tries to parse the resulting page instead of the search page
I'd suggest using an InitSpider and doing it like this:
# -*- coding: utf-8 -*-
import scrapy
from scrapy.spiders.init import InitSpider
class StrongbotSpider(InitSpider):
name = 'StrongBot'
login_url = 'https://www.strongflex.eu/en/login'
start_urls = [
'https://www.strongflex.eu/en/a3-96-03-8l-fwd/'
'298-221077b-front-wishbone-rear-bush-5902553512151.html'
'?search_query=221077&results=5'
]
def init_request(self):
return scrapy.Request(
url=self.login_url,
callback=self.login,
)
def login(self, response):
yield scrapy.FormRequest.from_response(
response=response,
formid='login_form',
formdata={
'email': '[email protected]',
'passwd': 'gorosucks',
},
callback=self.initialized,
)
def parse(self, response):
for item in response.css('div.pb-right-column.col-xs-12.col-sm-4.col-md-3'):
yield {
'Price': item.css('#our_price_display::text').extract()
}
RE: Python - Scrapy Login form - Baggelhsk95 - Oct-13-2018
im getting the price in zl, we can use request for euro?
RE: Python - Scrapy Login form - stranac - Oct-13-2018
Sure, just submit the form for currency selection (the dropdown on the top of the page).
That should set a cookie, and you should get the desired result.
RE: Python - Scrapy Login form - Baggelhsk95 - Oct-16-2018
i was trying to do the same thing with different website, but seems doesn't work as i wanted to....
# -*- coding: utf-8 -*-
import scrapy
class TabotSpider(scrapy.Spider):
name = 'TaBot'
login_url = 'https://www.tatechnix.de/tatechnix/gx/?language=en'
start_urls = ['https://www.tatechnix.de/tatechnix/gx/product_info.php?info=p44235_ta-technix-sport-suspension-kit-opel-astra-h-caravan-2-0t-1-7-1-9cdti--without-level-control-type-a-h-30-30mm.html',]
def init_request(self):
return scrapy.Request(
url=self.login_url,
callback=self.login,
)
def login(self, response):
yield scrapy.FormRequest.from_response(
response=response,
formid='loginbox',
formdata={
'text': 'example',
'password': 'example',
},
callback=self.initialized,
)
def parse(self, response):
for content in response.css('#gm_attr_calc_price'):
yield {
'Price' : content.css('span[itemprop="price"]::Text').extract()
}