Python Forum

hi,

I am trying to create a webscraper which will take links from csv file and scrape data.

but whenever i run this code it says object of nonetype has no len().

Below is my code:

import requests
from bs4 import BeautifulSoup
import time
import os
import csv

file = {}
final_data = []

def read_file():
    global file
    f = open("Data.csv")
    for line in f.readlines():
        data = line.split(",")
        file[data[0]]= data[1]

def writedata(alldata1, filename):
    with open("./"+filename, "w")as csvfile:
        csvfile = csv.writer(csvfile, delimiter=",")
        csvfile.writerow("")
        for i in range(0, len(alldata1)):
            csvfile.writerow(alldata1[i])

def parse_data():
    global file
    global final_data
    for data_obj in file.keys():
        link = file[data_obj]
        #print(link)
        data = getbyget(link, {})
        soup = BeautifulSoup(data, "html.parser")
        get_details = soup.find_all(class_="box2-body")[0]
        details = get_details.find_all(id="ctl00_ContentPlaceHolder1_Lblproposedenddt")
        name = get_details.find_all(id="ctl00_ContentPlaceHolder1_lblprojectname")
        date = ""
        names = ""
        sublist = []
        for more in details:
            date = more.text
            sublist.append(date)
        for nme in name:
            names = nme.text
            sublist.append(names)

        get_state = soup.find_all(class_="box2-body")[1]
        state = get_state.find_all(id="ctl00_ContentPlaceHolder1_lblState")
        city = get_state.find_all(id="ctl00_ContentPlaceHolder1_lbldistrict")
        cities = ""
        states = ""
        for statename in state:
            states = statename.text
            sublist.append(states)
        for citys in city:
            cities = citys.text
            sublist.append(cities)
            
        get_pmptername = soup.find_all(class_="box2-body")[2]
        promotername = get_pmptername.find_all(id="ctl00_ContentPlaceHolder1_Lblpromotername")
        geta = get_pmptername.find_all("a")
        alinks = ""
        promote = ""
        for promoter in promotername:
            promote = promoter.text
            sublist.append(promote)
        for linka in geta:
            alinks = "http://up-rera.in/"+ linka["href"]
            sublist.append(alinks)
            final_data.append(sublist)
            print(final_data)

def getbyget(url, values):
    res = requests.get(url, data=values)
    data = res.text
    return data

def main():
    read_file()
    datas = parse_data()
    writedata(datas, "UP_DATA.csv")                                                      
main()

error is receive is :

Error:
Traceback (most recent call last):
  File "C:\Users\prince.bhatia\Desktop\up_additional\up_rera_additional_details - Copy.py", line 80, in <module>
    main()
  File "C:\Users\prince.bhatia\Desktop\up_additional\up_rera_additional_details - Copy.py", line 79, in main
    writedata(datas, "UP_DATA.csv")
  File "C:\Users\prince.bhatia\Desktop\up_additional\up_rera_additional_details - Copy.py", line 21, in writedata
    for i in range(0, len(alldata1)):
TypeError: object of type 'NoneType' has no len()

i have append data into final_data so it should load data into csv but it throws error for no len(), can someone please help on this?

99% of the time, a TypeError involving the None object indicates that a function mistakenly returned None because of a missing return statement. In your case, there is no return statement in parse_data(), which means that datas has value None.

Perfect sir, thank you so much. This a great learning here.

final_data is a global variable. There is no need for return statement.
It's not recommended though. But your code looks like you have return statement - main function.

def parse_data():
    global file
    global final_data
    for data_obj in file.keys():
        link = file[data_obj]
        #print(link)
        data = getbyget(link, {})
        soup = BeautifulSoup(data, "html.parser")
        get_details = soup.find_all(class_="box2-body")[0]
        details = get_details.find_all(id="ctl00_ContentPlaceHolder1_Lblproposedenddt")
        name = get_details.find_all(id="ctl00_ContentPlaceHolder1_lblprojectname")
        date = ""
        names = ""
        sublist = []
        for more in details:
            date = more.text
            sublist.append(date)
        for nme in name:
            names = nme.text
            sublist.append(names)
 
        get_state = soup.find_all(class_="box2-body")[1]
        state = get_state.find_all(id="ctl00_ContentPlaceHolder1_lblState")
        city = get_state.find_all(id="ctl00_ContentPlaceHolder1_lbldistrict")
        cities = ""
        states = ""
        for statename in state:
            states = statename.text
            sublist.append(states)
        for citys in city:
            cities = citys.text
            sublist.append(cities)
             
        get_pmptername = soup.find_all(class_="box2-body")[2]
        promotername = get_pmptername.find_all(id="ctl00_ContentPlaceHolder1_Lblpromotername")
        geta = get_pmptername.find_all("a")
        alinks = ""
        promote = ""
        for promoter in promotername:
            promote = promoter.text
            sublist.append(promote)
        for linka in geta:
            alinks = "http://up-rera.in/"+ linka["href"]
            sublist.append(alinks)
            final_data.append(sublist)

    print(final_data)
    return final_data

However, I doubt printing final_data on every for loop iteration is what you want - line 69

For loop.
This is not how it's doing it in Python.

for i in range(0, len(alldata1)):
            csvfile.writerow(alldata1[i])

for i in alldata1:
            csvfile.writerow(i)

But the others for loops are alright.

in read_file you are opening a file for reading by using open. After that, you don't close the file. In the next function definition, you use the with statement which closes the file automatically.

Because of that and the for loops I think it will be good if you go back and recap(? I don't know if that is the right word) the basics of Python. Loops, IO, functions return, passing parameters and so on. Otherwise, you will write scripts, they will throw errors because of minor mistakes.