Python Forum

Full Version: Still learning, class var?
You're currently viewing a stripped down version of our content. View the full version with proper formatting.
Hello all, hope we have some patience for a beginner this morning.

I've written a weather scraper and it runs without any errors. My problem is one class_ has a variable of either high or low. My if statement returns a None value and I can't seem to figure it out. Any advice would be greatly appreciated.

time = current.find(class_='period-name').text
short_desc = current.find(class_='short-desc').text
temp = current.find(class_='temp temp-()')
if temp == ('low'):
		find(class_='temp temp-low').text
if temp == ('high'):
		find(class_='temp temp-high').text
weather.append(time)
weather.append(short_desc)
weather.append(temp)
~b4iknew
You need to provide enough code to recreate the problem, very difficult without.
Thanks for your reply Larz60+, as requested here is the entire code. It is only my "if" statement that isn't working like I thought.


#----  National Weather System by Python:
from user_agent import generate_user_agent
from bs4 import BeautifulSoup
import requests
 
headers = {'User-Agent': generate_user_agent(device_type='desktop', os=('mac', 'linux'))}
page_response = requests.get('https://forecast.weather.gov/MapClick.php?lat=32.47318000000007&lon=-100.40478999999999#.XC2EbM1MFEa/', timeout=5, headers=headers)
soup = BeautifulSoup(page_response.content, 'html.parser')

current = soup.find(id='current_conditions-summary')
forecast = current.find_all(class_='myforecast-current')
condition = forecast[0]

weather = []
period = current.find(class_='myforecast-current').text
temp_f = current.find(class_='myforecast-current-lrg').text
temp_c = current.find(class_='myforecast-current-sm').text
weather.append(period)
weather.append(temp_f)
weather.append(temp_c)
 
table = soup.find('table')
for row in table.find_all('tr'):
        key = (' '.join(td.text.strip() for td in row.find_all('td')))
        weather.append(key)

seven_day = soup.find(id='seven-day-forecast')
forecast_items = seven_day.find_all(class_='tombstone-container')
current = forecast_items[0]
 
time = current.find(class_='period-name').text
short_desc = current.find(class_='short-desc').text
temp = current.find(class_='temp temp-()')
if temp == ('low'):
		find(class_='temp temp-low').text
if temp == ('high'):
		find(class_='temp temp-high').text
weather.append(time)
weather.append(short_desc)
weather.append(temp)

with open('current-weather.dat', 'w') as f:
    for info in weather:
        f.write(f'{info}\n')

I'll take a look. later ...
Ok,

First of all, I reformatted your code a bit so that I could follow better, and added caching of URL so I wasn't hammering on site.

My findings after code listing
Modified code:
# Thread: https://python-forum.io/Thread-Still-learning-class-var
#----  National Weather System by Python:
from user_agent import generate_user_agent
from bs4 import BeautifulSoup
import requests
from pathlib import Path
import os

# Added cache capability, to turn off, call with cache=False
def get_page(url, cache=True):
    home = Path('.')
    print(home.resolve())
    cachefilename = home / 'weather_main.html'
    soup = None
    if cache and cachefilename.exists():
        with cachefilename.open('rb') as fp:
            soup = BeautifulSoup(fp.read(), 'html.parser')
    else:
        headers = {'User-Agent': generate_user_agent(device_type='desktop', os=('mac', 'linux'))}    
        page_response = requests.get(url, timeout=5, headers=headers)
        if page_response.status_code == 200:
            if cache:
                with cachefilename.open('wb') as fp:
                    fp.write(page_response.content)

            soup = BeautifulSoup(page_response.content, 'html.parser')
            if cache:
                with cachefilename.open('wb') as fp:
                    fp.write(page_response.content)
        else:
            print('Unaable to fetch page. status_code: {}'.format(page_response.status_code))
    return soup

def get_weather_info(url):
    soup = get_page(url)

    if soup:
        current = soup.find(id='current_conditions-summary')
        forecast = current.find_all(class_='myforecast-current')
        condition = forecast[0]
        
        weather = []
        period = current.find(class_='myforecast-current').text
        temp_f = current.find(class_='myforecast-current-lrg').text
        temp_c = current.find(class_='myforecast-current-sm').text
        weather.append(period)
        weather.append(temp_f)
        weather.append(temp_c)
        
        table = soup.find('table')
        # Remove following when working, for debug only
        print(f'\n================================= debug table =================================')
        print(f'type: {type(table)}, length: {len(table)}\n{table}')

        for row in table.find_all('tr'):
                key = (' '.join(td.text.strip() for td in row.find_all('td')))
                weather.append(key)
        
        seven_day = soup.find(id='seven-day-forecast')
        # Remove following when working, for debug only
        print(f'\n================================= debug seven_day =================================')
        print(f'seven_day: {seven_day}')

        forecast_items = seven_day.find_all(class_='tombstone-container')
        current = forecast_items[0]
        # Remove following when working, for debug only
        print(f'\n================================= debug current =================================')
        print(f'current: {current}')

        time = current.find(class_='period-name').text
        # Remove following when working, for debug only
        print(f'\n================================= debug time =================================')
        print(f'current: {time}')

        short_desc = current.find(class_='short-desc').text
        # Remove following when working, for debug only
        print(f'\n================================= debug short_desc =================================')
        print(f'current: {short_desc}')

        temp = current.find(class_='temp temp-()')
        # Remove following when working, for debug only
        print(f'\n================================= debug temp =================================')
        print(f'temp: {temp}')

        if temp == ('low'):
                find(class_='temp temp-low').text
        if temp == ('high'):
                find(class_='temp temp-high').text
        weather.append(time)
        weather.append(short_desc)
        weather.append(temp)
        
        with open('current-weather.dat', 'w') as f:
            for info in weather:
                f.write(f'{info}\n')


def main():
    # make sure CWD is source directory
    os.chdir(os.path.abspath(os.path.dirname(__file__)))
    url = 'https://forecast.weather.gov/MapClick.php?lat=32.47318000000007&lon=-100.40478999999999#.XC2EbM1MFEa/'
    get_weather_info(url)


if __name__ == '__main__':
    main()
output:
Output:
/media/larz60/Data-2TB/Projects/T-Z/T/TryStuff/src ================================= debug table ================================= type: <class 'bs4.element.Tag'>, length: 15 <table> <tr> <td class="text-right"><b>Humidity</b></td> <td>37%</td> </tr> <tr> <td class="text-right"><b>Wind Speed</b></td> <td>SW 8 mph</td> </tr> <tr> <td class="text-right"><b>Barometer</b></td> <td>30.02 in</td> </tr> <tr> <td class="text-right"><b>Dewpoint</b></td> <td>20°F (-7°C)</td> </tr> <tr> <td class="text-right"><b>Visibility</b></td> <td>10.00 mi</td> </tr> <tr><td class="text-right"><b>Wind Chill</b></td><td>41°F (5°C)</td></tr> <tr> <td class="text-right"><b>Last update</b></td> <td> 24 Jan 10:15 am CST </td> </tr> </table> ================================= debug seven_day ================================= seven_day: <div class="panel panel-default" id="seven-day-forecast"> <div class="panel-heading"> <b>Extended Forecast for</b> <h2 class="panel-title"> Sweetwater TX </h2> </div> <div class="panel-body" id="seven-day-forecast-body"> <div id="seven-day-forecast-container"><ul class="list-unstyled" id="seven-day-forecast-list"><li class="forecast-tombstone"> <div class="tombstone-container"> <p class="period-name">Today<br/><br/></p> <p><img alt="Today: Sunny, with a high near 53. West wind 10 to 15 mph becoming north northeast in the afternoon. Winds could gust as high as 25 mph. " class="forecast-icon" src="newimages/medium/few.png" title="Today: Sunny, with a high near 53. West wind 10 to 15 mph becoming north northeast in the afternoon. Winds could gust as high as 25 mph. "/></p><p class="short-desc">Sunny</p><p class="temp temp-high">High: 53 °F</p></div></li><li class="forecast-tombstone"> <div class="tombstone-container"> <p class="period-name">Tonight<br/><br/></p> <p><img alt="Tonight: Mostly clear, with a low around 25. North northeast wind 5 to 15 mph, with gusts as high as 25 mph. " class="forecast-icon" src="newimages/medium/nfew.png" title="Tonight: Mostly clear, with a low around 25. North northeast wind 5 to 15 mph, with gusts as high as 25 mph. "/></p><p class="short-desc">Mostly Clear</p><p class="temp temp-low">Low: 25 °F</p></div></li><li class="forecast-tombstone"> <div class="tombstone-container"> <p class="period-name">Friday<br/><br/></p> <p><img alt="Friday: Sunny, with a high near 50. Wind chill values between 19 and 29 early. East northeast wind 5 to 10 mph becoming west southwest in the afternoon. " class="forecast-icon" src="newimages/medium/few.png" title="Friday: Sunny, with a high near 50. Wind chill values between 19 and 29 early. East northeast wind 5 to 10 mph becoming west southwest in the afternoon. "/></p><p class="short-desc">Sunny</p><p class="temp temp-high">High: 50 °F</p></div></li><li class="forecast-tombstone"> <div class="tombstone-container"> <p class="period-name">Friday<br/>Night</p> <p><img alt="Friday Night: Partly cloudy, with a low around 33. South wind 5 to 10 mph. " class="forecast-icon" src="newimages/medium/nsct.png" title="Friday Night: Partly cloudy, with a low around 33. South wind 5 to 10 mph. "/></p><p class="short-desc">Partly Cloudy</p><p class="temp temp-low">Low: 33 °F</p></div></li><li class="forecast-tombstone"> <div class="tombstone-container"> <p class="period-name">Saturday<br/><br/></p> <p><img alt="Saturday: Partly sunny, with a high near 54. South southwest wind around 5 mph becoming northwest in the afternoon. " class="forecast-icon" src="newimages/medium/bkn.png" title="Saturday: Partly sunny, with a high near 54. South southwest wind around 5 mph becoming northwest in the afternoon. "/></p><p class="short-desc">Partly Sunny</p><p class="temp temp-high">High: 54 °F</p></div></li><li class="forecast-tombstone"> <div class="tombstone-container"> <p class="period-name">Saturday<br/>Night</p> <p><img alt="Saturday Night: Mostly clear, with a low around 35. Southwest wind 5 to 10 mph. " class="forecast-icon" src="newimages/medium/nfew.png" title="Saturday Night: Mostly clear, with a low around 35. Southwest wind 5 to 10 mph. "/></p><p class="short-desc">Mostly Clear</p><p class="temp temp-low">Low: 35 °F</p></div></li><li class="forecast-tombstone"> <div class="tombstone-container"> <p class="period-name">Sunday<br/><br/></p> <p><img alt="Sunday: Sunny, with a high near 62. West wind around 10 mph. " class="forecast-icon" src="newimages/medium/few.png" title="Sunday: Sunny, with a high near 62. West wind around 10 mph. "/></p><p class="short-desc">Sunny</p><p class="temp temp-high">High: 62 °F</p></div></li><li class="forecast-tombstone"> <div class="tombstone-container"> <p class="period-name">Sunday<br/>Night</p> <p><img alt="Sunday Night: Partly cloudy, with a low around 43. South southwest wind 10 to 15 mph, with gusts as high as 25 mph. " class="forecast-icon" src="newimages/medium/nsct.png" title="Sunday Night: Partly cloudy, with a low around 43. South southwest wind 10 to 15 mph, with gusts as high as 25 mph. "/></p><p class="short-desc">Partly Cloudy</p><p class="temp temp-low">Low: 43 °F</p></div></li><li class="forecast-tombstone"> <div class="tombstone-container"> <p class="period-name">Monday<br/><br/></p> <p><img alt="Monday: Mostly sunny, with a high near 57. Breezy, with a southwest wind 15 to 20 mph becoming north in the afternoon. Winds could gust as high as 30 mph. " class="forecast-icon" src="newimages/medium/wind_sct.png" title="Monday: Mostly sunny, with a high near 57. Breezy, with a southwest wind 15 to 20 mph becoming north in the afternoon. Winds could gust as high as 30 mph. "/></p><p class="short-desc">Mostly Sunny<br/>and Breezy</p><p class="temp temp-high">High: 57 °F</p></div></li></ul></div> <script type="text/javascript"> // equalize forecast heights $(function () { var maxh = 0; $(".forecast-tombstone .short-desc").each(function () { var h = $(this).height(); if (h > maxh) { maxh = h; } }); $(".forecast-tombstone .short-desc").height(maxh); }); </script> </div> </div> ================================= debug current ================================= current: <div class="tombstone-container"> <p class="period-name">Today<br/><br/></p> <p><img alt="Today: Sunny, with a high near 53. West wind 10 to 15 mph becoming north northeast in the afternoon. Winds could gust as high as 25 mph. " class="forecast-icon" src="newimages/medium/few.png" title="Today: Sunny, with a high near 53. West wind 10 to 15 mph becoming north northeast in the afternoon. Winds could gust as high as 25 mph. "/></p><p class="short-desc">Sunny</p><p class="temp temp-high">High: 53 °F</p></div> ================================= debug time ================================= current: Today ================================= debug short_desc ================================= current: Sunny temp: None
Here's what I have found out

Note that temp-high is in current, but not temp-low
but in seven_day under 'Tonight', nut not 'Today'

It is inconistant, so you need to accommodate for this

** Note ** I have left all debugging statements in the code as you may want to use them for further debugging
Best way to view is to save to file:
python weather_prog_name.py > results.txt
(Jan-24-2019, 03:26 PM)b4iknew Wrote: [ -> ]My problem is one class_ has a variable of either high or low.
Example with CSS Selector.
Now goes direct to values wanted,and p.temp.temp should work for both high and low.
Direct for only high would be p.temp.temp-high.
 
from user_agent import generate_user_agent
from bs4 import BeautifulSoup
import requests

headers = {'User-Agent': generate_user_agent(device_type='desktop', os=('mac', 'linux'))}
page_response = requests.get('https://forecast.weather.gov/MapClick.php?lat=32.47318000000007&lon=-100.40478999999999#.XC2EbM1MFEa/', timeout=5, headers=headers)
soup = BeautifulSoup(page_response.content, 'lxml')

temp = soup.select('p.temp.temp')[0]
print(f'Today temp is {temp.text}')
Output:
Today temp is High: 53 °F
Larz60+ thanks for your input, and snippsat , wow thanks allot for your insight. I did not know about the CSS Selector. With a slight modification too your post it worked like a charm. Thanks again and have a great day!

temp = current.select('p.temp.temp ')[0].text
weather.append(time)
weather.append(short_desc)
weather.append(temp)