+- Python Forum (https://python-forum.io)
+-- Forum: Python Coding (https://python-forum.io/forum-7.html)
+--- Forum: General Coding Help (https://python-forum.io/forum-8.html)
+--- Thread: Code to retrieve data from a website (/thread-37670.html)
Code to retrieve data from a website - charlie13255 - Jul-07-2022
Hi guys, I am trying to retrieve some data from the following link: https://quickstats.nass.usda.gov
I am somehow stuck on getting it as what I want to do is to download it automatically through a series of API codes and then export it to an excel. Could you maybe go through it and help me? thanks a lot.
import json
import pandas as pd
import requests
api_key = "API KEY AQUI"
commo_to_items = {
"SOYBEANS": ["SOYBEANS - YIELD, MEASURED IN BU / ACRE", "SOYBEANS - ACRES PLANTED", "SOYBEANS - ACRES HARVESTED", "SOYBEANS - PRODUCTION, MEASURED IN BU"],
"CORN": ["CORN, GRAIN - YIELD, MEASURED IN BU / ACRE", "CORN - ACRES PLANTED", "CORN, GRAIN - ACRES HARVESTED", "CORN, GRAIN - PRODUCTION, MEASURED IN BU"],
}
def get_data(commo, items, year):
data = {
"key": api_key,
"source_desc": "SURVEY",
"commodity_desc": commo,
"year": year,
"short_desc": items,
"agg_level_desc": "COUNTY",
"format": "JSON"
}
r = requests.get("http://quickstats.nass.usda.gov/api/api_GET/", params=data)
try:
df = pd.DataFrame(json.loads(r.text)["data"])[["county_name", "year", "statisticcat_desc", "location_desc", "unit_desc", "state_name", "commodity_desc", "Value"]]
df = df.rename(columns={
"county_name": "county",
"state_name": "state",
"location_desc": "location",
"unit_desc": "unit",
"commodity_desc": "commodity",
"Value": "value"
})
return df
except Exception as e:
print(commo)
print(e)
print(r.content)
return None
dfs = []
for year in range(2019, 2021):
print(year)
for commo, items in commo_to_items.items():
df = get_data(commo, items, year)
if df is not None:
dfs.append(df)
df = pd.concat(dfs)
df.to_csv("excel.csv",index=False)
BTW: the API KEY comes from the website and its a code that you need to get by registering within the website