I need to run a python script written by a previous developer.
When I run it on my Mac, I get the following
File "health_json_to_csv.py", line 31
output = write_with_prefix(input[0], output, f"Wages{prefix}Annual")
The script performs API calls to populate a csv file from a database. Here is the code minus the API keys:
Looking at the database where the data will be imported from the CSV vile, I think the prefixes could be 'National' and 'State' or perhaps 'Area'. I have tried running the script with these but no success so far.
When I run it on my Mac, I get the following
File "health_json_to_csv.py", line 31
output = write_with_prefix(input[0], output, f"Wages{prefix}Annual")
The script performs API calls to populate a csv file from a database. Here is the code minus the API keys:
import requests
import json
import csv
import time
ID = "xxxxxxxxxxxxxxxxxxxxxxxx"
headers = {'Authorization': "api key"}
def write_with_prefix(input, output, prefix):
""" iterate over input dict; flatten data to output dict with prefix """
for item in input.keys():
if input[item] is not None:
ret_key = prefix + item
output[ret_key] = input[item].strip()
return output
def flatten_wages(input, output, prefix):
""" take national/state/area wages; strip unneeded data and flatten remainder """
rate_type = input[0]["RateType"]
del input[0]["RateType"]
del input[0]["StFips"]
del input[0]["Area"]
if len(input) > 1:
del input[1]["RateType"]
del input[1]["StFips"]
del input[1]["Area"]
if rate_type == "Annual":
output = write_with_prefix(input[0], output, f"Wages{prefix}Annual")
if len(input) > 1:
output = write_with_prefix(input[1], output, f"Wages{prefix}Hourly")
else:
output = write_with_prefix(input[0], output, f"Wages{prefix}Hourly")
if len(input) > 1:
output = write_with_prefix(input[1], output, f"Wages{prefix}Annual")
return output
def flatten_ed_type(input, output):
""" Flatten education type dict in input to output dict """
matrix = {
"Less than high school diploma": "EdLessThanHighSchool",
"High school diploma or equivalent": "EdHighSchoolDiploma",
"Some college, no degree": "EdSomeCollege",
"Associate's degree": "EdAssociatesDegree",
"Bachelor's degree": "EdBachelorsDegree",
"Master's degree": "EdMastersDegree",
"Doctoral or professional degree": "EdDoctoralOrProfessionalDegree"
}
for item in input:
key = matrix[item["EducationLevel"]] + "Pct"
output[key] = item["Value"]
return output
def flatten_projections(input, output):
""" flatten state and national projections from input dict into output dict """
for data in input:
if data["StateName"] == "Washington":
prefix = "State"
else:
prefix = "National"
del data["Stfips"]
del data["StateName"]
output[prefix+"CurrentEstimatedEmployment"] = data["EstimatedEmployment"].strip()
del data["EstimatedEmployment"]
write_with_prefix(data, output, prefix)
return output
def prepare_for_write(data):
""" recieve a single occupational detail, manually flatten for csv translation"""
ret = {}
# flatten wage data
if len(data["Wages"]["NationalWagesList"]) > 0:
ret = flatten_wages(data["Wages"]["NationalWagesList"], ret, "National")
if len(data["Wages"]["StateWagesList"]) > 0:
ret = flatten_wages(data["Wages"]["StateWagesList"], ret, "State")
if len(data["Wages"]["BLSAreaWagesList"]) > 0:
ret = flatten_wages(data["Wages"]["BLSAreaWagesList"], ret, "Area")
ret["WageYear"] = data["Wages"]["WageYear"]
#flatten education
education = data["EducationTraining"]
ret = flatten_ed_type(education["EducationType"], ret)
#flatten projections
projections = data["Projections"]
ret["ProjectedYear"] = projections["ProjectedYear"]
ret = flatten_projections(projections["Projections"], ret)
# remove unneeded top-level items
del data["COSVideoURL"]
del data["RelatedOnetTitles"]
del data["TrainingPrograms"]
del data["Wages"]
del data["Projections"]
del data["EducationTraining"]
del data["SocInfo"]
del data["AlternateTitles"]
del data["StFips"]
del data["Location"]
ret = write_with_prefix(data, ret, '')
return ret
with open('health-codes') as f:
with open('data.csv', 'w') as csvfile:
# good to find a way to autogen these
fieldnames = [ "OnetTitle","OnetCode","OnetDescription",
"WagesNationalAnnualPct10", "WagesNationalAnnualPct25",
"WagesNationalAnnualMedian","WagesNationalAnnualPct75",
"WagesNationalAnnualPct90","WagesNationalAnnualAreaName",
"WagesNationalHourlyPct10","WagesNationalHourlyPct25",
"WagesNationalHourlyMedian","WagesNationalHourlyPct75",
"WagesNationalHourlyPct90","WagesNationalHourlyAreaName",
"WagesStateAnnualPct10","WagesStateAnnualPct25",
"WagesStateAnnualMedian","WagesStateAnnualPct75",
"WagesStateAnnualPct90","WagesStateAnnualAreaName",
"WagesStateHourlyPct10","WagesStateHourlyPct25",
"WagesStateHourlyMedian","WagesStateHourlyPct75",
"WagesStateHourlyPct90","WagesStateHourlyAreaName",
"WagesAreaAnnualPct10","WagesAreaAnnualPct25",
"WagesAreaAnnualMedian","WagesAreaAnnualPct75",
"WagesAreaAnnualPct90","WagesAreaAnnualAreaName",
"WagesAreaHourlyPct10","WagesAreaHourlyPct25",
"WagesAreaHourlyMedian","WagesAreaHourlyPct75",
"WagesAreaHourlyPct90","WagesAreaHourlyAreaName",
"WageYear","EdLessThanHighSchoolPct",
"EdHighSchoolDiplomaPct","EdSomeCollegePct",
"EdAssociatesDegreePct","EdBachelorsDegreePct",
"EdMastersDegreePct","EdDoctoralOrProfessionalDegreePct",
"ProjectedYear","StateCurrentEstimatedEmployment",
"StateProjectedEmployment","StatePerCentChange",
"StateProjectedAnnualJobOpening","NationalCurrentEstimatedEmployment",
"NationalProjectedEmployment","NationalPerCentChange",
"NationalProjectedAnnualJobOpening","BrightOutlook","Green",
"BrightOutlookCategory"]
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for line in f:
url = 'https://api.careeronestop.org/v1/occupation/' + ID + '/'+ line.strip() + '/98107?projectedEmployment=true&training=true&wages=true'
print (url)
response = requests.get(url, headers=headers)
data = json.loads(response.content)
prepared_data = prepare_for_write(data["OccupationDetail"][0])
writer.writerow(prepared_data)The file, 'health-codes' is in the same folder as the python script. Thank you for your assistance.Looking at the database where the data will be imported from the CSV vile, I think the prefixes could be 'National' and 'State' or perhaps 'Area'. I have tried running the script with these but no success so far.
