Python Forum
invalid literal for int() with base 10: '# NRECS: 1096\n'
Thread Rating:
  • 0 Vote(s) - 0 Average
  • 1
  • 2
  • 3
  • 4
  • 5
invalid literal for int() with base 10: '# NRECS: 1096\n'
#1
Hi all,

I am kinda new to Python, and would really appreciate some help with the following code. The purpose is to take individual variable data from the respective column from each of (258) files and convert into a NetCDF4 file. Part of the output file is as follows:
Output:
# NRECS: 1096 # DT: 24 # STARTDATE: 1999-01-01 00:00:00 # ALMA_OUTPUT: 0 # NVARS: 10 # YEAR MONTH DAY OUT_SNOW_COVER OUT_SURF_TEMP OUT_RUNOFF OUT_BASEFLOW OUT_SWE OUT_EVAP OUT_PREC 1999 01 01 0.0000 -0.6910 0.0000 1.7175 0.0000 1.2187 1.2250 1999 01 02 0.0000 -8.1983 0.0000 1.7042 0.0000 0.0132 0.0000 1999 01 03 0.0000 -13.7701 0.0000 1.6907 0.0000 0.0076 0.0000 1999 01 04 1.0000 -11.0906 0.0000 1.6772 6.1095 0.4404 7.4750 1999 01 05 1.0000 -7.4365 0.0000 1.6637 9.7234 0.6585 4.3000 1999 01 06 1.0000 -6.4047 0.0000 1.6501 12.1842 0.5672 3.0000 1999 01 07 1.0000 -9.1578 0.0000 1.6364 12.0282 0.5211 0.0000
The code is as follows:
# import dependencies
from __future__ import print_function
import sys
import os, string
# handle dates...
import datetime as dt
# NetCDF and Numeric
from netCDF4 import *
from numpy import *

def flux2nc(influxes,outpath,var=None,start_year=None,end_year=None):

    # building file list and sorted lat lon list
    dirin = os.path.dirname(influxes)

    try:
        file_list = os.listdir(dirin)
    except OSError:
        raise OSError('Input flux directory not valid, please fix path')

    lat_t = []
    lon_t = []
    lat = []
    lon = []

    try:
        for f in file_list:
            lat_t.append(float(str.split(f, "_")[1]))
            lon_t.append(float(str.split(f, "_")[2]))
    except ValueError:
        raise ValueError('Input path contains files that are not flux files')

    for i in lat_t:
        if i not in lat:
            lat.append(i)

    for i in lon_t:
        if i not in lon:
            lon.append(i)


    # putting in order. Lat should be from top to botom
    # lon from left to right
    lon.sort()
    lat.sort()
    lat.reverse()

    del(lat_t)
    del(lon_t)

    # if variable is not set, get it from user
    if var == None:

        #determining the parameter to use
        print("Choose output parameter")
        print("1 - SNOW_COVER")
        print("2 - SURFACE_TEMPERATURE")
        print("3 - Runoff")
        print("4 - Base flow")
        print("5 - Snow Water Equivalent")
        print("6 - EVAPORATION")
        print("7 - PRECIPITATION")
        varini = input('Choose output (1 a 7)>')

        #getting the collumn right
        if int (varini) < 7:
            var = varini + 2
        elif varini == 6:        #more than one soil layer...
            camada = input('which soil layer?>')
            var = varini + 2 + camada

    #set name of out_file. Named after parameter choice
    if var == 3:
        var_txt = "ppt"
        var_name = "Precipitation"
    elif var == 4:
        var_txt = "evap"
        var_name = "Evapotranspiration"
    elif var == 5:
        var_txt = "runoff"
        var_name = "Runoff"
    elif var == 6:
        var_txt = "base"
        var_name = "Baseflow"
    elif var == 7:
        var_txt = "swe"
        var_name = "Snow Water Equivalent"
    else:
        var_txt = "soilLyr"+str(camada)
        var_name = "Soil moisture, layer {0}".format(camada)

    # if the date information is not set get it from user
    if start_year == None:
        # for what date?
        start_year = input("Enter start year:")
    if end_year == None:
        end_year = input("End year:")

    # set date information in datetime object
    inidate = dt.date(start_year,1,1)
    enddate = dt.date(end_year,12,31)

    # calculate number of days in time series
    days = enddate.toordinal() - inidate.toordinal()+1

    #print "Gridding {0} data...".format(var_name)

    #
    # create array containig all data
    # This is going to be huge. Create an array with -9999 (NoData)
    # Then populate the array by reading each flux file
    #

    all_data = zeros([days,len(lat),len(lon)], dtype=float32)
    all_data[:,:,:] = -9999

    c = len(file_list)

    # for each file in list
    for f in file_list:
        # get lat & lon and it's index
        latitude = float(str.split(f, sep="_")[1])
        longitude = float(str.split(f, sep="_")[2])
        lat_id = lat.index(latitude)
        lon_id = lon.index(longitude)

        c = c -1

        infile = open(dirin+'/'+f, "r")
        lixo = infile.readlines()
        infile.close()
        dado = []

        for l in lixo:
            if int(string.split(l, sep="\t")[0]) in range(inidate.year, enddate.year+1):
                dado.append(float(str.split(l, sep="\t")[var]))
            # putting data inside array.
            # Since data has lat & lon fixed uses dimension [:,lat_index,lon_index]

        all_data[:,lat_id,lon_id] = dado

    del dado # del data to free memory for large datasets

    try:

        # open netCDF file for writing
        ncfile = Dataset(outpath+str(var_txt)+'_'+str(start_year)+".nc", "w")

        # set netCDF metadata information
        ncfile.Conventions = "CF-1.6"
        ncfile.title = "VIC hydrologic flux outputs"
        ncfile.source = 'VIC hydrologic model 4.2.d'
        ncfile.history = "Created using the script created by NASA SERVIR. " + dt.date.today().isoformat()
        ncfile.date_created = str(dt.datetime.now())
        ncfile.references = "N/A"
        ncfile.comment = "N/A"

        ncfile.start_date = inidate.isoformat()
        ncfile.end_date = enddate.isoformat()

        #create dimensions
        ncfile.createDimension("longitude", len(lon))
        ncfile.createDimension("latitude", len(lat))
        ncfile.createDimension("time", days)

        #create variables
        latvar = ncfile.createVariable("latitude", float, ("latitude",))
        latvar.long_name = "Latitude"
        latvar.units = "degrees_north"
        latvar[:] = lat

        lonvar = ncfile.createVariable("longitude", float, ("longitude",))
        lonvar.long_name = "Longitude"
        lonvar.units = "degrees_east"
        lonvar[:] = lon

        timevar = ncfile.createVariable("time", int, ("time",))
        timevar.long_name = "Time"
        timevar.units = "days since " + inidate.isoformat()
        timevar.calendar = 'gregorian'
        timevar[:] = range(0, days)

        # save gridded flux data to file
        data_var = ncfile.createVariable(var_txt, float, ("time","latitude","longitude"))
        data_var.long_name = var_name
        data_var.missing_value = -9999.0
        data_var.units = "mm"
        data_var[:] = all_data[:,:,:]

        # close the file
        ncfile.close()

    except IOError:
        raise IOError('Output path is not valid, please fix the path string')

    return

def main():
    # checking user input
    if len(sys.argv) != 3:
        print("Wrong user input")
        print("Convert VIC fluxes files to NetCDF")
        print("usage flux2cdf.py <vic flux dir> <out netcdf dir>")
        print("DIR INPUTS SHOULD CONTAIN TRAILING /")
        sys.exit()

    if sys.argv[1][-1] != "/":
        print("VIC FLUX DIR SHOULD CONTAIN TRAILING /")
        print("fixing it for you...")
        sys.argv[1] = sys.argv[1] + "/"

    print("IMPORTANT: "+sys.argv[1]+" SHOULD CONTAIN ONLY FLUXES FILES!!!")

    flux2nc(sys.argv[1],sys.argv[2])

    return

# Execute the main level program if run as standalone
if __name__ == "__main__":
    main()
I get the following error:
Error:
python3 flux2nc_a.py /mnt/d/Spring_2020/VIC/VIC_Output/ /mnt/d/Spring_2020/VIC/VIC_Output/NetCDF IMPORTANT: /mnt/d/Spring_2020/VIC/VIC_Output/ SHOULD CONTAIN ONLY FLUXES FILES!!! Choose output parameter 1 - SNOW_COVER 2 - SURFACE_TEMPERATURE 3 - Runoff 4 - Base flow 5 - Snow Water Equivalent 6 - EVAPORATION 7 - PRECIPITATION Choose output (1 a 7)>1 Enter start year:1999 End year:2000 Traceback (most recent call last): File "flux2nc_a.py", line 241, in <module> main() File "flux2nc_a.py", line 235, in main flux2nc(sys.argv[1],sys.argv[2]) File "flux2nc_a.py", line 156, in flux2nc if int(str.split(l, sep="\t")[0]) in range(inidate.year, enddate.year+1): ValueError: invalid literal for int() with base 10: '# NRECS: 1096\n'
Reply
#2
Your error traceback line numbers don't match code.
with the code you posted, and the inputs shown in error traceback, you would have received an error on line 100:
inidate = dt.date(start_year,1,1)

proof:
>>> import datetime as dt
>>> import os, string
>>> 
>>> start_year = input("Enter start year:")
Enter start year:1999
>>> start_year
'1999'
>>> inidate = dt.date(start_year,1,1)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: an integer is required (got type str)
>>>
you must first convert satrt_year to int int(start_year)
repost code with error messages that correspond to the code
Reply
#3
After making the changes suggested by Larz60+ including the header information to the code at line 158, I am now getting a different error. I am posting the code again for reference and to show the changes made at lines 119, 121 and 158.
#!/usr/bin/python

#----------------------------------------------------
# Program to convert VIC fluxes files to NetCDF file
# will ask the user wich variable he wants to export
# and also for wich years. Assumes there is data
# for the entire time period, from 1-jan to 31-dec
# SET UP FOR DAILY TIME STEP. FLUX FILE SHOUD NOT
# CONTAIN HOUR RECORD!!
#----------------------------------------------------

#------------------------------------------------
# Writen by Daniel de Castro Victoria
# [email protected] or [email protected]
# Needs python libraries Numeric and Scientific
# 03-dec-2004
#
# Script updated by Kel Markert
# [email protected] or [email protected]
#-------------------------------------------------

# import dependencies
from __future__ import print_function
import sys
import os, string
# handle dates...
import datetime as dt
# NetCDF and Numeric
from netCDF4 import *
from numpy import *

def flux2nc(influxes,outpath,var=None,start_year=None,end_year=None):

    # building file list and sorted lat lon list
    dirin = os.path.dirname(influxes)

    try:
        file_list = os.listdir(dirin)
    except OSError:
        raise OSError('Input flux directory not valid, please fix path')

    lat_t = []
    lon_t = []
    lat = []
    lon = []

    try:
        for f in file_list:
            lat_t.append(float(str.split(f, "_")[1]))
            lon_t.append(float(str.split(f, "_")[2]))
    except ValueError:
        raise ValueError('Input path contains files that are not flux files')

    for i in lat_t:
        if i not in lat:
            lat.append(i)

    for i in lon_t:
        if i not in lon:
            lon.append(i)


    # putting in order. Lat should be from top to botom
    # lon from left to right
    lon.sort()
    lat.sort()
    lat.reverse()

    del(lat_t)
    del(lon_t)

    # if variable is not set, get it from user
    if var == None:

        #determining the parameter to use
        print("Choose output parameter")
        print("1 - SNOW_COVER")
        print("2 - SURFACE_TEMPERATURE")
        print("3 - Runoff")
        print("4 - Base flow")
        print("5 - Snow Water Equivalent")
        print("6 - EVAPORATION")
        print("7 - PRECIPITATION")
        varini = int (input('Choose output (1 a 7)>'))

        #getting the collumn right
        if int (varini) < 7:
            var = varini + 2
        elif varini == 6:        #more than one soil layer...
            camada = input('which soil layer?>')
            var = varini + 2 + camada

    #set name of out_file. Named after parameter choice
    if var == 3:
        var_txt = "SCA"
        var_name = "SNOW_COVER"
    elif var == 4:
        var_txt = "surf_temp"
        var_name = "SURFACE_TEMPERATURE"
    elif var == 5:
        var_txt = "Runoff"
        var_name = "Runoff"
    elif var == 6:
        var_txt = "base"
        var_name = "Baseflow"
    elif var == 7:
        var_txt = "swe"
        var_name = "Snow Water Equivalent"
    elif var == 8:
        var_txt = "evap"
        var_name = "EVAPORATION"
    else:
        var_txt = "prec"+str(camada)
        var_name = "PRECIPITATION".format(camada)

    # if the date information is not set get it from user
    if start_year == None:
        # for what date?
        start_year = int (input("Enter start year:"))
    if end_year == None:
        end_year = int (input("End year:"))

    # set date information in datetime object
    inidate = dt.date(start_year,1,1)
    enddate = dt.date(end_year,12,31)

    # calculate number of days in time series
    days = enddate.toordinal() - inidate.toordinal()+1

    #print "Gridding {0} data...".format(var_name)

    #
    # create array containig all data
    # This is going to be huge. Create an array with -9999 (NoData)
    # Then populate the array by reading each flux file
    #

    all_data = zeros([days,len(lat),len(lon)], dtype=float32)
    all_data[:,:,:] = -9999

    c = len(file_list)

    # for each file in list
    for f in file_list:
        # get lat & lon and it's index
        latitude = float(str.split(f, sep="_")[1])
        longitude = float(str.split(f, sep="_")[2])
        lat_id = lat.index(latitude)
        lon_id = lon.index(longitude)

        c = c -1

        infile = open(dirin+'/'+f, "r")
        lixo = infile.readlines()
        infile.close()
        dado = []

        for l in lixo:
            if l.startswith('#'):
                print(l)
                continue
            # putting data inside array.
            # Since data has lat & lon fixed uses dimension [:,lat_index,lon_index]

        all_data[:,lat_id,lon_id] = dado

    del dado # del data to free memory for large datasets

    try:

        # open netCDF file for writing
        ncfile = Dataset(outpath+str(var_txt)+'_'+str(start_year)+".nc", "w")

        # set netCDF metadata information
        ncfile.Conventions = "CF-1.6"
        ncfile.title = "VIC hydrologic flux outputs"
        ncfile.source = 'VIC hydrologic model 4.2.d'
        ncfile.history = "Created using the script created by NASA SERVIR. " + dt.date.today().isoformat()
        ncfile.date_created = str(dt.datetime.now())
        ncfile.references = "N/A"
        ncfile.comment = "N/A"

        ncfile.start_date = inidate.isoformat()
        ncfile.end_date = enddate.isoformat()

        #create dimensions
        ncfile.createDimension("longitude", len(lon))
        ncfile.createDimension("latitude", len(lat))
        ncfile.createDimension("time", days)

        #create variables
        latvar = ncfile.createVariable("latitude", float, ("latitude",))
        latvar.long_name = "Latitude"
        latvar.units = "degrees_north"
        latvar[:] = lat

        lonvar = ncfile.createVariable("longitude", float, ("longitude",))
        lonvar.long_name = "Longitude"
        lonvar.units = "degrees_east"
        lonvar[:] = lon

        timevar = ncfile.createVariable("time", int, ("time",))
        timevar.long_name = "Time"
        timevar.units = "days since " + inidate.isoformat()
        timevar.calendar = 'gregorian'
        timevar[:] = range(0, days)

        # save gridded flux data to file
        data_var = ncfile.createVariable(var_txt, float, ("time","latitude","longitude"))
        data_var.long_name = var_name
        data_var.missing_value = -9999.0
        data_var.units = "mm"
        data_var[:] = all_data[:,:,:]

        # close the file
        ncfile.close()

    except IOError:
        raise IOError('Output path is not valid, please fix the path string')

    return

def main():
    # checking user input
    if len(sys.argv) != 3:
        print("Wrong user input")
        print("Convert VIC fluxes files to NetCDF")
        print("usage flux2cdf.py <vic flux dir> <out netcdf dir>")
        print("DIR INPUTS SHOULD CONTAIN TRAILING /")
        sys.exit()

    if sys.argv[1][-1] != "/":
        print("VIC FLUX DIR SHOULD CONTAIN TRAILING /")
        print("fixing it for you...")
        sys.argv[1] = sys.argv[1] + "/"

    print("IMPORTANT: "+sys.argv[1]+" SHOULD CONTAIN ONLY FLUXES FILES!!!")

    flux2nc(sys.argv[1],sys.argv[2])

    return

# Execute the main level program if run as standalone
if __name__ == "__main__":
    main()
Error:
# DT: 24 # STARTDATE: 1999-01-01 00:00:00 # ALMA_OUTPUT: 0 # NVARS: 10 # YEAR MONTH DAY OUT_SNOW_COVER OUT_SURF_TEMP OUT_RUNOFF OUT_BASEFLOW OUT_SWE OUT_EVAP OUT_PREC Traceback (most recent call last): File "flux2nc_b.py", line 245, in <module> main() File "flux2nc_b.py", line 239, in main flux2nc(sys.argv[1],sys.argv[2]) File "flux2nc_b.py", line 165, in flux2nc all_data[:,lat_id,lon_id] = dado ValueError: cannot copy sequence with size 0 to array axis with dimension 731
Reply
#4
This statement: all_data[:,lat_id,lon_id] = dado
doesn't make sense. you create a list (all_data) then immediately over write it with dado
If you want to merge two lists, use '+' list3 = list1 + list2

on line 164 add:
        print(f"dado: {dado} -- all_data: {all_data}")
Reply
#5
First of all thank you Laraz60+ for taking the time to help me with this. So I made the following changes to the code at line 164 as you said:
for l in lixo:
            if l.startswith('#'):
                print(l)
                continue
            # putting data inside array.
            # Since data has lat & lon fixed uses dimension [:,lat_index,lon_index]
        print(f"dado: {dado} -- all_data: {all_data}")
        all_data[:,lat_id,lon_id] = dado

    del dado # del data to free memory for large datasets
Now, I am getting a different output:
Output:
# DT: 24 # STARTDATE: 1999-01-01 00:00:00 # ALMA_OUTPUT: 0 # NVARS: 10 # YEAR MONTH DAY OUT_SNOW_COVER OUT_SURF_TEMP OUT_RUNOFF OUT_BASEFLOW OUT_SWE OUT_EVAP OUT_PREC dado: [] -- all_data: [[[-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] ..., [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.]] [[-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] ..., [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.]] [[-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] ..., [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.]] ..., [[-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] ..., [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.]] [[-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] ..., [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.]] [[-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] ..., [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.] [-9999. -9999. -9999. ..., -9999. -9999. -9999.]]]
The new error is as follows:
Error:
Traceback (most recent call last): File "flux2nc_edit.py", line 245, in <module> main() File "flux2nc_edit.py", line 239, in main flux2nc(sys.argv[1],sys.argv[2]) File "flux2nc_edit.py", line 165, in flux2nc all_data[:,lat_id,lon_id] = dado ValueError: cannot copy sequence with size 0 to array axis with dimension 731
I am also putting a sample of output text file used by this code for reference and better understanding of the data. So, the code picks up 258 of such text files and will convert them into a single NetCDF file based upon the variable selected. And I believe, the code will name the created NetCDF file according to the variable. I don't know how to display the text file properly, so I used the insert python option.
# NRECS: 1096
# DT: 24
# STARTDATE: 1999-01-01 00:00:00
# ALMA_OUTPUT: 0
# NVARS: 10
# YEAR	MONTH	DAY	OUT_SNOW_COVER	 OUT_SURF_TEMP	 OUT_RUNOFF	 OUT_BASEFLOW	 OUT_SWE	 OUT_EVAP	 OUT_PREC
1999	01	01	0.0000	 -0.6910	 0.0000	 1.7175	 0.0000	 1.2187	 1.2250
1999	01	02	0.0000	 -8.1983	 0.0000	 1.7042	 0.0000	 0.0132	 0.0000
1999	01	03	0.0000	 -13.7701	 0.0000	 1.6907	 0.0000	 0.0076	 0.0000
1999	01	04	1.0000	 -11.0906	 0.0000	 1.6772	 6.1095	 0.4404	 7.4750
1999	01	05	1.0000	 -7.4365	 0.0000	 1.6637	 9.7234	 0.6585	 4.3000
1999	01	06	1.0000	 -6.4047	 0.0000	 1.6501	 12.1842	 0.5672	 3.0000
1999	01	07	1.0000	 -9.1578	 0.0000	 1.6364	 12.0282	 0.5211	 0.0000
Reply
#6
your indentation is all messed up.
Things look like they are getting worse.
Reply
#7
(May-15-2020, 02:54 AM)Larz60+ Wrote: your indentation is all messed up.
Things look like they are getting worse.

Any help with this will be highly appreciated.
Thank you,
Reply
#8
I finally worked it using the following code but I had to make some changes to it.
 #!/usr/bin/env python

#----------------------------------------------------
# Program to convert VIC fluxes files to NetCDF file
# will ask the user wich variable he wants to export
# and also for wich years. Assumes there is data
# for the entire time period, from 1-jan to 31-dec
# SET UP FOR DAILY TIME STEP. FLUX FILE SHOUD NOT
# CONTAIN HOUR RECORD!!
#----------------------------------------------------

#------------------------------------------------
# Writen by Daniel de Castro Victoria
# [email protected] or [email protected]
# 03-dec-2004
#
# 13-mar-2018: Code update. Change libraries and treat
# header lines. Changes done by Stuart Smith (smit1770 at purdue dot edu)
#-------------------------------------------------

import os
import sys
# handle dates...
import datetime
# SciPy netCDF and NumPy
from scipy.io.netcdf import *
from numpy import *

# In case flux files contains header lines
# set the variable according to the number of lines
skip_lines = 6

# checking user input
print len(sys.argv)
if len(sys.argv) != 2:
    print "Wrong user input"
    print "Convert VIC fluxes files to NetCDF"
    print "usage flux2cdf.py <vic flux dir>"
    print "VIC FLUX DIR SHOULD CONTAIN TRAILING /"
    sys.exit()

if sys.argv[1][-1] != "/":
    print "VIC FLUX DIR SHOULD CONTAIN TRAILING /"
    print "fixing it for you..."
    sys.argv[1] = sys.argv[1] + "/"

print "IMPORTANT: "+sys.argv[1]+" SHOULD CONTAIN ONLY FLUXES FILES!!!"

# building file list and sorted lat lon list
file_list = os.listdir(sys.argv[1])

lat_t = []
lon_t = []
lat = []
lon = []

for f in file_list:
    lat_t.append(float(f.split("_")[1]))
    lon_t.append(float(f.split("_")[2]))

for i in lat_t:
    if i not in lat:
        lat.append(i)

for i in lon_t:
    if i not in lon:
        lon.append(i)


# putting in order. Lat should be from top to botom
# lon from left to rigth
lon.sort()
lat.sort()
lat.reverse()

del(lat_t)
del(lon_t)

#determining the parameter to use
print "Choose output parameter"
print "1 - Snow_Cover"
print "2 - Surface_Temperature"
print "3 - Runoff"
print "4 - Base flow"
print "5 - SWE"
print "6 - Precipitation"
print "7 - Evaporation"
print "8 - Soil Moisture"

varini = input('Choose output (1 a 8)>')

#getting the collumn right
if int (varini) < 8:
    var = varini + 2
elif varini == 8:        #more than one soil layer...
    camada = input('which soil layer?>')
    var = varini + 1 + camada

#set name of out_file. Named after parameter choice
if var == 3:
    var_txt = "Snow_Cover"
    var_name = "Snow_Cover"
elif var == 4:
    var_txt = "Surf_Temp"
    var_name = "Surface_Temperature"
elif var == 5:
    var_txt = "Runoff"
    var_name = "Runoff"
elif var == 6:
    var_txt = "base"
    var_name = "Baseflow"
elif var == 7:
    var_txt = "SWE"
    var_name = "SWE"
elif var == 8:
    var_txt = "Precipitation"
    var_name = "Precipitation"
elif var == 9:
    var_txt = "Evaporation"
    var_name = "Evaporation"
else:
    var_txt = "soil_"+str(camada)
    var_name = "Soil moisture, layer %i", camada

# for what date?
start_year = input("Enter start year:")
end_year = input("End year:")

inidate = datetime.date(start_year,1,1)
enddate = datetime.date(end_year,12,31)

days = enddate.toordinal() - inidate.toordinal()+1

print "Go grab a coffe, this could take a while..."

#
# create array containig all data
# This is going to be huge. Create an array with -9999 (NoData)
# Then populate the array by reading each flux file
#

all_data = zeros([days,len(lat),len(lon)], float)-9999

c = len(file_list)

# for each file in list
for f in file_list:
    # get lat & lon and it's index
    latitude = float(f.split("_")[1])
    longitude = float(f.split("_")[2])
    lat_id = lat.index(latitude)
    lon_id = lon.index(longitude)

    print "%i files to write." % c
    c = c -1

    infile = open(sys.argv[1]+f, "r")
    # here we skip the number of header lines
    # variable set at the begining of the code
    lixo = infile.readlines()[skip_lines:]
    infile.close()
    dado = []

    for l in lixo:
        if int(l.split("\t")[0]) in range(inidate.year, enddate.year+1):
            dado.append(float(l.split("\t")[var]))
        # putting data inside array.
        # Since data has lat & lon fixed uses dimension [:,lat_index,lon_index]

    all_data[:,lat_id,lon_id] = dado

#
# writing NetCDF
#

ncfile = netcdf_file(var_txt+".nc", "w")

ncfile.Conventions = "COARDS"
ncfile.history = "Created using flux2cdf.py. " + datetime.date.today().isoformat()
ncfile.production = "VIC output"

ncfile.start_date = inidate.isoformat()
ncfile.end_date = enddate.isoformat()

#create dimensions
ncfile.createDimension("X", len(lon))
ncfile.createDimension("Y", len(lat))
ncfile.createDimension("T", days)

#create variables
latvar = ncfile.createVariable("Y", "f4", ("Y",))
latvar.long_name = "Latitude"
latvar.units = "degrees_north"
latvar[:] = lat

lonvar = ncfile.createVariable("X", "f4", ("X",))
lonvar.long_name = "Longitude"
lonvar.units = "degrees_east"
lonvar[:] = lon

timevar = ncfile.createVariable("T", "f4", ("T",))
timevar.long_name = "Time"
timevar.units = "days since " + inidate.isoformat()
timevar[:] = range(0, days)

data_var = ncfile.createVariable(var_txt, "f4", ("T","Y","X"))
data_var.long_name = var_name+" calculated by VIC"
data_var.missing_value = -9999.0
data_var.units = "milimeters"
data_var[:] = all_data

ncfile.close()
Reply
#9
A final suggestion to make your programming easier to write and debug.
Learn how to use functions 'def' and pass attributes to and return values from each.
This will allow to to debug each function as written, before writing more code.

It's the secret to writing great code.
Once you're comfortable with functions, start learning cases.
Reply


Possibly Related Threads…
Thread Author Replies Views Last Post
  ValueError: invalid literal for int() with base 10: omega_elite 5 5,678 Dec-30-2020, 06:11 AM
Last Post: delonbest
  invalid literal for int() with base 10: '' mrsenorchuck 5 5,330 Apr-29-2020, 05:48 AM
Last Post: markfilan
  ValueError: invalid literal for int() with base 10: '\n' srisrinu 9 5,581 Apr-13-2020, 01:30 PM
Last Post: ibreeden
  zlib decompress error: invalid code lengths set / invalid block type DreamingInsanity 0 6,741 Mar-29-2020, 12:44 PM
Last Post: DreamingInsanity
  input-ValueError: invalid literal for int() jacklee26 2 2,510 Feb-21-2020, 01:27 PM
Last Post: ndc85430
  ValueError: invalid literal for int() with base 10: '0.5' emmapaw24 2 3,673 Feb-16-2020, 07:24 PM
Last Post: emmapaw24
  ValueError: invalid literal for int() with base 10: '' Jay123 7 7,217 Aug-05-2019, 02:43 PM
Last Post: Jay123
  ValueError: invalid literal for int() with base 10: '' ivinjjunior 6 9,069 Apr-20-2019, 05:37 PM
Last Post: keames
  Problem with "invalid literal for int() with base 10: '' jirkaj4 4 10,385 Jan-23-2018, 06:55 PM
Last Post: jirkaj4
  ValueError: invalid literal for int() with base 10: '[2,4,7,8,19]' rajeev1729 3 60,618 Sep-15-2017, 05:51 PM
Last Post: nilamo

Forum Jump:

User Panel Messages

Announcements
Announcement #1 8/1/2020
Announcement #2 8/2/2020
Announcement #3 8/6/2020