acgc.hysplit API documentation

# acgc.hysplit

Package of functions to read HYSPLIT output

Assumed directory structure for meteorological data. All directories are archived meteorology except for "forecast/" directory.

    metroot/
      forecast/     (All forecasts in subdirectories for each initialization date)
        YYYYMMDD/   (forecasts initialized on YYYYMMDD)
      gdas1/
      gdas0p5/
      gfs0p25/
      hrrr/
      nam12/        (what ARL calls nams)
      nam3/         (pieced forecast of NAM CONUS nest (3 km))

View Source

      1#!/usr/bin/env python3
      2'''Package of functions to read HYSPLIT output
      3
      4Assumed directory structure for meteorological data.
      5All directories are archived meteorology except for "forecast/" directory.
      6```
      7metroot/
      8  forecast/     (All forecasts in subdirectories for each initialization date)
      9    YYYYMMDD/   (forecasts initialized on YYYYMMDD)
     10  gdas1/
     11  gdas0p5/
     12  gfs0p25/
     13  hrrr/
     14  nam12/        (what ARL calls nams)
     15  nam3/         (pieced forecast of NAM CONUS nest (3 km))
     16```
     17'''
     18
     19import os
     20import glob
     21import datetime as dt
     22import warnings
     23import numpy  as np
     24import pandas as pd
     25from . import netcdf as nct
     26
     27METROOT = '/data/MetData/ARL/'
     28'''Default location for ARL met data'''
     29
     30def check_METROOT():
     31    '''Check if METROOT is a valid directory path'''
     32    if not os.path.isdir(METROOT):
     33        warnings.warn('\n'
     34            +'Directory with ARL meteorology data not found. '
     35            +f'METROOT is currently set to {METROOT}.\n'
     36            +'Use set_METROOT(path) to set the correct directory path.'
     37            )
     38
     39def set_METROOT(path='/data/MetData/ARL/'):
     40    '''Set METROOT, the directory for ARL meteorology data
     41
     42    Parameters
     43    ----------
     44    path : str or path
     45        absolute path'''
     46    # Enable access to module variable
     47    global METROOT
     48
     49    # Ensure that path contains a trailing '/'
     50    path = os.path.join(path,'')
     51
     52    # Ensure that path is a valid directory
     53    if not os.path.isdir(path):
     54        raise NotADirectoryError(f'{path} is not a directory. '
     55                                 +'It should be the path for ARL meteorology data' )
     56    # Set the path
     57    METROOT=path
     58
     59def tdump2nc( inFile, outFile, clobber=False, globalAtt=None,
     60             altIsMSL=False, dropOneTime=False, pack=False ):
     61    '''Convert a HYSPLIT tdump file to netCDF
     62    Works with single point or ensemble trajectories
     63
     64    Parameters
     65    ----------
     66    inFile : str
     67        name/path of HYSPLIT tdump file
     68    outFile : str
     69        name/path of netCDF file that will be created
     70    clobber : bool, default=False
     71        determines whether outFile will be overwrite any previous file
     72    globalAtt : dict, default=None
     73        If present, dict keys will be added to outFile as global attributes
     74    altIsMSL : bool, default=False
     75        Determines whether altitude in HYSPLIT tdump file is treated as altitude above sea level
     76        (altIsMSL=True) or altitude above ground (altIsMSL=False). In either case, the netCDF
     77        file will contain both altitude variables.
     78    dropOneTime : bool, default=False
     79        Kludge to address back trajectories that start 1 minute after the hour,
     80        due to CONTROL files created with write_control(... exacttime=False )
     81        set True only for trajectories using this setup.
     82    pack : bool, default=False
     83        NOT IMPLEMENTED
     84        determines whether variables in the netCDF file should be compressed with *lossy*
     85        integer packing.
     86    '''
     87    # Return if the file already exists and not set to clobber
     88    if os.path.exists(outFile) and clobber==False:
     89        return
     90
     91    # Trajectory points
     92    traj = read_tdump( inFile )
     93
     94    # Trajectory numbers; convert to int32
     95    tnums = traj.tnum.unique().astype('int32')
     96
     97    # Number of trajectories (usually 1 or 27)
     98    ntraj = len( tnums )
     99
    100    # Trajectory start time
    101    starttime = traj.time[0]
    102
    103    # Time along trajectory, hours since trajectory start
    104    ttime  = traj.thour.unique().astype('f4')
    105
    106    # Number of times along trajectory
    107    nttime = len( ttime )
    108
    109    # Empty arrays
    110    lat    = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    111    lon    = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    112    alt    = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    113    altTerr= np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    114    p      = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    115    T      = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    116    Q      = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    117    U      = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    118    V      = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    119    precip = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    120    zmix   = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    121    inBL   = np.zeros( (ntraj, nttime), np.int8 )    * -9
    122
    123    # Check if optional variables are present
    124    doP        = ('PRESSURE' in traj.columns)
    125    doTerr     = ('TERR_MSL' in traj.columns)
    126    doBL       = ('MIXDEPTH' in traj.columns)
    127    doT        = ('AIR_TEMP' in traj.columns)
    128    doQ        = ('SPCHUMID' in traj.columns)
    129    doU        = ('UWIND'    in traj.columns)
    130    doV        = ('VWIND'    in traj.columns)
    131    doPrecip   = ('RAINFALL' in traj.columns)
    132
    133    for t in tnums:
    134
    135        # Find entries for this trajectory
    136        idx = traj.tnum==t
    137
    138        # Number of times in this trajectory
    139        nt = np.sum(idx)
    140
    141        if dropOneTime:
    142            # Drop the second time element (at minute 0) to retain one point per hour
    143            # Find entries and Drop the second element
    144            tmpidx = np.where(traj.tnum==t)[0]
    145            idx = [tmpidx[0]]
    146            idx.extend(tmpidx[2:])
    147
    148        # Save the coordinates
    149        lat[t-1,:nt] = traj.lat[idx]
    150        lon[t-1,:nt] = traj.lon[idx]
    151        alt[t-1,:nt] = traj.alt[idx]
    152
    153        # Add optional variables
    154        if doP:
    155            p[t-1,:nt] = traj.PRESSURE[idx]
    156        if doT:
    157            T[t-1,:nt]      = traj.AIR_TEMP[idx]
    158        if doQ:
    159            Q[t-1,:nt]      = traj.SPCHUMID[idx]
    160        if doU:
    161            U[t-1,:nt]      = traj.UWIND[idx]
    162        if doV:
    163            V[t-1,:nt]      = traj.VWIND[idx]
    164        if doPrecip:
    165            precip[t-1,:nt] = traj.RAINFALL[idx]
    166        if doTerr:
    167            altTerr[t-1,:nt]= traj.TERR_MSL[idx]
    168        if doBL:
    169            inBL[t-1,:nt]   = traj.alt[idx] < traj.MIXDEPTH[idx]
    170            zmix[t-1,:nt]   = traj.MIXDEPTH[idx]
    171
    172    if altIsMSL:
    173        altName=     'altMSL'
    174        altLongName= 'altitude above mean sea level'
    175        if doTerr:
    176            alt2Name=     'altAGL'
    177            alt2LongName= 'altitude above ground level'
    178            alt2=         alt-altTerr
    179    else:
    180        altName=     'altAGL'
    181        altLongName= 'altitude above ground level'
    182        if doTerr:
    183            alt2Name=     'altMSL'
    184            alt2LongName= 'altitude above mean sea level'
    185            alt2=         alt+altTerr
    186
    187    # Put output variables into a list
    188    variables = [
    189        {'name':'lat',
    190            'long_name':'latitude of trajectory',
    191            'units':'degrees_north',
    192            'value':np.expand_dims(lat,axis=0),
    193            'fill_value':np.float32(np.nan)},
    194        {'name':'lon',
    195           'long_name':'longitude of trajectory',
    196           'units':'degrees_east',
    197           'value':np.expand_dims(lon, axis=0),
    198           'fill_value':np.float32(np.nan)},
    199        {'name':altName,
    200           'long_name':altLongName,
    201           'units':'m',
    202           'value':np.expand_dims(alt, axis=0),
    203           'fill_value':np.float32(np.nan)} ]
    204
    205    # Add optional variables to output list
    206    if doTerr:
    207        variables.append(
    208           {'name':'altTerr',
    209           'long_name':'altitude of terrain',
    210           'units':'m',
    211           'value':np.expand_dims(altTerr,axis=0),
    212           'fill_value':np.float32(np.nan)} )
    213        variables.append(
    214           {'name':alt2Name,
    215           'long_name':alt2LongName,
    216           'units':'m',
    217           'value':np.expand_dims(alt2,axis=0),
    218           'fill_value':np.float32(np.nan)} )
    219    if doP:
    220        variables.append(
    221            {'name':'p',
    222           'long_name':'pressure',
    223           'units':'hPa',
    224           'value':np.expand_dims(p,axis=0),
    225           'fill_value':np.float32(np.nan)} )
    226    if doT:
    227        variables.append(
    228            {'name':'T',
    229           'long_name':'temperature',
    230           'units':'K',
    231           'value':np.expand_dims(T,axis=0),
    232           'fill_value':np.float32(np.nan)} )
    233    if doQ:
    234        variables.append(
    235            {'name':'q',
    236           'long_name':'specific humidity',
    237           'units':'g/kg',
    238           'value':np.expand_dims(Q,axis=0),
    239           'fill_value':np.float32(np.nan)} )
    240    if doU:
    241        variables.append(
    242            {'name':'U',
    243           'long_name':'eastward wind speed',
    244           'units':'m/s',
    245           'value':np.expand_dims(U,axis=0),
    246           'fill_value':np.float32(np.nan)} )
    247    if doV:
    248        variables.append(
    249            {'name':'V',
    250           'long_name':'northward wind speed',
    251           'units':'m/s',
    252           'value':np.expand_dims(V,axis=0),
    253           'fill_value':np.float32(np.nan)} )
    254    if doPrecip:
    255        variables.append(
    256            {'name':'precipitation',
    257           'long_name':'precipitation',
    258           'units':'mm/hr',
    259           'value':np.expand_dims(precip,axis=0),
    260           'fill_value':np.float32(np.nan)} )
    261    if doBL:
    262        variables.append(
    263            {'name':'inBL',
    264           'long_name':'trajectory in boundary layer flag',
    265           'units':'unitless',
    266           'value':np.expand_dims(inBL,axis=0),
    267           'fill_value':-9} )
    268        variables.append(
    269            {'name':'mixdepth',
    270           'long_name':'boundary layer mixing depth',
    271           'units':'m',
    272           'value':np.expand_dims(zmix,axis=0),
    273           'fill_value':np.float32(np.nan)} )
    274
    275    # Add dimension information to all variables
    276    for v in range(len(variables)):
    277        variables[v]['dim_names'] = ['time','trajnum','trajtime']
    278
    279    # Construct global attributes
    280    # Start with default and add any provided by user input
    281    gAtt = {'Content': 'HYSPLIT trajectory'}
    282    if isinstance(globalAtt, dict):
    283        gAtt.update(globalAtt)
    284
    285    # Create the output file
    286    nct.write_geo_nc( outFile, variables,
    287        xDim={'name':'trajnum',
    288            'long_name':'trajectory number',
    289            'units':'unitless',
    290            'value':tnums},
    291        yDim={'name':'trajtime',
    292            'long_name':'time since trajectory start',
    293            'units':'hours',
    294            'value':ttime},
    295        tDim={'name':'time',
    296            'long_name':'time of trajectory start',
    297            'units':'hours since 2000-01-01 00:00:00',
    298            'calendar':'standard',
    299            'value':np.array([starttime]),
    300            'unlimited':True},
    301        globalAtt=gAtt,
    302        nc4=True, classic=True, clobber=clobber )
    303
    304def read_tdump(file):
    305    '''Read trajectory file output from HYSPLIT
    306
    307    Parameters
    308    ----------
    309    file : str
    310        name of trajectory file to read
    311
    312    Returns
    313    -------
    314    pandas.DataFrame
    315        DataFrame contains columns:
    316        - time : datetime object
    317        - year, month, day, hour, minute : floats, same as time
    318        - lat, lon, alt : trajectory location
    319        - thour : hours since trajectory initialization,
    320            negative for back trajectories
    321        - tnum : trajectory number tnum=1 for single trajectory,
    322            tnum=1-27 for trajectory ensemble
    323        - metnum : index number of met file used at this point in trajectory,
    324            see tdump file for corresponding file paths
    325        - fcasthr : hours since the meteorological dataset was initialized
    326    '''
    327
    328    # Open the file
    329    with open(file,"r",encoding="ascii") as fid:
    330
    331        # First line gives number of met fields
    332        nmet = int(fid.readline().split()[0])
    333
    334        # Skip the met file lines
    335        for i in range(nmet):
    336            next(fid)
    337
    338        # Number of trajectories
    339        ntraj = int(fid.readline().split()[0])
    340
    341        # Skip the next lines
    342        for i in range(ntraj):
    343            next(fid)
    344
    345        # Read the variables that are included
    346        line = fid.readline()
    347        nvar = int(line.split()[0])
    348        vnames = line.split()[1:]
    349
    350        # Read the data
    351        df = pd.read_csv( fid, delim_whitespace=True,
    352                          header=None, index_col=False, na_values=['NaN','********'],
    353                names=['tnum','metnum',
    354                       'year','month','day','hour','minute','fcasthr',
    355                       'thour','lat','lon','alt']+vnames )
    356
    357        # Convert 2-digit year to 4-digits
    358        df.loc[:,'year'] += 2000
    359
    360        # Convert to time
    361        df['time'] = pd.to_datetime( df[['year','month','day','hour','minute']] )
    362
    363        #print(df.iloc[-1,:])
    364        return df
    365
    366def _get_gdas1_filename( time ):
    367    '''Directory and File names for GDAS 1 degree meteorology, for given date'''
    368
    369    # Directory for GDAS 1 degree
    370    dirname = METROOT+'gdas1/'
    371
    372    # Filename template
    373    filetmp = 'gdas1.{mon:s}{yy:%y}.w{week:d}'
    374
    375    # week number in the month
    376    wnum = ((time.day-1) // 7) + 1
    377
    378    # GDAS 1 degree file
    379    filename = dirname + filetmp.format( mon=time.strftime("%b").lower(), yy=time, week=wnum )
    380
    381    return filename
    382
    383def _get_hrrr_filename( time ):
    384    '''Directory and File names for HRRR meteorology, for given date'''
    385
    386    # Directory
    387    dirname = METROOT+'hrrr/'
    388
    389    # There are four files per day containing these hours
    390    hourstrings = ['00-05','06-11','12-17','18-23']
    391
    392    filenames = [ '{:s}/{:%Y%m%d}_{:s}_hrrr'.format( dirname, time, hstr )
    393                  for hstr in hourstrings ]
    394    dirnames = [ dirname for i in range(4) ]
    395
    396    # Return
    397    return filenames
    398
    399def _get_met_filename( metmodel, time ):
    400    '''Directory and file names for given meteorology and date
    401
    402    Parameters
    403    ----------
    404    metmodel : str
    405        met model wanted: gdas1, gdas0p5, gfs0p25, nam12, nam3, hrrr
    406    time : pandas.Timestamp, datetime.date, datetime.datetime
    407        day of desired meteorology
    408
    409    Returns
    410    -------
    411    filename : str or list
    412        name of met file or files containing met data for input date
    413    '''
    414
    415    # Ensure that time is a datetime object
    416    if not isinstance( time, dt.date) :
    417        raise TypeError( "_get_met_filename: time must be a datetime object" )
    418
    419    # Directory and filename template
    420    doFormat=True
    421    if metmodel == 'gdas1':
    422        # Special treatment
    423        filename = _get_gdas1_filename( time )
    424    elif metmodel == 'gdas0p5':
    425        dirname  = METROOT+'gdas0p5/'
    426        filename = dirname+'{date:%Y%m%d}_gdas0p5'
    427    elif metmodel == 'gfs0p25':
    428        dirname  = METROOT+'gfs0p25/'
    429        filename = dirname+'{date:%Y%m%d}_gfs0p25'
    430    elif metmodel == 'nam3':
    431        dirname  = METROOT+'nam3/'
    432        filename = dirname+'{date:%Y%m%d}_hysplit.namsa.CONUS'
    433    elif metmodel == 'nam12':
    434        dirname  = METROOT+'nam12/'
    435        filename = dirname+'{date:%Y%m%d}_hysplit.t00z.namsa'
    436    elif metmodel == 'hrrr':
    437        # Special treatment
    438        filename = _get_hrrr_filename( time )
    439        doFormat=False
    440    else:
    441        raise NotImplementedError(
    442           f"_get_met_filename: {metmodel} unrecognized" )
    443
    444    # Build the filename
    445    if doFormat:
    446        filename = filename.format( date=time )
    447
    448    return filename
    449
    450def _get_archive_filelist( metmodels, time, useAll=True ):
    451    '''Get a list of met directories and files
    452    When there are two met version provided, the first result will be used
    453
    454
    455    Parameters
    456    ----------
    457    metmodels : list or str
    458        met models wanted: gdas1, gdas0p5, gfs0p25, nam12, nam3, hrrr
    459        commonly provide a list with both regional and global models
    460        e.g. ['hrrr','gfs0p25']
    461    time : pandas.Timestamp, datetime.date, datetime.datetime
    462        day of desired meteorology
    463    useAll : bool, default=True
    464        if True, then return files for all metmodels found
    465        if False, then return files only for the first metmodel found
    466
    467    Returns
    468    -------
    469    filename : list
    470        names of files containing met data for input date
    471    '''
    472
    473    if isinstance( metmodels, str ):
    474
    475        # If metmodels is a single string, then get value from appropriate function
    476        filename = _get_met_filename( metmodels, time )
    477
    478        # Convert to list, if isn't already
    479        if isinstance(filename, str):
    480            filename = [filename]
    481
    482    elif isinstance( metmodels, list ):
    483
    484        # If metmodels is a list, then get files for each met version in list
    485
    486        filename = []
    487
    488        # Loop over all the metmodelss
    489        # Use the first one with a file present
    490        for met in metmodels:
    491
    492            # Find filename for this met version
    493            f = _get_met_filename( met, time )
    494
    495            if useAll:
    496
    497                # Ensure that directory and file are lists so that we can use extend below
    498                if isinstance(f, str):
    499                    f = [f]
    500                elif isinstance(f, list):
    501                    pass
    502                else:
    503                    raise NotImplementedError(
    504                        'Variable expected to be string or list but is actually ',type(f) )
    505
    506                # Append to the list so that all can be used
    507                filename.extend(f)
    508
    509            else:
    510                # Use just the first met file that exists
    511                filename = f
    512                break
    513                # If the file exists, use this and exit;
    514                # otherwise keep looking
    515                #if isinstance( f, str ):
    516                #    if ( os.path.isfile( d+f ) ):
    517                #        dirname  = d
    518                #        filename = f
    519                #        break
    520                #elif isinstance( f, list ):
    521                #    if ( os.path.isfile( d[0]+f[0] ) ):
    522                #        dirname  = d
    523                #        filename = f
    524                #        break
    525
    526        # Raise an error if
    527        if filename is []:
    528            raise FileNotFoundError(
    529                "_get_archive_filename: no files found for " + ','.join(metmodels) )
    530
    531    else:
    532        raise TypeError( "_get_archive_filelist: metmodels must be a string or list" )
    533
    534    return filename
    535
    536def _get_hybrid_filelist( metmodels, time ):
    537    '''Get list of met files combining archive and forecast
    538
    539    Starting 1 day before 'time' and extending 10 days ahead, find all archived analysis
    540    files, then add forecast files
    541
    542    Parameters
    543    ----------
    544    metmodels : str
    545        IGNORED
    546        Current implementation uses nam3 for past meteorology and namsfCONUS for forecast
    547    time : datetime.datetime or pandas.Timestamp
    548        date of interest, should be near present.
    549        If farther in past, then use archived met. Distance future won't have any forecst available.
    550
    551    Returns
    552    -------
    553    metfiles : list
    554        file path names that are found
    555    '''
    556
    557    print( "Hybrid Archive/Forecast meteorology using NAM3 CONUS nest" )
    558
    559    archivemet = 'nam3'
    560    forecastmet = 'namsfCONUS'
    561
    562    # List of met directories and met files that will be used
    563    metfiles = []
    564
    565    # Loop over 10 days, because that's probably enough for FIREX-AQ
    566    for d in range(-1,10):
    567
    568        # date of met data
    569        metdate = time.date() + pd.Timedelta( d, "D" )
    570
    571        #dirname, filename = _get_archive_filelist( metmodel, metdate )
    572        filename = _get_met_filename( archivemet, metdate )
    573
    574        # Check if the file exists
    575        if os.path.isfile( filename ):
    576
    577            # Add the file, if it isn't already in the list
    578            if filename not in metfiles:
    579                metfiles.append( filename )
    580
    581        else:
    582
    583            # We need to get forecast meteorology
    584
    585            # Loop over forecast cycles
    586            for hr in [0,6,12,18]:
    587
    588                cy =  dt.datetime.combine( metdate, dt.time( hour=hr ) )
    589
    590                try:
    591                    # If this works, then the file exists
    592                    f = _get_forecast_filename( forecastmet, cy, partial=True )
    593                    # Add the first 6-hr forecast period
    594                    metfiles.append( f[0] )
    595                    # If we have a full forecast cycle, save it
    596                    if len(f)==8:
    597                        flast = f
    598                except FileNotFoundError:
    599                    # We have run out of forecast meteorology,
    600                    # So add the remainder of the prior forecast cycle and we're done
    601                    metfiles.extend( flast[1:] )
    602
    603                    return metfiles
    604
    605    return metfiles
    606
    607def _get_forecast_template( metmodel ):
    608    '''Get filename template for forecast meteorology
    609
    610    Parameters
    611    ----------
    612    metmodel : str
    613        name of forecast model: namsfCONUS, namf, gfs0p25, hrrr
    614
    615    Returns
    616    -------
    617    template : str
    618        template filename with {:%H} field for initialization time and ?? for forecast hour
    619    nexpected : int
    620        number of forecast files expected for each forecast initialization
    621    '''
    622
    623    if metmodel == 'namsfCONUS':
    624        # Filename template
    625        filetemplate = 'hysplit.t{:%H}z.namsf??.CONUS'
    626        nexpected = 8
    627    else:
    628        filetemplate = 'hysplit.t{:%H}z.'+metmodel
    629        nexpected = 1
    630
    631    return filetemplate, nexpected
    632
    633def _get_forecast_filename( metmodel, cycle, partial=False ):
    634    '''Find files for a particular met model and forecast cycle
    635
    636    Parameters
    637    ----------
    638    metmodel : str
    639        name of forecast model: namsfCONUS, namf, gfs0p25, hrrr
    640    cycle : datetime.datetime or pandas.Timestamp
    641        forecast initialization time UTC (date and hour)
    642    partial : bool, default=True
    643        With partial=True, function will raise error if some forecast files are missing
    644        With partial=False, function will return all forecast files that are found
    645
    646    Returns
    647    -------
    648    filenames : list
    649        file paths to all files that are found
    650    '''
    651
    652    dirname  = METROOT + f'forecast/{cycle:%Y%m%d}/'
    653    filename, nexpected  = _get_forecast_template( metmodel )
    654
    655    # Filename for this cycle
    656    filename = filename.format(cycle)
    657
    658    # Find all the files that match these criteria
    659    files = glob.glob( dirname + filename )
    660
    661    # Check if we found the expected number of files
    662    if (len(files) == nexpected) or (partial and len(files) >=1):
    663
    664        # When we find then, sort and combine into one list
    665        filenames = sorted( files )
    666
    667        # Return
    668        return filenames
    669
    670    # Raise an error if no forecasts are found
    671    raise FileNotFoundError('ARL forecast meteorology found' )
    672
    673def _get_forecast_filename_latest( metmodel ):
    674    '''Find files for the latest available forecast cycle for the requested met version.
    675
    676    Requires a complete set of forecast files for a cycle
    677
    678    Parameter
    679    ---------
    680    metmodel : str
    681        name of forecast model: namsfCONUS, namf, gfs0p25, hrrr
    682
    683    Returns
    684    -------
    685    filenames : list
    686        file paths to all files that are found
    687    '''
    688
    689    # Filename template for forecast files
    690    filetemplate, nexpected = _get_forecast_template( metmodel )
    691
    692    # Find all of the forecast directories, most recent first
    693    dirs = [item for item
    694            in sorted( glob.glob( METROOT+'forecast/????????' ), reverse=True )
    695            if os.path.isdir(item) ]
    696
    697    for d in dirs:
    698
    699        # Loop backwards over the forecast cycles
    700        for hh in [18,12,6,0]:
    701
    702            # Check if the forecast files exist
    703            files = glob.glob(d+'/'+filetemplate.format(dt.time(hh)))
    704            #'hysplit.t{:02d}z.{:s}'.format(hh,metsearch))
    705
    706            # Check if we found the expected number of files
    707            if len(files) == nexpected:
    708
    709                # When we find then, sort and combine into one list
    710                filenames = sorted( files )
    711
    712                # Return
    713                return filenames
    714
    715    # Raise an error if no forecasts are found
    716    raise FileNotFoundError('No ARL forecast meteorology found' )
    717
    718def _get_forecast_filelist( metmodels=None, cycle=None ):
    719    '''Get list of filenames for forecast cycle
    720
    721    This function calls itself recursively
    722
    723    Parameters
    724    ----------
    725    metmodels : str or list, default=['namsfCONUS','namf']
    726        name of forecast model: namsfCONUS, namf, gfs0p25, hrrr
    727    cycle : datetime.datetime or pandas.Timestamp or None
    728        forecast initialization time UTC (date and hour)
    729        if cycle=None, then will use the latest available forecast cycle
    730
    731    Returns
    732    -------
    733    filenames : list
    734        file paths to all files that are found
    735    '''
    736
    737    if metmodels is None:
    738        metmodels = ['namsfCONUS','namf']
    739
    740    # If metmodel is a single string, then get value from appropriate function
    741    if isinstance( metmodels, str ):
    742
    743        if cycle is None:
    744            filenames = _get_forecast_filename_latest( metmodels )
    745        else:
    746            filenames = _get_forecast_filename( metmodels, cycle )
    747
    748    else:
    749
    750        filenames = []
    751
    752        # Loop over the list of met types, combine them all
    753        for met in metmodels:
    754
    755            # Get directory and file names for one version
    756            f = _get_forecast_filelist( met, cycle )
    757
    758            # Combine them into one list
    759            filenames = filenames + f
    760
    761    return filenames
    762
    763def find_arl_metfiles( start_time, ndays, back=False, metmodels=None,
    764                       forecast=False, forecastcycle=None, hybrid=False ):
    765    '''Find ARL meteorology files for specified dates, models, forecast and archive
    766
    767    Files will be located for start_time and extending ndays forward or backward
    768
    769    Parameters
    770    ----------
    771    start_time : datetime.datetime or pandas.Timestamp
    772        start date and time for finding meteorology data files
    773    ndays : int
    774        number of days of files to retrieve
    775    back : bool, default=False
    776        specifies files should go ndays backward (back=True) from start_time
    777    metmodels : list or str, default=['gdas0p5','gdas1']
    778        meteorological models that will be used, in order of decreasing resolution and priority
    779    forecast : bool, default=False
    780        set forecast=True to use forecast meteorology for trajectory computation
    781        set forecast=False to use archived (past) meteorology for trajectory computation
    782        Note: hybrid=True will supercede forecast=True
    783    forecastcycle : datetime.datetime, pandas.Timestamp, or None, default=None
    784        if forecast=True, this sets the forecast initialization cycle that will be used
    785        set forecastcycle=None to use the latest available cycle for which files are found
    786        if forecast=False, this parameter has no effect
    787    hybrid : bool, default=False
    788        set hybrid=True for trajectories that use a combination of past archive and
    789        forecast meteorlogy. This supercedes forecast=True
    790
    791    Returns
    792    -------
    793    metfiles : list
    794        path to meteorology files meeting the criteria
    795    '''
    796
    797    check_METROOT()
    798
    799    if metmodels is None:
    800        metmodels = ['gdas0p5','gdas1']
    801
    802    # Find the meteorology files
    803    if hybrid is True:
    804
    805        # "Hybrid" combines past (archived) and future (forecast) meteorology
    806        if back:
    807            raise NotImplementedError( "Combined Analysis/Forecast meteorology "
    808                                   + "not supported for back trajectories" )
    809        metfiles = _get_hybrid_filelist( metmodels, start_time )
    810
    811    elif forecast is True:
    812        # Get the forecast meteorology
    813        metfiles = _get_forecast_filelist( metmodels, forecastcycle )
    814
    815        # Check if the forecast meteorology covers the entire trajectory duration
    816    else:
    817        # Use archived (past) meteorology
    818
    819        # Relative to trajectory start day,
    820        # we need meteorology for days d0 through d1
    821        if back:
    822            d0 = -ndays+1
    823            d1 = 1
    824            # For trajectories that start 23-0Z (nam) or 21-0Z (gfs0p25),
    825            # also need the next day to bracket first time step
    826            if   (('nam3'    in metmodels) and (start_time.hour==23)) \
    827              or (('gfs0p25' in metmodels) and (start_time.hour>=21)):
    828                d1 = 2
    829        else:
    830            d0 = 0
    831            d1 = ndays
    832            # For trajectories that start 00-01Z (nam) or 00-03Z (gfs0p25),
    833            # also need the prior day to bracket first time step
    834            if start_time.hour==0:
    835                d0 = -1
    836
    837        # Debug output
    838        # print('Initial Date Range',d0,d1)
    839        # datelist = np.unique( ( time +
    840        #   pd.TimedeltaIndex( np.sign(trajhours) * np.arange(0,np.abs(trajhours)+1),"H" ) ).date )
    841        # print(datelist)
    842
    843        # List of met directories and met files that will be used
    844        metfiles = []
    845        for d in range(d0,d1):
    846
    847            # date of met data
    848            metdate = start_time.date() + pd.Timedelta( d, "D" )
    849
    850            # Met data for a single day
    851            filename = _get_archive_filelist( metmodels, metdate )
    852
    853            # Add the files to the list
    854            metfiles.extend( filename )
    855        # Keep only the unique files
    856        metfiles = np.unique(metfiles)
    857
    858    if len(metfiles) <= 0:
    859        raise ValueError( 'Meteorology files not found for ' + str( metmodels ) )
    860
    861    return metfiles
    862
    863def write_control( time, lat, lon, alt, trajhours,
    864                   fname='CONTROL.000', clobber=False,
    865                   maxheight=15000., outdir='./', tfile='tdump',
    866                   metfiles=None, exacttime=True, **kwargs ):
    867    '''Write HYSPLIT control file for trajectory starting at designated time and coordinates
    868
    869    Parameters
    870    ----------
    871    time : datetime.datetime or pandas.Timestamp
    872        trajectory initialization time
    873    lat, lon : float or list
    874        trajectory initialization latitude and longitude in degrees
    875    alt : float or list
    876        trajectory initialization altitude in meters
    877        The setup.cfg determines whether this is above ground or above mean sea level
    878    trajhours : int
    879        desired trajectory duration in hours. Use negative for back trajectories
    880    fname : str, default='CONTROL.000'
    881        path and name for the file that will be written
    882    clobber : bool, default=False
    883        if clobber=True, then fname will be overwritten
    884    maxheight : float, default=15000
    885        terminate trajectories that exceed maxheight altitude in meters
    886    outdir : str, default='./'
    887        directory path where HYSPLIT output will be written
    888    tfile : str, default='tdump'
    889        name of the trajectory file that HYSPLIT will write
    890    metfiles : list or str, default=None
    891        paths to ARL meteorology files needed for the trajectory computation
    892        If metfiles=None, then find_arl_metfiles will be used to locate necessary files
    893    exacttime : bool, default=True
    894        It is not recommended to change this default, but keyword is retained for backward
    895        compatibility with some scripts. Setting exacttime=False will shift the actual
    896        start time of trajectories that begin at 00:00 UTC to 00:01 UTC to avoid reading
    897        an additional day of meteorological data.
    898    **kwargs
    899        kwargs will be passed to find_arl_metfiles to locate ARL meteorlogy files if metfiles=None
    900        These keywords should include metmodels and possibly forecast, forecastcycle, or hybrid
    901        See find_arl_metfiles for definitions of these parameters
    902    '''
    903
    904    check_METROOT()
    905
    906    if os.path.isfile( fname ) and (clobber is False):
    907        raise OSError( f'File exists. Set clobber=True to overwrite: {fname:s}' )
    908
    909    # Ensure that lat, lon, and alt are lists
    910    try:
    911        nlat = len( lat )
    912    except TypeError:
    913        lat = [ lat ]
    914        nlat = len( lat )
    915    try:
    916        nlon = len( lon )
    917    except TypeError:
    918        lon = [ lon ]
    919        nlon = len( lon )
    920    try:
    921        nalt = len( alt )
    922    except TypeError:
    923        alt = [ alt ]
    924        nalt = len( alt )
    925
    926    # Should add some type checking to ensure that lon, lat, alt
    927    # have conformable lengths
    928    if ( (nlat != nlon) or (nlat != nalt) ):
    929        raise ValueError( "lon, lat, alt must be conformable" )
    930
    931    # Number of days of met data
    932    ndays = int( np.ceil( np.abs( trajhours ) / 24 ) + 1 )
    933
    934    # Get met file names, if not provided
    935    if (metfiles is None) or (len(metfiles)==0):
    936        metfiles = find_arl_metfiles( time, ndays, back=trajhours<0, **kwargs )
    937
    938    # Number of met files
    939    nmet = len( metfiles )
    940
    941    # Runs will fail if the initial time is not bracketed by met data.
    942    # When met resolution changes on the first day, this condition may not be met.
    943    # Unless exacttime==True, the starting time will be shifted 1 minute to 00:01
    944    # to avoid reading in an entire extra day of met data.
    945    if (time.hour==0 and time.minute == 0 and not exacttime):
    946        time = time + pd.Timedelta( 1, "m" )
    947
    948    # Start date-time, formatted as YY MM DD HH {mm}
    949    startdate = time.strftime( "%y %m %d %H %M" )
    950
    951    # Write the CONTROL file
    952    with open( fname, 'w', encoding='ascii' ) as f:
    953
    954        f.write( startdate+'\n' )
    955        f.write( f'{nlat:d}\n' )
    956        for i in range(nlat):
    957            f.write( f'{lat[i]:<10.4f} {lon[i]:<10.4f} {alt[i]:<10.4f}\n' )
    958        f.write( f'{trajhours:d}\n' )
    959        f.write( "0\n" )
    960        f.write( f"{maxheight:<10.1f}\n" )
    961        f.write( f'{nmet:d}\n' )
    962        for file in metfiles:
    963            f.write( os.path.dirname(  file ) + '/\n' )
    964            f.write( os.path.basename( file ) + '\n'  )
    965        f.write( f'{outdir:s}\n' )
    966        f.write( f'{tfile:s}\n' )
    967
    968# Check METROOT path at import
    969check_METROOT()

METROOT = '/data/MetData/ARL/'

Default location for ARL met data

def check_METROOT(): View Source

    31def check_METROOT():
    32    '''Check if METROOT is a valid directory path'''
    33    if not os.path.isdir(METROOT):
    34        warnings.warn('\n'
    35            +'Directory with ARL meteorology data not found. '
    36            +f'METROOT is currently set to {METROOT}.\n'
    37            +'Use set_METROOT(path) to set the correct directory path.'
    38            )

Check if METROOT is a valid directory path

def set_METROOT(path='/data/MetData/ARL/'): View Source

    40def set_METROOT(path='/data/MetData/ARL/'):
    41    '''Set METROOT, the directory for ARL meteorology data
    42
    43    Parameters
    44    ----------
    45    path : str or path
    46        absolute path'''
    47    # Enable access to module variable
    48    global METROOT
    49
    50    # Ensure that path contains a trailing '/'
    51    path = os.path.join(path,'')
    52
    53    # Ensure that path is a valid directory
    54    if not os.path.isdir(path):
    55        raise NotADirectoryError(f'{path} is not a directory. '
    56                                 +'It should be the path for ARL meteorology data' )
    57    # Set the path
    58    METROOT=path

Set METROOT, the directory for ARL meteorology data

###### Parameters

- path (str or path): absolute path

def tdump2nc( inFile, outFile, clobber=False, globalAtt=None, altIsMSL=False, dropOneTime=False, pack=False): View Source

     60def tdump2nc( inFile, outFile, clobber=False, globalAtt=None,
     61             altIsMSL=False, dropOneTime=False, pack=False ):
     62    '''Convert a HYSPLIT tdump file to netCDF
     63    Works with single point or ensemble trajectories
     64
     65    Parameters
     66    ----------
     67    inFile : str
     68        name/path of HYSPLIT tdump file
     69    outFile : str
     70        name/path of netCDF file that will be created
     71    clobber : bool, default=False
     72        determines whether outFile will be overwrite any previous file
     73    globalAtt : dict, default=None
     74        If present, dict keys will be added to outFile as global attributes
     75    altIsMSL : bool, default=False
     76        Determines whether altitude in HYSPLIT tdump file is treated as altitude above sea level
     77        (altIsMSL=True) or altitude above ground (altIsMSL=False). In either case, the netCDF
     78        file will contain both altitude variables.
     79    dropOneTime : bool, default=False
     80        Kludge to address back trajectories that start 1 minute after the hour,
     81        due to CONTROL files created with write_control(... exacttime=False )
     82        set True only for trajectories using this setup.
     83    pack : bool, default=False
     84        NOT IMPLEMENTED
     85        determines whether variables in the netCDF file should be compressed with *lossy*
     86        integer packing.
     87    '''
     88    # Return if the file already exists and not set to clobber
     89    if os.path.exists(outFile) and clobber==False:
     90        return
     91
     92    # Trajectory points
     93    traj = read_tdump( inFile )
     94
     95    # Trajectory numbers; convert to int32
     96    tnums = traj.tnum.unique().astype('int32')
     97
     98    # Number of trajectories (usually 1 or 27)
     99    ntraj = len( tnums )
    100
    101    # Trajectory start time
    102    starttime = traj.time[0]
    103
    104    # Time along trajectory, hours since trajectory start
    105    ttime  = traj.thour.unique().astype('f4')
    106
    107    # Number of times along trajectory
    108    nttime = len( ttime )
    109
    110    # Empty arrays
    111    lat    = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    112    lon    = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    113    alt    = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    114    altTerr= np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    115    p      = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    116    T      = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    117    Q      = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    118    U      = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    119    V      = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    120    precip = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    121    zmix   = np.zeros( (ntraj, nttime), np.float32 ) * np.nan
    122    inBL   = np.zeros( (ntraj, nttime), np.int8 )    * -9
    123
    124    # Check if optional variables are present
    125    doP        = ('PRESSURE' in traj.columns)
    126    doTerr     = ('TERR_MSL' in traj.columns)
    127    doBL       = ('MIXDEPTH' in traj.columns)
    128    doT        = ('AIR_TEMP' in traj.columns)
    129    doQ        = ('SPCHUMID' in traj.columns)
    130    doU        = ('UWIND'    in traj.columns)
    131    doV        = ('VWIND'    in traj.columns)
    132    doPrecip   = ('RAINFALL' in traj.columns)
    133
    134    for t in tnums:
    135
    136        # Find entries for this trajectory
    137        idx = traj.tnum==t
    138
    139        # Number of times in this trajectory
    140        nt = np.sum(idx)
    141
    142        if dropOneTime:
    143            # Drop the second time element (at minute 0) to retain one point per hour
    144            # Find entries and Drop the second element
    145            tmpidx = np.where(traj.tnum==t)[0]
    146            idx = [tmpidx[0]]
    147            idx.extend(tmpidx[2:])
    148
    149        # Save the coordinates
    150        lat[t-1,:nt] = traj.lat[idx]
    151        lon[t-1,:nt] = traj.lon[idx]
    152        alt[t-1,:nt] = traj.alt[idx]
    153
    154        # Add optional variables
    155        if doP:
    156            p[t-1,:nt] = traj.PRESSURE[idx]
    157        if doT:
    158            T[t-1,:nt]      = traj.AIR_TEMP[idx]
    159        if doQ:
    160            Q[t-1,:nt]      = traj.SPCHUMID[idx]
    161        if doU:
    162            U[t-1,:nt]      = traj.UWIND[idx]
    163        if doV:
    164            V[t-1,:nt]      = traj.VWIND[idx]
    165        if doPrecip:
    166            precip[t-1,:nt] = traj.RAINFALL[idx]
    167        if doTerr:
    168            altTerr[t-1,:nt]= traj.TERR_MSL[idx]
    169        if doBL:
    170            inBL[t-1,:nt]   = traj.alt[idx] < traj.MIXDEPTH[idx]
    171            zmix[t-1,:nt]   = traj.MIXDEPTH[idx]
    172
    173    if altIsMSL:
    174        altName=     'altMSL'
    175        altLongName= 'altitude above mean sea level'
    176        if doTerr:
    177            alt2Name=     'altAGL'
    178            alt2LongName= 'altitude above ground level'
    179            alt2=         alt-altTerr
    180    else:
    181        altName=     'altAGL'
    182        altLongName= 'altitude above ground level'
    183        if doTerr:
    184            alt2Name=     'altMSL'
    185            alt2LongName= 'altitude above mean sea level'
    186            alt2=         alt+altTerr
    187
    188    # Put output variables into a list
    189    variables = [
    190        {'name':'lat',
    191            'long_name':'latitude of trajectory',
    192            'units':'degrees_north',
    193            'value':np.expand_dims(lat,axis=0),
    194            'fill_value':np.float32(np.nan)},
    195        {'name':'lon',
    196           'long_name':'longitude of trajectory',
    197           'units':'degrees_east',
    198           'value':np.expand_dims(lon, axis=0),
    199           'fill_value':np.float32(np.nan)},
    200        {'name':altName,
    201           'long_name':altLongName,
    202           'units':'m',
    203           'value':np.expand_dims(alt, axis=0),
    204           'fill_value':np.float32(np.nan)} ]
    205
    206    # Add optional variables to output list
    207    if doTerr:
    208        variables.append(
    209           {'name':'altTerr',
    210           'long_name':'altitude of terrain',
    211           'units':'m',
    212           'value':np.expand_dims(altTerr,axis=0),
    213           'fill_value':np.float32(np.nan)} )
    214        variables.append(
    215           {'name':alt2Name,
    216           'long_name':alt2LongName,
    217           'units':'m',
    218           'value':np.expand_dims(alt2,axis=0),
    219           'fill_value':np.float32(np.nan)} )
    220    if doP:
    221        variables.append(
    222            {'name':'p',
    223           'long_name':'pressure',
    224           'units':'hPa',
    225           'value':np.expand_dims(p,axis=0),
    226           'fill_value':np.float32(np.nan)} )
    227    if doT:
    228        variables.append(
    229            {'name':'T',
    230           'long_name':'temperature',
    231           'units':'K',
    232           'value':np.expand_dims(T,axis=0),
    233           'fill_value':np.float32(np.nan)} )
    234    if doQ:
    235        variables.append(
    236            {'name':'q',
    237           'long_name':'specific humidity',
    238           'units':'g/kg',
    239           'value':np.expand_dims(Q,axis=0),
    240           'fill_value':np.float32(np.nan)} )
    241    if doU:
    242        variables.append(
    243            {'name':'U',
    244           'long_name':'eastward wind speed',
    245           'units':'m/s',
    246           'value':np.expand_dims(U,axis=0),
    247           'fill_value':np.float32(np.nan)} )
    248    if doV:
    249        variables.append(
    250            {'name':'V',
    251           'long_name':'northward wind speed',
    252           'units':'m/s',
    253           'value':np.expand_dims(V,axis=0),
    254           'fill_value':np.float32(np.nan)} )
    255    if doPrecip:
    256        variables.append(
    257            {'name':'precipitation',
    258           'long_name':'precipitation',
    259           'units':'mm/hr',
    260           'value':np.expand_dims(precip,axis=0),
    261           'fill_value':np.float32(np.nan)} )
    262    if doBL:
    263        variables.append(
    264            {'name':'inBL',
    265           'long_name':'trajectory in boundary layer flag',
    266           'units':'unitless',
    267           'value':np.expand_dims(inBL,axis=0),
    268           'fill_value':-9} )
    269        variables.append(
    270            {'name':'mixdepth',
    271           'long_name':'boundary layer mixing depth',
    272           'units':'m',
    273           'value':np.expand_dims(zmix,axis=0),
    274           'fill_value':np.float32(np.nan)} )
    275
    276    # Add dimension information to all variables
    277    for v in range(len(variables)):
    278        variables[v]['dim_names'] = ['time','trajnum','trajtime']
    279
    280    # Construct global attributes
    281    # Start with default and add any provided by user input
    282    gAtt = {'Content': 'HYSPLIT trajectory'}
    283    if isinstance(globalAtt, dict):
    284        gAtt.update(globalAtt)
    285
    286    # Create the output file
    287    nct.write_geo_nc( outFile, variables,
    288        xDim={'name':'trajnum',
    289            'long_name':'trajectory number',
    290            'units':'unitless',
    291            'value':tnums},
    292        yDim={'name':'trajtime',
    293            'long_name':'time since trajectory start',
    294            'units':'hours',
    295            'value':ttime},
    296        tDim={'name':'time',
    297            'long_name':'time of trajectory start',
    298            'units':'hours since 2000-01-01 00:00:00',
    299            'calendar':'standard',
    300            'value':np.array([starttime]),
    301            'unlimited':True},
    302        globalAtt=gAtt,
    303        nc4=True, classic=True, clobber=clobber )

Convert a HYSPLIT tdump file to netCDF Works with single point or ensemble trajectories

###### Parameters

- inFile (str): name/path of HYSPLIT tdump file
- outFile (str): name/path of netCDF file that will be created
- clobber (bool, default=False): determines whether outFile will be overwrite any previous file
- globalAtt (dict, default=None): If present, dict keys will be added to outFile as global attributes
- altIsMSL (bool, default=False): Determines whether altitude in HYSPLIT tdump file is treated as altitude above sea level (altIsMSL=True) or altitude above ground (altIsMSL=False). In either case, the netCDF file will contain both altitude variables.
- dropOneTime (bool, default=False): Kludge to address back trajectories that start 1 minute after the hour, due to CONTROL files created with write_control(... exacttime=False ) set True only for trajectories using this setup.
- pack (bool, default=False): NOT IMPLEMENTED determines whether variables in the netCDF file should be compressed with lossy integer packing.

def read_tdump(file): View Source

    305def read_tdump(file):
    306    '''Read trajectory file output from HYSPLIT
    307
    308    Parameters
    309    ----------
    310    file : str
    311        name of trajectory file to read
    312
    313    Returns
    314    -------
    315    pandas.DataFrame
    316        DataFrame contains columns:
    317        - time : datetime object
    318        - year, month, day, hour, minute : floats, same as time
    319        - lat, lon, alt : trajectory location
    320        - thour : hours since trajectory initialization,
    321            negative for back trajectories
    322        - tnum : trajectory number tnum=1 for single trajectory,
    323            tnum=1-27 for trajectory ensemble
    324        - metnum : index number of met file used at this point in trajectory,
    325            see tdump file for corresponding file paths
    326        - fcasthr : hours since the meteorological dataset was initialized
    327    '''
    328
    329    # Open the file
    330    with open(file,"r",encoding="ascii") as fid:
    331
    332        # First line gives number of met fields
    333        nmet = int(fid.readline().split()[0])
    334
    335        # Skip the met file lines
    336        for i in range(nmet):
    337            next(fid)
    338
    339        # Number of trajectories
    340        ntraj = int(fid.readline().split()[0])
    341
    342        # Skip the next lines
    343        for i in range(ntraj):
    344            next(fid)
    345
    346        # Read the variables that are included
    347        line = fid.readline()
    348        nvar = int(line.split()[0])
    349        vnames = line.split()[1:]
    350
    351        # Read the data
    352        df = pd.read_csv( fid, delim_whitespace=True,
    353                          header=None, index_col=False, na_values=['NaN','********'],
    354                names=['tnum','metnum',
    355                       'year','month','day','hour','minute','fcasthr',
    356                       'thour','lat','lon','alt']+vnames )
    357
    358        # Convert 2-digit year to 4-digits
    359        df.loc[:,'year'] += 2000
    360
    361        # Convert to time
    362        df['time'] = pd.to_datetime( df[['year','month','day','hour','minute']] )
    363
    364        #print(df.iloc[-1,:])
    365        return df

Read trajectory file output from HYSPLIT

###### Parameters

- file (str): name of trajectory file to read

###### Returns

- pandas.DataFrame: DataFrame contains columns:
  - time : datetime object
  - year, month, day, hour, minute : floats, same as time
  - lat, lon, alt : trajectory location
  - thour : hours since trajectory initialization, negative for back trajectories
  - tnum : trajectory number tnum=1 for single trajectory, tnum=1-27 for trajectory ensemble
  - metnum : index number of met file used at this point in trajectory, see tdump file for corresponding file paths
  - fcasthr : hours since the meteorological dataset was initialized

def find_arl_metfiles( start_time, ndays, back=False, metmodels=None, forecast=False, forecastcycle=None, hybrid=False): View Source

    764def find_arl_metfiles( start_time, ndays, back=False, metmodels=None,
    765                       forecast=False, forecastcycle=None, hybrid=False ):
    766    '''Find ARL meteorology files for specified dates, models, forecast and archive
    767
    768    Files will be located for start_time and extending ndays forward or backward
    769
    770    Parameters
    771    ----------
    772    start_time : datetime.datetime or pandas.Timestamp
    773        start date and time for finding meteorology data files
    774    ndays : int
    775        number of days of files to retrieve
    776    back : bool, default=False
    777        specifies files should go ndays backward (back=True) from start_time
    778    metmodels : list or str, default=['gdas0p5','gdas1']
    779        meteorological models that will be used, in order of decreasing resolution and priority
    780    forecast : bool, default=False
    781        set forecast=True to use forecast meteorology for trajectory computation
    782        set forecast=False to use archived (past) meteorology for trajectory computation
    783        Note: hybrid=True will supercede forecast=True
    784    forecastcycle : datetime.datetime, pandas.Timestamp, or None, default=None
    785        if forecast=True, this sets the forecast initialization cycle that will be used
    786        set forecastcycle=None to use the latest available cycle for which files are found
    787        if forecast=False, this parameter has no effect
    788    hybrid : bool, default=False
    789        set hybrid=True for trajectories that use a combination of past archive and
    790        forecast meteorlogy. This supercedes forecast=True
    791
    792    Returns
    793    -------
    794    metfiles : list
    795        path to meteorology files meeting the criteria
    796    '''
    797
    798    check_METROOT()
    799
    800    if metmodels is None:
    801        metmodels = ['gdas0p5','gdas1']
    802
    803    # Find the meteorology files
    804    if hybrid is True:
    805
    806        # "Hybrid" combines past (archived) and future (forecast) meteorology
    807        if back:
    808            raise NotImplementedError( "Combined Analysis/Forecast meteorology "
    809                                   + "not supported for back trajectories" )
    810        metfiles = _get_hybrid_filelist( metmodels, start_time )
    811
    812    elif forecast is True:
    813        # Get the forecast meteorology
    814        metfiles = _get_forecast_filelist( metmodels, forecastcycle )
    815
    816        # Check if the forecast meteorology covers the entire trajectory duration
    817    else:
    818        # Use archived (past) meteorology
    819
    820        # Relative to trajectory start day,
    821        # we need meteorology for days d0 through d1
    822        if back:
    823            d0 = -ndays+1
    824            d1 = 1
    825            # For trajectories that start 23-0Z (nam) or 21-0Z (gfs0p25),
    826            # also need the next day to bracket first time step
    827            if   (('nam3'    in metmodels) and (start_time.hour==23)) \
    828              or (('gfs0p25' in metmodels) and (start_time.hour>=21)):
    829                d1 = 2
    830        else:
    831            d0 = 0
    832            d1 = ndays
    833            # For trajectories that start 00-01Z (nam) or 00-03Z (gfs0p25),
    834            # also need the prior day to bracket first time step
    835            if start_time.hour==0:
    836                d0 = -1
    837
    838        # Debug output
    839        # print('Initial Date Range',d0,d1)
    840        # datelist = np.unique( ( time +
    841        #   pd.TimedeltaIndex( np.sign(trajhours) * np.arange(0,np.abs(trajhours)+1),"H" ) ).date )
    842        # print(datelist)
    843
    844        # List of met directories and met files that will be used
    845        metfiles = []
    846        for d in range(d0,d1):
    847
    848            # date of met data
    849            metdate = start_time.date() + pd.Timedelta( d, "D" )
    850
    851            # Met data for a single day
    852            filename = _get_archive_filelist( metmodels, metdate )
    853
    854            # Add the files to the list
    855            metfiles.extend( filename )
    856        # Keep only the unique files
    857        metfiles = np.unique(metfiles)
    858
    859    if len(metfiles) <= 0:
    860        raise ValueError( 'Meteorology files not found for ' + str( metmodels ) )
    861
    862    return metfiles

Find ARL meteorology files for specified dates, models, forecast and archive

Files will be located for start_time and extending ndays forward or backward

###### Parameters

- start_time (datetime.datetime or pandas.Timestamp): start date and time for finding meteorology data files
- ndays (int): number of days of files to retrieve
- back (bool, default=False): specifies files should go ndays backward (back=True) from start_time
- metmodels (list or str, default=['gdas0p5','gdas1']): meteorological models that will be used, in order of decreasing resolution and priority
- forecast (bool, default=False): set forecast=True to use forecast meteorology for trajectory computation set forecast=False to use archived (past) meteorology for trajectory computation Note: hybrid=True will supercede forecast=True
- forecastcycle (datetime.datetime, pandas.Timestamp, or None, default=None): if forecast=True, this sets the forecast initialization cycle that will be used set forecastcycle=None to use the latest available cycle for which files are found if forecast=False, this parameter has no effect
- hybrid (bool, default=False): set hybrid=True for trajectories that use a combination of past archive and forecast meteorlogy. This supercedes forecast=True

###### Returns

- metfiles (list): path to meteorology files meeting the criteria

def write_control( time, lat, lon, alt, trajhours, fname='CONTROL.000', clobber=False, maxheight=15000.0, outdir='./', tfile='tdump', metfiles=None, exacttime=True, **kwargs): View Source

    864def write_control( time, lat, lon, alt, trajhours,
    865                   fname='CONTROL.000', clobber=False,
    866                   maxheight=15000., outdir='./', tfile='tdump',
    867                   metfiles=None, exacttime=True, **kwargs ):
    868    '''Write HYSPLIT control file for trajectory starting at designated time and coordinates
    869
    870    Parameters
    871    ----------
    872    time : datetime.datetime or pandas.Timestamp
    873        trajectory initialization time
    874    lat, lon : float or list
    875        trajectory initialization latitude and longitude in degrees
    876    alt : float or list
    877        trajectory initialization altitude in meters
    878        The setup.cfg determines whether this is above ground or above mean sea level
    879    trajhours : int
    880        desired trajectory duration in hours. Use negative for back trajectories
    881    fname : str, default='CONTROL.000'
    882        path and name for the file that will be written
    883    clobber : bool, default=False
    884        if clobber=True, then fname will be overwritten
    885    maxheight : float, default=15000
    886        terminate trajectories that exceed maxheight altitude in meters
    887    outdir : str, default='./'
    888        directory path where HYSPLIT output will be written
    889    tfile : str, default='tdump'
    890        name of the trajectory file that HYSPLIT will write
    891    metfiles : list or str, default=None
    892        paths to ARL meteorology files needed for the trajectory computation
    893        If metfiles=None, then find_arl_metfiles will be used to locate necessary files
    894    exacttime : bool, default=True
    895        It is not recommended to change this default, but keyword is retained for backward
    896        compatibility with some scripts. Setting exacttime=False will shift the actual
    897        start time of trajectories that begin at 00:00 UTC to 00:01 UTC to avoid reading
    898        an additional day of meteorological data.
    899    **kwargs
    900        kwargs will be passed to find_arl_metfiles to locate ARL meteorlogy files if metfiles=None
    901        These keywords should include metmodels and possibly forecast, forecastcycle, or hybrid
    902        See find_arl_metfiles for definitions of these parameters
    903    '''
    904
    905    check_METROOT()
    906
    907    if os.path.isfile( fname ) and (clobber is False):
    908        raise OSError( f'File exists. Set clobber=True to overwrite: {fname:s}' )
    909
    910    # Ensure that lat, lon, and alt are lists
    911    try:
    912        nlat = len( lat )
    913    except TypeError:
    914        lat = [ lat ]
    915        nlat = len( lat )
    916    try:
    917        nlon = len( lon )
    918    except TypeError:
    919        lon = [ lon ]
    920        nlon = len( lon )
    921    try:
    922        nalt = len( alt )
    923    except TypeError:
    924        alt = [ alt ]
    925        nalt = len( alt )
    926
    927    # Should add some type checking to ensure that lon, lat, alt
    928    # have conformable lengths
    929    if ( (nlat != nlon) or (nlat != nalt) ):
    930        raise ValueError( "lon, lat, alt must be conformable" )
    931
    932    # Number of days of met data
    933    ndays = int( np.ceil( np.abs( trajhours ) / 24 ) + 1 )
    934
    935    # Get met file names, if not provided
    936    if (metfiles is None) or (len(metfiles)==0):
    937        metfiles = find_arl_metfiles( time, ndays, back=trajhours<0, **kwargs )
    938
    939    # Number of met files
    940    nmet = len( metfiles )
    941
    942    # Runs will fail if the initial time is not bracketed by met data.
    943    # When met resolution changes on the first day, this condition may not be met.
    944    # Unless exacttime==True, the starting time will be shifted 1 minute to 00:01
    945    # to avoid reading in an entire extra day of met data.
    946    if (time.hour==0 and time.minute == 0 and not exacttime):
    947        time = time + pd.Timedelta( 1, "m" )
    948
    949    # Start date-time, formatted as YY MM DD HH {mm}
    950    startdate = time.strftime( "%y %m %d %H %M" )
    951
    952    # Write the CONTROL file
    953    with open( fname, 'w', encoding='ascii' ) as f:
    954
    955        f.write( startdate+'\n' )
    956        f.write( f'{nlat:d}\n' )
    957        for i in range(nlat):
    958            f.write( f'{lat[i]:<10.4f} {lon[i]:<10.4f} {alt[i]:<10.4f}\n' )
    959        f.write( f'{trajhours:d}\n' )
    960        f.write( "0\n" )
    961        f.write( f"{maxheight:<10.1f}\n" )
    962        f.write( f'{nmet:d}\n' )
    963        for file in metfiles:
    964            f.write( os.path.dirname(  file ) + '/\n' )
    965            f.write( os.path.basename( file ) + '\n'  )
    966        f.write( f'{outdir:s}\n' )
    967        f.write( f'{tfile:s}\n' )

Write HYSPLIT control file for trajectory starting at designated time and coordinates

###### Parameters

- time (datetime.datetime or pandas.Timestamp): trajectory initialization time
- lat, lon (float or list): trajectory initialization latitude and longitude in degrees
- alt (float or list): trajectory initialization altitude in meters The setup.cfg determines whether this is above ground or above mean sea level
- trajhours (int): desired trajectory duration in hours. Use negative for back trajectories
- fname (str, default='CONTROL.000'): path and name for the file that will be written
- clobber (bool, default=False): if clobber=True, then fname will be overwritten
- maxheight (float, default=15000): terminate trajectories that exceed maxheight altitude in meters
- outdir (str, default='./'): directory path where HYSPLIT output will be written
- tfile (str, default='tdump'): name of the trajectory file that HYSPLIT will write
- metfiles (list or str, default=None): paths to ARL meteorology files needed for the trajectory computation If metfiles=None, then find_arl_metfiles will be used to locate necessary files
- exacttime (bool, default=True): It is not recommended to change this default, but keyword is retained for backward compatibility with some scripts. Setting exacttime=False will shift the actual start time of trajectories that begin at 00:00 UTC to 00:01 UTC to avoid reading an additional day of meteorological data.
- **kwargs: kwargs will be passed to find_arl_metfiles to locate ARL meteorlogy files if metfiles=None These keywords should include metmodels and possibly forecast, forecastcycle, or hybrid See find_arl_metfiles for definitions of these parameters