"""
 Since the NOAAPort feed of HRRR data does not have radiation, we should
 download this manually from NCEP

Run at 40 AFTER for the previous hour

"""
import urllib2
import sys
import datetime
import os
import time


def fetch(valid):
    """ Fetch the radiation data for this timestamp
    80:54371554:d=2014101002:ULWRF:top of atmosphere:anl:
    81:56146124:d=2014101002:DSWRF:surface:anl:
    """

    uri = valid.strftime(("http://www.ftp.ncep.noaa.gov/data/nccf/"
                          "nonoperational/com/hrrr/prod/hrrr.%Y%m%d/hrrr.t%Hz."
                          "wrfprsf00.grib2.idx"))
    data = urllib2.urlopen(uri, timeout=30)

    offsets = []
    neednext = False
    for line in data:
        tokens = line.split(":")
        if neednext:
            offsets[-1].append(int(tokens[1]))
            neednext = False
        if tokens[3] in ['ULWRF', 'DSWRF']:
            offsets.append([int(tokens[1]), ])
            neednext = True
        # Save soil temp and water at surface, 10cm and 40cm
        if tokens[3] in ['TSOIL', 'SOILW']:
            if tokens[4] in ['0-0 m below ground',
                             '0.01-0.01 m below ground',
                             '0.04-0.04 m below ground']:
                offsets.append([int(tokens[1]), ])
                neednext = True

    outfn = valid.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/model/hrrr/"
                            "%H/hrrr.t%Hz.3kmf00.grib2"))
    outdir = os.path.dirname(outfn)
    if not os.path.isdir(outdir):
        os.makedirs(outdir, mode=0775)  # make sure LDM can then write to dir
    output = open(outfn, 'ab', 0664)

    req = urllib2.Request(uri[:-4])
    if len(offsets) != 8:
        print("download_hrrr_rad warning, found %s gribs for %s" % (
                                                    len(offsets), valid))
    for pr in offsets:
        req.headers['Range'] = 'bytes=%s-%s' % (pr[0], pr[1])
        attempt = 0
        f = None
        while attempt < 10:
            try:
                f = urllib2.urlopen(req, timeout=30)
            except:
                time.sleep(10)
                attempt += 1
                continue
            attempt = 10
        if f is None:
            print("download_hrrr.py failure for uri: %s" % (uri,))
        else:
            output.write(f.read())

    output.close()


def main():
    """ Go Main Go"""
    ts = datetime.datetime.utcnow() - datetime.timedelta(hours=1)
    if len(sys.argv) == 5:
        ts = datetime.datetime(int(sys.argv[1]), int(sys.argv[2]),
                               int(sys.argv[3]), int(sys.argv[4]))
    fetch(ts)


if __name__ == '__main__':
    os.umask(0002)
    main()

@@ -212,16 +212,679 @@
 rt time%0A
+import pygrib%0A%0A%0Adef need_to_run(valid):%0A    %22%22%22 Check to see if we already have the radiation data we need%22%22%22%0A    gribfn = valid.strftime((%22/mesonet/ARCHIVE/data/%25Y/%25m/%25d/model/hrrr/%22%0A                             %22%25H/hrrr.t%25Hz.3kmf00.grib2%22))%0A    if not os.path.isfile(gribfn):%0A        return True%0A    try:%0A        grbs = pygrib.open(gribfn)%0A        for name in %5B'Downward short-wave radiation flux',%0A                     'Upward long-wave radiation flux'%5D:%0A            grbs.select(name=name)%0A        # print(%22%25s had everything we desired!%22 %25 (gribfn, ))%0A        return False%0A    except Exception, _:%0A        # print gribfn, _%0A        return True%0A    return True%0A
 %0A%0Adef fe
@@ -1057,16 +1057,16 @@
 nl:%0A
+
     %22%22%22%0A
 %0A   
@@ -1061,17 +1061,16 @@
     %22%22%22%0A
-%0A
     uri 
@@ -3043,68 +3043,17 @@
    t
-s = datetime.datetime.utcnow() - datetime.timedelta(hours=1)
+imes = %5B%5D
 %0A   
@@ -3085,20 +3085,28 @@
        t
-s = 
+imes.append(
 datetime
@@ -3182,16 +3182,24 @@
         
+    
+    
 int(sys.
@@ -3225,17 +3225,344 @@
 rgv%5B4%5D))
-%0A
+)%0A    else:%0A        times.append(datetime.datetime.utcnow() - datetime.timedelta(hours=1))%0A        times.append(datetime.datetime.utcnow() - datetime.timedelta(hours=6))%0A        times.append(datetime.datetime.utcnow() - datetime.timedelta(hours=24))%0A    for ts in times:%0A        if not need_to_run(ts):%0A            continue%0A    
     fetc

add some more intelligence to HRRR solar download