X-Git-Url: https://yuggoth.org/gitweb?a=blobdiff_plain;ds=sidebyside;f=weather.py;h=78fd79c58375a0dbf43be90644a4377df87b14b6;hb=8a37eddc0667410e77de9d1441d90532440a0329;hp=47870ec51273b455bf6290d0b595d2ef3e727eff;hpb=f7fc45b44d9ce028485c9de0467c68fcf55c17a8;p=weather.git diff --git a/weather.py b/weather.py index 47870ec..78fd79c 100644 --- a/weather.py +++ b/weather.py @@ -1,12 +1,12 @@ """Contains various object definitions needed by the weather utility.""" weather_copyright = """\ -# Copyright (c) 2006-2012 Jeremy Stanley . Permission to +# Copyright (c) 2006-2020 Jeremy Stanley . Permission to # use, copy, modify, and distribute this software is granted under terms # provided in the LICENSE file distributed with this software. #""" -weather_version = "2.0" +weather_version = "2.4" radian_to_km = 6372.795484 radian_to_mi = 3959.871528 @@ -692,6 +692,8 @@ def data_index(path): os.stat(candidate).st_mtime ) break + if filename in datafiles: + break return datafiles def guess( @@ -742,7 +744,10 @@ def guess( datafile = datafiles[dataname][0] if datafile.endswith(".gz"): import gzip - stations.readfp( gzip.open(datafile) ) + if pyversion("3"): + stations.read_string( + gzip.open(datafile).read().decode("utf-8") ) + else: stations.readfp( gzip.open(datafile) ) else: stations.read(datafile) else: @@ -758,7 +763,9 @@ def guess( datafile = datafiles[dataname][0] if datafile.endswith(".gz"): import gzip - zones.readfp( gzip.open(datafile) ) + if pyversion("3"): + zones.read_string( gzip.open(datafile).read().decode("utf-8") ) + else: zones.readfp( gzip.open(datafile) ) else: zones.read(datafile) else: @@ -782,7 +789,10 @@ def guess( datafile = datafiles[dataname][0] if datafile.endswith(".gz"): import gzip - airports.readfp( gzip.open(datafile) ) + if pyversion("3"): + airports.read_string( + gzip.open(datafile).read().decode("utf-8") ) + else: airports.readfp( gzip.open(datafile) ) else: airports.read(datafile) else: @@ -868,7 +878,10 @@ def guess( datafile = datafiles[dataname][0] if datafile.endswith(".gz"): import gzip - zctas.readfp( gzip.open(datafile) ) + if pyversion("3"): + zctas.read_string( + gzip.open(datafile).read().decode("utf-8") ) + else: zctas.readfp( gzip.open(datafile) ) else: zctas.read(datafile) else: @@ -923,7 +936,10 @@ def guess( datafile = datafiles[dataname][0] if datafile.endswith(".gz"): import gzip - places.readfp( gzip.open(datafile) ) + if pyversion("3"): + places.read_string( + gzip.open(datafile).read().decode("utf-8") ) + else: places.readfp( gzip.open(datafile) ) else: places.read(datafile) else: @@ -1193,25 +1209,26 @@ def gecos(formatted): return tuple(coordinates) def correlate(): - import codecs, datetime, hashlib, os, re, sys, tarfile, time, zipfile + import codecs, csv, datetime, hashlib, os, re, sys, tarfile, time, zipfile if pyversion("3"): import configparser else: import ConfigParser as configparser - gcounties_an = "Gaz_counties_national.zip" - gcounties_fn = "Gaz_counties_national.txt" - gcousubs_an = "Gaz_cousubs_national.zip" - gcousubs_fn = "Gaz_cousubs_national.txt" - gplaces_an = "Gaz_places_national.zip" - gplaces_fn = "Gaz_places_national.txt" - gzcta_an = "Gaz_zcta_national.zip" - gzcta_fn = "Gaz_zcta_national.txt" for filename in os.listdir("."): - if re.match("bp[0-9][0-9][a-z][a-z][0-9][0-9].dbx$", filename): + if re.match("[0-9]{4}_Gaz_counties_national.zip$", filename): + gcounties_an = filename + gcounties_fn = filename[:-4] + ".txt" + elif re.match("[0-9]{4}_Gaz_cousubs_national.zip$", filename): + gcousubs_an = filename + gcousubs_fn = filename[:-4] + ".txt" + elif re.match("[0-9]{4}_Gaz_place_national.zip$", filename): + gplace_an = filename + gplace_fn = filename[:-4] + ".txt" + elif re.match("[0-9]{4}_Gaz_zcta_national.zip$", filename): + gzcta_an = filename + gzcta_fn = filename[:-4] + ".txt" + elif re.match("bp[0-9]{2}[a-z]{2}[0-9]{2}.dbx$", filename): cpfzcf_fn = filename - break nsdcccc_fn = "nsd_cccc.txt" - zcatalog_an = "zonecatalog.curr.tar" - metartbl_fn = "metar.tbl" - coopact_fn = "COOP-ACT.TXT" + ourairports_fn = "airports.csv" overrides_fn = "overrides.conf" overrideslog_fn = "overrides.log" slist_fn = "slist" @@ -1226,25 +1243,19 @@ def correlate(): %s # generated by %s on %s from these public domain sources: # -# http://www.census.gov/geo/www/gazetteer/gazetteer2010.html +# https://www.census.gov/geographies/reference-files/time-series/geo/gazetteer-files.html # %s %s %s # %s %s %s # %s %s %s # %s %s %s # -# http://www.weather.gov/geodata/catalog/wsom/html/cntyzone.htm +# https://www.weather.gov/gis/ZoneCounty/ # %s %s %s # -# http://weather.noaa.gov/data/nsd_cccc.txt +# https://tgftp.nws.noaa.gov/data/ # %s %s %s # -# http://weather.noaa.gov/pub/data/zonecatalog.curr.tar -# %s %s %s -# -# http://www.nco.ncep.noaa.gov/pmb/codes/nwprod/dictionaries/metar.tbl -# %s %s %s -# -# ftp://ftp.ncdc.noaa.gov/pub/data/inventories/COOP-ACT.TXT +# https://ourairports.com/data/ # %s %s %s # # ...and these manually-generated or hand-compiled adjustments: @@ -1267,11 +1278,11 @@ def correlate(): datetime.datetime.fromtimestamp( os.path.getmtime(gcousubs_an) ) ), gcousubs_an, - hashlib.md5( open(gplaces_an, "rb").read() ).hexdigest(), + hashlib.md5( open(gplace_an, "rb").read() ).hexdigest(), datetime.date.isoformat( - datetime.datetime.fromtimestamp( os.path.getmtime(gplaces_an) ) + datetime.datetime.fromtimestamp( os.path.getmtime(gplace_an) ) ), - gplaces_an, + gplace_an, hashlib.md5( open(gzcta_an, "rb").read() ).hexdigest(), datetime.date.isoformat( datetime.datetime.fromtimestamp( os.path.getmtime(gzcta_an) ) @@ -1287,21 +1298,11 @@ def correlate(): datetime.datetime.fromtimestamp( os.path.getmtime(nsdcccc_fn) ) ), nsdcccc_fn, - hashlib.md5( open(zcatalog_an, "rb").read() ).hexdigest(), - datetime.date.isoformat( - datetime.datetime.fromtimestamp( os.path.getmtime(zcatalog_an) ) - ), - zcatalog_an, - hashlib.md5( open(metartbl_fn, "rb").read() ).hexdigest(), - datetime.date.isoformat( - datetime.datetime.fromtimestamp( os.path.getmtime(metartbl_fn) ) - ), - metartbl_fn, - hashlib.md5( open(coopact_fn, "rb").read() ).hexdigest(), + hashlib.md5( open(ourairports_fn, "rb").read() ).hexdigest(), datetime.date.isoformat( - datetime.datetime.fromtimestamp( os.path.getmtime(coopact_fn) ) + datetime.datetime.fromtimestamp( os.path.getmtime(ourairports_fn) ) ), - coopact_fn, + ourairports_fn, hashlib.md5( open(overrides_fn, "rb").read() ).hexdigest(), datetime.date.isoformat( datetime.datetime.fromtimestamp( os.path.getmtime(overrides_fn) ) @@ -1328,9 +1329,9 @@ def correlate(): sys.stdout.flush() count = 0 gcounties = zipfile.ZipFile(gcounties_an).open(gcounties_fn, "rU") - columns = gcounties.readline().decode("latin1").strip().split("\t") + columns = gcounties.readline().decode("utf-8").strip().split("\t") for line in gcounties: - fields = line.decode("latin1").strip().split("\t") + fields = line.decode("utf-8").strip().split("\t") f_geoid = fields[ columns.index("GEOID") ].strip() f_name = fields[ columns.index("NAME") ].strip() f_usps = fields[ columns.index("USPS") ].strip() @@ -1351,9 +1352,9 @@ def correlate(): sys.stdout.flush() count = 0 gcousubs = zipfile.ZipFile(gcousubs_an).open(gcousubs_fn, "rU") - columns = gcousubs.readline().decode("latin1").strip().split("\t") + columns = gcousubs.readline().decode("utf-8").strip().split("\t") for line in gcousubs: - fields = line.decode("latin1").strip().split("\t") + fields = line.decode("utf-8").strip().split("\t") f_geoid = fields[ columns.index("GEOID") ].strip() f_name = fields[ columns.index("NAME") ].strip() f_usps = fields[ columns.index("USPS") ].strip() @@ -1369,14 +1370,14 @@ def correlate(): count += 1 gcousubs.close() print("done (%s lines)." % count) - message = "Reading %s:%s..." % (gplaces_an, gplaces_fn) + message = "Reading %s:%s..." % (gplace_an, gplace_fn) sys.stdout.write(message) sys.stdout.flush() count = 0 - gplaces = zipfile.ZipFile(gplaces_an).open(gplaces_fn, "rU") - columns = gplaces.readline().decode("latin1").strip().split("\t") - for line in gplaces: - fields = line.decode("latin1").strip().split("\t") + gplace = zipfile.ZipFile(gplace_an).open(gplace_fn, "rU") + columns = gplace.readline().decode("utf-8").strip().split("\t") + for line in gplace: + fields = line.decode("utf-8").strip().split("\t") f_geoid = fields[ columns.index("GEOID") ].strip() f_name = fields[ columns.index("NAME") ].strip() f_usps = fields[ columns.index("USPS") ].strip() @@ -1390,57 +1391,28 @@ def correlate(): ) places[fips]["description"] = "%s, %s" % (f_name, f_usps) count += 1 - gplaces.close() + gplace.close() print("done (%s lines)." % count) message = "Reading %s..." % slist_fn sys.stdout.write(message) sys.stdout.flush() count = 0 - slist = codecs.open(slist_fn, "rU") + slist = codecs.open(slist_fn, "rU", "utf-8") for line in slist: icao = line.split("#")[0].strip() if icao: stations[icao] = { - "metar": "http://weather.noaa.gov/pub/data/observations/"\ + "metar": "https://tgftp.nws.noaa.gov/data/observations/"\ + "metar/decoded/%s.TXT" % icao.upper() } count += 1 slist.close() print("done (%s lines)." % count) - message = "Reading %s..." % metartbl_fn - sys.stdout.write(message) - sys.stdout.flush() - count = 0 - metartbl = codecs.open(metartbl_fn, "rU") - for line in metartbl: - icao = line[:4].strip().lower() - if icao in stations: - description = [] - name = " ".join( - line[16:48].replace("_", " ").strip().title().split() - ) - if name: description.append(name) - st = line[49:51].strip() - if st: description.append(st) - cn = line[52:54].strip() - if cn: description.append(cn) - if description: - stations[icao]["description"] = ", ".join(description) - lat = line[55:60].strip() - if lat: - lat = int(lat)/100.0 - lon = line[61:67].strip() - if lon: - lon = int(lon)/100.0 - stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) ) - count += 1 - metartbl.close() - print("done (%s lines)." % count) message = "Reading %s..." % nsdcccc_fn sys.stdout.write(message) sys.stdout.flush() count = 0 - nsdcccc = codecs.open(nsdcccc_fn, "rU", "latin1") + nsdcccc = codecs.open(nsdcccc_fn, "rU", "utf-8") for line in nsdcccc: line = str(line) fields = line.split(";") @@ -1465,44 +1437,49 @@ def correlate(): count += 1 nsdcccc.close() print("done (%s lines)." % count) - message = "Reading %s..." % coopact_fn + message = "Reading %s..." % ourairports_fn sys.stdout.write(message) sys.stdout.flush() count = 0 - coopact = open(coopact_fn) - for line in coopact: - icao = line[33:37].strip().lower() + ourairports = open(ourairports_fn, "rU") + for row in csv.reader(ourairports): + icao = row[12].decode('utf-8').lower() if icao in stations: - iata = line[22:26].strip().lower() + iata = row[13].decode('utf-8').lower() if len(iata) == 3: airports[iata] = { "station": icao } if "description" not in stations[icao]: description = [] - name = " ".join( line[99:129].strip().title().split() ) + name = row[3].decode('utf-8') if name: description.append(name) - st = line[59:61].strip() - if st: description.append(st) - country = " ".join( line[38:58].strip().title().split() ) - if country: description.append(country) + municipality = row[10].decode('utf-8') + if municipality: description.append(municipality) + region = row[9].decode('utf-8') + country = row[8].decode('utf-8') + if region: + if "-" in region: + c,r = region.split("-", 1) + if c == country: region = r + description.append(region) + if country: + description.append(country) if description: stations[icao]["description"] = ", ".join(description) if "location" not in stations[icao]: - lat = line[130:139].strip() + lat = row[4].decode('utf-8') if lat: - lat = lat.replace(" ", "-") - lon = line[140:150].strip() + lon = row[5].decode('utf-8') if lon: - lon = lon.replace(" ", "-") stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) ) count += 1 - coopact.close() + ourairports.close() print("done (%s lines)." % count) message = "Reading %s..." % zlist_fn sys.stdout.write(message) sys.stdout.flush() count = 0 - zlist = codecs.open(zlist_fn, "rU") + zlist = codecs.open(zlist_fn, "rU", "utf-8") for line in zlist: line = line.split("#")[0].strip() if line: @@ -1510,68 +1487,76 @@ def correlate(): count += 1 zlist.close() print("done (%s lines)." % count) - message = "Reading %s:*..." % zcatalog_an - sys.stdout.write(message) - sys.stdout.flush() - count = 0 - zcatalog = tarfile.open(zcatalog_an) - for entry in zcatalog.getmembers(): - if entry.isfile(): - fnmatch = re.match( - r"([a-z]+z[0-9]+)\.txt$", - os.path.basename(entry.name) - ) - if fnmatch: - zone = fnmatch.group(1) - if zone in zones: - data = zcatalog.extractfile(entry).readlines() - description = data[0].decode("ascii").strip() - zones[zone]["description"] = description - for line in data[1:]: - line = line.decode("latin1").strip() - urimatch = re.match("/webdocs/(.+):(.+) for ", line) - if urimatch: - uritype = urimatch.group(2).lower().replace(" ","_") - zones[zone][uritype] \ - = "http://weather.noaa.gov/%s" \ - % urimatch.group(1) - count += 1 - zcatalog.close() - print("done (%s files)." % count) message = "Reading %s..." % cpfzcf_fn sys.stdout.write(message) sys.stdout.flush() count = 0 cpfz = {} - cpfzcf = open(cpfzcf_fn) + cpfzcf = codecs.open(cpfzcf_fn, "rU", "utf-8") for line in cpfzcf: - fields = line.split("|") + fields = line.strip().split("|") if len(fields) == 11 \ and fields[0] and fields[1] and fields[9] and fields[10]: zone = "z".join( fields[:2] ).lower() if zone in zones: - zones[zone]["centroid"] = gecos( ",".join( fields[9:] ) ) - elif fields[6]: state = fields[0] - description = fields[3] - county = fields[5] + if state: + zones[zone]["coastal_flood_statement"] = ( + "https://tgftp.nws.noaa.gov/data/watches_warnings/" + "flood/coastal/%s/%s.txt" % (state.lower(), zone)) + zones[zone]["flash_flood_statement"] = ( + "https://tgftp.nws.noaa.gov/data/watches_warnings/" + "flash_flood/statement/%s/%s.txt" + % (state.lower(), zone)) + zones[zone]["flash_flood_warning"] = ( + "https://tgftp.nws.noaa.gov/data/watches_warnings/" + "flash_flood/warning/%s/%s.txt" + % (state.lower(), zone)) + zones[zone]["flash_flood_watch"] = ( + "https://tgftp.nws.noaa.gov/data/watches_warnings/" + "flash_flood/watch/%s/%s.txt" % (state.lower(), zone)) + zones[zone]["flood_statement"] = ( + "https://tgftp.nws.noaa.gov/data/watches_warnings/" + "flood/statement/%s/%s.txt" % (state.lower(), zone)) + zones[zone]["flood_warning"] = ( + "https://tgftp.nws.noaa.gov/data/watches_warnings/" + "flood/warning/%s/%s.txt" % (state.lower(), zone)) + zones[zone]["severe_thunderstorm_warning"] = ( + "https://tgftp.nws.noaa.gov/data/watches_warnings/" + "thunderstorm/%s/%s.txt" % (state.lower(), zone)) + zones[zone]["severe_weather_statement"] = ( + "https://tgftp.nws.noaa.gov/data/watches_warnings/" + "severe_weather_stmt/%s/%s.txt" + % (state.lower(), zone)) + zones[zone]["short_term_forecast"] = ( + "https://tgftp.nws.noaa.gov/data/forecasts/nowcast/" + "%s/%s.txt" % (state.lower(), zone)) + zones[zone]["special_weather_statement"] = ( + "https://tgftp.nws.noaa.gov/data/watches_warnings/" + "special_weather_stmt/%s/%s.txt" + % (state.lower(), zone)) + zones[zone]["state_forecast"] = ( + "https://tgftp.nws.noaa.gov/data/forecasts/state/" + "%s/%s.txt" % (state.lower(), zone)) + zones[zone]["urgent_weather_message"] = ( + "https://tgftp.nws.noaa.gov/data/watches_warnings/" + "non_precip/%s/%s.txt" % (state.lower(), zone)) + zones[zone]["zone_forecast"] = ( + "https://tgftp.nws.noaa.gov/data/forecasts/zone/" + "%s/%s.txt" % (state.lower(), zone)) + description = fields[3].strip() fips = "fips%s"%fields[6] - possible = [ - "%s, %s" % (county, state), - "%s County, %s" % (county, state), - ] - if description.endswith(" Counties"): - description = description[:-9] - for addition in description.split(" and "): - possible.append( "%s, %s" % (addition, state) ) - possible.append( "%s County, %s" % (addition, state) ) - if fips in places and "centroid" in places[fips]: - for candidate in zones: - if "centroid" not in zones[candidate] and \ - "description" in zones[candidate] and \ - zones[candidate]["description"] in possible: - zones[candidate]["centroid"] = \ - places[fips]["centroid"] + county = fields[5] + if county: + if description.endswith(county): + description += " County" + else: + description += ", %s County" % county + description += ", %s, US" % state + zones[zone]["description"] = description + zones[zone]["centroid"] = gecos( ",".join( fields[9:11] ) ) + if fips in places and not zones[zone]["centroid"]: + zones[zone]["centroid"] = places[fips]["centroid"] count += 1 cpfzcf.close() print("done (%s lines)." % count) @@ -1580,9 +1565,9 @@ def correlate(): sys.stdout.flush() count = 0 gzcta = zipfile.ZipFile(gzcta_an).open(gzcta_fn, "rU") - columns = gzcta.readline().decode("latin1").strip().split("\t") + columns = gzcta.readline().decode("utf-8").strip().split("\t") for line in gzcta: - fields = line.decode("latin1").strip().split("\t") + fields = line.decode("utf-8").strip().split("\t") f_geoid = fields[ columns.index("GEOID") ].strip() f_intptlat = fields[ columns.index("INTPTLAT") ].strip() f_intptlong = fields[ columns.index("INTPTLONG") ].strip() @@ -1909,6 +1894,13 @@ def correlate(): for airport in sorted( airports.keys() ): airports_fd.write("\n\n[%s]" % airport) for key, value in sorted( airports[airport].items() ): + if type(value) is float: value = "%.7f"%value + elif type(value) is tuple: + elements = [] + for element in value: + if type(element) is float: elements.append("%.7f"%element) + else: elements.append( repr(element) ) + value = "(%s)"%", ".join(elements) airports_fd.write( "\n%s = %s" % (key, value) ) count += 1 airports_fd.write("\n") @@ -1925,6 +1917,13 @@ def correlate(): for fips in sorted( places.keys() ): places_fd.write("\n\n[%s]" % fips) for key, value in sorted( places[fips].items() ): + if type(value) is float: value = "%.7f"%value + elif type(value) is tuple: + elements = [] + for element in value: + if type(element) is float: elements.append("%.7f"%element) + else: elements.append( repr(element) ) + value = "(%s)"%", ".join(elements) places_fd.write( "\n%s = %s" % (key, value) ) count += 1 places_fd.write("\n") @@ -1936,11 +1935,18 @@ def correlate(): count = 0 if os.path.exists(stations_fn): os.rename(stations_fn, "%s_old"%stations_fn) - stations_fd = codecs.open(stations_fn, "w", "utf8") + stations_fd = codecs.open(stations_fn, "w", "utf-8") stations_fd.write(header) for station in sorted( stations.keys() ): stations_fd.write("\n\n[%s]" % station) for key, value in sorted( stations[station].items() ): + if type(value) is float: value = "%.7f"%value + elif type(value) is tuple: + elements = [] + for element in value: + if type(element) is float: elements.append("%.7f"%element) + else: elements.append( repr(element) ) + value = "(%s)"%", ".join(elements) stations_fd.write( "\n%s = %s" % (key, value) ) count += 1 stations_fd.write("\n") @@ -1957,6 +1963,13 @@ def correlate(): for zcta in sorted( zctas.keys() ): zctas_fd.write("\n\n[%s]" % zcta) for key, value in sorted( zctas[zcta].items() ): + if type(value) is float: value = "%.7f"%value + elif type(value) is tuple: + elements = [] + for element in value: + if type(element) is float: elements.append("%.7f"%element) + else: elements.append( repr(element) ) + value = "(%s)"%", ".join(elements) zctas_fd.write( "\n%s = %s" % (key, value) ) count += 1 zctas_fd.write("\n") @@ -1973,6 +1986,13 @@ def correlate(): for zone in sorted( zones.keys() ): zones_fd.write("\n\n[%s]" % zone) for key, value in sorted( zones[zone].items() ): + if type(value) is float: value = "%.7f"%value + elif type(value) is tuple: + elements = [] + for element in value: + if type(element) is float: elements.append("%.7f"%element) + else: elements.append( repr(element) ) + value = "(%s)"%", ".join(elements) zones_fd.write( "\n%s = %s" % (key, value) ) count += 1 zones_fd.write("\n")