weather.py

   1 """Contains various object definitions needed by the weather utility."""
   2
   3 weather_copyright = """\
   4 # Copyright (c) 2006-2024 Jeremy Stanley <fungi@yuggoth.org>. Permission to
   5 # use, copy, modify, and distribute this software is granted under terms
   6 # provided in the LICENSE file distributed with this software.
   7 #"""
   8
   9 weather_version = "2.4.4"
  10
  11 radian_to_km = 6372.795484
  12 radian_to_mi = 3959.871528
  13
  14 def pyversion(ref=None):
  15     """Determine the Python version and optionally compare to a reference."""
  16     import platform
  17     ver = platform.python_version()
  18     if ref:
  19         return [
  20             int(x) for x in ver.split(".")[:2]
  21         ] >= [
  22             int(x) for x in ref.split(".")[:2]
  23         ]
  24     else: return ver
  25
  26 class Selections:
  27     """An object to contain selection data."""
  28     def __init__(self):
  29         """Store the config, options and arguments."""
  30         self.config = get_config()
  31         self.options, self.arguments = get_options(self.config)
  32         if self.get_bool("cache") and self.get_bool("cache_search") \
  33             and not self.get_bool("longlist"):
  34             integrate_search_cache(
  35                 self.config,
  36                 self.get("cachedir"),
  37                 self.get("setpath")
  38             )
  39         if not self.arguments:
  40             if "id" in self.options.__dict__ \
  41                 and self.options.__dict__["id"]:
  42                 self.arguments.append( self.options.__dict__["id"] )
  43                 del( self.options.__dict__["id"] )
  44                 import sys
  45                 message = "WARNING: the --id option is deprecated and will eventually be removed\n"
  46                 sys.stderr.write(message)
  47             elif "city" in self.options.__dict__ \
  48                 and self.options.__dict__["city"] \
  49                 and "st" in self.options.__dict__ \
  50                 and self.options.__dict__["st"]:
  51                 self.arguments.append(
  52                     "^%s city, %s" % (
  53                         self.options.__dict__["city"],
  54                         self.options.__dict__["st"]
  55                     )
  56                 )
  57                 del( self.options.__dict__["city"] )
  58                 del( self.options.__dict__["st"] )
  59                 import sys
  60                 message = "WARNING: the --city/--st options are deprecated and will eventually be removed\n"
  61                 sys.stderr.write(message)
  62     def get(self, option, argument=None):
  63         """Retrieve data from the config or options."""
  64         if argument:
  65             if self.config.has_section(argument) and (
  66                 self.config.has_option(argument, "city") \
  67                     or self.config.has_option(argument, "id") \
  68                     or self.config.has_option(argument, "st")
  69             ):
  70                 self.config.remove_section(argument)
  71                 import sys
  72                 message = "WARNING: the city/id/st options are now unsupported in aliases\n"
  73                 sys.stderr.write(message)
  74             if not self.config.has_section(argument):
  75                 guessed = guess(
  76                     argument,
  77                     path=self.get("setpath"),
  78                     info=self.get("info"),
  79                     cache_search=(
  80                         self.get("cache") and self.get("cache_search")
  81                     ),
  82                     cachedir=self.get("cachedir"),
  83                     quiet=self.get_bool("quiet")
  84                 )
  85                 self.config.add_section(argument)
  86                 for item in guessed.items():
  87                     self.config.set(argument, *item)
  88             if self.config.has_option(argument, option):
  89                 return self.config.get(argument, option)
  90         if option in self.options.__dict__:
  91             return self.options.__dict__[option]
  92         import sys
  93         message = "WARNING: no URI defined for %s\n" % option
  94         sys.stderr.write(message)
  95         return None
  96     def get_bool(self, option, argument=None):
  97         """Get data and coerce to a boolean if necessary."""
  98         # Mimic configparser's getboolean() method by treating
  99         # false/no/off/0 as False and true/yes/on/1 as True values,
 100         # case-insensitively
 101         value = self.get(option, argument)
 102         if isinstance(value, bool):
 103             return value
 104         if isinstance(value, str):
 105             vlower = value.lower()
 106             if vlower in ('false', 'no', 'off', '0'):
 107                 return False
 108             elif vlower in ('true', 'yes', 'on', '1'):
 109                 return True
 110         raise ValueError("Not a boolean: %s" % value)
 111     def getint(self, option, argument=None):
 112         """Get data and coerce to an integer if necessary."""
 113         value = self.get(option, argument)
 114         if value: return int(value)
 115         else: return 0
 116
 117 def average(coords):
 118     """Average a list of coordinates."""
 119     x = 0
 120     y = 0
 121     for coord in coords:
 122         x += coord[0]
 123         y += coord[1]
 124     count = len(coords)
 125     return (x/count, y/count)
 126
 127 def filter_units(line, units="imperial"):
 128     """Filter or convert units in a line of text between US/UK and metric."""
 129     import re
 130     # filter lines with both pressures in the form of "X inches (Y hPa)" or
 131     # "X in. Hg (Y hPa)"
 132     dual_p = re.match(
 133         r"(.* )(\d*(\.\d+)? (inches|in\. Hg)) \((\d*(\.\d+)? hPa)\)(.*)",
 134         line
 135     )
 136     if dual_p:
 137         preamble, in_hg, i_fr, i_un, hpa, h_fr, trailer = dual_p.groups()
 138         if units == "imperial": line = preamble + in_hg + trailer
 139         elif units == "metric": line = preamble + hpa + trailer
 140     # filter lines with both temperatures in the form of "X F (Y C)"
 141     dual_t = re.match(
 142         r"(.* )(-?\d*(\.\d+)? F) \((-?\d*(\.\d+)? C)\)(.*)",
 143         line
 144     )
 145     if dual_t:
 146         preamble, fahrenheit, f_fr, celsius, c_fr, trailer = dual_t.groups()
 147         if units == "imperial": line = preamble + fahrenheit + trailer
 148         elif units == "metric": line = preamble + celsius + trailer
 149     # if metric is desired, convert distances in the form of "X mile(s)" to
 150     # "Y kilometer(s)"
 151     if units == "metric":
 152         imperial_d = re.match(
 153             r"(.* )(\d+)( mile\(s\))(.*)",
 154             line
 155         )
 156         if imperial_d:
 157             preamble, mi, m_u, trailer = imperial_d.groups()
 158             line = preamble + str(int(round(int(mi)*1.609344))) \
 159                 + " kilometer(s)" + trailer
 160     # filter speeds in the form of "X MPH (Y KT)" to just "X MPH"; if metric is
 161     # desired, convert to "Z KPH"
 162     imperial_s = re.match(
 163         r"(.* )(\d+)( MPH)( \(\d+ KT\))(.*)",
 164         line
 165     )
 166     if imperial_s:
 167         preamble, mph, m_u, kt, trailer = imperial_s.groups()
 168         if units == "imperial": line = preamble + mph + m_u + trailer
 169         elif units == "metric":
 170             line = preamble + str(int(round(int(mph)*1.609344))) + " KPH" + \
 171                 trailer
 172     imperial_s = re.match(
 173         r"(.* )(\d+)( MPH)( \(\d+ KT\))(.*)",
 174         line
 175     )
 176     if imperial_s:
 177         preamble, mph, m_u, kt, trailer = imperial_s.groups()
 178         if units == "imperial": line = preamble + mph + m_u + trailer
 179         elif units == "metric":
 180             line = preamble + str(int(round(int(mph)*1.609344))) + " KPH" + \
 181                 trailer
 182     # if imperial is desired, qualify given forcast temperatures like "X F"; if
 183     # metric is desired, convert to "Y C"
 184     imperial_t = re.match(
 185         r"(.* )(High |high |Low |low )(\d+)(\.|,)(.*)",
 186         line
 187     )
 188     if imperial_t:
 189         preamble, parameter, fahrenheit, sep, trailer = imperial_t.groups()
 190         if units == "imperial":
 191             line = preamble + parameter + fahrenheit + " F" + sep + trailer
 192         elif units == "metric":
 193             line = preamble + parameter \
 194                 + str(int(round((int(fahrenheit)-32)*5/9))) + " C" + sep \
 195                 + trailer
 196     # hand off the resulting line
 197     return line
 198
 199 def get_uri(
 200     uri,
 201     ignore_fail=False,
 202     cache_data=False,
 203     cacheage=900,
 204     cachedir="."
 205 ):
 206     """Return a string containing the results of a URI GET."""
 207     if pyversion("3"):
 208         import urllib, urllib.error, urllib.request
 209         URLError = urllib.error.URLError
 210         urlopen = urllib.request.urlopen
 211     else:
 212         import urllib2 as urllib
 213         URLError = urllib.URLError
 214         urlopen = urllib.urlopen
 215     import os, time
 216     if cache_data:
 217         dcachedir = os.path.join( os.path.expanduser(cachedir), "datacache" )
 218         if not os.path.exists(dcachedir):
 219             try: os.makedirs(dcachedir)
 220             except (IOError, OSError): pass
 221         dcache_fn = os.path.join(
 222             dcachedir,
 223             uri.split(":",1)[1].replace("/","_")
 224         )
 225     now = time.time()
 226     if cache_data and os.access(dcache_fn, os.R_OK) \
 227         and now-cacheage < os.stat(dcache_fn).st_mtime <= now:
 228         dcache_fd = open(dcache_fn)
 229         data = dcache_fd.read()
 230         dcache_fd.close()
 231     else:
 232         try:
 233             data = urlopen(uri).read().decode("utf-8")
 234         except URLError:
 235             if ignore_fail: return ""
 236             import os, sys
 237             sys.stderr.write("%s error: failed to retrieve\n   %s\n\n" % (
 238                 os.path.basename( sys.argv[0] ), uri))
 239             raise
 240         # Some data sources are HTML with the plain text wrapped in pre tags
 241         if "<pre>" in data:
 242             data = data[data.find("<pre>")+5:data.find("</pre>")]
 243         if cache_data:
 244             try:
 245                 import codecs
 246                 dcache_fd = codecs.open(dcache_fn, "w", "utf-8")
 247                 dcache_fd.write(data)
 248                 dcache_fd.close()
 249             except (IOError, OSError): pass
 250     return data
 251
 252 def get_metar(
 253     uri=None,
 254     verbose=False,
 255     quiet=False,
 256     headers=None,
 257     imperial=False,
 258     metric=False,
 259     cache_data=False,
 260     cacheage=900,
 261     cachedir="."
 262 ):
 263     """Return a summarized METAR for the specified station."""
 264     if not uri:
 265         import os, sys
 266         message = "%s error: METAR URI required for conditions\n" % \
 267             os.path.basename( sys.argv[0] )
 268         sys.stderr.write(message)
 269         sys.exit(1)
 270     metar = get_uri(
 271         uri,
 272         cache_data=cache_data,
 273         cacheage=cacheage,
 274         cachedir=cachedir
 275     )
 276     if pyversion("3") and type(metar) is bytes: metar = metar.decode("utf-8")
 277     if verbose: return metar
 278     else:
 279         import re
 280         lines = metar.split("\n")
 281         if not headers:
 282             headers = \
 283                 "relative_humidity," \
 284                 + "precipitation_last_hour," \
 285                 + "sky conditions," \
 286                 + "temperature," \
 287                 + "heat index," \
 288                 + "windchill," \
 289                 + "weather," \
 290                 + "wind"
 291         headerlist = headers.lower().replace("_"," ").split(",")
 292         output = []
 293         if not quiet:
 294             title = "Current conditions at %s"
 295             place = lines[0].split(", ")
 296             if len(place) > 1:
 297                 place = "%s, %s" % ( place[0].title(), place[1] )
 298             else: place = "<UNKNOWN>"
 299             output.append(title%place)
 300             output.append("Last updated " + lines[1])
 301         header_match = False
 302         for header in headerlist:
 303             for line in lines:
 304                 if line.lower().startswith(header + ":"):
 305                     if re.match(r".*:\d+$", line): line = line[:line.rfind(":")]
 306                     if imperial: line = filter_units(line, units="imperial")
 307                     elif metric: line = filter_units(line, units="metric")
 308                     if quiet: output.append(line)
 309                     else: output.append("   " + line)
 310                     header_match = True
 311         if not header_match:
 312             output.append(
 313                 "(no conditions matched your header list, try with --verbose)"
 314             )
 315         return "\n".join(output)
 316
 317 def get_alert(
 318     uri=None,
 319     verbose=False,
 320     quiet=False,
 321     cache_data=False,
 322     cacheage=900,
 323     cachedir="."
 324 ):
 325     """Return alert notice for the specified URI."""
 326     if not uri:
 327         return ""
 328     alert = get_uri(
 329         uri,
 330         ignore_fail=True,
 331         cache_data=cache_data,
 332         cacheage=cacheage,
 333         cachedir=cachedir
 334     ).strip()
 335     if pyversion("3") and type(alert) is bytes: alert = alert.decode("utf-8")
 336     if alert:
 337         if verbose: return alert
 338         else:
 339             import re
 340             if re.search(r"\nNational Weather Service", alert):
 341                 muted = True
 342             else:
 343                 muted = False
 344             expirycheck = re.search(r"Expires:([0-9]{12})", alert)
 345             if expirycheck:
 346                 # only report alerts and forecasts that expired less than
 347                 # offset ago
 348                 import datetime, zoneinfo
 349                 expiration = datetime.datetime.fromisoformat(
 350                     "%sT%sZ" % (expirycheck[1][:8], expirycheck[1][-4:]))
 351                 now = datetime.datetime.now(tz=zoneinfo.ZoneInfo("UTC"))
 352                 # TODO: make this offset configurable
 353                 if now - expiration > datetime.timedelta(hours=1):
 354                     return ""
 355             lines = alert.split("\n")
 356             output = []
 357             for line in lines:
 358                 if muted and line.startswith("National Weather Service"):
 359                     muted = False
 360                     line = ""
 361                 elif line == "&&":
 362                     line = ""
 363                 elif line == "$$":
 364                     muted = True
 365                 if line and not muted:
 366                     if quiet: output.append(line)
 367                     else: output.append("   " + line)
 368             return "\n".join(output)
 369
 370 def get_options(config):
 371     """Parse the options passed on the command line."""
 372
 373     # for optparse's builtin -h/--help option
 374     usage = \
 375         "usage: %prog [options] [alias1|search1 [alias2|search2 [...]]]"
 376
 377     # for optparse's builtin --version option
 378     verstring = "%prog " + weather_version
 379
 380     # create the parser
 381     import optparse
 382     option_parser = optparse.OptionParser(usage=usage, version=verstring)
 383     # separate options object from list of arguments and return both
 384
 385     # the -a/--alert option
 386     if config.has_option("default", "alert"):
 387         default_alert = config.getboolean("default", "alert")
 388     else: default_alert = False
 389     option_parser.add_option("-a", "--alert",
 390         dest="alert",
 391         action="store_true",
 392         default=default_alert,
 393         help="include local alert notices")
 394
 395     # the --atypes option
 396     if config.has_option("default", "atypes"):
 397         default_atypes = config.get("default", "atypes")
 398     else:
 399         default_atypes = \
 400             "coastal_flood_statement," \
 401             + "flash_flood_statement," \
 402             + "flash_flood_warning," \
 403             + "flash_flood_watch," \
 404             + "flood_warning," \
 405             + "severe_thunderstorm_warning," \
 406             + "severe_weather_statement," \
 407             + "special_weather_statement," \
 408             + "tornado," \
 409             + "urgent_weather_message"
 410     option_parser.add_option("--atypes",
 411         dest="atypes",
 412         default=default_atypes,
 413         help="list of alert notification types to display")
 414
 415     # the --build-sets option
 416     option_parser.add_option("--build-sets",
 417         dest="build_sets",
 418         action="store_true",
 419         default=False,
 420         help="(re)build location correlation sets")
 421
 422     # the --cacheage option
 423     if config.has_option("default", "cacheage"):
 424         default_cacheage = config.getint("default", "cacheage")
 425     else: default_cacheage = 900
 426     option_parser.add_option("--cacheage",
 427         dest="cacheage",
 428         default=default_cacheage,
 429         help="duration in seconds to refresh cached data")
 430
 431     # the --cachedir option
 432     if config.has_option("default", "cachedir"):
 433         default_cachedir = config.get("default", "cachedir")
 434     else: default_cachedir = "~/.weather"
 435     option_parser.add_option("--cachedir",
 436         dest="cachedir",
 437         default=default_cachedir,
 438         help="directory for storing cached searches and data")
 439
 440     # the -f/--forecast option
 441     if config.has_option("default", "forecast"):
 442         default_forecast = config.getboolean("default", "forecast")
 443     else: default_forecast = False
 444     option_parser.add_option("-f", "--forecast",
 445         dest="forecast",
 446         action="store_true",
 447         default=default_forecast,
 448         help="include a local forecast")
 449
 450     # the --headers option
 451     if config.has_option("default", "headers"):
 452         default_headers = config.get("default", "headers")
 453     else:
 454         default_headers = \
 455             "temperature," \
 456             + "relative_humidity," \
 457             + "wind," \
 458             + "heat_index," \
 459             + "windchill," \
 460             + "weather," \
 461             + "sky_conditions," \
 462             + "precipitation_last_hour"
 463     option_parser.add_option("--headers",
 464         dest="headers",
 465         default=default_headers,
 466         help="list of conditions headers to display")
 467
 468     # the --imperial option
 469     if config.has_option("default", "imperial"):
 470         default_imperial = config.getboolean("default", "imperial")
 471     else: default_imperial = False
 472     option_parser.add_option("--imperial",
 473         dest="imperial",
 474         action="store_true",
 475         default=default_imperial,
 476         help="filter/convert conditions for US/UK units")
 477
 478     # the --info option
 479     option_parser.add_option("--info",
 480         dest="info",
 481         action="store_true",
 482         default=False,
 483         help="output detailed information for your search")
 484
 485     # the -l/--list option
 486     option_parser.add_option("-l", "--list",
 487         dest="list",
 488         action="store_true",
 489         default=False,
 490         help="list all configured aliases and cached searches")
 491
 492     # the --longlist option
 493     option_parser.add_option("--longlist",
 494         dest="longlist",
 495         action="store_true",
 496         default=False,
 497         help="display details of all configured aliases")
 498
 499     # the -m/--metric option
 500     if config.has_option("default", "metric"):
 501         default_metric = config.getboolean("default", "metric")
 502     else: default_metric = False
 503     option_parser.add_option("-m", "--metric",
 504         dest="metric",
 505         action="store_true",
 506         default=default_metric,
 507         help="filter/convert conditions for metric units")
 508
 509     # the -n/--no-conditions option
 510     if config.has_option("default", "conditions"):
 511         default_conditions = config.getboolean("default", "conditions")
 512     else: default_conditions = True
 513     option_parser.add_option("-n", "--no-conditions",
 514         dest="conditions",
 515         action="store_false",
 516         default=default_conditions,
 517         help="disable output of current conditions")
 518
 519     # the --no-cache option
 520     if config.has_option("default", "cache"):
 521         default_cache = config.getboolean("default", "cache")
 522     else: default_cache = True
 523     option_parser.add_option("--no-cache",
 524         dest="cache",
 525         action="store_false",
 526         default=True,
 527         help="disable all caching (searches and data)")
 528
 529     # the --no-cache-data option
 530     if config.has_option("default", "cache_data"):
 531         default_cache_data = config.getboolean("default", "cache_data")
 532     else: default_cache_data = True
 533     option_parser.add_option("--no-cache-data",
 534         dest="cache_data",
 535         action="store_false",
 536         default=True,
 537         help="disable retrieved data caching")
 538
 539     # the --no-cache-search option
 540     if config.has_option("default", "cache_search"):
 541         default_cache_search = config.getboolean("default", "cache_search")
 542     else: default_cache_search = True
 543     option_parser.add_option("--no-cache-search",
 544         dest="cache_search",
 545         action="store_false",
 546         default=True,
 547         help="disable search result caching")
 548
 549     # the -q/--quiet option
 550     if config.has_option("default", "quiet"):
 551         default_quiet = config.getboolean("default", "quiet")
 552     else: default_quiet = False
 553     option_parser.add_option("-q", "--quiet",
 554         dest="quiet",
 555         action="store_true",
 556         default=default_quiet,
 557         help="skip preambles and don't indent")
 558
 559     # the --setpath option
 560     if config.has_option("default", "setpath"):
 561         default_setpath = config.get("default", "setpath")
 562     else: default_setpath = ".:~/.weather"
 563     option_parser.add_option("--setpath",
 564         dest="setpath",
 565         default=default_setpath,
 566         help="directory search path for correlation sets")
 567
 568     # the -v/--verbose option
 569     if config.has_option("default", "verbose"):
 570         default_verbose = config.getboolean("default", "verbose")
 571     else: default_verbose = False
 572     option_parser.add_option("-v", "--verbose",
 573         dest="verbose",
 574         action="store_true",
 575         default=default_verbose,
 576         help="show full decoded feeds")
 577
 578     # deprecated options
 579     if config.has_option("default", "city"):
 580         default_city = config.get("default", "city")
 581     else: default_city = ""
 582     option_parser.add_option("-c", "--city",
 583         dest="city",
 584         default=default_city,
 585         help=optparse.SUPPRESS_HELP)
 586     if config.has_option("default", "id"):
 587         default_id = config.get("default", "id")
 588     else: default_id = ""
 589     option_parser.add_option("-i", "--id",
 590         dest="id",
 591         default=default_id,
 592         help=optparse.SUPPRESS_HELP)
 593     if config.has_option("default", "st"):
 594         default_st = config.get("default", "st")
 595     else: default_st = ""
 596     option_parser.add_option("-s", "--st",
 597         dest="st",
 598         default=default_st,
 599         help=optparse.SUPPRESS_HELP)
 600
 601     options, arguments = option_parser.parse_args()
 602     return options, arguments
 603
 604 def get_config():
 605     """Parse the aliases and configuration."""
 606     if pyversion("3"): import configparser
 607     else: import ConfigParser as configparser
 608     config = configparser.ConfigParser()
 609     import os
 610     rcfiles = [
 611         "/etc/weatherrc",
 612         "/etc/weather/weatherrc",
 613         os.path.expanduser("~/.weather/weatherrc"),
 614         os.path.expanduser("~/.weatherrc"),
 615         "weatherrc"
 616         ]
 617     for rcfile in rcfiles:
 618         if os.access(rcfile, os.R_OK):
 619             if pyversion("3"):
 620                 config.read(rcfile, encoding="utf-8")
 621             else:
 622                 config.read(rcfile)
 623     for section in config.sections():
 624         if section != section.lower():
 625             if config.has_section(section.lower()):
 626                 config.remove_section(section.lower())
 627             config.add_section(section.lower())
 628             for option,value in config.items(section):
 629                 config.set(section.lower(), option, value)
 630     return config
 631
 632 def integrate_search_cache(config, cachedir, setpath):
 633     """Add cached search results into the configuration."""
 634     if pyversion("3"): import configparser
 635     else: import ConfigParser as configparser
 636     import os, time
 637     scache_fn = os.path.join( os.path.expanduser(cachedir), "searches" )
 638     if not os.access(scache_fn, os.R_OK): return config
 639     scache_fd = open(scache_fn)
 640     created = float( scache_fd.readline().split(":")[1].strip().split()[0] )
 641     scache_fd.close()
 642     now = time.time()
 643     datafiles = data_index(setpath)
 644     if datafiles:
 645         data_freshness = sorted(
 646             [ x[1] for x in datafiles.values() ],
 647             reverse=True
 648         )[0]
 649     else: data_freshness = now
 650     if created < data_freshness <= now:
 651         try:
 652             os.remove(scache_fn)
 653             print( "[clearing outdated %s]" % scache_fn )
 654         except (IOError, OSError):
 655             pass
 656         return config
 657     scache = configparser.ConfigParser()
 658     if pyversion("3"):
 659         scache.read(scache_fn, encoding="utf-8")
 660     else:
 661         scache.read(scache_fn)
 662     for section in scache.sections():
 663         if not config.has_section(section):
 664             config.add_section(section)
 665             for option,value in scache.items(section):
 666                 config.set(section, option, value)
 667     return config
 668
 669 def list_aliases(config, detail=False):
 670     """Return a formatted list of aliases defined in the config."""
 671     if detail:
 672         output = "\n# configured alias details..."
 673         for section in sorted(config.sections()):
 674             output += "\n\n[%s]" % section
 675             for item in sorted(config.items(section)):
 676                 output += "\n%s = %s" % item
 677         output += "\n"
 678     else:
 679         output = "configured aliases and cached searches..."
 680         for section in sorted(config.sections()):
 681             if config.has_option(section, "description"):
 682                 description = config.get(section, "description")
 683             else: description = "(no description provided)"
 684             output += "\n   %s: %s" % (section, description)
 685     return output
 686
 687 def data_index(path):
 688     import os
 689     datafiles = {}
 690     for filename in ("airports", "places", "stations", "zctas", "zones"):
 691         for dirname in path.split(":"):
 692             for extension in ("", ".gz", ".txt"):
 693                 candidate = os.path.expanduser(
 694                     os.path.join( dirname, "".join( (filename, extension) ) )
 695                 )
 696                 if os.path.exists(candidate):
 697                     datafiles[filename] = (
 698                         candidate,
 699                         os.stat(candidate).st_mtime
 700                     )
 701                     break
 702             if filename in datafiles:
 703                 break
 704     return datafiles
 705
 706 def guess(
 707     expression,
 708     path=".",
 709     max_results=20,
 710     info=False,
 711     cache_search=False,
 712     cacheage=900,
 713     cachedir=".",
 714     quiet=False
 715 ):
 716     """Find URIs using airport, gecos, placename, station, ZCTA/ZIP, zone."""
 717     import codecs, datetime, time, os, re, sys
 718     if pyversion("3"): import configparser
 719     else: import ConfigParser as configparser
 720     datafiles = data_index(path)
 721     if re.match("[A-Za-z]{3}$", expression): searchtype = "airport"
 722     elif re.match("[A-Za-z0-9]{4}$", expression): searchtype = "station"
 723     elif re.match("[A-Za-z]{2}[Zz][0-9]{3}$", expression): searchtype = "zone"
 724     elif re.match("[0-9]{5}$", expression): searchtype = "ZCTA"
 725     elif re.match(
 726         r"[\+-]?\d+(\.\d+)?(-\d+){,2}[ENSWensw]?, *[\+-]?\d+(\.\d+)?(-\d+){,2}[ENSWensw]?$",
 727         expression
 728     ):
 729         searchtype = "coordinates"
 730     elif re.match(r"(FIPS|fips)\d+$", expression): searchtype = "FIPS"
 731     else:
 732         searchtype = "name"
 733         cache_search = False
 734     if cache_search: action = "caching"
 735     else: action = "using"
 736     if info:
 737         scores = [
 738             (0.005, "bad"),
 739             (0.025, "poor"),
 740             (0.160, "suspect"),
 741             (0.500, "mediocre"),
 742             (0.840, "good"),
 743             (0.975, "great"),
 744             (0.995, "excellent"),
 745             (1.000, "ideal"),
 746         ]
 747     if not quiet: print("Searching via %s..."%searchtype)
 748     stations = configparser.ConfigParser()
 749     dataname = "stations"
 750     if dataname in datafiles:
 751         datafile = datafiles[dataname][0]
 752         if datafile.endswith(".gz"):
 753             import gzip
 754             if pyversion("3"):
 755                 stations.read_string(
 756                     gzip.open(datafile).read().decode("utf-8") )
 757             else: stations.read_file( gzip.open(datafile) )
 758         else:
 759             if pyversion("3"):
 760                 stations.read(datafile, encoding="utf-8")
 761             else:
 762                 stations.read(datafile)
 763     else:
 764         message = "%s error: can't find \"%s\" data file\n" % (
 765             os.path.basename( sys.argv[0] ),
 766             dataname
 767         )
 768         sys.stderr.write(message)
 769         exit(1)
 770     zones = configparser.ConfigParser()
 771     dataname = "zones"
 772     if dataname in datafiles:
 773         datafile = datafiles[dataname][0]
 774         if datafile.endswith(".gz"):
 775             import gzip
 776             if pyversion("3"):
 777                 zones.read_string( gzip.open(datafile).read().decode("utf-8") )
 778             else: zones.read_file( gzip.open(datafile) )
 779         else:
 780             if pyversion("3"):
 781                 zones.read(datafile, encoding="utf-8")
 782             else:
 783                 zones.read(datafile)
 784     else:
 785         message = "%s error: can't find \"%s\" data file\n" % (
 786             os.path.basename( sys.argv[0] ),
 787             dataname
 788         )
 789         sys.stderr.write(message)
 790         exit(1)
 791     search = None
 792     station = ("", 0)
 793     zone = ("", 0)
 794     dataset = None
 795     possibilities = []
 796     uris = {}
 797     if searchtype == "airport":
 798         expression = expression.lower()
 799         airports = configparser.ConfigParser()
 800         dataname = "airports"
 801         if dataname in datafiles:
 802             datafile = datafiles[dataname][0]
 803             if datafile.endswith(".gz"):
 804                 import gzip
 805                 if pyversion("3"):
 806                     airports.read_string(
 807                         gzip.open(datafile).read().decode("utf-8") )
 808                 else: airports.read_file( gzip.open(datafile) )
 809             else:
 810                 if pyversion("3"):
 811                     airports.read(datafile, encoding="utf-8")
 812                 else:
 813                     airports.read(datafile)
 814         else:
 815             message = "%s error: can't find \"%s\" data file\n" % (
 816                 os.path.basename( sys.argv[0] ),
 817                 dataname
 818             )
 819             sys.stderr.write(message)
 820             exit(1)
 821         if airports.has_section(expression) \
 822             and airports.has_option(expression, "station"):
 823             search = (expression, "IATA/FAA airport code %s" % expression)
 824             station = ( airports.get(expression, "station"), 0 )
 825             if stations.has_option(station[0], "zone"):
 826                 zone = eval( stations.get(station[0], "zone") )
 827                 dataset = stations
 828             if not ( info or quiet ) \
 829                 and stations.has_option( station[0], "description" ):
 830                 print(
 831                     "[%s result %s]" % (
 832                         action,
 833                         stations.get(station[0], "description")
 834                     )
 835                 )
 836         else:
 837             message = "No IATA/FAA airport code \"%s\" in the %s file.\n" % (
 838                 expression,
 839                 datafiles["airports"][0]
 840             )
 841             sys.stderr.write(message)
 842             exit(1)
 843     elif searchtype == "station":
 844         expression = expression.lower()
 845         if stations.has_section(expression):
 846             station = (expression, 0)
 847             if not search:
 848                 search = (expression, "ICAO station code %s" % expression)
 849             if stations.has_option(expression, "zone"):
 850                 zone = eval( stations.get(expression, "zone") )
 851                 dataset = stations
 852             if not ( info or quiet ) \
 853                 and stations.has_option(expression, "description"):
 854                 print(
 855                     "[%s result %s]" % (
 856                         action,
 857                         stations.get(expression, "description")
 858                     )
 859                 )
 860         else:
 861             message = "No ICAO weather station \"%s\" in the %s file.\n" % (
 862                 expression,
 863                 datafiles["stations"][0]
 864             )
 865             sys.stderr.write(message)
 866             exit(1)
 867     elif searchtype == "zone":
 868         expression = expression.lower()
 869         if zones.has_section(expression) \
 870             and zones.has_option(expression, "station"):
 871             zone = (expression, 0)
 872             station = eval( zones.get(expression, "station") )
 873             dataset = zones
 874             search = (expression, "NWS/NOAA weather zone %s" % expression)
 875             if not ( info or quiet ) \
 876                 and zones.has_option(expression, "description"):
 877                 print(
 878                     "[%s result %s]" % (
 879                         action,
 880                         zones.get(expression, "description")
 881                     )
 882                 )
 883         else:
 884             message = "No usable NWS weather zone \"%s\" in the %s file.\n" % (
 885                 expression,
 886                 datafiles["zones"][0]
 887             )
 888             sys.stderr.write(message)
 889             exit(1)
 890     elif searchtype == "ZCTA":
 891         zctas = configparser.ConfigParser()
 892         dataname = "zctas"
 893         if dataname in datafiles:
 894             datafile = datafiles[dataname][0]
 895             if datafile.endswith(".gz"):
 896                 import gzip
 897                 if pyversion("3"):
 898                     zctas.read_string(
 899                         gzip.open(datafile).read().decode("utf-8") )
 900                 else: zctas.read_file( gzip.open(datafile) )
 901             else:
 902                 if pyversion("3"):
 903                     zctas.read(datafile, encoding="utf-8")
 904                 else:
 905                     zctas.read(datafile)
 906         else:
 907             message = "%s error: can't find \"%s\" data file\n" % (
 908                 os.path.basename( sys.argv[0] ),
 909                 dataname
 910             )
 911             sys.stderr.write(message)
 912             exit(1)
 913         dataset = zctas
 914         if zctas.has_section(expression) \
 915             and zctas.has_option(expression, "station"):
 916             station = eval( zctas.get(expression, "station") )
 917             search = (expression, "Census ZCTA (ZIP code) %s" % expression)
 918             if zctas.has_option(expression, "zone"):
 919                 zone = eval( zctas.get(expression, "zone") )
 920         else:
 921             message = "No census ZCTA (ZIP code) \"%s\" in the %s file.\n" % (
 922                 expression,
 923                 datafiles["zctas"][0]
 924             )
 925             sys.stderr.write(message)
 926             exit(1)
 927     elif searchtype == "coordinates":
 928         search = (expression, "Geographic coordinates %s" % expression)
 929         stationtable = {}
 930         for station in stations.sections():
 931             if stations.has_option(station, "location"):
 932                 stationtable[station] = {
 933                     "location": eval( stations.get(station, "location") )
 934                 }
 935         station = closest( gecos(expression), stationtable, "location", 0.1 )
 936         if not station[0]:
 937             message = "No ICAO weather station found near %s.\n" % expression
 938             sys.stderr.write(message)
 939             exit(1)
 940         zonetable = {}
 941         for zone in zones.sections():
 942             if zones.has_option(zone, "centroid"):
 943                 zonetable[zone] = {
 944                     "centroid": eval( zones.get(zone, "centroid") )
 945                 }
 946         zone = closest( gecos(expression), zonetable, "centroid", 0.1 )
 947         if not zone[0]:
 948             message = "No NWS weather zone near %s; forecasts unavailable.\n" \
 949                 % expression
 950             sys.stderr.write(message)
 951     elif searchtype in ("FIPS", "name"):
 952         places = configparser.ConfigParser()
 953         dataname = "places"
 954         if dataname in datafiles:
 955             datafile = datafiles[dataname][0]
 956             if datafile.endswith(".gz"):
 957                 import gzip
 958                 if pyversion("3"):
 959                     places.read_string(
 960                         gzip.open(datafile).read().decode("utf-8") )
 961                 else: places.read_file( gzip.open(datafile) )
 962             else:
 963                 if pyversion("3"):
 964                     places.read(datafile, encoding="utf-8")
 965                 else:
 966                     places.read(datafile)
 967         else:
 968             message = "%s error: can't find \"%s\" data file\n" % (
 969                 os.path.basename( sys.argv[0] ),
 970                 dataname
 971             )
 972             sys.stderr.write(message)
 973             exit(1)
 974         dataset = places
 975         place = expression.lower()
 976         if places.has_section(place) and places.has_option(place, "station"):
 977             station = eval( places.get(place, "station") )
 978             search = (expression, "Census Place %s" % expression)
 979             if places.has_option(place, "description"):
 980                 search = (
 981                     search[0],
 982                     search[1] + ", %s" % places.get(place, "description")
 983                 )
 984             if places.has_option(place, "zone"):
 985                 zone = eval( places.get(place, "zone") )
 986             if not ( info or quiet ) \
 987                 and places.has_option(place, "description"):
 988                 print(
 989                     "[%s result %s]" % (
 990                         action,
 991                         places.get(place, "description")
 992                     )
 993                 )
 994         else:
 995             for place in places.sections():
 996                 if places.has_option(place, "description") \
 997                     and places.has_option(place, "station") \
 998                     and re.search(
 999                         expression,
1000                         places.get(place, "description"),
1001                         re.I
1002                     ):
1003                         possibilities.append(place)
1004             for place in stations.sections():
1005                 if stations.has_option(place, "description") \
1006                     and re.search(
1007                         expression,
1008                         stations.get(place, "description"),
1009                         re.I
1010                     ):
1011                         possibilities.append(place)
1012             for place in zones.sections():
1013                 if zones.has_option(place, "description") \
1014                     and zones.has_option(place, "station") \
1015                     and re.search(
1016                         expression,
1017                         zones.get(place, "description"),
1018                         re.I
1019                     ):
1020                         possibilities.append(place)
1021             if len(possibilities) == 1:
1022                 place = possibilities[0]
1023                 if places.has_section(place):
1024                     station = eval( places.get(place, "station") )
1025                     description = places.get(place, "description")
1026                     if places.has_option(place, "zone"):
1027                         zone = eval( places.get(place, "zone" ) )
1028                     search = ( expression, "%s: %s" % (place, description) )
1029                 elif stations.has_section(place):
1030                     station = (place, 0.0)
1031                     description = stations.get(place, "description")
1032                     if stations.has_option(place, "zone"):
1033                         zone = eval( stations.get(place, "zone" ) )
1034                     search = ( expression, "ICAO station code %s" % place )
1035                 elif zones.has_section(place):
1036                     station = eval( zones.get(place, "station") )
1037                     description = zones.get(place, "description")
1038                     zone = (place, 0.0)
1039                     search = ( expression, "NWS/NOAA weather zone %s" % place )
1040                 if not ( info or quiet ):
1041                     print( "[%s result %s]" % (action, description) )
1042             if not possibilities and not station[0]:
1043                 message = "No FIPS code/census area match in the %s file.\n" % (
1044                     datafiles["places"][0]
1045                 )
1046                 sys.stderr.write(message)
1047                 exit(1)
1048     if station[0]:
1049         uris["metar"] = stations.get( station[0], "metar" )
1050         if zone[0]:
1051             for key,value in zones.items( zone[0] ):
1052                 if key not in ("centroid", "description", "station"):
1053                     uris[key] = value
1054     elif possibilities:
1055         count = len(possibilities)
1056         if count <= max_results:
1057             print( "Your search is ambiguous, returning %s matches:" % count )
1058             for place in sorted(possibilities):
1059                 if places.has_section(place):
1060                     print(
1061                         "   [%s] %s" % (
1062                             place,
1063                             places.get(place, "description")
1064                         )
1065                     )
1066                 elif stations.has_section(place):
1067                     print(
1068                         "   [%s] %s" % (
1069                             place,
1070                             stations.get(place, "description")
1071                         )
1072                     )
1073                 elif zones.has_section(place):
1074                     print(
1075                         "   [%s] %s" % (
1076                             place,
1077                             zones.get(place, "description")
1078                         )
1079                     )
1080         else:
1081             print(
1082                 "Your search is too ambiguous, returning %s matches." % count
1083             )
1084         exit(0)
1085     if info:
1086         stationlist = []
1087         zonelist = []
1088         if dataset:
1089             for section in dataset.sections():
1090                 if dataset.has_option(section, "station"):
1091                     stationlist.append(
1092                         eval( dataset.get(section, "station") )[1]
1093                     )
1094                 if dataset.has_option(section, "zone"):
1095                     zonelist.append( eval( dataset.get(section, "zone") )[1] )
1096         stationlist.sort()
1097         zonelist.sort()
1098         scount = len(stationlist)
1099         zcount = len(zonelist)
1100         sranks = []
1101         zranks = []
1102         for score in scores:
1103             if stationlist:
1104                 sranks.append( stationlist[ int( (1-score[0]) * scount ) ] )
1105             if zonelist:
1106                 zranks.append( zonelist[ int( (1-score[0]) * zcount ) ] )
1107         description = search[1]
1108         uris["description"] = description
1109         print(
1110             "%s\n%s" % ( description, "-" * len(description) )
1111         )
1112         print(
1113             "%s: %s" % (
1114                 station[0],
1115                 stations.get( station[0], "description" )
1116             )
1117         )
1118         km = radian_to_km*station[1]
1119         mi = radian_to_mi*station[1]
1120         if sranks and not description.startswith("ICAO station code "):
1121             for index in range(0, len(scores)):
1122                 if station[1] >= sranks[index]:
1123                     score = scores[index][1]
1124                     break
1125             print(
1126                 "   (proximity %s, %.3gkm, %.3gmi)" % ( score, km, mi )
1127             )
1128         elif searchtype == "coordinates":
1129             print( "   (%.3gkm, %.3gmi)" % (km, mi) )
1130         if zone[0]:
1131             print(
1132                 "%s: %s" % ( zone[0], zones.get( zone[0], "description" ) )
1133             )
1134         km = radian_to_km*zone[1]
1135         mi = radian_to_mi*zone[1]
1136         if zranks and not description.startswith("NWS/NOAA weather zone "):
1137             for index in range(0, len(scores)):
1138                 if zone[1] >= zranks[index]:
1139                     score = scores[index][1]
1140                     break
1141             print(
1142                 "   (proximity %s, %.3gkm, %.3gmi)" % ( score, km, mi )
1143             )
1144         elif searchtype == "coordinates" and zone[0]:
1145             print( "   (%.3gkm, %.3gmi)" % (km, mi) )
1146     if cache_search:
1147         now = time.time()
1148         nowstamp = "%s (%s)" % (
1149             now,
1150             datetime.datetime.isoformat(
1151                 datetime.datetime.fromtimestamp(now),
1152                 " "
1153             )
1154         )
1155         search_cache = ["\n"]
1156         search_cache.append( "[%s]\n" % search[0] )
1157         search_cache.append( "cached = %s\n" % nowstamp )
1158         for uriname in sorted(uris.keys()):
1159             search_cache.append( "%s = %s\n" % ( uriname, uris[uriname] ) )
1160         real_cachedir = os.path.expanduser(cachedir)
1161         if not os.path.exists(real_cachedir):
1162             try: os.makedirs(real_cachedir)
1163             except (IOError, OSError): pass
1164         scache_fn = os.path.join(real_cachedir, "searches")
1165         if not os.path.exists(scache_fn):
1166             then = sorted(
1167                     [ x[1] for x in datafiles.values() ],
1168                     reverse=True
1169                 )[0]
1170             thenstamp = "%s (%s)" % (
1171                 then,
1172                 datetime.datetime.isoformat(
1173                     datetime.datetime.fromtimestamp(then),
1174                     " "
1175                 )
1176             )
1177             search_cache.insert(
1178                 0,
1179                 "# based on data files from: %s\n" % thenstamp
1180             )
1181         try:
1182             scache_existing = configparser.ConfigParser()
1183             if pyversion("3"):
1184                 scache_existing.read(scache_fn, encoding="utf-8")
1185             else:
1186                 scache_existing.read(scache_fn)
1187             if not scache_existing.has_section(search[0]):
1188                 scache_fd = codecs.open(scache_fn, "a", "utf-8")
1189                 scache_fd.writelines(search_cache)
1190                 scache_fd.close()
1191         except (IOError, OSError): pass
1192     if not info:
1193         return(uris)
1194
1195 def closest(position, nodes, fieldname, angle=None):
1196     import math
1197     if not angle: angle = 2*math.pi
1198     match = None
1199     for name in nodes:
1200         if fieldname in nodes[name]:
1201             node = nodes[name][fieldname]
1202             if node and abs( position[0]-node[0] ) < angle:
1203                 if abs( position[1]-node[1] ) < angle \
1204                     or abs( abs( position[1]-node[1] ) - 2*math.pi ) < angle:
1205                     if position == node:
1206                         angle = 0
1207                         match = name
1208                     else:
1209                         candidate = math.acos(
1210                             math.sin( position[0] ) * math.sin( node[0] ) \
1211                                 + math.cos( position[0] ) \
1212                                 * math.cos( node[0] ) \
1213                                 * math.cos( position[1] - node[1] )
1214                             )
1215                         if candidate < angle:
1216                             angle = candidate
1217                             match = name
1218     if match: match = str(match)
1219     return (match, angle)
1220
1221 def gecos(formatted):
1222     import math, re
1223     coordinates = formatted.split(",")
1224     for coordinate in range(0, 2):
1225         degrees, foo, minutes, bar, seconds, hemisphere = re.match(
1226             r"([\+-]?\d+\.?\d*)(-(\d+))?(-(\d+))?([ensw]?)$",
1227             coordinates[coordinate].strip().lower()
1228         ).groups()
1229         value = float(degrees)
1230         if minutes: value += float(minutes)/60
1231         if seconds: value += float(seconds)/3600
1232         if hemisphere and hemisphere in "sw": value *= -1
1233         coordinates[coordinate] = math.radians(value)
1234     return tuple(coordinates)
1235
1236 def correlate():
1237     import codecs, csv, datetime, hashlib, os, re, sys, time, zipfile, zoneinfo
1238     if pyversion("3"): import configparser
1239     else: import ConfigParser as configparser
1240     for filename in os.listdir("."):
1241         if re.match("[0-9]{4}_Gaz_counties_national.zip$", filename):
1242             gcounties_an = filename
1243             gcounties_fn = filename[:-4] + ".txt"
1244         elif re.match("[0-9]{4}_Gaz_cousubs_national.zip$", filename):
1245             gcousubs_an = filename
1246             gcousubs_fn = filename[:-4] + ".txt"
1247         elif re.match("[0-9]{4}_Gaz_place_national.zip$", filename):
1248             gplace_an = filename
1249             gplace_fn = filename[:-4] + ".txt"
1250         elif re.match("[0-9]{4}_Gaz_zcta_national.zip$", filename):
1251             gzcta_an = filename
1252             gzcta_fn = filename[:-4] + ".txt"
1253         elif re.match("bp[0-9]{2}[a-z]{2}[0-9]{2}.dbx$", filename):
1254             cpfzcf_fn = filename
1255     nsdcccc_fn = "nsd_cccc.txt"
1256     ourairports_fn = "airports.csv"
1257     overrides_fn = "overrides.conf"
1258     overrideslog_fn = "overrides.log"
1259     slist_fn = "slist"
1260     zlist_fn = "zlist"
1261     qalog_fn = "qa.log"
1262     airports_fn = "airports"
1263     places_fn = "places"
1264     stations_fn = "stations"
1265     zctas_fn = "zctas"
1266     zones_fn = "zones"
1267     header = """\
1268 %s
1269 # generated by %s on %s from these public domain sources:
1270 #
1271 # https://www.census.gov/geographies/reference-files/time-series/geo/gazetteer-files.html
1272 # %s %s %s
1273 # %s %s %s
1274 # %s %s %s
1275 # %s %s %s
1276 #
1277 # https://www.weather.gov/gis/ZoneCounty/
1278 # %s %s %s
1279 #
1280 # https://tgftp.nws.noaa.gov/data/
1281 # %s %s %s
1282 #
1283 # https://ourairports.com/data/
1284 # %s %s %s
1285 #
1286 # ...and these manually-generated or hand-compiled adjustments:
1287 # %s %s %s
1288 # %s %s %s
1289 # %s %s %s\
1290 """ % (
1291         weather_copyright,
1292         os.path.basename( sys.argv[0] ),
1293         datetime.date.isoformat(
1294             datetime.datetime.utcfromtimestamp( int(os.environ.get('SOURCE_DATE_EPOCH', time.time())) )
1295         ),
1296         hashlib.md5( open(gcounties_an, "rb").read() ).hexdigest(),
1297         datetime.date.isoformat(
1298             datetime.datetime.utcfromtimestamp( os.path.getmtime(gcounties_an) )
1299         ),
1300         gcounties_an,
1301         hashlib.md5( open(gcousubs_an, "rb").read() ).hexdigest(),
1302         datetime.date.isoformat(
1303             datetime.datetime.utcfromtimestamp( os.path.getmtime(gcousubs_an) )
1304         ),
1305         gcousubs_an,
1306         hashlib.md5( open(gplace_an, "rb").read() ).hexdigest(),
1307         datetime.date.isoformat(
1308             datetime.datetime.utcfromtimestamp( os.path.getmtime(gplace_an) )
1309         ),
1310         gplace_an,
1311         hashlib.md5( open(gzcta_an, "rb").read() ).hexdigest(),
1312         datetime.date.isoformat(
1313             datetime.datetime.utcfromtimestamp( os.path.getmtime(gzcta_an) )
1314         ),
1315         gzcta_an,
1316         hashlib.md5( open(cpfzcf_fn, "rb").read() ).hexdigest(),
1317         datetime.date.isoformat(
1318             datetime.datetime.utcfromtimestamp( os.path.getmtime(cpfzcf_fn) )
1319         ),
1320         cpfzcf_fn,
1321         hashlib.md5( open(nsdcccc_fn, "rb").read() ).hexdigest(),
1322         datetime.date.isoformat(
1323             datetime.datetime.utcfromtimestamp( os.path.getmtime(nsdcccc_fn) )
1324         ),
1325         nsdcccc_fn,
1326         hashlib.md5( open(ourairports_fn, "rb").read() ).hexdigest(),
1327         datetime.date.isoformat(
1328             datetime.datetime.utcfromtimestamp( os.path.getmtime(ourairports_fn) )
1329         ),
1330         ourairports_fn,
1331         hashlib.md5( open(overrides_fn, "rb").read() ).hexdigest(),
1332         datetime.date.isoformat(
1333             datetime.datetime.utcfromtimestamp( os.path.getmtime(overrides_fn) )
1334         ),
1335         overrides_fn,
1336         hashlib.md5( open(slist_fn, "rb").read() ).hexdigest(),
1337         datetime.date.isoformat(
1338             datetime.datetime.utcfromtimestamp( os.path.getmtime(slist_fn) )
1339         ),
1340         slist_fn,
1341         hashlib.md5( open(zlist_fn, "rb").read() ).hexdigest(),
1342         datetime.date.isoformat(
1343             datetime.datetime.utcfromtimestamp( os.path.getmtime(zlist_fn) )
1344         ),
1345         zlist_fn
1346     )
1347     airports = {}
1348     places = {}
1349     stations = {}
1350     zctas = {}
1351     zones = {}
1352     message = "Reading %s:%s..." % (gcounties_an, gcounties_fn)
1353     sys.stdout.write(message)
1354     sys.stdout.flush()
1355     count = 0
1356     gcounties = zipfile.ZipFile(gcounties_an).open(gcounties_fn, "r")
1357     columns = gcounties.readline().decode("utf-8").strip().split("\t")
1358     for line in gcounties:
1359         fields = line.decode("utf-8").strip().split("\t")
1360         f_geoid = fields[ columns.index("GEOID") ].strip()
1361         f_name = fields[ columns.index("NAME") ].strip()
1362         f_usps = fields[ columns.index("USPS") ].strip()
1363         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1364         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1365         if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
1366             fips = "fips%s" % f_geoid
1367             if fips not in places: places[fips] = {}
1368             places[fips]["centroid"] = gecos(
1369                 "%s,%s" % (f_intptlat, f_intptlong)
1370             )
1371             places[fips]["description"] = "%s, %s" % (f_name, f_usps)
1372             count += 1
1373     gcounties.close()
1374     print("done (%s lines)." % count)
1375     message = "Reading %s:%s..." % (gcousubs_an, gcousubs_fn)
1376     sys.stdout.write(message)
1377     sys.stdout.flush()
1378     count = 0
1379     gcousubs = zipfile.ZipFile(gcousubs_an).open(gcousubs_fn, "r")
1380     columns = gcousubs.readline().decode("utf-8").strip().split("\t")
1381     for line in gcousubs:
1382         fields = line.decode("utf-8").strip().split("\t")
1383         f_geoid = fields[ columns.index("GEOID") ].strip()
1384         f_name = fields[ columns.index("NAME") ].strip()
1385         f_usps = fields[ columns.index("USPS") ].strip()
1386         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1387         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1388         if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
1389             fips = "fips%s" % f_geoid
1390             if fips not in places: places[fips] = {}
1391             places[fips]["centroid"] = gecos(
1392                 "%s,%s" % (f_intptlat, f_intptlong)
1393             )
1394             places[fips]["description"] = "%s, %s" % (f_name, f_usps)
1395             count += 1
1396     gcousubs.close()
1397     print("done (%s lines)." % count)
1398     message = "Reading %s:%s..." % (gplace_an, gplace_fn)
1399     sys.stdout.write(message)
1400     sys.stdout.flush()
1401     count = 0
1402     gplace = zipfile.ZipFile(gplace_an).open(gplace_fn, "r")
1403     columns = gplace.readline().decode("utf-8").strip().split("\t")
1404     for line in gplace:
1405         fields = line.decode("utf-8").strip().split("\t")
1406         f_geoid = fields[ columns.index("GEOID") ].strip()
1407         f_name = fields[ columns.index("NAME") ].strip()
1408         f_usps = fields[ columns.index("USPS") ].strip()
1409         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1410         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1411         if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
1412             fips = "fips%s" % f_geoid
1413             if fips not in places: places[fips] = {}
1414             places[fips]["centroid"] = gecos(
1415                 "%s,%s" % (f_intptlat, f_intptlong)
1416             )
1417             places[fips]["description"] = "%s, %s" % (f_name, f_usps)
1418             count += 1
1419     gplace.close()
1420     print("done (%s lines)." % count)
1421     message = "Reading %s..." % slist_fn
1422     sys.stdout.write(message)
1423     sys.stdout.flush()
1424     count = 0
1425     slist = codecs.open(slist_fn, "r", "utf-8")
1426     for line in slist:
1427         icao = line.split("#")[0].strip()
1428         if icao:
1429             stations[icao] = {
1430                 "metar": "https://tgftp.nws.noaa.gov/data/observations/"\
1431                     + "metar/decoded/%s.TXT" % icao.upper()
1432             }
1433             count += 1
1434     slist.close()
1435     print("done (%s lines)." % count)
1436     message = "Reading %s..." % nsdcccc_fn
1437     sys.stdout.write(message)
1438     sys.stdout.flush()
1439     count = 0
1440     nsdcccc = codecs.open(nsdcccc_fn, "r", "utf-8")
1441     for line in nsdcccc:
1442         line = str(line)
1443         fields = line.split(";")
1444         icao = fields[0].strip().lower()
1445         if icao in stations:
1446             description = []
1447             name = " ".join( fields[3].strip().title().split() )
1448             if name: description.append(name)
1449             st = fields[4].strip()
1450             if st: description.append(st)
1451             country = " ".join( fields[5].strip().title().split() )
1452             if country: description.append(country)
1453             if description:
1454                 stations[icao]["description"] = ", ".join(description)
1455             lat, lon = fields[7:9]
1456             if lat and lon:
1457                 stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) )
1458             elif "location" not in stations[icao]:
1459                 lat, lon = fields[5:7]
1460                 if lat and lon:
1461                     stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) )
1462         count += 1
1463     nsdcccc.close()
1464     print("done (%s lines)." % count)
1465     message = "Reading %s..." % ourairports_fn
1466     sys.stdout.write(message)
1467     sys.stdout.flush()
1468     count = 0
1469     ourairports = open(ourairports_fn, "r")
1470     for row in csv.reader(ourairports):
1471         icao = row[12].lower()
1472         if icao in stations:
1473             iata = row[13].lower()
1474             if len(iata) == 3: airports[iata] = { "station": icao }
1475             if "description" not in stations[icao]:
1476                 description = []
1477                 name = row[3]
1478                 if name: description.append(name)
1479                 municipality = row[10]
1480                 if municipality: description.append(municipality)
1481                 region = row[9]
1482                 country = row[8]
1483                 if region:
1484                     if "-" in region:
1485                         c,r = region.split("-", 1)
1486                         if c == country: region = r
1487                     description.append(region)
1488                 if country:
1489                     description.append(country)
1490                 if description:
1491                     stations[icao]["description"] = ", ".join(description)
1492             if "location" not in stations[icao]:
1493                 lat = row[4]
1494                 if lat:
1495                     lon = row[5]
1496                     if lon:
1497                         stations[icao]["location"] = gecos(
1498                             "%s,%s" % (lat, lon)
1499                         )
1500         count += 1
1501     ourairports.close()
1502     print("done (%s lines)." % count)
1503     message = "Reading %s..." % zlist_fn
1504     sys.stdout.write(message)
1505     sys.stdout.flush()
1506     count = 0
1507     zlist = codecs.open(zlist_fn, "r", "utf-8")
1508     for line in zlist:
1509         line = line.split("#")[0].strip()
1510         if line:
1511             zones[line] = {}
1512             count += 1
1513     zlist.close()
1514     print("done (%s lines)." % count)
1515     message = "Reading %s..." % cpfzcf_fn
1516     sys.stdout.write(message)
1517     sys.stdout.flush()
1518     count = 0
1519     cpfz = {}
1520     cpfzcf = codecs.open(cpfzcf_fn, "r", "utf-8")
1521     for line in cpfzcf:
1522         fields = line.strip().split("|")
1523         if len(fields) == 11 \
1524             and fields[0] and fields[1] and fields[9] and fields[10]:
1525             zone = "z".join( fields[:2] ).lower()
1526             if zone in zones:
1527                 state = fields[0]
1528                 description = fields[3].strip()
1529                 fips = "fips%s"%fields[6]
1530                 countycode = "%sc%s" % (state.lower(), fips[-3:])
1531                 if state:
1532                     zones[zone]["coastal_flood_statement"] = (
1533                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1534                         "flood/coastal/%s/%s.txt" % (state.lower(), zone))
1535                     zones[zone]["flash_flood_statement"] = (
1536                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1537                         "flash_flood/statement/%s/%s.txt"
1538                         % (state.lower(), countycode))
1539                     zones[zone]["flash_flood_warning"] = (
1540                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1541                         "flash_flood/warning/%s/%s.txt"
1542                         % (state.lower(), countycode))
1543                     zones[zone]["flash_flood_watch"] = (
1544                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1545                         "flash_flood/watch/%s/%s.txt" % (state.lower(), zone))
1546                     zones[zone]["flood_warning"] = (
1547                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1548                         "flood/warning/%s/%s.txt"
1549                         % (state.lower(), countycode))
1550                     zones[zone]["severe_thunderstorm_warning"] = (
1551                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1552                         "thunderstorm/%s/%s.txt" % (state.lower(), countycode))
1553                     zones[zone]["severe_weather_statement"] = (
1554                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1555                         "severe_weather_stmt/%s/%s.txt"
1556                         % (state.lower(), countycode))
1557                     zones[zone]["short_term_forecast"] = (
1558                         "https://tgftp.nws.noaa.gov/data/forecasts/nowcast/"
1559                         "%s/%s.txt" % (state.lower(), zone))
1560                     zones[zone]["special_weather_statement"] = (
1561                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1562                         "special_weather_stmt/%s/%s.txt"
1563                         % (state.lower(), zone))
1564                     zones[zone]["state_forecast"] = (
1565                         "https://tgftp.nws.noaa.gov/data/forecasts/state/"
1566                         "%s/%s.txt" % (state.lower(), zone))
1567                     zones[zone]["tornado"] = (
1568                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1569                         "tornado/%s/%s.txt" % (state.lower(), countycode))
1570                     zones[zone]["urgent_weather_message"] = (
1571                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1572                         "non_precip/%s/%s.txt" % (state.lower(), zone))
1573                     zones[zone]["zone_forecast"] = (
1574                         "https://tgftp.nws.noaa.gov/data/forecasts/zone/"
1575                         "%s/%s.txt" % (state.lower(), zone))
1576                 tzcode = fields[7]
1577                 if tzcode == "A":
1578                     zones[zone]["tz"] = "US/Alaska"
1579                 elif tzcode == "AH":
1580                     zones[zone]["tz"] = "US/Aleutian"
1581                 elif tzcode in ("C", "CE", "CM"):
1582                     zones[zone]["tz"] = "US/Central"
1583                 elif tzcode in ("E", "e"):
1584                     zones[zone]["tz"] = "US/Eastern"
1585                 elif tzcode == "F":
1586                     zones[zone]["tz"] = "Pacific/Guadalcanal"
1587                 elif tzcode == "G":
1588                     zones[zone]["tz"] = "Pacific/Guam"
1589                 elif tzcode == "H":
1590                     zones[zone]["tz"] = "US/Hawaii"
1591                 elif tzcode == "J":
1592                     zones[zone]["tz"] = "Japan"
1593                 elif tzcode == "K":
1594                     zones[zone]["tz"] = "Pacific/Kwajalein"
1595                 elif tzcode in ("M", "MC", "MP"):
1596                     zones[zone]["tz"] = "US/Mountain"
1597                 elif tzcode == "m":
1598                     zones[zone]["tz"] = "US/Arizona"
1599                 elif tzcode == "P":
1600                     zones[zone]["tz"] = "US/Pacific"
1601                 elif tzcode == "S":
1602                     zones[zone]["tz"] = "US/Samoa"
1603                 elif tzcode == "V":
1604                     zones[zone]["tz"] = "America/Virgin"
1605                 else:
1606                     zones[zone]["tz"] = ""
1607                 county = fields[5]
1608                 if county:
1609                     if description.endswith(county):
1610                         description += " County"
1611                     else:
1612                         description += ", %s County" % county
1613                 description += ", %s, US" % state
1614                 zones[zone]["description"] = description
1615                 zones[zone]["centroid"] = gecos( ",".join( fields[9:11] ) )
1616                 if fips in places and not zones[zone]["centroid"]:
1617                     zones[zone]["centroid"] = places[fips]["centroid"]
1618         count += 1
1619     cpfzcf.close()
1620     print("done (%s lines)." % count)
1621     message = "Reading %s:%s..." % (gzcta_an, gzcta_fn)
1622     sys.stdout.write(message)
1623     sys.stdout.flush()
1624     count = 0
1625     gzcta = zipfile.ZipFile(gzcta_an).open(gzcta_fn, "r")
1626     columns = gzcta.readline().decode("utf-8").strip().split("\t")
1627     for line in gzcta:
1628         fields = line.decode("utf-8").strip().split("\t")
1629         f_geoid = fields[ columns.index("GEOID") ].strip()
1630         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1631         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1632         if f_geoid and f_intptlat and f_intptlong:
1633             if f_geoid not in zctas: zctas[f_geoid] = {}
1634             zctas[f_geoid]["centroid"] = gecos(
1635                 "%s,%s" % (f_intptlat, f_intptlong)
1636             )
1637             count += 1
1638     gzcta.close()
1639     print("done (%s lines)." % count)
1640     message = "Reading %s..." % overrides_fn
1641     sys.stdout.write(message)
1642     sys.stdout.flush()
1643     count = 0
1644     added = 0
1645     removed = 0
1646     changed = 0
1647     overrides = configparser.ConfigParser()
1648     overrides.read_file( codecs.open(overrides_fn, "r", "utf8") )
1649     overrideslog = []
1650     for section in overrides.sections():
1651         addopt = 0
1652         chgopt = 0
1653         if section.startswith("-"):
1654             section = section[1:]
1655             delete = True
1656         else: delete = False
1657         if re.match("[A-Za-z]{3}$", section):
1658             if delete:
1659                 if section in airports:
1660                     del( airports[section] )
1661                     logact = "removed airport %s" % section
1662                     removed += 1
1663                 else:
1664                     logact = "tried to remove nonexistent airport %s" % section
1665             else:
1666                 if section in airports:
1667                     logact = "changed airport %s" % section
1668                     changed += 1
1669                 else:
1670                     airports[section] = {}
1671                     logact = "added airport %s" % section
1672                     added += 1
1673                 for key,value in overrides.items(section):
1674                     if key in airports[section]: chgopt += 1
1675                     else: addopt += 1
1676                     if key in ("centroid", "location"):
1677                         airports[section][key] = eval(value)
1678                     else:
1679                         airports[section][key] = value
1680                 if addopt and chgopt:
1681                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1682                 elif addopt: logact += " (+%s options)" % addopt
1683                 elif chgopt: logact += " (!%s options)" % chgopt
1684         elif re.match("[A-Za-z0-9]{4}$", section):
1685             if delete:
1686                 if section in stations:
1687                     del( stations[section] )
1688                     logact = "removed station %s" % section
1689                     removed += 1
1690                 else:
1691                     logact = "tried to remove nonexistent station %s" % section
1692             else:
1693                 if section in stations:
1694                     logact = "changed station %s" % section
1695                     changed += 1
1696                 else:
1697                     stations[section] = {}
1698                     logact = "added station %s" % section
1699                     added += 1
1700                 for key,value in overrides.items(section):
1701                     if key in stations[section]: chgopt += 1
1702                     else: addopt += 1
1703                     if key in ("centroid", "location"):
1704                         stations[section][key] = eval(value)
1705                     else:
1706                         stations[section][key] = value
1707                 if addopt and chgopt:
1708                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1709                 elif addopt: logact += " (+%s options)" % addopt
1710                 elif chgopt: logact += " (!%s options)" % chgopt
1711         elif re.match("[0-9]{5}$", section):
1712             if delete:
1713                 if section in zctas:
1714                     del( zctas[section] )
1715                     logact = "removed zcta %s" % section
1716                     removed += 1
1717                 else:
1718                     logact = "tried to remove nonexistent zcta %s" % section
1719             else:
1720                 if section in zctas:
1721                     logact = "changed zcta %s" % section
1722                     changed += 1
1723                 else:
1724                     zctas[section] = {}
1725                     logact = "added zcta %s" % section
1726                     added += 1
1727                 for key,value in overrides.items(section):
1728                     if key in zctas[section]: chgopt += 1
1729                     else: addopt += 1
1730                     if key in ("centroid", "location"):
1731                         zctas[section][key] = eval(value)
1732                     else:
1733                         zctas[section][key] = value
1734                 if addopt and chgopt:
1735                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1736                 elif addopt: logact += " (+%s options)" % addopt
1737                 elif chgopt: logact += " (!%s options)" % chgopt
1738         elif re.match("[A-Za-z]{2}[Zz][0-9]{3}$", section):
1739             if delete:
1740                 if section in zones:
1741                     del( zones[section] )
1742                     logact = "removed zone %s" % section
1743                     removed += 1
1744                 else:
1745                     logact = "tried to remove nonexistent zone %s" % section
1746             else:
1747                 if section in zones:
1748                     logact = "changed zone %s" % section
1749                     changed += 1
1750                 else:
1751                     zones[section] = {}
1752                     logact = "added zone %s" % section
1753                     added += 1
1754                 for key,value in overrides.items(section):
1755                     if key in zones[section]: chgopt += 1
1756                     else: addopt += 1
1757                     if key in ("centroid", "location"):
1758                         zones[section][key] = eval(value)
1759                     else:
1760                         zones[section][key] = value
1761                 if addopt and chgopt:
1762                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1763                 elif addopt: logact += " (+%s options)" % addopt
1764                 elif chgopt: logact += " (!%s options)" % chgopt
1765         elif re.match("fips[0-9]+$", section):
1766             if delete:
1767                 if section in places:
1768                     del( places[section] )
1769                     logact = "removed place %s" % section
1770                     removed += 1
1771                 else:
1772                     logact = "tried to remove nonexistent place %s" % section
1773             else:
1774                 if section in places:
1775                     logact = "changed place %s" % section
1776                     changed += 1
1777                 else:
1778                     places[section] = {}
1779                     logact = "added place %s" % section
1780                     added += 1
1781                 for key,value in overrides.items(section):
1782                     if key in places[section]: chgopt += 1
1783                     else: addopt += 1
1784                     if key in ("centroid", "location"):
1785                         places[section][key] = eval(value)
1786                     else:
1787                         places[section][key] = value
1788                 if addopt and chgopt:
1789                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1790                 elif addopt: logact += " (+%s options)" % addopt
1791                 elif chgopt: logact += " (!%s options)" % chgopt
1792         count += 1
1793         overrideslog.append("%s\n" % logact)
1794     overrideslog.sort()
1795     if os.path.exists(overrideslog_fn):
1796         os.rename(overrideslog_fn, "%s_old"%overrideslog_fn)
1797     overrideslog_fd = codecs.open(overrideslog_fn, "w", "utf8")
1798     import time
1799     overrideslog_fd.write(
1800         '# Copyright (c) %s Jeremy Stanley <fungi@yuggoth.org>. Permission to\n'
1801         '# use, copy, modify, and distribute this software is granted under terms\n'
1802         '# provided in the LICENSE file distributed with this software.\n\n'
1803         % time.gmtime().tm_year)
1804     overrideslog_fd.writelines(overrideslog)
1805     overrideslog_fd.close()
1806     print("done (%s overridden sections: +%s/-%s/!%s)." % (
1807         count,
1808         added,
1809         removed,
1810         changed
1811     ) )
1812     estimate = 2*len(places) + len(stations) + 2*len(zctas) + len(zones)
1813     print(
1814         "Correlating places, stations, ZCTAs and zones (upper bound is %s):" % \
1815             estimate
1816     )
1817     count = 0
1818     milestones = list( range(51) )
1819     message = "   "
1820     sys.stdout.write(message)
1821     sys.stdout.flush()
1822     for fips in places:
1823         centroid = places[fips]["centroid"]
1824         if centroid:
1825             station = closest(centroid, stations, "location", 0.1)
1826         if station[0]:
1827             places[fips]["station"] = station
1828             count += 1
1829             if not count%100:
1830                 level = int(50*count/estimate)
1831                 if level in milestones:
1832                     for remaining in milestones[:milestones.index(level)+1]:
1833                         if remaining%5:
1834                             message = "."
1835                             sys.stdout.write(message)
1836                             sys.stdout.flush()
1837                         else:
1838                             message = "%s%%" % (remaining*2,)
1839                             sys.stdout.write(message)
1840                             sys.stdout.flush()
1841                         milestones.remove(remaining)
1842         if centroid:
1843             zone = closest(centroid, zones, "centroid", 0.1)
1844         if zone[0]:
1845             places[fips]["zone"] = zone
1846             count += 1
1847             if not count%100:
1848                 level = int(50*count/estimate)
1849                 if level in milestones:
1850                     for remaining in milestones[:milestones.index(level)+1]:
1851                         if remaining%5:
1852                             message = "."
1853                             sys.stdout.write(message)
1854                             sys.stdout.flush()
1855                         else:
1856                             message = "%s%%" % (remaining*2,)
1857                             sys.stdout.write(message)
1858                             sys.stdout.flush()
1859                         milestones.remove(remaining)
1860     for station in stations:
1861         if "location" in stations[station]:
1862             location = stations[station]["location"]
1863             if location:
1864                 zone = closest(location, zones, "centroid", 0.1)
1865             if zone[0]:
1866                 stations[station]["zone"] = zone
1867                 count += 1
1868                 if not count%100:
1869                     level = int(50*count/estimate)
1870                     if level in milestones:
1871                         for remaining in milestones[:milestones.index(level)+1]:
1872                             if remaining%5:
1873                                 message = "."
1874                                 sys.stdout.write(message)
1875                                 sys.stdout.flush()
1876                             else:
1877                                 message = "%s%%" % (remaining*2,)
1878                                 sys.stdout.write(message)
1879                                 sys.stdout.flush()
1880                             milestones.remove(remaining)
1881     for zcta in zctas.keys():
1882         centroid = zctas[zcta]["centroid"]
1883         if centroid:
1884             station = closest(centroid, stations, "location", 0.1)
1885         if station[0]:
1886             zctas[zcta]["station"] = station
1887             count += 1
1888             if not count%100:
1889                 level = int(50*count/estimate)
1890                 if level in milestones:
1891                     for remaining in milestones[ : milestones.index(level)+1 ]:
1892                         if remaining%5:
1893                             message = "."
1894                             sys.stdout.write(message)
1895                             sys.stdout.flush()
1896                         else:
1897                             message = "%s%%" % (remaining*2,)
1898                             sys.stdout.write(message)
1899                             sys.stdout.flush()
1900                         milestones.remove(remaining)
1901         if centroid:
1902             zone = closest(centroid, zones, "centroid", 0.1)
1903         if zone[0]:
1904             zctas[zcta]["zone"] = zone
1905             count += 1
1906             if not count%100:
1907                 level = int(50*count/estimate)
1908                 if level in milestones:
1909                     for remaining in milestones[:milestones.index(level)+1]:
1910                         if remaining%5:
1911                             message = "."
1912                             sys.stdout.write(message)
1913                             sys.stdout.flush()
1914                         else:
1915                             message = "%s%%" % (remaining*2,)
1916                             sys.stdout.write(message)
1917                             sys.stdout.flush()
1918                         milestones.remove(remaining)
1919     for zone in zones.keys():
1920         if "centroid" in zones[zone]:
1921             centroid = zones[zone]["centroid"]
1922             if centroid:
1923                 station = closest(centroid, stations, "location", 0.1)
1924             if station[0]:
1925                 zones[zone]["station"] = station
1926                 count += 1
1927                 if not count%100:
1928                     level = int(50*count/estimate)
1929                     if level in milestones:
1930                         for remaining in milestones[:milestones.index(level)+1]:
1931                             if remaining%5:
1932                                 message = "."
1933                                 sys.stdout.write(message)
1934                                 sys.stdout.flush()
1935                             else:
1936                                 message = "%s%%" % (remaining*2,)
1937                                 sys.stdout.write(message)
1938                                 sys.stdout.flush()
1939                             milestones.remove(remaining)
1940     for remaining in milestones:
1941         if remaining%5:
1942             message = "."
1943             sys.stdout.write(message)
1944             sys.stdout.flush()
1945         else:
1946             message = "%s%%" % (remaining*2,)
1947             sys.stdout.write(message)
1948             sys.stdout.flush()
1949     print("\n   done (%s correlations)." % count)
1950     message = "Writing %s..." % airports_fn
1951     sys.stdout.write(message)
1952     sys.stdout.flush()
1953     count = 0
1954     if os.path.exists(airports_fn):
1955         os.rename(airports_fn, "%s_old"%airports_fn)
1956     airports_fd = codecs.open(airports_fn, "w", "utf8")
1957     airports_fd.write(header)
1958     for airport in sorted( airports.keys() ):
1959         airports_fd.write("\n\n[%s]" % airport)
1960         for key, value in sorted( airports[airport].items() ):
1961             if type(value) is float: value = "%.7f"%value
1962             elif type(value) is tuple:
1963                 elements = []
1964                 for element in value:
1965                     if type(element) is float: elements.append("%.7f"%element)
1966                     else: elements.append( repr(element) )
1967                 value = "(%s)"%", ".join(elements)
1968             airports_fd.write( "\n%s = %s" % (key, value) )
1969         count += 1
1970     airports_fd.write("\n")
1971     airports_fd.close()
1972     print("done (%s sections)." % count)
1973     message = "Writing %s..." % places_fn
1974     sys.stdout.write(message)
1975     sys.stdout.flush()
1976     count = 0
1977     if os.path.exists(places_fn):
1978         os.rename(places_fn, "%s_old"%places_fn)
1979     places_fd = codecs.open(places_fn, "w", "utf8")
1980     places_fd.write(header)
1981     for fips in sorted( places.keys() ):
1982         places_fd.write("\n\n[%s]" % fips)
1983         for key, value in sorted( places[fips].items() ):
1984             if type(value) is float: value = "%.7f"%value
1985             elif type(value) is tuple:
1986                 elements = []
1987                 for element in value:
1988                     if type(element) is float: elements.append("%.7f"%element)
1989                     else: elements.append( repr(element) )
1990                 value = "(%s)"%", ".join(elements)
1991             places_fd.write( "\n%s = %s" % (key, value) )
1992         count += 1
1993     places_fd.write("\n")
1994     places_fd.close()
1995     print("done (%s sections)." % count)
1996     message = "Writing %s..." % stations_fn
1997     sys.stdout.write(message)
1998     sys.stdout.flush()
1999     count = 0
2000     if os.path.exists(stations_fn):
2001         os.rename(stations_fn, "%s_old"%stations_fn)
2002     stations_fd = codecs.open(stations_fn, "w", "utf-8")
2003     stations_fd.write(header)
2004     for station in sorted( stations.keys() ):
2005         stations_fd.write("\n\n[%s]" % station)
2006         for key, value in sorted( stations[station].items() ):
2007             if type(value) is float: value = "%.7f"%value
2008             elif type(value) is tuple:
2009                 elements = []
2010                 for element in value:
2011                     if type(element) is float: elements.append("%.7f"%element)
2012                     else: elements.append( repr(element) )
2013                 value = "(%s)"%", ".join(elements)
2014             if type(value) is bytes:
2015                 value = value.decode("utf-8")
2016             stations_fd.write( "\n%s = %s" % (key, value) )
2017         count += 1
2018     stations_fd.write("\n")
2019     stations_fd.close()
2020     print("done (%s sections)." % count)
2021     message = "Writing %s..." % zctas_fn
2022     sys.stdout.write(message)
2023     sys.stdout.flush()
2024     count = 0
2025     if os.path.exists(zctas_fn):
2026         os.rename(zctas_fn, "%s_old"%zctas_fn)
2027     zctas_fd = codecs.open(zctas_fn, "w", "utf8")
2028     zctas_fd.write(header)
2029     for zcta in sorted( zctas.keys() ):
2030         zctas_fd.write("\n\n[%s]" % zcta)
2031         for key, value in sorted( zctas[zcta].items() ):
2032             if type(value) is float: value = "%.7f"%value
2033             elif type(value) is tuple:
2034                 elements = []
2035                 for element in value:
2036                     if type(element) is float: elements.append("%.7f"%element)
2037                     else: elements.append( repr(element) )
2038                 value = "(%s)"%", ".join(elements)
2039             zctas_fd.write( "\n%s = %s" % (key, value) )
2040         count += 1
2041     zctas_fd.write("\n")
2042     zctas_fd.close()
2043     print("done (%s sections)." % count)
2044     message = "Writing %s..." % zones_fn
2045     sys.stdout.write(message)
2046     sys.stdout.flush()
2047     count = 0
2048     if os.path.exists(zones_fn):
2049         os.rename(zones_fn, "%s_old"%zones_fn)
2050     zones_fd = codecs.open(zones_fn, "w", "utf8")
2051     zones_fd.write(header)
2052     for zone in sorted( zones.keys() ):
2053         zones_fd.write("\n\n[%s]" % zone)
2054         for key, value in sorted( zones[zone].items() ):
2055             if type(value) is float: value = "%.7f"%value
2056             elif type(value) is tuple:
2057                 elements = []
2058                 for element in value:
2059                     if type(element) is float: elements.append("%.7f"%element)
2060                     else: elements.append( repr(element) )
2061                 value = "(%s)"%", ".join(elements)
2062             zones_fd.write( "\n%s = %s" % (key, value) )
2063         count += 1
2064     zones_fd.write("\n")
2065     zones_fd.close()
2066     print("done (%s sections)." % count)
2067     message = "Starting QA check..."
2068     sys.stdout.write(message)
2069     sys.stdout.flush()
2070     airports = configparser.ConfigParser()
2071     if pyversion("3"):
2072         airports.read(airports_fn, encoding="utf-8")
2073     else:
2074         airports.read(airports_fn)
2075     places = configparser.ConfigParser()
2076     if pyversion("3"):
2077         places.read(places_fn, encoding="utf-8")
2078     else:
2079         places.read(places_fn)
2080     stations = configparser.ConfigParser()
2081     if pyversion("3"):
2082         stations.read(stations_fn, encoding="utf-8")
2083     else:
2084         stations.read(stations_fn)
2085     zctas = configparser.ConfigParser()
2086     if pyversion("3"):
2087         zctas.read(zctas_fn, encoding="utf-8")
2088     else:
2089         zctas.read(zctas_fn)
2090     zones = configparser.ConfigParser()
2091     if pyversion("3"):
2092         zones.read(zones_fn, encoding="utf-8")
2093     else:
2094         zones.read(zones_fn)
2095     qalog = []
2096     places_nocentroid = 0
2097     places_nodescription = 0
2098     for place in sorted( places.sections() ):
2099         if not places.has_option(place, "centroid"):
2100             qalog.append("%s: no centroid\n" % place)
2101             places_nocentroid += 1
2102         if not places.has_option(place, "description"):
2103             qalog.append("%s: no description\n" % place)
2104             places_nodescription += 1
2105     stations_nodescription = 0
2106     stations_nolocation = 0
2107     stations_nometar = 0
2108     for station in sorted( stations.sections() ):
2109         if not stations.has_option(station, "description"):
2110             qalog.append("%s: no description\n" % station)
2111             stations_nodescription += 1
2112         if not stations.has_option(station, "location"):
2113             qalog.append("%s: no location\n" % station)
2114             stations_nolocation += 1
2115         if not stations.has_option(station, "metar"):
2116             qalog.append("%s: no metar\n" % station)
2117             stations_nometar += 1
2118     airports_badstation = 0
2119     airports_nostation = 0
2120     for airport in sorted( airports.sections() ):
2121         if not airports.has_option(airport, "station"):
2122             qalog.append("%s: no station\n" % airport)
2123             airports_nostation += 1
2124         else:
2125             station = airports.get(airport, "station")
2126             if station not in stations.sections():
2127                 qalog.append( "%s: bad station %s\n" % (airport, station) )
2128                 airports_badstation += 1
2129     zctas_nocentroid = 0
2130     for zcta in sorted( zctas.sections() ):
2131         if not zctas.has_option(zcta, "centroid"):
2132             qalog.append("%s: no centroid\n" % zcta)
2133             zctas_nocentroid += 1
2134     zones_nocentroid = 0
2135     zones_nodescription = 0
2136     zones_notz = 0
2137     zones_noforecast = 0
2138     zones_overlapping = 0
2139     zonetable = {}
2140     for zone in zones.sections():
2141         if zones.has_option(zone, "centroid"):
2142             zonetable[zone] = {
2143                 "centroid": eval( zones.get(zone, "centroid") )
2144             }
2145     for zone in sorted( zones.sections() ):
2146         if zones.has_option(zone, "centroid"):
2147             zonetable_local = zonetable.copy()
2148             del( zonetable_local[zone] )
2149             centroid = eval( zones.get(zone, "centroid") )
2150             if centroid:
2151                 nearest = closest(centroid, zonetable_local, "centroid", 0.1)
2152             if nearest[1]*radian_to_km < 1:
2153                 qalog.append( "%s: within one km of %s\n" % (
2154                     zone,
2155                     nearest[0]
2156                 ) )
2157                 zones_overlapping += 1
2158         else:
2159             qalog.append("%s: no centroid\n" % zone)
2160             zones_nocentroid += 1
2161         if not zones.has_option(zone, "description"):
2162             qalog.append("%s: no description\n" % zone)
2163             zones_nodescription += 1
2164         if not zones.has_option(zone, "tz") or not zones.get(
2165                 zone, "tz") in zoneinfo.available_timezones():
2166             qalog.append("%s: no time zone\n" % zone)
2167             zones_notz += 1
2168         if not zones.has_option(zone, "zone_forecast"):
2169             qalog.append("%s: no forecast\n" % zone)
2170             zones_noforecast += 1
2171     if os.path.exists(qalog_fn):
2172         os.rename(qalog_fn, "%s_old"%qalog_fn)
2173     qalog_fd = codecs.open(qalog_fn, "w", "utf8")
2174     import time
2175     qalog_fd.write(
2176         '# Copyright (c) %s Jeremy Stanley <fungi@yuggoth.org>. Permission to\n'
2177         '# use, copy, modify, and distribute this software is granted under terms\n'
2178         '# provided in the LICENSE file distributed with this software.\n\n'
2179         % time.gmtime().tm_year)
2180     qalog_fd.writelines(qalog)
2181     qalog_fd.close()
2182     if qalog:
2183         print("issues found (see %s for details):"%qalog_fn)
2184         if airports_badstation:
2185             print("   %s airports with invalid station"%airports_badstation)
2186         if airports_nostation:
2187             print("   %s airports with no station"%airports_nostation)
2188         if places_nocentroid:
2189             print("   %s places with no centroid"%places_nocentroid)
2190         if places_nodescription:
2191             print("   %s places with no description"%places_nodescription)
2192         if stations_nodescription:
2193             print("   %s stations with no description"%stations_nodescription)
2194         if stations_nolocation:
2195             print("   %s stations with no location"%stations_nolocation)
2196         if stations_nometar:
2197             print("   %s stations with no METAR"%stations_nometar)
2198         if zctas_nocentroid:
2199             print("   %s ZCTAs with no centroid"%zctas_nocentroid)
2200         if zones_nocentroid:
2201             print("   %s zones with no centroid"%zones_nocentroid)
2202         if zones_nodescription:
2203             print("   %s zones with no description"%zones_nodescription)
2204         if zones_notz:
2205             print("   %s zones with no time zone"%zones_notz)
2206         if zones_noforecast:
2207             print("   %s zones with no forecast"%zones_noforecast)
2208         if zones_overlapping:
2209             print("   %s zones within one km of another"%zones_overlapping)
2210     else: print("no issues found.")
2211     print("Indexing complete!")