weather.py

   1 """Contains various object definitions needed by the weather utility."""
   2
   3 weather_copyright = """\
   4 # Copyright (c) 2006-2024 Jeremy Stanley <fungi@yuggoth.org>. Permission to
   5 # use, copy, modify, and distribute this software is granted under terms
   6 # provided in the LICENSE file distributed with this software.
   7 #"""
   8
   9 weather_version = "2.4.4"
  10
  11 radian_to_km = 6372.795484
  12 radian_to_mi = 3959.871528
  13
  14 class Selections:
  15     """An object to contain selection data."""
  16     def __init__(self):
  17         """Store the config, options and arguments."""
  18         self.config = get_config()
  19         self.options, self.arguments = get_options(self.config)
  20         if self.get_bool("cache") and self.get_bool("cache_search") \
  21             and not self.get_bool("longlist"):
  22             integrate_search_cache(
  23                 self.config,
  24                 self.get("cachedir"),
  25                 self.get("setpath")
  26             )
  27         if not self.arguments:
  28             if "id" in self.options.__dict__ \
  29                 and self.options.__dict__["id"]:
  30                 self.arguments.append( self.options.__dict__["id"] )
  31                 del( self.options.__dict__["id"] )
  32                 import sys
  33                 message = "WARNING: the --id option is deprecated and will eventually be removed\n"
  34                 sys.stderr.write(message)
  35             elif "city" in self.options.__dict__ \
  36                 and self.options.__dict__["city"] \
  37                 and "st" in self.options.__dict__ \
  38                 and self.options.__dict__["st"]:
  39                 self.arguments.append(
  40                     "^%s city, %s" % (
  41                         self.options.__dict__["city"],
  42                         self.options.__dict__["st"]
  43                     )
  44                 )
  45                 del( self.options.__dict__["city"] )
  46                 del( self.options.__dict__["st"] )
  47                 import sys
  48                 message = "WARNING: the --city/--st options are deprecated and will eventually be removed\n"
  49                 sys.stderr.write(message)
  50     def get(self, option, argument=None):
  51         """Retrieve data from the config or options."""
  52         if argument:
  53             if self.config.has_section(argument) and (
  54                 self.config.has_option(argument, "city") \
  55                     or self.config.has_option(argument, "id") \
  56                     or self.config.has_option(argument, "st")
  57             ):
  58                 self.config.remove_section(argument)
  59                 import sys
  60                 message = "WARNING: the city/id/st options are now unsupported in aliases\n"
  61                 sys.stderr.write(message)
  62             if not self.config.has_section(argument):
  63                 guessed = guess(
  64                     argument,
  65                     path=self.get("setpath"),
  66                     info=self.get("info"),
  67                     cache_search=(
  68                         self.get("cache") and self.get("cache_search")
  69                     ),
  70                     cachedir=self.get("cachedir"),
  71                     quiet=self.get_bool("quiet")
  72                 )
  73                 self.config.add_section(argument)
  74                 for item in guessed.items():
  75                     self.config.set(argument, *item)
  76             if self.config.has_option(argument, option):
  77                 return self.config.get(argument, option)
  78         if option in self.options.__dict__:
  79             return self.options.__dict__[option]
  80         import sys
  81         message = "WARNING: no URI defined for %s\n" % option
  82         sys.stderr.write(message)
  83         return None
  84     def get_bool(self, option, argument=None):
  85         """Get data and coerce to a boolean if necessary."""
  86         # Mimic configparser's getboolean() method by treating
  87         # false/no/off/0 as False and true/yes/on/1 as True values,
  88         # case-insensitively
  89         value = self.get(option, argument)
  90         if isinstance(value, bool):
  91             return value
  92         if isinstance(value, str):
  93             vlower = value.lower()
  94             if vlower in ('false', 'no', 'off', '0'):
  95                 return False
  96             elif vlower in ('true', 'yes', 'on', '1'):
  97                 return True
  98         raise ValueError("Not a boolean: %s" % value)
  99     def getint(self, option, argument=None):
 100         """Get data and coerce to an integer if necessary."""
 101         value = self.get(option, argument)
 102         if value: return int(value)
 103         else: return 0
 104
 105 def average(coords):
 106     """Average a list of coordinates."""
 107     x = 0
 108     y = 0
 109     for coord in coords:
 110         x += coord[0]
 111         y += coord[1]
 112     count = len(coords)
 113     return (x/count, y/count)
 114
 115 def filter_units(line, units="imperial"):
 116     """Filter or convert units in a line of text between US/UK and metric."""
 117     import re
 118     # filter lines with both pressures in the form of "X inches (Y hPa)" or
 119     # "X in. Hg (Y hPa)"
 120     dual_p = re.match(
 121         r"(.* )(\d*(\.\d+)? (inches|in\. Hg)) \((\d*(\.\d+)? hPa)\)(.*)",
 122         line
 123     )
 124     if dual_p:
 125         preamble, in_hg, i_fr, i_un, hpa, h_fr, trailer = dual_p.groups()
 126         if units == "imperial": line = preamble + in_hg + trailer
 127         elif units == "metric": line = preamble + hpa + trailer
 128     # filter lines with both temperatures in the form of "X F (Y C)"
 129     dual_t = re.match(
 130         r"(.* )(-?\d*(\.\d+)? F) \((-?\d*(\.\d+)? C)\)(.*)",
 131         line
 132     )
 133     if dual_t:
 134         preamble, fahrenheit, f_fr, celsius, c_fr, trailer = dual_t.groups()
 135         if units == "imperial": line = preamble + fahrenheit + trailer
 136         elif units == "metric": line = preamble + celsius + trailer
 137     # if metric is desired, convert distances in the form of "X mile(s)" to
 138     # "Y kilometer(s)"
 139     if units == "metric":
 140         imperial_d = re.match(
 141             r"(.* )(\d+)( mile\(s\))(.*)",
 142             line
 143         )
 144         if imperial_d:
 145             preamble, mi, m_u, trailer = imperial_d.groups()
 146             line = preamble + str(int(round(int(mi)*1.609344))) \
 147                 + " kilometer(s)" + trailer
 148     # filter speeds in the form of "X MPH (Y KT)" to just "X MPH"; if metric is
 149     # desired, convert to "Z KPH"
 150     imperial_s = re.match(
 151         r"(.* )(\d+)( MPH)( \(\d+ KT\))(.*)",
 152         line
 153     )
 154     if imperial_s:
 155         preamble, mph, m_u, kt, trailer = imperial_s.groups()
 156         if units == "imperial": line = preamble + mph + m_u + trailer
 157         elif units == "metric":
 158             line = preamble + str(int(round(int(mph)*1.609344))) + " KPH" + \
 159                 trailer
 160     imperial_s = re.match(
 161         r"(.* )(\d+)( MPH)( \(\d+ KT\))(.*)",
 162         line
 163     )
 164     if imperial_s:
 165         preamble, mph, m_u, kt, trailer = imperial_s.groups()
 166         if units == "imperial": line = preamble + mph + m_u + trailer
 167         elif units == "metric":
 168             line = preamble + str(int(round(int(mph)*1.609344))) + " KPH" + \
 169                 trailer
 170     # if imperial is desired, qualify given forcast temperatures like "X F"; if
 171     # metric is desired, convert to "Y C"
 172     imperial_t = re.match(
 173         r"(.* )(High |high |Low |low )(\d+)(\.|,)(.*)",
 174         line
 175     )
 176     if imperial_t:
 177         preamble, parameter, fahrenheit, sep, trailer = imperial_t.groups()
 178         if units == "imperial":
 179             line = preamble + parameter + fahrenheit + " F" + sep + trailer
 180         elif units == "metric":
 181             line = preamble + parameter \
 182                 + str(int(round((int(fahrenheit)-32)*5/9))) + " C" + sep \
 183                 + trailer
 184     # hand off the resulting line
 185     return line
 186
 187 def get_uri(
 188     uri,
 189     ignore_fail=False,
 190     cache_data=False,
 191     cacheage=900,
 192     cachedir="."
 193 ):
 194     """Return a string containing the results of a URI GET."""
 195     import os, time, urllib, urllib.error, urllib.request
 196     if cache_data:
 197         dcachedir = os.path.join( os.path.expanduser(cachedir), "datacache" )
 198         if not os.path.exists(dcachedir):
 199             try: os.makedirs(dcachedir)
 200             except (IOError, OSError): pass
 201         dcache_fn = os.path.join(
 202             dcachedir,
 203             uri.split(":",1)[1].replace("/","_")
 204         )
 205     now = time.time()
 206     if cache_data and os.access(dcache_fn, os.R_OK) \
 207         and now-cacheage < os.stat(dcache_fn).st_mtime <= now:
 208         dcache_fd = open(dcache_fn)
 209         data = dcache_fd.read()
 210         dcache_fd.close()
 211     else:
 212         try:
 213             data = urllib.request.urlopen(uri).read().decode("utf-8")
 214         except urllib.error.URLError:
 215             if ignore_fail: return ""
 216             import os, sys
 217             sys.stderr.write("%s error: failed to retrieve\n   %s\n\n" % (
 218                 os.path.basename( sys.argv[0] ), uri))
 219             raise
 220         # Some data sources are HTML with the plain text wrapped in pre tags
 221         if "<pre>" in data:
 222             data = data[data.find("<pre>")+5:data.find("</pre>")]
 223         if cache_data:
 224             try:
 225                 import codecs
 226                 dcache_fd = codecs.open(dcache_fn, "w", "utf-8")
 227                 dcache_fd.write(data)
 228                 dcache_fd.close()
 229             except (IOError, OSError): pass
 230     return data
 231
 232 def get_metar(
 233     uri=None,
 234     verbose=False,
 235     quiet=False,
 236     headers=None,
 237     imperial=False,
 238     metric=False,
 239     cache_data=False,
 240     cacheage=900,
 241     cachedir="."
 242 ):
 243     """Return a summarized METAR for the specified station."""
 244     if not uri:
 245         import os, sys
 246         message = "%s error: METAR URI required for conditions\n" % \
 247             os.path.basename( sys.argv[0] )
 248         sys.stderr.write(message)
 249         sys.exit(1)
 250     metar = get_uri(
 251         uri,
 252         cache_data=cache_data,
 253         cacheage=cacheage,
 254         cachedir=cachedir
 255     )
 256     if type(metar) is bytes: metar = metar.decode("utf-8")
 257     if verbose: return metar
 258     else:
 259         import re
 260         lines = metar.split("\n")
 261         if not headers:
 262             headers = \
 263                 "relative_humidity," \
 264                 + "precipitation_last_hour," \
 265                 + "sky conditions," \
 266                 + "temperature," \
 267                 + "heat index," \
 268                 + "windchill," \
 269                 + "weather," \
 270                 + "wind"
 271         headerlist = headers.lower().replace("_"," ").split(",")
 272         output = []
 273         if not quiet:
 274             title = "Current conditions at %s"
 275             place = lines[0].split(", ")
 276             if len(place) > 1:
 277                 place = "%s, %s" % ( place[0].title(), place[1] )
 278             else: place = "<UNKNOWN>"
 279             output.append(title%place)
 280             output.append("Last updated " + lines[1])
 281         header_match = False
 282         for header in headerlist:
 283             for line in lines:
 284                 if line.lower().startswith(header + ":"):
 285                     if re.match(r".*:\d+$", line): line = line[:line.rfind(":")]
 286                     if imperial: line = filter_units(line, units="imperial")
 287                     elif metric: line = filter_units(line, units="metric")
 288                     if quiet: output.append(line)
 289                     else: output.append("   " + line)
 290                     header_match = True
 291         if not header_match:
 292             output.append(
 293                 "(no conditions matched your header list, try with --verbose)"
 294             )
 295         return "\n".join(output)
 296
 297 def get_alert(
 298     uri=None,
 299     verbose=False,
 300     quiet=False,
 301     cache_data=False,
 302     cacheage=900,
 303     cachedir=".",
 304     delay=1
 305 ):
 306     """Return alert notice for the specified URI."""
 307     if not uri:
 308         return ""
 309     alert = get_uri(
 310         uri,
 311         ignore_fail=True,
 312         cache_data=cache_data,
 313         cacheage=cacheage,
 314         cachedir=cachedir
 315     ).strip()
 316     if type(alert) is bytes: alert = alert.decode("utf-8")
 317     if alert:
 318         if verbose: return alert
 319         else:
 320             import re
 321             if re.search(r"\nNational Weather Service", alert):
 322                 muted = True
 323             else:
 324                 muted = False
 325             expirycheck = re.search(r"Expires:([0-9]{12})", alert)
 326             if expirycheck:
 327                 # only report alerts and forecasts that expired less than delay
 328                 # hours ago
 329                 import datetime, zoneinfo
 330                 expiration = datetime.datetime.fromisoformat(
 331                     "%sT%sZ" % (expirycheck[1][:8], expirycheck[1][-4:]))
 332                 now = datetime.datetime.now(tz=zoneinfo.ZoneInfo("UTC"))
 333                 if now - expiration > datetime.timedelta(hours=delay):
 334                     return ""
 335             lines = alert.split("\n")
 336             output = []
 337             for line in lines:
 338                 if muted and line.startswith("National Weather Service"):
 339                     muted = False
 340                     line = ""
 341                 elif line == "&&":
 342                     line = ""
 343                 elif line == "$$":
 344                     muted = True
 345                 if line and not muted:
 346                     if quiet: output.append(line)
 347                     else: output.append("   " + line)
 348             return "\n".join(output)
 349
 350 def get_options(config):
 351     """Parse the options passed on the command line."""
 352
 353     # for optparse's builtin -h/--help option
 354     usage = \
 355         "usage: %prog [options] [alias1|search1 [alias2|search2 [...]]]"
 356
 357     # for optparse's builtin --version option
 358     verstring = "%prog " + weather_version
 359
 360     # create the parser
 361     import optparse
 362     option_parser = optparse.OptionParser(usage=usage, version=verstring)
 363     # separate options object from list of arguments and return both
 364
 365     # the -a/--alert option
 366     if config.has_option("default", "alert"):
 367         default_alert = config.getboolean("default", "alert")
 368     else: default_alert = False
 369     option_parser.add_option("-a", "--alert",
 370         dest="alert",
 371         action="store_true",
 372         default=default_alert,
 373         help="include local alert notices")
 374
 375     # the --atypes option
 376     if config.has_option("default", "atypes"):
 377         default_atypes = config.get("default", "atypes")
 378     else:
 379         default_atypes = \
 380             "coastal_flood_statement," \
 381             + "flash_flood_statement," \
 382             + "flash_flood_warning," \
 383             + "flash_flood_watch," \
 384             + "flood_warning," \
 385             + "severe_thunderstorm_warning," \
 386             + "severe_weather_statement," \
 387             + "special_weather_statement," \
 388             + "tornado," \
 389             + "urgent_weather_message"
 390     option_parser.add_option("--atypes",
 391         dest="atypes",
 392         default=default_atypes,
 393         help="list of alert notification types to display")
 394
 395     # the --build-sets option
 396     option_parser.add_option("--build-sets",
 397         dest="build_sets",
 398         action="store_true",
 399         default=False,
 400         help="(re)build location correlation sets")
 401
 402     # the --cacheage option
 403     if config.has_option("default", "cacheage"):
 404         default_cacheage = config.getint("default", "cacheage")
 405     else: default_cacheage = 900
 406     option_parser.add_option("--cacheage",
 407         dest="cacheage",
 408         default=default_cacheage,
 409         help="duration in seconds to refresh cached data")
 410
 411     # the --cachedir option
 412     if config.has_option("default", "cachedir"):
 413         default_cachedir = config.get("default", "cachedir")
 414     else: default_cachedir = "~/.weather"
 415     option_parser.add_option("--cachedir",
 416         dest="cachedir",
 417         default=default_cachedir,
 418         help="directory for storing cached searches and data")
 419
 420     # the --delay option
 421     if config.has_option("default", "delay"):
 422         default_delay = config.getint("default", "delay")
 423     else: default_delay = 1
 424     option_parser.add_option("--delay",
 425         dest="delay",
 426         default=default_delay,
 427         help="hours to delay alert and forecast expiration")
 428
 429     # the -f/--forecast option
 430     if config.has_option("default", "forecast"):
 431         default_forecast = config.getboolean("default", "forecast")
 432     else: default_forecast = False
 433     option_parser.add_option("-f", "--forecast",
 434         dest="forecast",
 435         action="store_true",
 436         default=default_forecast,
 437         help="include a local forecast")
 438
 439     # the --headers option
 440     if config.has_option("default", "headers"):
 441         default_headers = config.get("default", "headers")
 442     else:
 443         default_headers = \
 444             "temperature," \
 445             + "relative_humidity," \
 446             + "wind," \
 447             + "heat_index," \
 448             + "windchill," \
 449             + "weather," \
 450             + "sky_conditions," \
 451             + "precipitation_last_hour"
 452     option_parser.add_option("--headers",
 453         dest="headers",
 454         default=default_headers,
 455         help="list of conditions headers to display")
 456
 457     # the --imperial option
 458     if config.has_option("default", "imperial"):
 459         default_imperial = config.getboolean("default", "imperial")
 460     else: default_imperial = False
 461     option_parser.add_option("--imperial",
 462         dest="imperial",
 463         action="store_true",
 464         default=default_imperial,
 465         help="filter/convert conditions for US/UK units")
 466
 467     # the --info option
 468     option_parser.add_option("--info",
 469         dest="info",
 470         action="store_true",
 471         default=False,
 472         help="output detailed information for your search")
 473
 474     # the -l/--list option
 475     option_parser.add_option("-l", "--list",
 476         dest="list",
 477         action="store_true",
 478         default=False,
 479         help="list all configured aliases and cached searches")
 480
 481     # the --longlist option
 482     option_parser.add_option("--longlist",
 483         dest="longlist",
 484         action="store_true",
 485         default=False,
 486         help="display details of all configured aliases")
 487
 488     # the -m/--metric option
 489     if config.has_option("default", "metric"):
 490         default_metric = config.getboolean("default", "metric")
 491     else: default_metric = False
 492     option_parser.add_option("-m", "--metric",
 493         dest="metric",
 494         action="store_true",
 495         default=default_metric,
 496         help="filter/convert conditions for metric units")
 497
 498     # the -n/--no-conditions option
 499     if config.has_option("default", "conditions"):
 500         default_conditions = config.getboolean("default", "conditions")
 501     else: default_conditions = True
 502     option_parser.add_option("-n", "--no-conditions",
 503         dest="conditions",
 504         action="store_false",
 505         default=default_conditions,
 506         help="disable output of current conditions")
 507
 508     # the --no-cache option
 509     if config.has_option("default", "cache"):
 510         default_cache = config.getboolean("default", "cache")
 511     else: default_cache = True
 512     option_parser.add_option("--no-cache",
 513         dest="cache",
 514         action="store_false",
 515         default=True,
 516         help="disable all caching (searches and data)")
 517
 518     # the --no-cache-data option
 519     if config.has_option("default", "cache_data"):
 520         default_cache_data = config.getboolean("default", "cache_data")
 521     else: default_cache_data = True
 522     option_parser.add_option("--no-cache-data",
 523         dest="cache_data",
 524         action="store_false",
 525         default=True,
 526         help="disable retrieved data caching")
 527
 528     # the --no-cache-search option
 529     if config.has_option("default", "cache_search"):
 530         default_cache_search = config.getboolean("default", "cache_search")
 531     else: default_cache_search = True
 532     option_parser.add_option("--no-cache-search",
 533         dest="cache_search",
 534         action="store_false",
 535         default=True,
 536         help="disable search result caching")
 537
 538     # the -q/--quiet option
 539     if config.has_option("default", "quiet"):
 540         default_quiet = config.getboolean("default", "quiet")
 541     else: default_quiet = False
 542     option_parser.add_option("-q", "--quiet",
 543         dest="quiet",
 544         action="store_true",
 545         default=default_quiet,
 546         help="skip preambles and don't indent")
 547
 548     # the --setpath option
 549     if config.has_option("default", "setpath"):
 550         default_setpath = config.get("default", "setpath")
 551     else: default_setpath = ".:~/.weather"
 552     option_parser.add_option("--setpath",
 553         dest="setpath",
 554         default=default_setpath,
 555         help="directory search path for correlation sets")
 556
 557     # the -v/--verbose option
 558     if config.has_option("default", "verbose"):
 559         default_verbose = config.getboolean("default", "verbose")
 560     else: default_verbose = False
 561     option_parser.add_option("-v", "--verbose",
 562         dest="verbose",
 563         action="store_true",
 564         default=default_verbose,
 565         help="show full decoded feeds")
 566
 567     # deprecated options
 568     if config.has_option("default", "city"):
 569         default_city = config.get("default", "city")
 570     else: default_city = ""
 571     option_parser.add_option("-c", "--city",
 572         dest="city",
 573         default=default_city,
 574         help=optparse.SUPPRESS_HELP)
 575     if config.has_option("default", "id"):
 576         default_id = config.get("default", "id")
 577     else: default_id = ""
 578     option_parser.add_option("-i", "--id",
 579         dest="id",
 580         default=default_id,
 581         help=optparse.SUPPRESS_HELP)
 582     if config.has_option("default", "st"):
 583         default_st = config.get("default", "st")
 584     else: default_st = ""
 585     option_parser.add_option("-s", "--st",
 586         dest="st",
 587         default=default_st,
 588         help=optparse.SUPPRESS_HELP)
 589
 590     options, arguments = option_parser.parse_args()
 591     return options, arguments
 592
 593 def get_config():
 594     """Parse the aliases and configuration."""
 595     import configparser, os
 596     config = configparser.ConfigParser()
 597     rcfiles = [
 598         "/etc/weatherrc",
 599         "/etc/weather/weatherrc",
 600         os.path.expanduser("~/.weather/weatherrc"),
 601         os.path.expanduser("~/.weatherrc"),
 602         "weatherrc"
 603         ]
 604     for rcfile in rcfiles:
 605         if os.access(rcfile, os.R_OK):
 606             config.read(rcfile, encoding="utf-8")
 607     for section in config.sections():
 608         if section != section.lower():
 609             if config.has_section(section.lower()):
 610                 config.remove_section(section.lower())
 611             config.add_section(section.lower())
 612             for option,value in config.items(section):
 613                 config.set(section.lower(), option, value)
 614     return config
 615
 616 def integrate_search_cache(config, cachedir, setpath):
 617     """Add cached search results into the configuration."""
 618     import configparser, os, time
 619     scache_fn = os.path.join( os.path.expanduser(cachedir), "searches" )
 620     if not os.access(scache_fn, os.R_OK): return config
 621     scache_fd = open(scache_fn)
 622     created = float( scache_fd.readline().split(":")[1].strip().split()[0] )
 623     scache_fd.close()
 624     now = time.time()
 625     datafiles = data_index(setpath)
 626     if datafiles:
 627         data_freshness = sorted(
 628             [ x[1] for x in datafiles.values() ],
 629             reverse=True
 630         )[0]
 631     else: data_freshness = now
 632     if created < data_freshness <= now:
 633         try:
 634             os.remove(scache_fn)
 635             print( "[clearing outdated %s]" % scache_fn )
 636         except (IOError, OSError):
 637             pass
 638         return config
 639     scache = configparser.ConfigParser()
 640     scache.read(scache_fn, encoding="utf-8")
 641     for section in scache.sections():
 642         if not config.has_section(section):
 643             config.add_section(section)
 644             for option,value in scache.items(section):
 645                 config.set(section, option, value)
 646     return config
 647
 648 def list_aliases(config, detail=False):
 649     """Return a formatted list of aliases defined in the config."""
 650     if detail:
 651         output = "\n# configured alias details..."
 652         for section in sorted(config.sections()):
 653             output += "\n\n[%s]" % section
 654             for item in sorted(config.items(section)):
 655                 output += "\n%s = %s" % item
 656         output += "\n"
 657     else:
 658         output = "configured aliases and cached searches..."
 659         for section in sorted(config.sections()):
 660             if config.has_option(section, "description"):
 661                 description = config.get(section, "description")
 662             else: description = "(no description provided)"
 663             output += "\n   %s: %s" % (section, description)
 664     return output
 665
 666 def data_index(path):
 667     import os
 668     datafiles = {}
 669     for filename in ("airports", "places", "stations", "zctas", "zones"):
 670         for dirname in path.split(":"):
 671             for extension in ("", ".gz", ".txt"):
 672                 candidate = os.path.expanduser(
 673                     os.path.join( dirname, "".join( (filename, extension) ) )
 674                 )
 675                 if os.path.exists(candidate):
 676                     datafiles[filename] = (
 677                         candidate,
 678                         os.stat(candidate).st_mtime
 679                     )
 680                     break
 681             if filename in datafiles:
 682                 break
 683     return datafiles
 684
 685 def guess(
 686     expression,
 687     path=".",
 688     max_results=20,
 689     info=False,
 690     cache_search=False,
 691     cacheage=900,
 692     cachedir=".",
 693     quiet=False
 694 ):
 695     """Find URIs using airport, gecos, placename, station, ZCTA/ZIP, zone."""
 696     import codecs, configparser, datetime, time, os, re, sys
 697     datafiles = data_index(path)
 698     if re.match("[A-Za-z]{3}$", expression): searchtype = "airport"
 699     elif re.match("[A-Za-z0-9]{4}$", expression): searchtype = "station"
 700     elif re.match("[A-Za-z]{2}[Zz][0-9]{3}$", expression): searchtype = "zone"
 701     elif re.match("[0-9]{5}$", expression): searchtype = "ZCTA"
 702     elif re.match(
 703         r"[\+-]?\d+(\.\d+)?(-\d+){,2}[ENSWensw]?, *[\+-]?\d+(\.\d+)?(-\d+){,2}[ENSWensw]?$",
 704         expression
 705     ):
 706         searchtype = "coordinates"
 707     elif re.match(r"(FIPS|fips)\d+$", expression): searchtype = "FIPS"
 708     else:
 709         searchtype = "name"
 710         cache_search = False
 711     if cache_search: action = "caching"
 712     else: action = "using"
 713     if info:
 714         scores = [
 715             (0.005, "bad"),
 716             (0.025, "poor"),
 717             (0.160, "suspect"),
 718             (0.500, "mediocre"),
 719             (0.840, "good"),
 720             (0.975, "great"),
 721             (0.995, "excellent"),
 722             (1.000, "ideal"),
 723         ]
 724     if not quiet: print("Searching via %s..."%searchtype)
 725     stations = configparser.ConfigParser()
 726     dataname = "stations"
 727     if dataname in datafiles:
 728         datafile = datafiles[dataname][0]
 729         if datafile.endswith(".gz"):
 730             import gzip
 731             stations.read_string( gzip.open(datafile).read().decode("utf-8") )
 732         else:
 733             stations.read(datafile, encoding="utf-8")
 734     else:
 735         message = "%s error: can't find \"%s\" data file\n" % (
 736             os.path.basename( sys.argv[0] ),
 737             dataname
 738         )
 739         sys.stderr.write(message)
 740         exit(1)
 741     zones = configparser.ConfigParser()
 742     dataname = "zones"
 743     if dataname in datafiles:
 744         datafile = datafiles[dataname][0]
 745         if datafile.endswith(".gz"):
 746             import gzip
 747             zones.read_string( gzip.open(datafile).read().decode("utf-8") )
 748         else:
 749             zones.read(datafile, encoding="utf-8")
 750     else:
 751         message = "%s error: can't find \"%s\" data file\n" % (
 752             os.path.basename( sys.argv[0] ),
 753             dataname
 754         )
 755         sys.stderr.write(message)
 756         exit(1)
 757     search = None
 758     station = ("", 0)
 759     zone = ("", 0)
 760     dataset = None
 761     possibilities = []
 762     uris = {}
 763     if searchtype == "airport":
 764         expression = expression.lower()
 765         airports = configparser.ConfigParser()
 766         dataname = "airports"
 767         if dataname in datafiles:
 768             datafile = datafiles[dataname][0]
 769             if datafile.endswith(".gz"):
 770                 import gzip
 771                 airports.read_string(
 772                     gzip.open(datafile).read().decode("utf-8") )
 773             else:
 774                 airports.read(datafile, encoding="utf-8")
 775         else:
 776             message = "%s error: can't find \"%s\" data file\n" % (
 777                 os.path.basename( sys.argv[0] ),
 778                 dataname
 779             )
 780             sys.stderr.write(message)
 781             exit(1)
 782         if airports.has_section(expression) \
 783             and airports.has_option(expression, "station"):
 784             search = (expression, "IATA/FAA airport code %s" % expression)
 785             station = ( airports.get(expression, "station"), 0 )
 786             if stations.has_option(station[0], "zone"):
 787                 zone = eval( stations.get(station[0], "zone") )
 788                 dataset = stations
 789             if not ( info or quiet ) \
 790                 and stations.has_option( station[0], "description" ):
 791                 print(
 792                     "[%s result %s]" % (
 793                         action,
 794                         stations.get(station[0], "description")
 795                     )
 796                 )
 797         else:
 798             message = "No IATA/FAA airport code \"%s\" in the %s file.\n" % (
 799                 expression,
 800                 datafiles["airports"][0]
 801             )
 802             sys.stderr.write(message)
 803             exit(1)
 804     elif searchtype == "station":
 805         expression = expression.lower()
 806         if stations.has_section(expression):
 807             station = (expression, 0)
 808             if not search:
 809                 search = (expression, "ICAO station code %s" % expression)
 810             if stations.has_option(expression, "zone"):
 811                 zone = eval( stations.get(expression, "zone") )
 812                 dataset = stations
 813             if not ( info or quiet ) \
 814                 and stations.has_option(expression, "description"):
 815                 print(
 816                     "[%s result %s]" % (
 817                         action,
 818                         stations.get(expression, "description")
 819                     )
 820                 )
 821         else:
 822             message = "No ICAO weather station \"%s\" in the %s file.\n" % (
 823                 expression,
 824                 datafiles["stations"][0]
 825             )
 826             sys.stderr.write(message)
 827             exit(1)
 828     elif searchtype == "zone":
 829         expression = expression.lower()
 830         if zones.has_section(expression) \
 831             and zones.has_option(expression, "station"):
 832             zone = (expression, 0)
 833             station = eval( zones.get(expression, "station") )
 834             dataset = zones
 835             search = (expression, "NWS/NOAA weather zone %s" % expression)
 836             if not ( info or quiet ) \
 837                 and zones.has_option(expression, "description"):
 838                 print(
 839                     "[%s result %s]" % (
 840                         action,
 841                         zones.get(expression, "description")
 842                     )
 843                 )
 844         else:
 845             message = "No usable NWS weather zone \"%s\" in the %s file.\n" % (
 846                 expression,
 847                 datafiles["zones"][0]
 848             )
 849             sys.stderr.write(message)
 850             exit(1)
 851     elif searchtype == "ZCTA":
 852         zctas = configparser.ConfigParser()
 853         dataname = "zctas"
 854         if dataname in datafiles:
 855             datafile = datafiles[dataname][0]
 856             if datafile.endswith(".gz"):
 857                 import gzip
 858                 zctas.read_string( gzip.open(datafile).read().decode("utf-8") )
 859             else:
 860                 zctas.read(datafile, encoding="utf-8")
 861         else:
 862             message = "%s error: can't find \"%s\" data file\n" % (
 863                 os.path.basename( sys.argv[0] ),
 864                 dataname
 865             )
 866             sys.stderr.write(message)
 867             exit(1)
 868         dataset = zctas
 869         if zctas.has_section(expression) \
 870             and zctas.has_option(expression, "station"):
 871             station = eval( zctas.get(expression, "station") )
 872             search = (expression, "Census ZCTA (ZIP code) %s" % expression)
 873             if zctas.has_option(expression, "zone"):
 874                 zone = eval( zctas.get(expression, "zone") )
 875         else:
 876             message = "No census ZCTA (ZIP code) \"%s\" in the %s file.\n" % (
 877                 expression,
 878                 datafiles["zctas"][0]
 879             )
 880             sys.stderr.write(message)
 881             exit(1)
 882     elif searchtype == "coordinates":
 883         search = (expression, "Geographic coordinates %s" % expression)
 884         stationtable = {}
 885         for station in stations.sections():
 886             if stations.has_option(station, "location"):
 887                 stationtable[station] = {
 888                     "location": eval( stations.get(station, "location") )
 889                 }
 890         station = closest( gecos(expression), stationtable, "location", 0.1 )
 891         if not station[0]:
 892             message = "No ICAO weather station found near %s.\n" % expression
 893             sys.stderr.write(message)
 894             exit(1)
 895         zonetable = {}
 896         for zone in zones.sections():
 897             if zones.has_option(zone, "centroid"):
 898                 zonetable[zone] = {
 899                     "centroid": eval( zones.get(zone, "centroid") )
 900                 }
 901         zone = closest( gecos(expression), zonetable, "centroid", 0.1 )
 902         if not zone[0]:
 903             message = "No NWS weather zone near %s; forecasts unavailable.\n" \
 904                 % expression
 905             sys.stderr.write(message)
 906     elif searchtype in ("FIPS", "name"):
 907         places = configparser.ConfigParser()
 908         dataname = "places"
 909         if dataname in datafiles:
 910             datafile = datafiles[dataname][0]
 911             if datafile.endswith(".gz"):
 912                 import gzip
 913                 places.read_string( gzip.open(datafile).read().decode("utf-8") )
 914             else:
 915                 places.read(datafile, encoding="utf-8")
 916         else:
 917             message = "%s error: can't find \"%s\" data file\n" % (
 918                 os.path.basename( sys.argv[0] ),
 919                 dataname
 920             )
 921             sys.stderr.write(message)
 922             exit(1)
 923         dataset = places
 924         place = expression.lower()
 925         if places.has_section(place) and places.has_option(place, "station"):
 926             station = eval( places.get(place, "station") )
 927             search = (expression, "Census Place %s" % expression)
 928             if places.has_option(place, "description"):
 929                 search = (
 930                     search[0],
 931                     search[1] + ", %s" % places.get(place, "description")
 932                 )
 933             if places.has_option(place, "zone"):
 934                 zone = eval( places.get(place, "zone") )
 935             if not ( info or quiet ) \
 936                 and places.has_option(place, "description"):
 937                 print(
 938                     "[%s result %s]" % (
 939                         action,
 940                         places.get(place, "description")
 941                     )
 942                 )
 943         else:
 944             for place in places.sections():
 945                 if places.has_option(place, "description") \
 946                     and places.has_option(place, "station") \
 947                     and re.search(
 948                         expression,
 949                         places.get(place, "description"),
 950                         re.I
 951                     ):
 952                         possibilities.append(place)
 953             for place in stations.sections():
 954                 if stations.has_option(place, "description") \
 955                     and re.search(
 956                         expression,
 957                         stations.get(place, "description"),
 958                         re.I
 959                     ):
 960                         possibilities.append(place)
 961             for place in zones.sections():
 962                 if zones.has_option(place, "description") \
 963                     and zones.has_option(place, "station") \
 964                     and re.search(
 965                         expression,
 966                         zones.get(place, "description"),
 967                         re.I
 968                     ):
 969                         possibilities.append(place)
 970             if len(possibilities) == 1:
 971                 place = possibilities[0]
 972                 if places.has_section(place):
 973                     station = eval( places.get(place, "station") )
 974                     description = places.get(place, "description")
 975                     if places.has_option(place, "zone"):
 976                         zone = eval( places.get(place, "zone" ) )
 977                     search = ( expression, "%s: %s" % (place, description) )
 978                 elif stations.has_section(place):
 979                     station = (place, 0.0)
 980                     description = stations.get(place, "description")
 981                     if stations.has_option(place, "zone"):
 982                         zone = eval( stations.get(place, "zone" ) )
 983                     search = ( expression, "ICAO station code %s" % place )
 984                 elif zones.has_section(place):
 985                     station = eval( zones.get(place, "station") )
 986                     description = zones.get(place, "description")
 987                     zone = (place, 0.0)
 988                     search = ( expression, "NWS/NOAA weather zone %s" % place )
 989                 if not ( info or quiet ):
 990                     print( "[%s result %s]" % (action, description) )
 991             if not possibilities and not station[0]:
 992                 message = "No FIPS code/census area match in the %s file.\n" % (
 993                     datafiles["places"][0]
 994                 )
 995                 sys.stderr.write(message)
 996                 exit(1)
 997     if station[0]:
 998         uris["metar"] = stations.get( station[0], "metar" )
 999         if zone[0]:
1000             for key,value in zones.items( zone[0] ):
1001                 if key not in ("centroid", "description", "station"):
1002                     uris[key] = value
1003     elif possibilities:
1004         count = len(possibilities)
1005         if count <= max_results:
1006             print( "Your search is ambiguous, returning %s matches:" % count )
1007             for place in sorted(possibilities):
1008                 if places.has_section(place):
1009                     print(
1010                         "   [%s] %s" % (
1011                             place,
1012                             places.get(place, "description")
1013                         )
1014                     )
1015                 elif stations.has_section(place):
1016                     print(
1017                         "   [%s] %s" % (
1018                             place,
1019                             stations.get(place, "description")
1020                         )
1021                     )
1022                 elif zones.has_section(place):
1023                     print(
1024                         "   [%s] %s" % (
1025                             place,
1026                             zones.get(place, "description")
1027                         )
1028                     )
1029         else:
1030             print(
1031                 "Your search is too ambiguous, returning %s matches." % count
1032             )
1033         exit(0)
1034     if info:
1035         stationlist = []
1036         zonelist = []
1037         if dataset:
1038             for section in dataset.sections():
1039                 if dataset.has_option(section, "station"):
1040                     stationlist.append(
1041                         eval( dataset.get(section, "station") )[1]
1042                     )
1043                 if dataset.has_option(section, "zone"):
1044                     zonelist.append( eval( dataset.get(section, "zone") )[1] )
1045         stationlist.sort()
1046         zonelist.sort()
1047         scount = len(stationlist)
1048         zcount = len(zonelist)
1049         sranks = []
1050         zranks = []
1051         for score in scores:
1052             if stationlist:
1053                 sranks.append( stationlist[ int( (1-score[0]) * scount ) ] )
1054             if zonelist:
1055                 zranks.append( zonelist[ int( (1-score[0]) * zcount ) ] )
1056         description = search[1]
1057         uris["description"] = description
1058         print(
1059             "%s\n%s" % ( description, "-" * len(description) )
1060         )
1061         print(
1062             "%s: %s" % (
1063                 station[0],
1064                 stations.get( station[0], "description" )
1065             )
1066         )
1067         km = radian_to_km*station[1]
1068         mi = radian_to_mi*station[1]
1069         if sranks and not description.startswith("ICAO station code "):
1070             for index in range(0, len(scores)):
1071                 if station[1] >= sranks[index]:
1072                     score = scores[index][1]
1073                     break
1074             print(
1075                 "   (proximity %s, %.3gkm, %.3gmi)" % ( score, km, mi )
1076             )
1077         elif searchtype == "coordinates":
1078             print( "   (%.3gkm, %.3gmi)" % (km, mi) )
1079         if zone[0]:
1080             print(
1081                 "%s: %s" % ( zone[0], zones.get( zone[0], "description" ) )
1082             )
1083         km = radian_to_km*zone[1]
1084         mi = radian_to_mi*zone[1]
1085         if zranks and not description.startswith("NWS/NOAA weather zone "):
1086             for index in range(0, len(scores)):
1087                 if zone[1] >= zranks[index]:
1088                     score = scores[index][1]
1089                     break
1090             print(
1091                 "   (proximity %s, %.3gkm, %.3gmi)" % ( score, km, mi )
1092             )
1093         elif searchtype == "coordinates" and zone[0]:
1094             print( "   (%.3gkm, %.3gmi)" % (km, mi) )
1095     if cache_search:
1096         now = time.time()
1097         nowstamp = "%s (%s)" % (
1098             now,
1099             datetime.datetime.isoformat(
1100                 datetime.datetime.fromtimestamp(now),
1101                 " "
1102             )
1103         )
1104         search_cache = ["\n"]
1105         search_cache.append( "[%s]\n" % search[0] )
1106         search_cache.append( "cached = %s\n" % nowstamp )
1107         for uriname in sorted(uris.keys()):
1108             search_cache.append( "%s = %s\n" % ( uriname, uris[uriname] ) )
1109         real_cachedir = os.path.expanduser(cachedir)
1110         if not os.path.exists(real_cachedir):
1111             try: os.makedirs(real_cachedir)
1112             except (IOError, OSError): pass
1113         scache_fn = os.path.join(real_cachedir, "searches")
1114         if not os.path.exists(scache_fn):
1115             then = sorted(
1116                     [ x[1] for x in datafiles.values() ],
1117                     reverse=True
1118                 )[0]
1119             thenstamp = "%s (%s)" % (
1120                 then,
1121                 datetime.datetime.isoformat(
1122                     datetime.datetime.fromtimestamp(then),
1123                     " "
1124                 )
1125             )
1126             search_cache.insert(
1127                 0,
1128                 "# based on data files from: %s\n" % thenstamp
1129             )
1130         try:
1131             scache_existing = configparser.ConfigParser()
1132             scache_existing.read(scache_fn, encoding="utf-8")
1133             if not scache_existing.has_section(search[0]):
1134                 scache_fd = codecs.open(scache_fn, "a", "utf-8")
1135                 scache_fd.writelines(search_cache)
1136                 scache_fd.close()
1137         except (IOError, OSError): pass
1138     if not info:
1139         return(uris)
1140
1141 def closest(position, nodes, fieldname, angle=None):
1142     import math
1143     if not angle: angle = 2*math.pi
1144     match = None
1145     for name in nodes:
1146         if fieldname in nodes[name]:
1147             node = nodes[name][fieldname]
1148             if node and abs( position[0]-node[0] ) < angle:
1149                 if abs( position[1]-node[1] ) < angle \
1150                     or abs( abs( position[1]-node[1] ) - 2*math.pi ) < angle:
1151                     if position == node:
1152                         angle = 0
1153                         match = name
1154                     else:
1155                         candidate = math.acos(
1156                             math.sin( position[0] ) * math.sin( node[0] ) \
1157                                 + math.cos( position[0] ) \
1158                                 * math.cos( node[0] ) \
1159                                 * math.cos( position[1] - node[1] )
1160                             )
1161                         if candidate < angle:
1162                             angle = candidate
1163                             match = name
1164     if match: match = str(match)
1165     return (match, angle)
1166
1167 def gecos(formatted):
1168     import math, re
1169     coordinates = formatted.split(",")
1170     for coordinate in range(0, 2):
1171         degrees, foo, minutes, bar, seconds, hemisphere = re.match(
1172             r"([\+-]?\d+\.?\d*)(-(\d+))?(-(\d+))?([ensw]?)$",
1173             coordinates[coordinate].strip().lower()
1174         ).groups()
1175         value = float(degrees)
1176         if minutes: value += float(minutes)/60
1177         if seconds: value += float(seconds)/3600
1178         if hemisphere and hemisphere in "sw": value *= -1
1179         coordinates[coordinate] = math.radians(value)
1180     return tuple(coordinates)
1181
1182 def correlate():
1183     import codecs, configparser, csv, datetime, hashlib, os, re, sys, time
1184     import zipfile, zoneinfo
1185     for filename in os.listdir("."):
1186         if re.match("[0-9]{4}_Gaz_counties_national.zip$", filename):
1187             gcounties_an = filename
1188             gcounties_fn = filename[:-4] + ".txt"
1189         elif re.match("[0-9]{4}_Gaz_cousubs_national.zip$", filename):
1190             gcousubs_an = filename
1191             gcousubs_fn = filename[:-4] + ".txt"
1192         elif re.match("[0-9]{4}_Gaz_place_national.zip$", filename):
1193             gplace_an = filename
1194             gplace_fn = filename[:-4] + ".txt"
1195         elif re.match("[0-9]{4}_Gaz_zcta_national.zip$", filename):
1196             gzcta_an = filename
1197             gzcta_fn = filename[:-4] + ".txt"
1198         elif re.match("bp[0-9]{2}[a-z]{2}[0-9]{2}.dbx$", filename):
1199             cpfzcf_fn = filename
1200     nsdcccc_fn = "nsd_cccc.txt"
1201     ourairports_fn = "airports.csv"
1202     overrides_fn = "overrides.conf"
1203     overrideslog_fn = "overrides.log"
1204     slist_fn = "slist"
1205     zlist_fn = "zlist"
1206     qalog_fn = "qa.log"
1207     airports_fn = "airports"
1208     places_fn = "places"
1209     stations_fn = "stations"
1210     zctas_fn = "zctas"
1211     zones_fn = "zones"
1212     header = """\
1213 %s
1214 # generated by %s on %s from these public domain sources:
1215 #
1216 # https://www.census.gov/geographies/reference-files/time-series/geo/gazetteer-files.html
1217 # %s %s %s
1218 # %s %s %s
1219 # %s %s %s
1220 # %s %s %s
1221 #
1222 # https://www.weather.gov/gis/ZoneCounty/
1223 # %s %s %s
1224 #
1225 # https://tgftp.nws.noaa.gov/data/
1226 # %s %s %s
1227 #
1228 # https://ourairports.com/data/
1229 # %s %s %s
1230 #
1231 # ...and these manually-generated or hand-compiled adjustments:
1232 # %s %s %s
1233 # %s %s %s
1234 # %s %s %s\
1235 """ % (
1236         weather_copyright,
1237         os.path.basename( sys.argv[0] ),
1238         datetime.date.isoformat(
1239             datetime.datetime.utcfromtimestamp( int(os.environ.get('SOURCE_DATE_EPOCH', time.time())) )
1240         ),
1241         hashlib.md5( open(gcounties_an, "rb").read() ).hexdigest(),
1242         datetime.date.isoformat(
1243             datetime.datetime.utcfromtimestamp( os.path.getmtime(gcounties_an) )
1244         ),
1245         gcounties_an,
1246         hashlib.md5( open(gcousubs_an, "rb").read() ).hexdigest(),
1247         datetime.date.isoformat(
1248             datetime.datetime.utcfromtimestamp( os.path.getmtime(gcousubs_an) )
1249         ),
1250         gcousubs_an,
1251         hashlib.md5( open(gplace_an, "rb").read() ).hexdigest(),
1252         datetime.date.isoformat(
1253             datetime.datetime.utcfromtimestamp( os.path.getmtime(gplace_an) )
1254         ),
1255         gplace_an,
1256         hashlib.md5( open(gzcta_an, "rb").read() ).hexdigest(),
1257         datetime.date.isoformat(
1258             datetime.datetime.utcfromtimestamp( os.path.getmtime(gzcta_an) )
1259         ),
1260         gzcta_an,
1261         hashlib.md5( open(cpfzcf_fn, "rb").read() ).hexdigest(),
1262         datetime.date.isoformat(
1263             datetime.datetime.utcfromtimestamp( os.path.getmtime(cpfzcf_fn) )
1264         ),
1265         cpfzcf_fn,
1266         hashlib.md5( open(nsdcccc_fn, "rb").read() ).hexdigest(),
1267         datetime.date.isoformat(
1268             datetime.datetime.utcfromtimestamp( os.path.getmtime(nsdcccc_fn) )
1269         ),
1270         nsdcccc_fn,
1271         hashlib.md5( open(ourairports_fn, "rb").read() ).hexdigest(),
1272         datetime.date.isoformat(
1273             datetime.datetime.utcfromtimestamp( os.path.getmtime(ourairports_fn) )
1274         ),
1275         ourairports_fn,
1276         hashlib.md5( open(overrides_fn, "rb").read() ).hexdigest(),
1277         datetime.date.isoformat(
1278             datetime.datetime.utcfromtimestamp( os.path.getmtime(overrides_fn) )
1279         ),
1280         overrides_fn,
1281         hashlib.md5( open(slist_fn, "rb").read() ).hexdigest(),
1282         datetime.date.isoformat(
1283             datetime.datetime.utcfromtimestamp( os.path.getmtime(slist_fn) )
1284         ),
1285         slist_fn,
1286         hashlib.md5( open(zlist_fn, "rb").read() ).hexdigest(),
1287         datetime.date.isoformat(
1288             datetime.datetime.utcfromtimestamp( os.path.getmtime(zlist_fn) )
1289         ),
1290         zlist_fn
1291     )
1292     airports = {}
1293     places = {}
1294     stations = {}
1295     zctas = {}
1296     zones = {}
1297     message = "Reading %s:%s..." % (gcounties_an, gcounties_fn)
1298     sys.stdout.write(message)
1299     sys.stdout.flush()
1300     count = 0
1301     gcounties = zipfile.ZipFile(gcounties_an).open(gcounties_fn, "r")
1302     columns = gcounties.readline().decode("utf-8").strip().split("\t")
1303     for line in gcounties:
1304         fields = line.decode("utf-8").strip().split("\t")
1305         f_geoid = fields[ columns.index("GEOID") ].strip()
1306         f_name = fields[ columns.index("NAME") ].strip()
1307         f_usps = fields[ columns.index("USPS") ].strip()
1308         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1309         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1310         if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
1311             fips = "fips%s" % f_geoid
1312             if fips not in places: places[fips] = {}
1313             places[fips]["centroid"] = gecos(
1314                 "%s,%s" % (f_intptlat, f_intptlong)
1315             )
1316             places[fips]["description"] = "%s, %s" % (f_name, f_usps)
1317             count += 1
1318     gcounties.close()
1319     print("done (%s lines)." % count)
1320     message = "Reading %s:%s..." % (gcousubs_an, gcousubs_fn)
1321     sys.stdout.write(message)
1322     sys.stdout.flush()
1323     count = 0
1324     gcousubs = zipfile.ZipFile(gcousubs_an).open(gcousubs_fn, "r")
1325     columns = gcousubs.readline().decode("utf-8").strip().split("\t")
1326     for line in gcousubs:
1327         fields = line.decode("utf-8").strip().split("\t")
1328         f_geoid = fields[ columns.index("GEOID") ].strip()
1329         f_name = fields[ columns.index("NAME") ].strip()
1330         f_usps = fields[ columns.index("USPS") ].strip()
1331         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1332         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1333         if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
1334             fips = "fips%s" % f_geoid
1335             if fips not in places: places[fips] = {}
1336             places[fips]["centroid"] = gecos(
1337                 "%s,%s" % (f_intptlat, f_intptlong)
1338             )
1339             places[fips]["description"] = "%s, %s" % (f_name, f_usps)
1340             count += 1
1341     gcousubs.close()
1342     print("done (%s lines)." % count)
1343     message = "Reading %s:%s..." % (gplace_an, gplace_fn)
1344     sys.stdout.write(message)
1345     sys.stdout.flush()
1346     count = 0
1347     gplace = zipfile.ZipFile(gplace_an).open(gplace_fn, "r")
1348     columns = gplace.readline().decode("utf-8").strip().split("\t")
1349     for line in gplace:
1350         fields = line.decode("utf-8").strip().split("\t")
1351         f_geoid = fields[ columns.index("GEOID") ].strip()
1352         f_name = fields[ columns.index("NAME") ].strip()
1353         f_usps = fields[ columns.index("USPS") ].strip()
1354         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1355         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1356         if f_geoid and f_name and f_usps and f_intptlat and f_intptlong:
1357             fips = "fips%s" % f_geoid
1358             if fips not in places: places[fips] = {}
1359             places[fips]["centroid"] = gecos(
1360                 "%s,%s" % (f_intptlat, f_intptlong)
1361             )
1362             places[fips]["description"] = "%s, %s" % (f_name, f_usps)
1363             count += 1
1364     gplace.close()
1365     print("done (%s lines)." % count)
1366     message = "Reading %s..." % slist_fn
1367     sys.stdout.write(message)
1368     sys.stdout.flush()
1369     count = 0
1370     slist = codecs.open(slist_fn, "r", "utf-8")
1371     for line in slist:
1372         icao = line.split("#")[0].strip()
1373         if icao:
1374             stations[icao] = {
1375                 "metar": "https://tgftp.nws.noaa.gov/data/observations/"\
1376                     + "metar/decoded/%s.TXT" % icao.upper()
1377             }
1378             count += 1
1379     slist.close()
1380     print("done (%s lines)." % count)
1381     message = "Reading %s..." % nsdcccc_fn
1382     sys.stdout.write(message)
1383     sys.stdout.flush()
1384     count = 0
1385     nsdcccc = codecs.open(nsdcccc_fn, "r", "utf-8")
1386     for line in nsdcccc:
1387         line = str(line)
1388         fields = line.split(";")
1389         icao = fields[0].strip().lower()
1390         if icao in stations:
1391             description = []
1392             name = " ".join( fields[3].strip().title().split() )
1393             if name: description.append(name)
1394             st = fields[4].strip()
1395             if st: description.append(st)
1396             country = " ".join( fields[5].strip().title().split() )
1397             if country: description.append(country)
1398             if description:
1399                 stations[icao]["description"] = ", ".join(description)
1400             lat, lon = fields[7:9]
1401             if lat and lon:
1402                 stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) )
1403             elif "location" not in stations[icao]:
1404                 lat, lon = fields[5:7]
1405                 if lat and lon:
1406                     stations[icao]["location"] = gecos( "%s,%s" % (lat, lon) )
1407         count += 1
1408     nsdcccc.close()
1409     print("done (%s lines)." % count)
1410     message = "Reading %s..." % ourairports_fn
1411     sys.stdout.write(message)
1412     sys.stdout.flush()
1413     count = 0
1414     ourairports = open(ourairports_fn, "r")
1415     for row in csv.reader(ourairports):
1416         icao = row[12].lower()
1417         if icao in stations:
1418             iata = row[13].lower()
1419             if len(iata) == 3: airports[iata] = { "station": icao }
1420             if "description" not in stations[icao]:
1421                 description = []
1422                 name = row[3]
1423                 if name: description.append(name)
1424                 municipality = row[10]
1425                 if municipality: description.append(municipality)
1426                 region = row[9]
1427                 country = row[8]
1428                 if region:
1429                     if "-" in region:
1430                         c,r = region.split("-", 1)
1431                         if c == country: region = r
1432                     description.append(region)
1433                 if country:
1434                     description.append(country)
1435                 if description:
1436                     stations[icao]["description"] = ", ".join(description)
1437             if "location" not in stations[icao]:
1438                 lat = row[4]
1439                 if lat:
1440                     lon = row[5]
1441                     if lon:
1442                         stations[icao]["location"] = gecos(
1443                             "%s,%s" % (lat, lon)
1444                         )
1445         count += 1
1446     ourairports.close()
1447     print("done (%s lines)." % count)
1448     message = "Reading %s..." % zlist_fn
1449     sys.stdout.write(message)
1450     sys.stdout.flush()
1451     count = 0
1452     zlist = codecs.open(zlist_fn, "r", "utf-8")
1453     for line in zlist:
1454         line = line.split("#")[0].strip()
1455         if line:
1456             zones[line] = {}
1457             count += 1
1458     zlist.close()
1459     print("done (%s lines)." % count)
1460     message = "Reading %s..." % cpfzcf_fn
1461     sys.stdout.write(message)
1462     sys.stdout.flush()
1463     count = 0
1464     cpfz = {}
1465     cpfzcf = codecs.open(cpfzcf_fn, "r", "utf-8")
1466     for line in cpfzcf:
1467         fields = line.strip().split("|")
1468         if len(fields) == 11 \
1469             and fields[0] and fields[1] and fields[9] and fields[10]:
1470             zone = "z".join( fields[:2] ).lower()
1471             if zone in zones:
1472                 state = fields[0]
1473                 description = fields[3].strip()
1474                 fips = "fips%s"%fields[6]
1475                 countycode = "%sc%s" % (state.lower(), fips[-3:])
1476                 if state:
1477                     zones[zone]["coastal_flood_statement"] = (
1478                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1479                         "flood/coastal/%s/%s.txt" % (state.lower(), zone))
1480                     zones[zone]["flash_flood_statement"] = (
1481                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1482                         "flash_flood/statement/%s/%s.txt"
1483                         % (state.lower(), countycode))
1484                     zones[zone]["flash_flood_warning"] = (
1485                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1486                         "flash_flood/warning/%s/%s.txt"
1487                         % (state.lower(), countycode))
1488                     zones[zone]["flash_flood_watch"] = (
1489                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1490                         "flash_flood/watch/%s/%s.txt" % (state.lower(), zone))
1491                     zones[zone]["flood_warning"] = (
1492                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1493                         "flood/warning/%s/%s.txt"
1494                         % (state.lower(), countycode))
1495                     zones[zone]["severe_thunderstorm_warning"] = (
1496                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1497                         "thunderstorm/%s/%s.txt" % (state.lower(), countycode))
1498                     zones[zone]["severe_weather_statement"] = (
1499                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1500                         "severe_weather_stmt/%s/%s.txt"
1501                         % (state.lower(), countycode))
1502                     zones[zone]["short_term_forecast"] = (
1503                         "https://tgftp.nws.noaa.gov/data/forecasts/nowcast/"
1504                         "%s/%s.txt" % (state.lower(), zone))
1505                     zones[zone]["special_weather_statement"] = (
1506                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1507                         "special_weather_stmt/%s/%s.txt"
1508                         % (state.lower(), zone))
1509                     zones[zone]["state_forecast"] = (
1510                         "https://tgftp.nws.noaa.gov/data/forecasts/state/"
1511                         "%s/%s.txt" % (state.lower(), zone))
1512                     zones[zone]["tornado"] = (
1513                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1514                         "tornado/%s/%s.txt" % (state.lower(), countycode))
1515                     zones[zone]["urgent_weather_message"] = (
1516                         "https://tgftp.nws.noaa.gov/data/watches_warnings/"
1517                         "non_precip/%s/%s.txt" % (state.lower(), zone))
1518                     zones[zone]["zone_forecast"] = (
1519                         "https://tgftp.nws.noaa.gov/data/forecasts/zone/"
1520                         "%s/%s.txt" % (state.lower(), zone))
1521                 tzcode = fields[7]
1522                 if tzcode == "A":
1523                     zones[zone]["tz"] = "US/Alaska"
1524                 elif tzcode == "AH":
1525                     zones[zone]["tz"] = "US/Aleutian"
1526                 elif tzcode in ("C", "CE", "CM"):
1527                     zones[zone]["tz"] = "US/Central"
1528                 elif tzcode in ("E", "e"):
1529                     zones[zone]["tz"] = "US/Eastern"
1530                 elif tzcode == "F":
1531                     zones[zone]["tz"] = "Pacific/Guadalcanal"
1532                 elif tzcode == "G":
1533                     zones[zone]["tz"] = "Pacific/Guam"
1534                 elif tzcode == "H":
1535                     zones[zone]["tz"] = "US/Hawaii"
1536                 elif tzcode == "J":
1537                     zones[zone]["tz"] = "Japan"
1538                 elif tzcode == "K":
1539                     zones[zone]["tz"] = "Pacific/Kwajalein"
1540                 elif tzcode in ("M", "MC", "MP"):
1541                     zones[zone]["tz"] = "US/Mountain"
1542                 elif tzcode == "m":
1543                     zones[zone]["tz"] = "US/Arizona"
1544                 elif tzcode == "P":
1545                     zones[zone]["tz"] = "US/Pacific"
1546                 elif tzcode == "S":
1547                     zones[zone]["tz"] = "US/Samoa"
1548                 elif tzcode == "V":
1549                     zones[zone]["tz"] = "America/Virgin"
1550                 else:
1551                     zones[zone]["tz"] = ""
1552                 county = fields[5]
1553                 if county:
1554                     if description.endswith(county):
1555                         description += " County"
1556                     else:
1557                         description += ", %s County" % county
1558                 description += ", %s, US" % state
1559                 zones[zone]["description"] = description
1560                 zones[zone]["centroid"] = gecos( ",".join( fields[9:11] ) )
1561                 if fips in places and not zones[zone]["centroid"]:
1562                     zones[zone]["centroid"] = places[fips]["centroid"]
1563         count += 1
1564     cpfzcf.close()
1565     print("done (%s lines)." % count)
1566     message = "Reading %s:%s..." % (gzcta_an, gzcta_fn)
1567     sys.stdout.write(message)
1568     sys.stdout.flush()
1569     count = 0
1570     gzcta = zipfile.ZipFile(gzcta_an).open(gzcta_fn, "r")
1571     columns = gzcta.readline().decode("utf-8").strip().split("\t")
1572     for line in gzcta:
1573         fields = line.decode("utf-8").strip().split("\t")
1574         f_geoid = fields[ columns.index("GEOID") ].strip()
1575         f_intptlat = fields[ columns.index("INTPTLAT") ].strip()
1576         f_intptlong = fields[ columns.index("INTPTLONG") ].strip()
1577         if f_geoid and f_intptlat and f_intptlong:
1578             if f_geoid not in zctas: zctas[f_geoid] = {}
1579             zctas[f_geoid]["centroid"] = gecos(
1580                 "%s,%s" % (f_intptlat, f_intptlong)
1581             )
1582             count += 1
1583     gzcta.close()
1584     print("done (%s lines)." % count)
1585     message = "Reading %s..." % overrides_fn
1586     sys.stdout.write(message)
1587     sys.stdout.flush()
1588     count = 0
1589     added = 0
1590     removed = 0
1591     changed = 0
1592     overrides = configparser.ConfigParser()
1593     overrides.read_file( codecs.open(overrides_fn, "r", "utf8") )
1594     overrideslog = []
1595     for section in overrides.sections():
1596         addopt = 0
1597         chgopt = 0
1598         if section.startswith("-"):
1599             section = section[1:]
1600             delete = True
1601         else: delete = False
1602         if re.match("[A-Za-z]{3}$", section):
1603             if delete:
1604                 if section in airports:
1605                     del( airports[section] )
1606                     logact = "removed airport %s" % section
1607                     removed += 1
1608                 else:
1609                     logact = "tried to remove nonexistent airport %s" % section
1610             else:
1611                 if section in airports:
1612                     logact = "changed airport %s" % section
1613                     changed += 1
1614                 else:
1615                     airports[section] = {}
1616                     logact = "added airport %s" % section
1617                     added += 1
1618                 for key,value in overrides.items(section):
1619                     if key in airports[section]: chgopt += 1
1620                     else: addopt += 1
1621                     if key in ("centroid", "location"):
1622                         airports[section][key] = eval(value)
1623                     else:
1624                         airports[section][key] = value
1625                 if addopt and chgopt:
1626                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1627                 elif addopt: logact += " (+%s options)" % addopt
1628                 elif chgopt: logact += " (!%s options)" % chgopt
1629         elif re.match("[A-Za-z0-9]{4}$", section):
1630             if delete:
1631                 if section in stations:
1632                     del( stations[section] )
1633                     logact = "removed station %s" % section
1634                     removed += 1
1635                 else:
1636                     logact = "tried to remove nonexistent station %s" % section
1637             else:
1638                 if section in stations:
1639                     logact = "changed station %s" % section
1640                     changed += 1
1641                 else:
1642                     stations[section] = {}
1643                     logact = "added station %s" % section
1644                     added += 1
1645                 for key,value in overrides.items(section):
1646                     if key in stations[section]: chgopt += 1
1647                     else: addopt += 1
1648                     if key in ("centroid", "location"):
1649                         stations[section][key] = eval(value)
1650                     else:
1651                         stations[section][key] = value
1652                 if addopt and chgopt:
1653                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1654                 elif addopt: logact += " (+%s options)" % addopt
1655                 elif chgopt: logact += " (!%s options)" % chgopt
1656         elif re.match("[0-9]{5}$", section):
1657             if delete:
1658                 if section in zctas:
1659                     del( zctas[section] )
1660                     logact = "removed zcta %s" % section
1661                     removed += 1
1662                 else:
1663                     logact = "tried to remove nonexistent zcta %s" % section
1664             else:
1665                 if section in zctas:
1666                     logact = "changed zcta %s" % section
1667                     changed += 1
1668                 else:
1669                     zctas[section] = {}
1670                     logact = "added zcta %s" % section
1671                     added += 1
1672                 for key,value in overrides.items(section):
1673                     if key in zctas[section]: chgopt += 1
1674                     else: addopt += 1
1675                     if key in ("centroid", "location"):
1676                         zctas[section][key] = eval(value)
1677                     else:
1678                         zctas[section][key] = value
1679                 if addopt and chgopt:
1680                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1681                 elif addopt: logact += " (+%s options)" % addopt
1682                 elif chgopt: logact += " (!%s options)" % chgopt
1683         elif re.match("[A-Za-z]{2}[Zz][0-9]{3}$", section):
1684             if delete:
1685                 if section in zones:
1686                     del( zones[section] )
1687                     logact = "removed zone %s" % section
1688                     removed += 1
1689                 else:
1690                     logact = "tried to remove nonexistent zone %s" % section
1691             else:
1692                 if section in zones:
1693                     logact = "changed zone %s" % section
1694                     changed += 1
1695                 else:
1696                     zones[section] = {}
1697                     logact = "added zone %s" % section
1698                     added += 1
1699                 for key,value in overrides.items(section):
1700                     if key in zones[section]: chgopt += 1
1701                     else: addopt += 1
1702                     if key in ("centroid", "location"):
1703                         zones[section][key] = eval(value)
1704                     else:
1705                         zones[section][key] = value
1706                 if addopt and chgopt:
1707                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1708                 elif addopt: logact += " (+%s options)" % addopt
1709                 elif chgopt: logact += " (!%s options)" % chgopt
1710         elif re.match("fips[0-9]+$", section):
1711             if delete:
1712                 if section in places:
1713                     del( places[section] )
1714                     logact = "removed place %s" % section
1715                     removed += 1
1716                 else:
1717                     logact = "tried to remove nonexistent place %s" % section
1718             else:
1719                 if section in places:
1720                     logact = "changed place %s" % section
1721                     changed += 1
1722                 else:
1723                     places[section] = {}
1724                     logact = "added place %s" % section
1725                     added += 1
1726                 for key,value in overrides.items(section):
1727                     if key in places[section]: chgopt += 1
1728                     else: addopt += 1
1729                     if key in ("centroid", "location"):
1730                         places[section][key] = eval(value)
1731                     else:
1732                         places[section][key] = value
1733                 if addopt and chgopt:
1734                     logact += " (+%s/!%s options)" % (addopt, chgopt)
1735                 elif addopt: logact += " (+%s options)" % addopt
1736                 elif chgopt: logact += " (!%s options)" % chgopt
1737         count += 1
1738         overrideslog.append("%s\n" % logact)
1739     overrideslog.sort()
1740     if os.path.exists(overrideslog_fn):
1741         os.rename(overrideslog_fn, "%s_old"%overrideslog_fn)
1742     overrideslog_fd = codecs.open(overrideslog_fn, "w", "utf8")
1743     import time
1744     overrideslog_fd.write(
1745         '# Copyright (c) %s Jeremy Stanley <fungi@yuggoth.org>. Permission to\n'
1746         '# use, copy, modify, and distribute this software is granted under terms\n'
1747         '# provided in the LICENSE file distributed with this software.\n\n'
1748         % time.gmtime().tm_year)
1749     overrideslog_fd.writelines(overrideslog)
1750     overrideslog_fd.close()
1751     print("done (%s overridden sections: +%s/-%s/!%s)." % (
1752         count,
1753         added,
1754         removed,
1755         changed
1756     ) )
1757     estimate = 2*len(places) + len(stations) + 2*len(zctas) + len(zones)
1758     print(
1759         "Correlating places, stations, ZCTAs and zones (upper bound is %s):" % \
1760             estimate
1761     )
1762     count = 0
1763     milestones = list( range(51) )
1764     message = "   "
1765     sys.stdout.write(message)
1766     sys.stdout.flush()
1767     for fips in places:
1768         centroid = places[fips]["centroid"]
1769         if centroid:
1770             station = closest(centroid, stations, "location", 0.1)
1771         if station[0]:
1772             places[fips]["station"] = station
1773             count += 1
1774             if not count%100:
1775                 level = int(50*count/estimate)
1776                 if level in milestones:
1777                     for remaining in milestones[:milestones.index(level)+1]:
1778                         if remaining%5:
1779                             message = "."
1780                             sys.stdout.write(message)
1781                             sys.stdout.flush()
1782                         else:
1783                             message = "%s%%" % (remaining*2,)
1784                             sys.stdout.write(message)
1785                             sys.stdout.flush()
1786                         milestones.remove(remaining)
1787         if centroid:
1788             zone = closest(centroid, zones, "centroid", 0.1)
1789         if zone[0]:
1790             places[fips]["zone"] = zone
1791             count += 1
1792             if not count%100:
1793                 level = int(50*count/estimate)
1794                 if level in milestones:
1795                     for remaining in milestones[:milestones.index(level)+1]:
1796                         if remaining%5:
1797                             message = "."
1798                             sys.stdout.write(message)
1799                             sys.stdout.flush()
1800                         else:
1801                             message = "%s%%" % (remaining*2,)
1802                             sys.stdout.write(message)
1803                             sys.stdout.flush()
1804                         milestones.remove(remaining)
1805     for station in stations:
1806         if "location" in stations[station]:
1807             location = stations[station]["location"]
1808             if location:
1809                 zone = closest(location, zones, "centroid", 0.1)
1810             if zone[0]:
1811                 stations[station]["zone"] = zone
1812                 count += 1
1813                 if not count%100:
1814                     level = int(50*count/estimate)
1815                     if level in milestones:
1816                         for remaining in milestones[:milestones.index(level)+1]:
1817                             if remaining%5:
1818                                 message = "."
1819                                 sys.stdout.write(message)
1820                                 sys.stdout.flush()
1821                             else:
1822                                 message = "%s%%" % (remaining*2,)
1823                                 sys.stdout.write(message)
1824                                 sys.stdout.flush()
1825                             milestones.remove(remaining)
1826     for zcta in zctas.keys():
1827         centroid = zctas[zcta]["centroid"]
1828         if centroid:
1829             station = closest(centroid, stations, "location", 0.1)
1830         if station[0]:
1831             zctas[zcta]["station"] = station
1832             count += 1
1833             if not count%100:
1834                 level = int(50*count/estimate)
1835                 if level in milestones:
1836                     for remaining in milestones[ : milestones.index(level)+1 ]:
1837                         if remaining%5:
1838                             message = "."
1839                             sys.stdout.write(message)
1840                             sys.stdout.flush()
1841                         else:
1842                             message = "%s%%" % (remaining*2,)
1843                             sys.stdout.write(message)
1844                             sys.stdout.flush()
1845                         milestones.remove(remaining)
1846         if centroid:
1847             zone = closest(centroid, zones, "centroid", 0.1)
1848         if zone[0]:
1849             zctas[zcta]["zone"] = zone
1850             count += 1
1851             if not count%100:
1852                 level = int(50*count/estimate)
1853                 if level in milestones:
1854                     for remaining in milestones[:milestones.index(level)+1]:
1855                         if remaining%5:
1856                             message = "."
1857                             sys.stdout.write(message)
1858                             sys.stdout.flush()
1859                         else:
1860                             message = "%s%%" % (remaining*2,)
1861                             sys.stdout.write(message)
1862                             sys.stdout.flush()
1863                         milestones.remove(remaining)
1864     for zone in zones.keys():
1865         if "centroid" in zones[zone]:
1866             centroid = zones[zone]["centroid"]
1867             if centroid:
1868                 station = closest(centroid, stations, "location", 0.1)
1869             if station[0]:
1870                 zones[zone]["station"] = station
1871                 count += 1
1872                 if not count%100:
1873                     level = int(50*count/estimate)
1874                     if level in milestones:
1875                         for remaining in milestones[:milestones.index(level)+1]:
1876                             if remaining%5:
1877                                 message = "."
1878                                 sys.stdout.write(message)
1879                                 sys.stdout.flush()
1880                             else:
1881                                 message = "%s%%" % (remaining*2,)
1882                                 sys.stdout.write(message)
1883                                 sys.stdout.flush()
1884                             milestones.remove(remaining)
1885     for remaining in milestones:
1886         if remaining%5:
1887             message = "."
1888             sys.stdout.write(message)
1889             sys.stdout.flush()
1890         else:
1891             message = "%s%%" % (remaining*2,)
1892             sys.stdout.write(message)
1893             sys.stdout.flush()
1894     print("\n   done (%s correlations)." % count)
1895     message = "Writing %s..." % airports_fn
1896     sys.stdout.write(message)
1897     sys.stdout.flush()
1898     count = 0
1899     if os.path.exists(airports_fn):
1900         os.rename(airports_fn, "%s_old"%airports_fn)
1901     airports_fd = codecs.open(airports_fn, "w", "utf8")
1902     airports_fd.write(header)
1903     for airport in sorted( airports.keys() ):
1904         airports_fd.write("\n\n[%s]" % airport)
1905         for key, value in sorted( airports[airport].items() ):
1906             if type(value) is float: value = "%.7f"%value
1907             elif type(value) is tuple:
1908                 elements = []
1909                 for element in value:
1910                     if type(element) is float: elements.append("%.7f"%element)
1911                     else: elements.append( repr(element) )
1912                 value = "(%s)"%", ".join(elements)
1913             airports_fd.write( "\n%s = %s" % (key, value) )
1914         count += 1
1915     airports_fd.write("\n")
1916     airports_fd.close()
1917     print("done (%s sections)." % count)
1918     message = "Writing %s..." % places_fn
1919     sys.stdout.write(message)
1920     sys.stdout.flush()
1921     count = 0
1922     if os.path.exists(places_fn):
1923         os.rename(places_fn, "%s_old"%places_fn)
1924     places_fd = codecs.open(places_fn, "w", "utf8")
1925     places_fd.write(header)
1926     for fips in sorted( places.keys() ):
1927         places_fd.write("\n\n[%s]" % fips)
1928         for key, value in sorted( places[fips].items() ):
1929             if type(value) is float: value = "%.7f"%value
1930             elif type(value) is tuple:
1931                 elements = []
1932                 for element in value:
1933                     if type(element) is float: elements.append("%.7f"%element)
1934                     else: elements.append( repr(element) )
1935                 value = "(%s)"%", ".join(elements)
1936             places_fd.write( "\n%s = %s" % (key, value) )
1937         count += 1
1938     places_fd.write("\n")
1939     places_fd.close()
1940     print("done (%s sections)." % count)
1941     message = "Writing %s..." % stations_fn
1942     sys.stdout.write(message)
1943     sys.stdout.flush()
1944     count = 0
1945     if os.path.exists(stations_fn):
1946         os.rename(stations_fn, "%s_old"%stations_fn)
1947     stations_fd = codecs.open(stations_fn, "w", "utf-8")
1948     stations_fd.write(header)
1949     for station in sorted( stations.keys() ):
1950         stations_fd.write("\n\n[%s]" % station)
1951         for key, value in sorted( stations[station].items() ):
1952             if type(value) is float: value = "%.7f"%value
1953             elif type(value) is tuple:
1954                 elements = []
1955                 for element in value:
1956                     if type(element) is float: elements.append("%.7f"%element)
1957                     else: elements.append( repr(element) )
1958                 value = "(%s)"%", ".join(elements)
1959             if type(value) is bytes:
1960                 value = value.decode("utf-8")
1961             stations_fd.write( "\n%s = %s" % (key, value) )
1962         count += 1
1963     stations_fd.write("\n")
1964     stations_fd.close()
1965     print("done (%s sections)." % count)
1966     message = "Writing %s..." % zctas_fn
1967     sys.stdout.write(message)
1968     sys.stdout.flush()
1969     count = 0
1970     if os.path.exists(zctas_fn):
1971         os.rename(zctas_fn, "%s_old"%zctas_fn)
1972     zctas_fd = codecs.open(zctas_fn, "w", "utf8")
1973     zctas_fd.write(header)
1974     for zcta in sorted( zctas.keys() ):
1975         zctas_fd.write("\n\n[%s]" % zcta)
1976         for key, value in sorted( zctas[zcta].items() ):
1977             if type(value) is float: value = "%.7f"%value
1978             elif type(value) is tuple:
1979                 elements = []
1980                 for element in value:
1981                     if type(element) is float: elements.append("%.7f"%element)
1982                     else: elements.append( repr(element) )
1983                 value = "(%s)"%", ".join(elements)
1984             zctas_fd.write( "\n%s = %s" % (key, value) )
1985         count += 1
1986     zctas_fd.write("\n")
1987     zctas_fd.close()
1988     print("done (%s sections)." % count)
1989     message = "Writing %s..." % zones_fn
1990     sys.stdout.write(message)
1991     sys.stdout.flush()
1992     count = 0
1993     if os.path.exists(zones_fn):
1994         os.rename(zones_fn, "%s_old"%zones_fn)
1995     zones_fd = codecs.open(zones_fn, "w", "utf8")
1996     zones_fd.write(header)
1997     for zone in sorted( zones.keys() ):
1998         zones_fd.write("\n\n[%s]" % zone)
1999         for key, value in sorted( zones[zone].items() ):
2000             if type(value) is float: value = "%.7f"%value
2001             elif type(value) is tuple:
2002                 elements = []
2003                 for element in value:
2004                     if type(element) is float: elements.append("%.7f"%element)
2005                     else: elements.append( repr(element) )
2006                 value = "(%s)"%", ".join(elements)
2007             zones_fd.write( "\n%s = %s" % (key, value) )
2008         count += 1
2009     zones_fd.write("\n")
2010     zones_fd.close()
2011     print("done (%s sections)." % count)
2012     message = "Starting QA check..."
2013     sys.stdout.write(message)
2014     sys.stdout.flush()
2015     airports = configparser.ConfigParser()
2016     airports.read(airports_fn, encoding="utf-8")
2017     places = configparser.ConfigParser()
2018     places.read(places_fn, encoding="utf-8")
2019     stations = configparser.ConfigParser()
2020     stations.read(stations_fn, encoding="utf-8")
2021     zctas = configparser.ConfigParser()
2022     zctas.read(zctas_fn, encoding="utf-8")
2023     zones = configparser.ConfigParser()
2024     zones.read(zones_fn, encoding="utf-8")
2025     qalog = []
2026     places_nocentroid = 0
2027     places_nodescription = 0
2028     for place in sorted( places.sections() ):
2029         if not places.has_option(place, "centroid"):
2030             qalog.append("%s: no centroid\n" % place)
2031             places_nocentroid += 1
2032         if not places.has_option(place, "description"):
2033             qalog.append("%s: no description\n" % place)
2034             places_nodescription += 1
2035     stations_nodescription = 0
2036     stations_nolocation = 0
2037     stations_nometar = 0
2038     for station in sorted( stations.sections() ):
2039         if not stations.has_option(station, "description"):
2040             qalog.append("%s: no description\n" % station)
2041             stations_nodescription += 1
2042         if not stations.has_option(station, "location"):
2043             qalog.append("%s: no location\n" % station)
2044             stations_nolocation += 1
2045         if not stations.has_option(station, "metar"):
2046             qalog.append("%s: no metar\n" % station)
2047             stations_nometar += 1
2048     airports_badstation = 0
2049     airports_nostation = 0
2050     for airport in sorted( airports.sections() ):
2051         if not airports.has_option(airport, "station"):
2052             qalog.append("%s: no station\n" % airport)
2053             airports_nostation += 1
2054         else:
2055             station = airports.get(airport, "station")
2056             if station not in stations.sections():
2057                 qalog.append( "%s: bad station %s\n" % (airport, station) )
2058                 airports_badstation += 1
2059     zctas_nocentroid = 0
2060     for zcta in sorted( zctas.sections() ):
2061         if not zctas.has_option(zcta, "centroid"):
2062             qalog.append("%s: no centroid\n" % zcta)
2063             zctas_nocentroid += 1
2064     zones_nocentroid = 0
2065     zones_nodescription = 0
2066     zones_notz = 0
2067     zones_noforecast = 0
2068     zones_overlapping = 0
2069     zonetable = {}
2070     for zone in zones.sections():
2071         if zones.has_option(zone, "centroid"):
2072             zonetable[zone] = {
2073                 "centroid": eval( zones.get(zone, "centroid") )
2074             }
2075     for zone in sorted( zones.sections() ):
2076         if zones.has_option(zone, "centroid"):
2077             zonetable_local = zonetable.copy()
2078             del( zonetable_local[zone] )
2079             centroid = eval( zones.get(zone, "centroid") )
2080             if centroid:
2081                 nearest = closest(centroid, zonetable_local, "centroid", 0.1)
2082             if nearest[1]*radian_to_km < 1:
2083                 qalog.append( "%s: within one km of %s\n" % (
2084                     zone,
2085                     nearest[0]
2086                 ) )
2087                 zones_overlapping += 1
2088         else:
2089             qalog.append("%s: no centroid\n" % zone)
2090             zones_nocentroid += 1
2091         if not zones.has_option(zone, "description"):
2092             qalog.append("%s: no description\n" % zone)
2093             zones_nodescription += 1
2094         if not zones.has_option(zone, "tz") or not zones.get(
2095                 zone, "tz") in zoneinfo.available_timezones():
2096             qalog.append("%s: no time zone\n" % zone)
2097             zones_notz += 1
2098         if not zones.has_option(zone, "zone_forecast"):
2099             qalog.append("%s: no forecast\n" % zone)
2100             zones_noforecast += 1
2101     if os.path.exists(qalog_fn):
2102         os.rename(qalog_fn, "%s_old"%qalog_fn)
2103     qalog_fd = codecs.open(qalog_fn, "w", "utf8")
2104     import time
2105     qalog_fd.write(
2106         '# Copyright (c) %s Jeremy Stanley <fungi@yuggoth.org>. Permission to\n'
2107         '# use, copy, modify, and distribute this software is granted under terms\n'
2108         '# provided in the LICENSE file distributed with this software.\n\n'
2109         % time.gmtime().tm_year)
2110     qalog_fd.writelines(qalog)
2111     qalog_fd.close()
2112     if qalog:
2113         print("issues found (see %s for details):"%qalog_fn)
2114         if airports_badstation:
2115             print("   %s airports with invalid station"%airports_badstation)
2116         if airports_nostation:
2117             print("   %s airports with no station"%airports_nostation)
2118         if places_nocentroid:
2119             print("   %s places with no centroid"%places_nocentroid)
2120         if places_nodescription:
2121             print("   %s places with no description"%places_nodescription)
2122         if stations_nodescription:
2123             print("   %s stations with no description"%stations_nodescription)
2124         if stations_nolocation:
2125             print("   %s stations with no location"%stations_nolocation)
2126         if stations_nometar:
2127             print("   %s stations with no METAR"%stations_nometar)
2128         if zctas_nocentroid:
2129             print("   %s ZCTAs with no centroid"%zctas_nocentroid)
2130         if zones_nocentroid:
2131             print("   %s zones with no centroid"%zones_nocentroid)
2132         if zones_nodescription:
2133             print("   %s zones with no description"%zones_nodescription)
2134         if zones_notz:
2135             print("   %s zones with no time zone"%zones_notz)
2136         if zones_noforecast:
2137             print("   %s zones with no forecast"%zones_noforecast)
2138         if zones_overlapping:
2139             print("   %s zones within one km of another"%zones_overlapping)
2140     else: print("no issues found.")
2141     print("Indexing complete!")