#!/usr/bin/env python3 """Auto-update OSM planet PBF and rebuild the maxspeed grid. Designed to be triggered weekly (Friday 00:00). Checks the RSS feed for a new planet release. If none found yet, retries every 5 minutes until one appears. Then downloads the PBF, verifies MD5, rebuilds the grid, and atomically swaps the old grid directory for the new one. """ import os, sys, time, hashlib, shutil, signal, subprocess, xml.etree.ElementTree as ET from urllib.request import urlopen, Request from pathlib import Path from datetime import datetime, timezone DATA_DIR = Path(__file__).resolve().parent PBF_DIR = DATA_DIR RSS_URL = "https://planet.openstreetmap.org/pbf/planet-pbf-rss.xml" BUILD_SCRIPT = DATA_DIR / "build_world.py" CHECK_INTERVAL = 5 * 60 # 5 min between retries when waiting for a new release POLL_TIMEOUT = 6 * 3600 # give up after 6 hours if nothing shows up LOG_PREFIX = "[auto-update]" def log(msg): ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S") line = f"{LOG_PREFIX} [{ts}] {msg}" print(line, flush=True) with open(DATA_DIR / "auto-update.log", "a") as f: f.write(line + "\n") def fetch_rss(): """Fetch RSS and return the latest planet filename + pubDate.""" req = Request(RSS_URL, headers={"User-Agent": "zektyc-auto-updater/1.0"}) with urlopen(req, timeout=30) as resp: xml_data = resp.read() root = ET.fromstring(xml_data) items = root.findall(".//item") if not items: return None, None item = items[0] title = (item.findtext("title") or "").strip() pub_date = (item.findtext("pubDate") or "").strip() # title is like "planet-260817.osm.pbf.torrent" filename = title.replace(".torrent", "") return filename, pub_date def get_current_pbf(): """Return the basename of the planet PBF currently in use.""" # Check which PBF file exists in PBF_DIR for f in sorted(PBF_DIR.glob("planet-*.osm.pbf"), reverse=True): if f.stat().st_size > 10_000_000_000: # > 10 GB = real planet return f.name return None def md5_file(path): h = hashlib.md5() with open(path, "rb") as f: for chunk in iter(lambda: f.read(1024 * 1024), b""): h.update(chunk) return h.hexdigest() def fetch_md5(filename): """Download the .md5 file from planet.openstreetmap.org.""" url = f"https://planet.openstreetmap.org/pbf/{filename}.md5" try: req = Request(url, headers={"User-Agent": "zektyc-auto-updater/1.0"}) with urlopen(req, timeout=30) as resp: text = resp.read().decode().strip() # format: "hash filename" return text.split()[0] except Exception as e: log(f"WARNING: could not fetch MD5: {e}") return None def download_pbf(filename): """Download the planet PBF with resume support.""" dest = PBF_DIR / filename url = f"https://planet.openstreetmap.org/pbf/{filename}" existing = dest.stat().st_size if dest.exists() else 0 headers = {"User-Agent": "zektyc-auto-updater/1.0"} if existing > 0: headers["Range"] = f"bytes={existing}-" log(f"Resuming download from {existing / 1e9:.1f} GB") req = Request(url, headers=headers) with urlopen(req, timeout=600) as resp: mode = "ab" if existing > 0 and resp.status == 206 else "wb" if mode == "wb": existing = 0 total = int(resp.headers.get("Content-Length", 0)) + existing with open(dest, mode) as f: downloaded = existing t0 = time.time() while True: chunk = resp.read(1024 * 1024) if not chunk: break f.write(chunk) downloaded += len(chunk) elapsed = time.time() - t0 if elapsed > 0 and downloaded > existing: speed = (downloaded - existing) / elapsed eta = (total - downloaded) / speed if speed > 0 else 0 log(f" {downloaded / 1e9:.2f} / {total / 1e9:.2f} GB " f"({speed / 1e6:.1f} MB/s, ETA {int(eta // 60)}m{int(eta % 60)}s)") return dest def kill_existing_build(): """Kill any running build_world.py process.""" try: result = subprocess.run( ["pgrep", "-f", "build_world.py"], capture_output=True, text=True, timeout=5 ) for pid_str in result.stdout.strip().split("\n"): pid_str = pid_str.strip() if pid_str: pid = int(pid_str) log(f"Killing existing build process PID {pid}") os.kill(pid, signal.SIGTERM) time.sleep(2) try: os.kill(pid, signal.SIGKILL) except OSError: pass except Exception as e: log(f"WARNING: could not kill existing build: {e}") def wait_for_build(): """Wait for any running build_world.py to finish.""" while True: result = subprocess.run( ["pgrep", "-f", "build_world.py"], capture_output=True, text=True, timeout=5 ) if not result.stdout.strip(): return True log("Waiting for existing build to finish...") time.sleep(30) return False def run_build(pbf_path, outdir): """Run build_world.py.""" log(f"Starting build: {pbf_path} -> {outdir}") proc = subprocess.Popen( [sys.executable, "-u", str(BUILD_SCRIPT), str(pbf_path), str(str(outdir))], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=str(DATA_DIR), ) with open(DATA_DIR / "build-planet.log", "a") as logf: for line in iter(proc.stdout.readline, b""): decoded = line.decode(errors="replace") logf.write(decoded) logf.flush() # Also echo last chars for our log proc.wait() if proc.returncode != 0: log(f"ERROR: build_world.py exited with code {proc.returncode}") return False log("Build completed successfully") return True def atomic_swap(new_grid, target_grid): """Atomically replace target_grid with new_grid.""" backup = target_grid.parent / (target_grid.name + "-old") if backup.exists(): shutil.rmtree(backup) if target_grid.exists(): target_grid.rename(backup) new_grid.rename(target_grid) log(f"Swapped {target_grid.name} (backup: {backup.name})") # Clean old backup after a short delay (keep for 1 hour for safety) # Actually let's keep it, it's useful if something goes wrong def main(): log("=" * 60) log("Auto-update started") current_pbf = get_current_pbf() log(f"Current PBF: {current_pbf or 'none'}") # Step 1: Check RSS for new release t_start = time.time() new_filename = None while True: try: filename, pub_date = fetch_rss() if filename and filename != current_pbf: log(f"New release found: {filename} (published: {pub_date})") new_filename = filename break elif filename: log(f"No new release yet (latest: {filename}, current: {current_pbf})") else: log("Could not parse RSS feed") except Exception as e: log(f"RSS check failed: {e}") elapsed = time.time() - t_start if elapsed > POLL_TIMEOUT: log(f"No new release after {elapsed/3600:.1f}h, giving up") return log(f"Retrying in {CHECK_INTERVAL // 60} minutes...") time.sleep(CHECK_INTERVAL) # Step 2: Kill any existing build kill_existing_build() wait_for_build() # Step 3: Download new PBF log(f"Downloading {new_filename}...") pbf_path = download_pbf(new_filename) log(f"Download complete: {pbf_path}") # Step 4: Verify MD5 expected_md5 = fetch_md5(new_filename) if expected_md5: log(f"Verifying MD5 (expected: {expected_md5})") actual_md5 = md5_file(pbf_path) log(f"Actual MD5: {actual_md5}") if actual_md5 != expected_md5: log("ERROR: MD5 mismatch! Aborting.") return log("MD5 verified OK") else: log("WARNING: skipping MD5 verification") # Step 5: Build new grid grid_tmp = PBF_DIR / "grid-planet-new" if grid_tmp.exists(): shutil.rmtree(grid_tmp) grid_target = PBF_DIR / "grid-planet" if not run_build(pbf_path, grid_tmp): log("ERROR: Build failed, keeping old grid") if grid_tmp.exists(): shutil.rmtree(grid_tmp) return # Step 6: Atomic swap atomic_swap(grid_tmp, grid_target) # Step 7: Update .env symlink (if needed) env_path = Path(__file__).resolve().parent.parent.parent / ".env" if env_path.exists(): content = env_path.read_text() if "grid-europe" in content: # Don't change .env — it might be intentionally pointing to grid-europe log("NOTE: .env still points to grid-europe, not auto-updating") elif "grid-planet" in content: log(".env already points to grid-planet") log("Auto-update complete!") log("=" * 60) if __name__ == "__main__": signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) main()