zektyc/data/maxspeed/auto_update.py

#!/usr/bin/env python3
"""Auto-update OSM planet PBF and rebuild the maxspeed grid.

Designed to be triggered weekly (Friday 00:00). Checks the RSS feed for a new
planet release. If none found yet, retries every 5 minutes until one appears.
Then downloads the PBF, verifies MD5, rebuilds the grid, and atomically
swaps the old grid directory for the new one.
"""
import os, sys, time, hashlib, shutil, signal, subprocess, xml.etree.ElementTree as ET
from urllib.request import urlopen, Request
from pathlib import Path
from datetime import datetime, timezone

DATA_DIR = Path(__file__).resolve().parent
PBF_DIR = DATA_DIR
RSS_URL = "https://planet.openstreetmap.org/pbf/planet-pbf-rss.xml"
BUILD_SCRIPT = DATA_DIR / "build_world.py"
CHECK_INTERVAL = 5 * 60  # 5 min between retries when waiting for a new release
POLL_TIMEOUT = 6 * 3600  # give up after 6 hours if nothing shows up
LOG_PREFIX = "[auto-update]"

def log(msg):
    ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    line = f"{LOG_PREFIX} [{ts}] {msg}"
    print(line, flush=True)
    with open(DATA_DIR / "auto-update.log", "a") as f:
        f.write(line + "\n")

def fetch_rss():
    """Fetch RSS and return the latest planet filename + pubDate."""
    req = Request(RSS_URL, headers={"User-Agent": "zektyc-auto-updater/1.0"})
    with urlopen(req, timeout=30) as resp:
        xml_data = resp.read()
    root = ET.fromstring(xml_data)
    items = root.findall(".//item")
    if not items:
        return None, None
    item = items[0]
    title = (item.findtext("title") or "").strip()
    pub_date = (item.findtext("pubDate") or "").strip()
    # title is like "planet-260817.osm.pbf.torrent"
    filename = title.replace(".torrent", "")
    return filename, pub_date

def get_current_pbf():
    """Return the basename of the planet PBF currently in use."""
    # Check which PBF file exists in PBF_DIR
    for f in sorted(PBF_DIR.glob("planet-*.osm.pbf"), reverse=True):
        if f.stat().st_size > 10_000_000_000:  # > 10 GB = real planet
            return f.name
    return None

def md5_file(path):
    h = hashlib.md5()
    with open(path, "rb") as f:
        for chunk in iter(lambda: f.read(1024 * 1024), b""):
            h.update(chunk)
    return h.hexdigest()

def fetch_md5(filename):
    """Download the .md5 file from planet.openstreetmap.org."""
    url = f"https://planet.openstreetmap.org/pbf/{filename}.md5"
    try:
        req = Request(url, headers={"User-Agent": "zektyc-auto-updater/1.0"})
        with urlopen(req, timeout=30) as resp:
            text = resp.read().decode().strip()
        # format: "hash  filename"
        return text.split()[0]
    except Exception as e:
        log(f"WARNING: could not fetch MD5: {e}")
        return None

def download_pbf(filename):
    """Download the planet PBF with resume support."""
    dest = PBF_DIR / filename
    url = f"https://planet.openstreetmap.org/pbf/{filename}"
    existing = dest.stat().st_size if dest.exists() else 0
    headers = {"User-Agent": "zektyc-auto-updater/1.0"}
    if existing > 0:
        headers["Range"] = f"bytes={existing}-"
        log(f"Resuming download from {existing / 1e9:.1f} GB")
    req = Request(url, headers=headers)
    with urlopen(req, timeout=600) as resp:
        mode = "ab" if existing > 0 and resp.status == 206 else "wb"
        if mode == "wb":
            existing = 0
        total = int(resp.headers.get("Content-Length", 0)) + existing
        with open(dest, mode) as f:
            downloaded = existing
            t0 = time.time()
            while True:
                chunk = resp.read(1024 * 1024)
                if not chunk:
                    break
                f.write(chunk)
                downloaded += len(chunk)
                elapsed = time.time() - t0
                if elapsed > 0 and downloaded > existing:
                    speed = (downloaded - existing) / elapsed
                    eta = (total - downloaded) / speed if speed > 0 else 0
                    log(f"  {downloaded / 1e9:.2f} / {total / 1e9:.2f} GB "
                        f"({speed / 1e6:.1f} MB/s, ETA {int(eta // 60)}m{int(eta % 60)}s)")
    return dest

def kill_existing_build():
    """Kill any running build_world.py process."""
    try:
        result = subprocess.run(
            ["pgrep", "-f", "build_world.py"],
            capture_output=True, text=True, timeout=5
        )
        for pid_str in result.stdout.strip().split("\n"):
            pid_str = pid_str.strip()
            if pid_str:
                pid = int(pid_str)
                log(f"Killing existing build process PID {pid}")
                os.kill(pid, signal.SIGTERM)
                time.sleep(2)
                try:
                    os.kill(pid, signal.SIGKILL)
                except OSError:
                    pass
    except Exception as e:
        log(f"WARNING: could not kill existing build: {e}")

def wait_for_build():
    """Wait for any running build_world.py to finish."""
    while True:
        result = subprocess.run(
            ["pgrep", "-f", "build_world.py"],
            capture_output=True, text=True, timeout=5
        )
        if not result.stdout.strip():
            return True
        log("Waiting for existing build to finish...")
        time.sleep(30)
    return False

def run_build(pbf_path, outdir):
    """Run build_world.py."""
    log(f"Starting build: {pbf_path} -> {outdir}")
    proc = subprocess.Popen(
        [sys.executable, "-u", str(BUILD_SCRIPT), str(pbf_path), str(str(outdir))],
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        cwd=str(DATA_DIR),
    )
    with open(DATA_DIR / "build-planet.log", "a") as logf:
        for line in iter(proc.stdout.readline, b""):
            decoded = line.decode(errors="replace")
            logf.write(decoded)
            logf.flush()
            # Also echo last chars for our log
        proc.wait()
    if proc.returncode != 0:
        log(f"ERROR: build_world.py exited with code {proc.returncode}")
        return False
    log("Build completed successfully")
    return True

def atomic_swap(new_grid, target_grid):
    """Atomically replace target_grid with new_grid."""
    backup = target_grid.parent / (target_grid.name + "-old")
    if backup.exists():
        shutil.rmtree(backup)
    if target_grid.exists():
        target_grid.rename(backup)
    new_grid.rename(target_grid)
    log(f"Swapped {target_grid.name} (backup: {backup.name})")
    # Clean old backup after a short delay (keep for 1 hour for safety)
    # Actually let's keep it, it's useful if something goes wrong

def main():
    log("=" * 60)
    log("Auto-update started")

    current_pbf = get_current_pbf()
    log(f"Current PBF: {current_pbf or 'none'}")

    # Step 1: Check RSS for new release
    t_start = time.time()
    new_filename = None
    while True:
        try:
            filename, pub_date = fetch_rss()
            if filename and filename != current_pbf:
                log(f"New release found: {filename} (published: {pub_date})")
                new_filename = filename
                break
            elif filename:
                log(f"No new release yet (latest: {filename}, current: {current_pbf})")
            else:
                log("Could not parse RSS feed")
        except Exception as e:
            log(f"RSS check failed: {e}")

        elapsed = time.time() - t_start
        if elapsed > POLL_TIMEOUT:
            log(f"No new release after {elapsed/3600:.1f}h, giving up")
            return
        log(f"Retrying in {CHECK_INTERVAL // 60} minutes...")
        time.sleep(CHECK_INTERVAL)

    # Step 2: Kill any existing build
    kill_existing_build()
    wait_for_build()

    # Step 3: Download new PBF
    log(f"Downloading {new_filename}...")
    pbf_path = download_pbf(new_filename)
    log(f"Download complete: {pbf_path}")

    # Step 4: Verify MD5
    expected_md5 = fetch_md5(new_filename)
    if expected_md5:
        log(f"Verifying MD5 (expected: {expected_md5})")
        actual_md5 = md5_file(pbf_path)
        log(f"Actual MD5: {actual_md5}")
        if actual_md5 != expected_md5:
            log("ERROR: MD5 mismatch! Aborting.")
            return
        log("MD5 verified OK")
    else:
        log("WARNING: skipping MD5 verification")

    # Step 5: Build new grid
    grid_tmp = PBF_DIR / "grid-planet-new"
    if grid_tmp.exists():
        shutil.rmtree(grid_tmp)
    grid_target = PBF_DIR / "grid-planet"

    if not run_build(pbf_path, grid_tmp):
        log("ERROR: Build failed, keeping old grid")
        if grid_tmp.exists():
            shutil.rmtree(grid_tmp)
        return

    # Step 6: Atomic swap
    atomic_swap(grid_tmp, grid_target)

    # Step 7: Update .env symlink (if needed)
    env_path = Path(__file__).resolve().parent.parent.parent / ".env"
    if env_path.exists():
        content = env_path.read_text()
        if "grid-europe" in content:
            # Don't change .env — it might be intentionally pointing to grid-europe
            log("NOTE: .env still points to grid-europe, not auto-updating")
        elif "grid-planet" in content:
            log(".env already points to grid-planet")

    log("Auto-update complete!")
    log("=" * 60)

if __name__ == "__main__":
    signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
    main()