chore: checkpoint initial — Zektyc dataset, avant humanisation
- site vitrine + fingerprint test + zmap + z-panel - remplacement duckdns -> riricdev.tail5ea5cd.ts.net (canonical/og/alternates/docs) - suppression du token DuckDNS (plus utilisé)
This commit is contained in:
riricdev
commit
8e7ec41f18
104 changed files with 19256 additions and 0 deletions
256
data/maxspeed/auto_update.py
Executable file
256
data/maxspeed/auto_update.py
Executable file
|
|
@ -0,0 +1,256 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Auto-update OSM planet PBF and rebuild the maxspeed grid.
|
||||
|
||||
Designed to be triggered weekly (Friday 00:00). Checks the RSS feed for a new
|
||||
planet release. If none found yet, retries every 5 minutes until one appears.
|
||||
Then downloads the PBF, verifies MD5, rebuilds the grid, and atomically
|
||||
swaps the old grid directory for the new one.
|
||||
"""
|
||||
import os, sys, time, hashlib, shutil, signal, subprocess, xml.etree.ElementTree as ET
|
||||
from urllib.request import urlopen, Request
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
|
||||
DATA_DIR = Path(__file__).resolve().parent
|
||||
PBF_DIR = DATA_DIR
|
||||
RSS_URL = "https://planet.openstreetmap.org/pbf/planet-pbf-rss.xml"
|
||||
BUILD_SCRIPT = DATA_DIR / "build_world.py"
|
||||
CHECK_INTERVAL = 5 * 60 # 5 min between retries when waiting for a new release
|
||||
POLL_TIMEOUT = 6 * 3600 # give up after 6 hours if nothing shows up
|
||||
LOG_PREFIX = "[auto-update]"
|
||||
|
||||
def log(msg):
|
||||
ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
line = f"{LOG_PREFIX} [{ts}] {msg}"
|
||||
print(line, flush=True)
|
||||
with open(DATA_DIR / "auto-update.log", "a") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
def fetch_rss():
|
||||
"""Fetch RSS and return the latest planet filename + pubDate."""
|
||||
req = Request(RSS_URL, headers={"User-Agent": "zektyc-auto-updater/1.0"})
|
||||
with urlopen(req, timeout=30) as resp:
|
||||
xml_data = resp.read()
|
||||
root = ET.fromstring(xml_data)
|
||||
items = root.findall(".//item")
|
||||
if not items:
|
||||
return None, None
|
||||
item = items[0]
|
||||
title = (item.findtext("title") or "").strip()
|
||||
pub_date = (item.findtext("pubDate") or "").strip()
|
||||
# title is like "planet-260817.osm.pbf.torrent"
|
||||
filename = title.replace(".torrent", "")
|
||||
return filename, pub_date
|
||||
|
||||
def get_current_pbf():
|
||||
"""Return the basename of the planet PBF currently in use."""
|
||||
# Check which PBF file exists in PBF_DIR
|
||||
for f in sorted(PBF_DIR.glob("planet-*.osm.pbf"), reverse=True):
|
||||
if f.stat().st_size > 10_000_000_000: # > 10 GB = real planet
|
||||
return f.name
|
||||
return None
|
||||
|
||||
def md5_file(path):
|
||||
h = hashlib.md5()
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
def fetch_md5(filename):
|
||||
"""Download the .md5 file from planet.openstreetmap.org."""
|
||||
url = f"https://planet.openstreetmap.org/pbf/{filename}.md5"
|
||||
try:
|
||||
req = Request(url, headers={"User-Agent": "zektyc-auto-updater/1.0"})
|
||||
with urlopen(req, timeout=30) as resp:
|
||||
text = resp.read().decode().strip()
|
||||
# format: "hash filename"
|
||||
return text.split()[0]
|
||||
except Exception as e:
|
||||
log(f"WARNING: could not fetch MD5: {e}")
|
||||
return None
|
||||
|
||||
def download_pbf(filename):
|
||||
"""Download the planet PBF with resume support."""
|
||||
dest = PBF_DIR / filename
|
||||
url = f"https://planet.openstreetmap.org/pbf/{filename}"
|
||||
existing = dest.stat().st_size if dest.exists() else 0
|
||||
headers = {"User-Agent": "zektyc-auto-updater/1.0"}
|
||||
if existing > 0:
|
||||
headers["Range"] = f"bytes={existing}-"
|
||||
log(f"Resuming download from {existing / 1e9:.1f} GB")
|
||||
req = Request(url, headers=headers)
|
||||
with urlopen(req, timeout=600) as resp:
|
||||
mode = "ab" if existing > 0 and resp.status == 206 else "wb"
|
||||
if mode == "wb":
|
||||
existing = 0
|
||||
total = int(resp.headers.get("Content-Length", 0)) + existing
|
||||
with open(dest, mode) as f:
|
||||
downloaded = existing
|
||||
t0 = time.time()
|
||||
while True:
|
||||
chunk = resp.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
f.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
elapsed = time.time() - t0
|
||||
if elapsed > 0 and downloaded > existing:
|
||||
speed = (downloaded - existing) / elapsed
|
||||
eta = (total - downloaded) / speed if speed > 0 else 0
|
||||
log(f" {downloaded / 1e9:.2f} / {total / 1e9:.2f} GB "
|
||||
f"({speed / 1e6:.1f} MB/s, ETA {int(eta // 60)}m{int(eta % 60)}s)")
|
||||
return dest
|
||||
|
||||
def kill_existing_build():
|
||||
"""Kill any running build_world.py process."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["pgrep", "-f", "build_world.py"],
|
||||
capture_output=True, text=True, timeout=5
|
||||
)
|
||||
for pid_str in result.stdout.strip().split("\n"):
|
||||
pid_str = pid_str.strip()
|
||||
if pid_str:
|
||||
pid = int(pid_str)
|
||||
log(f"Killing existing build process PID {pid}")
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
time.sleep(2)
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
except OSError:
|
||||
pass
|
||||
except Exception as e:
|
||||
log(f"WARNING: could not kill existing build: {e}")
|
||||
|
||||
def wait_for_build():
|
||||
"""Wait for any running build_world.py to finish."""
|
||||
while True:
|
||||
result = subprocess.run(
|
||||
["pgrep", "-f", "build_world.py"],
|
||||
capture_output=True, text=True, timeout=5
|
||||
)
|
||||
if not result.stdout.strip():
|
||||
return True
|
||||
log("Waiting for existing build to finish...")
|
||||
time.sleep(30)
|
||||
return False
|
||||
|
||||
def run_build(pbf_path, outdir):
|
||||
"""Run build_world.py."""
|
||||
log(f"Starting build: {pbf_path} -> {outdir}")
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "-u", str(BUILD_SCRIPT), str(pbf_path), str(str(outdir))],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
cwd=str(DATA_DIR),
|
||||
)
|
||||
with open(DATA_DIR / "build-planet.log", "a") as logf:
|
||||
for line in iter(proc.stdout.readline, b""):
|
||||
decoded = line.decode(errors="replace")
|
||||
logf.write(decoded)
|
||||
logf.flush()
|
||||
# Also echo last chars for our log
|
||||
proc.wait()
|
||||
if proc.returncode != 0:
|
||||
log(f"ERROR: build_world.py exited with code {proc.returncode}")
|
||||
return False
|
||||
log("Build completed successfully")
|
||||
return True
|
||||
|
||||
def atomic_swap(new_grid, target_grid):
|
||||
"""Atomically replace target_grid with new_grid."""
|
||||
backup = target_grid.parent / (target_grid.name + "-old")
|
||||
if backup.exists():
|
||||
shutil.rmtree(backup)
|
||||
if target_grid.exists():
|
||||
target_grid.rename(backup)
|
||||
new_grid.rename(target_grid)
|
||||
log(f"Swapped {target_grid.name} (backup: {backup.name})")
|
||||
# Clean old backup after a short delay (keep for 1 hour for safety)
|
||||
# Actually let's keep it, it's useful if something goes wrong
|
||||
|
||||
def main():
|
||||
log("=" * 60)
|
||||
log("Auto-update started")
|
||||
|
||||
current_pbf = get_current_pbf()
|
||||
log(f"Current PBF: {current_pbf or 'none'}")
|
||||
|
||||
# Step 1: Check RSS for new release
|
||||
t_start = time.time()
|
||||
new_filename = None
|
||||
while True:
|
||||
try:
|
||||
filename, pub_date = fetch_rss()
|
||||
if filename and filename != current_pbf:
|
||||
log(f"New release found: {filename} (published: {pub_date})")
|
||||
new_filename = filename
|
||||
break
|
||||
elif filename:
|
||||
log(f"No new release yet (latest: {filename}, current: {current_pbf})")
|
||||
else:
|
||||
log("Could not parse RSS feed")
|
||||
except Exception as e:
|
||||
log(f"RSS check failed: {e}")
|
||||
|
||||
elapsed = time.time() - t_start
|
||||
if elapsed > POLL_TIMEOUT:
|
||||
log(f"No new release after {elapsed/3600:.1f}h, giving up")
|
||||
return
|
||||
log(f"Retrying in {CHECK_INTERVAL // 60} minutes...")
|
||||
time.sleep(CHECK_INTERVAL)
|
||||
|
||||
# Step 2: Kill any existing build
|
||||
kill_existing_build()
|
||||
wait_for_build()
|
||||
|
||||
# Step 3: Download new PBF
|
||||
log(f"Downloading {new_filename}...")
|
||||
pbf_path = download_pbf(new_filename)
|
||||
log(f"Download complete: {pbf_path}")
|
||||
|
||||
# Step 4: Verify MD5
|
||||
expected_md5 = fetch_md5(new_filename)
|
||||
if expected_md5:
|
||||
log(f"Verifying MD5 (expected: {expected_md5})")
|
||||
actual_md5 = md5_file(pbf_path)
|
||||
log(f"Actual MD5: {actual_md5}")
|
||||
if actual_md5 != expected_md5:
|
||||
log("ERROR: MD5 mismatch! Aborting.")
|
||||
return
|
||||
log("MD5 verified OK")
|
||||
else:
|
||||
log("WARNING: skipping MD5 verification")
|
||||
|
||||
# Step 5: Build new grid
|
||||
grid_tmp = PBF_DIR / "grid-planet-new"
|
||||
if grid_tmp.exists():
|
||||
shutil.rmtree(grid_tmp)
|
||||
grid_target = PBF_DIR / "grid-planet"
|
||||
|
||||
if not run_build(pbf_path, grid_tmp):
|
||||
log("ERROR: Build failed, keeping old grid")
|
||||
if grid_tmp.exists():
|
||||
shutil.rmtree(grid_tmp)
|
||||
return
|
||||
|
||||
# Step 6: Atomic swap
|
||||
atomic_swap(grid_tmp, grid_target)
|
||||
|
||||
# Step 7: Update .env symlink (if needed)
|
||||
env_path = Path(__file__).resolve().parent.parent.parent / ".env"
|
||||
if env_path.exists():
|
||||
content = env_path.read_text()
|
||||
if "grid-europe" in content:
|
||||
# Don't change .env — it might be intentionally pointing to grid-europe
|
||||
log("NOTE: .env still points to grid-europe, not auto-updating")
|
||||
elif "grid-planet" in content:
|
||||
log(".env already points to grid-planet")
|
||||
|
||||
log("Auto-update complete!")
|
||||
log("=" * 60)
|
||||
|
||||
if __name__ == "__main__":
|
||||
signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
|
||||
main()
|
||||
Loading…
Reference in a new issue