chore: checkpoint initial — Zektyc dataset, avant humanisation
- site vitrine + fingerprint test + zmap + z-panel - remplacement duckdns -> riricdev.tail5ea5cd.ts.net (canonical/og/alternates/docs) - suppression du token DuckDNS (plus utilisé)
This commit is contained in:
riricdev
commit
8e7ec41f18
104 changed files with 19256 additions and 0 deletions
256
data/maxspeed/auto_update.py
Executable file
256
data/maxspeed/auto_update.py
Executable file
|
|
@ -0,0 +1,256 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Auto-update OSM planet PBF and rebuild the maxspeed grid.
|
||||
|
||||
Designed to be triggered weekly (Friday 00:00). Checks the RSS feed for a new
|
||||
planet release. If none found yet, retries every 5 minutes until one appears.
|
||||
Then downloads the PBF, verifies MD5, rebuilds the grid, and atomically
|
||||
swaps the old grid directory for the new one.
|
||||
"""
|
||||
import os, sys, time, hashlib, shutil, signal, subprocess, xml.etree.ElementTree as ET
|
||||
from urllib.request import urlopen, Request
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
|
||||
DATA_DIR = Path(__file__).resolve().parent
|
||||
PBF_DIR = DATA_DIR
|
||||
RSS_URL = "https://planet.openstreetmap.org/pbf/planet-pbf-rss.xml"
|
||||
BUILD_SCRIPT = DATA_DIR / "build_world.py"
|
||||
CHECK_INTERVAL = 5 * 60 # 5 min between retries when waiting for a new release
|
||||
POLL_TIMEOUT = 6 * 3600 # give up after 6 hours if nothing shows up
|
||||
LOG_PREFIX = "[auto-update]"
|
||||
|
||||
def log(msg):
|
||||
ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
line = f"{LOG_PREFIX} [{ts}] {msg}"
|
||||
print(line, flush=True)
|
||||
with open(DATA_DIR / "auto-update.log", "a") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
def fetch_rss():
|
||||
"""Fetch RSS and return the latest planet filename + pubDate."""
|
||||
req = Request(RSS_URL, headers={"User-Agent": "zektyc-auto-updater/1.0"})
|
||||
with urlopen(req, timeout=30) as resp:
|
||||
xml_data = resp.read()
|
||||
root = ET.fromstring(xml_data)
|
||||
items = root.findall(".//item")
|
||||
if not items:
|
||||
return None, None
|
||||
item = items[0]
|
||||
title = (item.findtext("title") or "").strip()
|
||||
pub_date = (item.findtext("pubDate") or "").strip()
|
||||
# title is like "planet-260817.osm.pbf.torrent"
|
||||
filename = title.replace(".torrent", "")
|
||||
return filename, pub_date
|
||||
|
||||
def get_current_pbf():
|
||||
"""Return the basename of the planet PBF currently in use."""
|
||||
# Check which PBF file exists in PBF_DIR
|
||||
for f in sorted(PBF_DIR.glob("planet-*.osm.pbf"), reverse=True):
|
||||
if f.stat().st_size > 10_000_000_000: # > 10 GB = real planet
|
||||
return f.name
|
||||
return None
|
||||
|
||||
def md5_file(path):
|
||||
h = hashlib.md5()
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
def fetch_md5(filename):
|
||||
"""Download the .md5 file from planet.openstreetmap.org."""
|
||||
url = f"https://planet.openstreetmap.org/pbf/{filename}.md5"
|
||||
try:
|
||||
req = Request(url, headers={"User-Agent": "zektyc-auto-updater/1.0"})
|
||||
with urlopen(req, timeout=30) as resp:
|
||||
text = resp.read().decode().strip()
|
||||
# format: "hash filename"
|
||||
return text.split()[0]
|
||||
except Exception as e:
|
||||
log(f"WARNING: could not fetch MD5: {e}")
|
||||
return None
|
||||
|
||||
def download_pbf(filename):
|
||||
"""Download the planet PBF with resume support."""
|
||||
dest = PBF_DIR / filename
|
||||
url = f"https://planet.openstreetmap.org/pbf/{filename}"
|
||||
existing = dest.stat().st_size if dest.exists() else 0
|
||||
headers = {"User-Agent": "zektyc-auto-updater/1.0"}
|
||||
if existing > 0:
|
||||
headers["Range"] = f"bytes={existing}-"
|
||||
log(f"Resuming download from {existing / 1e9:.1f} GB")
|
||||
req = Request(url, headers=headers)
|
||||
with urlopen(req, timeout=600) as resp:
|
||||
mode = "ab" if existing > 0 and resp.status == 206 else "wb"
|
||||
if mode == "wb":
|
||||
existing = 0
|
||||
total = int(resp.headers.get("Content-Length", 0)) + existing
|
||||
with open(dest, mode) as f:
|
||||
downloaded = existing
|
||||
t0 = time.time()
|
||||
while True:
|
||||
chunk = resp.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
f.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
elapsed = time.time() - t0
|
||||
if elapsed > 0 and downloaded > existing:
|
||||
speed = (downloaded - existing) / elapsed
|
||||
eta = (total - downloaded) / speed if speed > 0 else 0
|
||||
log(f" {downloaded / 1e9:.2f} / {total / 1e9:.2f} GB "
|
||||
f"({speed / 1e6:.1f} MB/s, ETA {int(eta // 60)}m{int(eta % 60)}s)")
|
||||
return dest
|
||||
|
||||
def kill_existing_build():
|
||||
"""Kill any running build_world.py process."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["pgrep", "-f", "build_world.py"],
|
||||
capture_output=True, text=True, timeout=5
|
||||
)
|
||||
for pid_str in result.stdout.strip().split("\n"):
|
||||
pid_str = pid_str.strip()
|
||||
if pid_str:
|
||||
pid = int(pid_str)
|
||||
log(f"Killing existing build process PID {pid}")
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
time.sleep(2)
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
except OSError:
|
||||
pass
|
||||
except Exception as e:
|
||||
log(f"WARNING: could not kill existing build: {e}")
|
||||
|
||||
def wait_for_build():
|
||||
"""Wait for any running build_world.py to finish."""
|
||||
while True:
|
||||
result = subprocess.run(
|
||||
["pgrep", "-f", "build_world.py"],
|
||||
capture_output=True, text=True, timeout=5
|
||||
)
|
||||
if not result.stdout.strip():
|
||||
return True
|
||||
log("Waiting for existing build to finish...")
|
||||
time.sleep(30)
|
||||
return False
|
||||
|
||||
def run_build(pbf_path, outdir):
|
||||
"""Run build_world.py."""
|
||||
log(f"Starting build: {pbf_path} -> {outdir}")
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "-u", str(BUILD_SCRIPT), str(pbf_path), str(str(outdir))],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
cwd=str(DATA_DIR),
|
||||
)
|
||||
with open(DATA_DIR / "build-planet.log", "a") as logf:
|
||||
for line in iter(proc.stdout.readline, b""):
|
||||
decoded = line.decode(errors="replace")
|
||||
logf.write(decoded)
|
||||
logf.flush()
|
||||
# Also echo last chars for our log
|
||||
proc.wait()
|
||||
if proc.returncode != 0:
|
||||
log(f"ERROR: build_world.py exited with code {proc.returncode}")
|
||||
return False
|
||||
log("Build completed successfully")
|
||||
return True
|
||||
|
||||
def atomic_swap(new_grid, target_grid):
|
||||
"""Atomically replace target_grid with new_grid."""
|
||||
backup = target_grid.parent / (target_grid.name + "-old")
|
||||
if backup.exists():
|
||||
shutil.rmtree(backup)
|
||||
if target_grid.exists():
|
||||
target_grid.rename(backup)
|
||||
new_grid.rename(target_grid)
|
||||
log(f"Swapped {target_grid.name} (backup: {backup.name})")
|
||||
# Clean old backup after a short delay (keep for 1 hour for safety)
|
||||
# Actually let's keep it, it's useful if something goes wrong
|
||||
|
||||
def main():
|
||||
log("=" * 60)
|
||||
log("Auto-update started")
|
||||
|
||||
current_pbf = get_current_pbf()
|
||||
log(f"Current PBF: {current_pbf or 'none'}")
|
||||
|
||||
# Step 1: Check RSS for new release
|
||||
t_start = time.time()
|
||||
new_filename = None
|
||||
while True:
|
||||
try:
|
||||
filename, pub_date = fetch_rss()
|
||||
if filename and filename != current_pbf:
|
||||
log(f"New release found: {filename} (published: {pub_date})")
|
||||
new_filename = filename
|
||||
break
|
||||
elif filename:
|
||||
log(f"No new release yet (latest: {filename}, current: {current_pbf})")
|
||||
else:
|
||||
log("Could not parse RSS feed")
|
||||
except Exception as e:
|
||||
log(f"RSS check failed: {e}")
|
||||
|
||||
elapsed = time.time() - t_start
|
||||
if elapsed > POLL_TIMEOUT:
|
||||
log(f"No new release after {elapsed/3600:.1f}h, giving up")
|
||||
return
|
||||
log(f"Retrying in {CHECK_INTERVAL // 60} minutes...")
|
||||
time.sleep(CHECK_INTERVAL)
|
||||
|
||||
# Step 2: Kill any existing build
|
||||
kill_existing_build()
|
||||
wait_for_build()
|
||||
|
||||
# Step 3: Download new PBF
|
||||
log(f"Downloading {new_filename}...")
|
||||
pbf_path = download_pbf(new_filename)
|
||||
log(f"Download complete: {pbf_path}")
|
||||
|
||||
# Step 4: Verify MD5
|
||||
expected_md5 = fetch_md5(new_filename)
|
||||
if expected_md5:
|
||||
log(f"Verifying MD5 (expected: {expected_md5})")
|
||||
actual_md5 = md5_file(pbf_path)
|
||||
log(f"Actual MD5: {actual_md5}")
|
||||
if actual_md5 != expected_md5:
|
||||
log("ERROR: MD5 mismatch! Aborting.")
|
||||
return
|
||||
log("MD5 verified OK")
|
||||
else:
|
||||
log("WARNING: skipping MD5 verification")
|
||||
|
||||
# Step 5: Build new grid
|
||||
grid_tmp = PBF_DIR / "grid-planet-new"
|
||||
if grid_tmp.exists():
|
||||
shutil.rmtree(grid_tmp)
|
||||
grid_target = PBF_DIR / "grid-planet"
|
||||
|
||||
if not run_build(pbf_path, grid_tmp):
|
||||
log("ERROR: Build failed, keeping old grid")
|
||||
if grid_tmp.exists():
|
||||
shutil.rmtree(grid_tmp)
|
||||
return
|
||||
|
||||
# Step 6: Atomic swap
|
||||
atomic_swap(grid_tmp, grid_target)
|
||||
|
||||
# Step 7: Update .env symlink (if needed)
|
||||
env_path = Path(__file__).resolve().parent.parent.parent / ".env"
|
||||
if env_path.exists():
|
||||
content = env_path.read_text()
|
||||
if "grid-europe" in content:
|
||||
# Don't change .env — it might be intentionally pointing to grid-europe
|
||||
log("NOTE: .env still points to grid-europe, not auto-updating")
|
||||
elif "grid-planet" in content:
|
||||
log(".env already points to grid-planet")
|
||||
|
||||
log("Auto-update complete!")
|
||||
log("=" * 60)
|
||||
|
||||
if __name__ == "__main__":
|
||||
signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
|
||||
main()
|
||||
138
data/maxspeed/build_band.cpp
Normal file
138
data/maxspeed/build_band.cpp
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
#include <osmium/io/any_input.hpp>
|
||||
#include <osmium/visitor.hpp>
|
||||
#include <osmium/handler/node_locations_for_ways.hpp>
|
||||
#include <osmium/index/map/sparse_file_array.hpp>
|
||||
#include <osmium/osm/way.hpp>
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <cmath>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <algorithm>
|
||||
#include <filesystem>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
using index_type = osmium::index::map::SparseFileArray<osmium::unsigned_object_id_type, osmium::Location>;
|
||||
using location_handler_type = osmium::handler::NodeLocationsForWays<index_type>;
|
||||
|
||||
static constexpr double CELL = 0.5;
|
||||
static constexpr double LON_MIN = -180.0;
|
||||
static constexpr double LAT_MIN = -60.0;
|
||||
static constexpr double LON_MAX = 180.0;
|
||||
static constexpr double LAT_MAX = 85.0;
|
||||
|
||||
struct RawPoint {
|
||||
int32_t col;
|
||||
int32_t row;
|
||||
uint16_t lonq;
|
||||
uint16_t latq;
|
||||
uint8_t speed;
|
||||
};
|
||||
|
||||
static inline uint16_t quantize_lon(double lon) {
|
||||
return static_cast<uint16_t>((lon - LON_MIN) / (LON_MAX - LON_MIN) * 65535.0);
|
||||
}
|
||||
|
||||
static inline uint16_t quantize_lat(double lat) {
|
||||
return static_cast<uint16_t>((lat - LAT_MIN) / (LAT_MAX - LAT_MIN) * 65535.0);
|
||||
}
|
||||
|
||||
static inline uint8_t parse_speed_val(const char* s) {
|
||||
if (!s || !*s) return 0;
|
||||
char* end = nullptr;
|
||||
double v = strtod(s, &end);
|
||||
if (end != s && v > 0 && v <= 255) return static_cast<uint8_t>(v);
|
||||
if (strstr(s, "mph")) {
|
||||
v = strtod(s, &end);
|
||||
if (end != s) {
|
||||
v *= 1.609;
|
||||
if (v > 0 && v <= 255) return static_cast<uint8_t>(v);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
class StreamHandler : public osmium::handler::Handler {
|
||||
public:
|
||||
FILE* out;
|
||||
uint64_t way_count = 0;
|
||||
uint64_t point_count = 0;
|
||||
|
||||
explicit StreamHandler(FILE* f) : out(f) {}
|
||||
|
||||
void way(const osmium::Way& way) {
|
||||
const char* highway = way.tags().get_value_by_key("highway");
|
||||
if (!highway) return;
|
||||
|
||||
const char* ms = way.tags().get_value_by_key("maxspeed");
|
||||
uint8_t speed = ms ? parse_speed_val(ms) : 0;
|
||||
if (speed == 0) return;
|
||||
|
||||
way_count++;
|
||||
|
||||
for (const auto& wn : way.nodes()) {
|
||||
if (!wn.location().valid()) continue;
|
||||
|
||||
double lon = wn.location().lon();
|
||||
double lat = wn.location().lat();
|
||||
|
||||
if (lon < LON_MIN || lon >= LON_MAX || lat < LAT_MIN || lat >= LAT_MAX) continue;
|
||||
|
||||
RawPoint rp;
|
||||
rp.col = static_cast<int32_t>((lon - LON_MIN) / CELL);
|
||||
rp.row = static_cast<int32_t>((lat - LAT_MIN) / CELL);
|
||||
rp.lonq = quantize_lon(lon);
|
||||
rp.latq = quantize_lat(lat);
|
||||
rp.speed = speed;
|
||||
|
||||
fwrite(&rp, sizeof(RawPoint), 1, out);
|
||||
point_count++;
|
||||
|
||||
if (point_count % 5000000 == 0) {
|
||||
fprintf(stderr, " %lu points (%lu ways)...\n", point_count, way_count);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
if (argc < 3) {
|
||||
fprintf(stderr, "Usage: %s <input.pbf> <output.raw>\n", argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const char* input = argv[1];
|
||||
const char* outpath = argv[2];
|
||||
|
||||
fs::create_directories(fs::path(outpath).parent_path());
|
||||
|
||||
FILE* out = fopen(outpath, "wb");
|
||||
if (!out) { fprintf(stderr, "Cannot open %s\n", outpath); return 1; }
|
||||
|
||||
std::string idx_path = std::string(outpath) + ".idx";
|
||||
int idx_fd = ::open(idx_path.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644);
|
||||
if (idx_fd == -1) { fprintf(stderr, "Cannot create index\n"); fclose(out); return 1; }
|
||||
|
||||
index_type index(idx_fd);
|
||||
location_handler_type location_handler(index);
|
||||
StreamHandler handler(out);
|
||||
|
||||
fprintf(stderr, "Reading %s...\n", input);
|
||||
osmium::io::Reader reader(input);
|
||||
osmium::apply(reader, location_handler, handler);
|
||||
reader.close();
|
||||
|
||||
fclose(out);
|
||||
::close(idx_fd);
|
||||
unlink(idx_path.c_str());
|
||||
|
||||
fprintf(stderr, "Ways with maxspeed: %lu\n", handler.way_count);
|
||||
fprintf(stderr, "Points written: %lu\n", handler.point_count);
|
||||
fprintf(stderr, "Output: %s\n", outpath);
|
||||
return 0;
|
||||
}
|
||||
245
data/maxspeed/build_global.cpp
Normal file
245
data/maxspeed/build_global.cpp
Normal file
|
|
@ -0,0 +1,245 @@
|
|||
#include <osmium/io/any_input.hpp>
|
||||
#include <osmium/visitor.hpp>
|
||||
#include <osmium/handler/node_locations_for_ways.hpp>
|
||||
#include <osmium/index/map/sparse_file_array.hpp>
|
||||
#include <osmium/osm/way.hpp>
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <cmath>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <algorithm>
|
||||
#include <filesystem>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
using index_type = osmium::index::map::SparseFileArray<osmium::unsigned_object_id_type, osmium::Location>;
|
||||
using location_handler_type = osmium::handler::NodeLocationsForWays<index_type>;
|
||||
|
||||
static constexpr double CELL = 0.5;
|
||||
static constexpr int SUB = 32;
|
||||
static constexpr double LON_MIN = -180.0;
|
||||
static constexpr double LAT_MIN = -60.0;
|
||||
static constexpr double LON_MAX = 180.0;
|
||||
static constexpr double LAT_MAX = 85.0;
|
||||
|
||||
struct RawPoint {
|
||||
int32_t col;
|
||||
int32_t row;
|
||||
uint16_t lonq;
|
||||
uint16_t latq;
|
||||
uint8_t speed;
|
||||
}; // 11 bytes
|
||||
|
||||
static inline uint16_t quantize_lon(double lon) {
|
||||
return static_cast<uint16_t>((lon - LON_MIN) / (LON_MAX - LON_MIN) * 65535.0);
|
||||
}
|
||||
|
||||
static inline uint16_t quantize_lat(double lat) {
|
||||
return static_cast<uint16_t>((lat - LAT_MIN) / (LAT_MAX - LAT_MIN) * 65535.0);
|
||||
}
|
||||
|
||||
static inline uint8_t parse_speed_val(const char* s) {
|
||||
if (!s || !*s) return 0;
|
||||
char* end = nullptr;
|
||||
double v = strtod(s, &end);
|
||||
if (end != s && v > 0 && v <= 255) return static_cast<uint8_t>(v);
|
||||
if (strstr(s, "mph")) {
|
||||
v = strtod(s, &end);
|
||||
if (end != s) {
|
||||
v *= 1.609;
|
||||
if (v > 0 && v <= 255) return static_cast<uint8_t>(v);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct Point64 {
|
||||
uint16_t lonq;
|
||||
uint16_t latq;
|
||||
uint8_t speed;
|
||||
};
|
||||
|
||||
class SpeedHandler : public osmium::handler::Handler {
|
||||
public:
|
||||
FILE* tmpfile;
|
||||
uint64_t way_count = 0;
|
||||
uint64_t point_count = 0;
|
||||
|
||||
explicit SpeedHandler(FILE* f) : tmpfile(f) {}
|
||||
|
||||
void way(const osmium::Way& way) {
|
||||
const char* highway = way.tags().get_value_by_key("highway");
|
||||
if (!highway) return;
|
||||
|
||||
const char* ms = way.tags().get_value_by_key("maxspeed");
|
||||
uint8_t speed = ms ? parse_speed_val(ms) : 0;
|
||||
if (speed == 0) return;
|
||||
|
||||
way_count++;
|
||||
|
||||
for (const auto& wn : way.nodes()) {
|
||||
if (!wn.location().valid()) continue;
|
||||
|
||||
double lon = wn.location().lon();
|
||||
double lat = wn.location().lat();
|
||||
|
||||
if (lon < LON_MIN || lon >= LON_MAX || lat < LAT_MIN || lat >= LAT_MAX) continue;
|
||||
|
||||
int32_t col = static_cast<int32_t>((lon - LON_MIN) / CELL);
|
||||
int32_t row = static_cast<int32_t>((lat - LAT_MIN) / CELL);
|
||||
|
||||
RawPoint rp;
|
||||
rp.col = col;
|
||||
rp.row = row;
|
||||
rp.lonq = quantize_lon(lon);
|
||||
rp.latq = quantize_lat(lat);
|
||||
rp.speed = speed;
|
||||
|
||||
fwrite(&rp, sizeof(RawPoint), 1, tmpfile);
|
||||
point_count++;
|
||||
|
||||
if (point_count % 10000000 == 0) {
|
||||
fprintf(stderr, " %lu points (%lu ways)...\n", point_count, way_count);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
static bool compare_by_cell(const RawPoint& a, const RawPoint& b) {
|
||||
if (a.col != b.col) return a.col < b.col;
|
||||
return a.row < b.row;
|
||||
}
|
||||
|
||||
static void split_and_write(const char* tmppath, const std::string& outdir) {
|
||||
fprintf(stderr, "Reading temp file into memory for sort...\n");
|
||||
|
||||
FILE* f = fopen(tmppath, "rb");
|
||||
fseek(f, 0, SEEK_END);
|
||||
long fsize = ftell(f);
|
||||
fseek(f, 0, SEEK_SET);
|
||||
|
||||
uint64_t npoints = fsize / sizeof(RawPoint);
|
||||
fprintf(stderr, "Sorting %lu points (%.1f MB)...\n", npoints, fsize / 1048576.0);
|
||||
|
||||
std::vector<RawPoint> points(npoints);
|
||||
fread(points.data(), sizeof(RawPoint), npoints, f);
|
||||
fclose(f);
|
||||
|
||||
std::sort(points.begin(), points.end(), compare_by_cell);
|
||||
fprintf(stderr, "Sort done. Writing cells...\n");
|
||||
|
||||
fs::create_directories(outdir);
|
||||
|
||||
std::string index_path = outdir + "/index.bin";
|
||||
FILE* index = fopen(index_path.c_str(), "wb");
|
||||
uint32_t magic = 0x5A4D4150;
|
||||
fwrite(&magic, 4, 1, index);
|
||||
uint16_t version = 2;
|
||||
fwrite(&version, 2, 1, index);
|
||||
uint16_t sub = SUB;
|
||||
fwrite(&sub, 2, 1, index);
|
||||
fwrite(&CELL, 8, 1, index);
|
||||
fwrite(&LON_MIN, 8, 1, index);
|
||||
fwrite(&LAT_MIN, 8, 1, index);
|
||||
fwrite(&LON_MAX, 8, 1, index);
|
||||
fwrite(&LAT_MAX, 8, 1, index);
|
||||
|
||||
uint32_t num_cells = 0;
|
||||
fseek(index, 36, SEEK_SET);
|
||||
fwrite(&num_cells, 4, 1, index);
|
||||
fseek(index, 0, SEEK_END);
|
||||
|
||||
int32_t cur_col = -1, cur_row = -1;
|
||||
FILE* cur_file = nullptr;
|
||||
uint32_t cell_points = 0;
|
||||
std::string cur_key;
|
||||
|
||||
for (uint64_t i = 0; i <= npoints; i++) {
|
||||
bool new_cell = (i == npoints || points[i].col != cur_col || points[i].row != cur_row);
|
||||
|
||||
if (new_cell && cur_file) {
|
||||
fclose(cur_file);
|
||||
|
||||
uint32_t klen = static_cast<uint32_t>(cur_key.size());
|
||||
fwrite(&klen, 4, 1, index);
|
||||
fwrite(cur_key.data(), 1, klen, index);
|
||||
fwrite(&cell_points, 4, 1, index);
|
||||
num_cells++;
|
||||
|
||||
if (num_cells % 1000 == 0) {
|
||||
fprintf(stderr, " %u cells written...\n", num_cells);
|
||||
}
|
||||
}
|
||||
|
||||
if (i == npoints) break;
|
||||
|
||||
if (new_cell) {
|
||||
cur_col = points[i].col;
|
||||
cur_row = points[i].row;
|
||||
cur_key = std::to_string(cur_col) + "_" + std::to_string(cur_row);
|
||||
std::string cellfile = outdir + "/cell_" + cur_key + ".bin";
|
||||
cur_file = fopen(cellfile.c_str(), "wb");
|
||||
cell_points = 0;
|
||||
}
|
||||
|
||||
Point64 p;
|
||||
p.lonq = points[i].lonq;
|
||||
p.latq = points[i].latq;
|
||||
p.speed = points[i].speed;
|
||||
fwrite(&p, sizeof(Point64), 1, cur_file);
|
||||
cell_points++;
|
||||
}
|
||||
|
||||
fseek(index, 36, SEEK_SET);
|
||||
fwrite(&num_cells, 4, 1, index);
|
||||
fclose(index);
|
||||
|
||||
fprintf(stderr, "Total cells: %u\n", num_cells);
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
if (argc < 3) {
|
||||
fprintf(stderr, "Usage: %s <input.pbf> <output_dir>\n", argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const char* input = argv[1];
|
||||
const char* outdir = argv[2];
|
||||
|
||||
std::string tmppath = std::string(outdir) + "/_points.tmp";
|
||||
fs::create_directories(outdir);
|
||||
|
||||
fprintf(stderr, "Step 1: Reading PBF, writing raw points to temp file...\n");
|
||||
FILE* tmpf = fopen(tmppath.c_str(), "w+b");
|
||||
if (!tmpf) { fprintf(stderr, "Cannot create temp file\n"); return 1; }
|
||||
|
||||
std::string idx_path = std::string(outdir) + "/_node_index.tmp";
|
||||
int idx_fd = ::open(idx_path.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644);
|
||||
if (idx_fd == -1) { fprintf(stderr, "Cannot create index temp file\n"); return 1; }
|
||||
|
||||
index_type index(idx_fd);
|
||||
location_handler_type location_handler(index);
|
||||
SpeedHandler handler(tmpf);
|
||||
|
||||
osmium::io::Reader reader(input);
|
||||
osmium::apply(reader, location_handler, handler);
|
||||
reader.close();
|
||||
fclose(tmpf);
|
||||
|
||||
fprintf(stderr, "\nWays with maxspeed: %lu\n", handler.way_count);
|
||||
fprintf(stderr, "Total points: %lu\n", handler.point_count);
|
||||
|
||||
fprintf(stderr, "Step 2: Sort + write grid cells...\n");
|
||||
split_and_write(tmppath.c_str(), outdir);
|
||||
|
||||
::close(idx_fd);
|
||||
unlink(idx_path.c_str());
|
||||
unlink(tmppath.c_str());
|
||||
fprintf(stderr, "Done! Grid written to %s\n", outdir);
|
||||
return 0;
|
||||
}
|
||||
273
data/maxspeed/build_maxspeed.py
Normal file
273
data/maxspeed/build_maxspeed.py
Normal file
|
|
@ -0,0 +1,273 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Construit une grille binaire des maxspeed OSM (par route, échantillonnée en points).
|
||||
|
||||
Pipeline :
|
||||
osmium tags-filter <extrait>.osm.pbf w/highway -o <extrait>-roads.osm.pbf
|
||||
python3 build_maxspeed.py <extrait>-roads.osm.pbf <sortie> [--cache=<file>]
|
||||
|
||||
Sortie : <sortie>/grid.dat + <sortie>/index.json + <sortie>/meta.json
|
||||
- Chaque cellule = 0.5° x 0.5° (lat, lon).
|
||||
- Blob par cellule (gzip) : int32 count ; 32x32 sous-cellules => int32 offset[1024] ;
|
||||
puis count points de (lonq:int16, latq:int16, speed:int8) rangés par sous-cellule.
|
||||
- lonq/latq : quantifiés sur 0..65535 dans la cellule (precision ~0.8 m).
|
||||
"""
|
||||
import sys, os, json, gzip, math, struct, re
|
||||
import osmium
|
||||
import sqlite3
|
||||
|
||||
CELL = 0.5 # degrés par cellule
|
||||
SUB = 32 # sous-cellules par dimension (grille 32x32)
|
||||
STEP_M_DEFAULT = 40.0 # pas d'échantillonnage le long des routes (m)
|
||||
STEP_M = STEP_M_DEFAULT
|
||||
|
||||
# maxspeed textuel -> km/h (valeurs par défaut OSM)
|
||||
ZONE = {
|
||||
"none": None,
|
||||
"signals": None,
|
||||
"variable": None,
|
||||
"walk": 5,
|
||||
"living_street": 20,
|
||||
"urban": 50, "rural": 80, "trunk": 110, "motorway": 130,
|
||||
}
|
||||
# codes pays -> (urban, rural) km/h ; motorway sans maxspeed = None (ex. DE)
|
||||
CC = {
|
||||
"FR": (50, 80), "BE": (50, 90), "DE": (50, 100), "NL": (50, 100),
|
||||
"GB": (48, 96), "ES": (50, 90), "IT": (50, 90), "CH": (50, 80),
|
||||
"AT": (50, 100), "LU": (50, 90), "PT": (50, 90), "PL": (50, 90),
|
||||
"CZ": (50, 90), "IE": (50, 80), "DK": (50, 80), "SE": (50, 70),
|
||||
"NO": (50, 80), "FI": (50, 100), "RO": (50, 90), "HU": (50, 90),
|
||||
"HR": (50, 90), "SI": (50, 90), "SK": (50, 90), "LT": (50, 90),
|
||||
"LV": (50, 90), "EE": (50, 90), "BG": (50, 90), "GR": (50, 90),
|
||||
"UA": (50, 90), "RU": (60, 90), "TR": (50, 90), "RS": (50, 80),
|
||||
"MK": (50, 80), "AL": (40, 90), "BA": (50, 80), "ME": (50, 80),
|
||||
"IS": (50, 90),
|
||||
}
|
||||
# Les highways de service/privées sans maxspeed sont exclues : on ne garde que les
|
||||
# ways marqués maxspeed (ou avec une valeur par défaut de zone reconnue).
|
||||
|
||||
def parse_speed(v):
|
||||
if not v:
|
||||
return None
|
||||
v = v.strip().lower()
|
||||
if v in ZONE:
|
||||
return ZONE[v]
|
||||
m = __import__("re").match(r"^([0-9]+)(?:\s*(mph|km/h|kmh))?\b", v)
|
||||
if m:
|
||||
n = int(m.group(1))
|
||||
if m.group(2) == "mph":
|
||||
return round(n * 1.609344)
|
||||
return n
|
||||
# code pays type FR:rural / DE:urban
|
||||
if ":" in v:
|
||||
cc, kind = v.split(":", 1)
|
||||
cc = cc.upper()
|
||||
if cc in CC and kind in ("urban", "rural"):
|
||||
return CC[cc][0 if kind == "urban" else 1]
|
||||
if cc in CC and kind == "motorway":
|
||||
return None
|
||||
return None
|
||||
|
||||
class MaxSpeedHandler(osmium.SimpleHandler):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.points = {} # (cellx, celly) -> dict subcell -> list of (lonq, latq, speed)
|
||||
self.skipped = 0
|
||||
|
||||
def way(self, w):
|
||||
if "highway" not in w.tags:
|
||||
return
|
||||
speed = parse_speed(w.tags.get("maxspeed", ""))
|
||||
if speed is None:
|
||||
return
|
||||
locs = []
|
||||
for n in w.nodes:
|
||||
loc = n.location
|
||||
if loc is not None:
|
||||
locs.append((loc.lon, loc.lat))
|
||||
if len(locs) < 2:
|
||||
return
|
||||
self.process_locs(locs, speed)
|
||||
|
||||
def process_locs(self, locs, speed):
|
||||
if STEP_M <= 0:
|
||||
for (lon, lat) in locs:
|
||||
self.add_point(lon, lat, speed)
|
||||
return
|
||||
prev = locs[0]
|
||||
lon, lat = prev
|
||||
acc = 0.0
|
||||
emit = [(lon, lat)]
|
||||
for node in locs[1:]:
|
||||
a = haversine(prev[1], prev[0], node[1], node[0])
|
||||
acc += a
|
||||
prev = node
|
||||
while acc >= STEP_M:
|
||||
f = 1.0 - (acc - STEP_M) / a if a > 0 else 0.0
|
||||
ilon = lon + (node[0] - lon) * f
|
||||
ilat = lat + (node[1] - lat) * f
|
||||
emit.append((ilon, ilat))
|
||||
acc -= STEP_M
|
||||
lon, lat = node[0], node[1]
|
||||
if (lon, lat) != emit[-1]:
|
||||
emit.append((lon, lat))
|
||||
for (el, ea) in emit:
|
||||
self.add_point(el, ea, speed)
|
||||
|
||||
def add_point(self, lon, lat, speed):
|
||||
cx = int(math.floor(lon / CELL))
|
||||
cy = int(math.floor(lat / CELL))
|
||||
lonq = int(round((lon - cx * CELL) / CELL * 65535))
|
||||
latq = int(round((lat - cy * CELL) / CELL * 65535))
|
||||
lonq = max(0, min(65535, lonq)); latq = max(0, min(65535, latq))
|
||||
sx = min(SUB - 1, lonq * SUB // 65536)
|
||||
sy = min(SUB - 1, latq * SUB // 65536)
|
||||
key = (cx, cy)
|
||||
cell = self.points.get(key)
|
||||
if cell is None:
|
||||
cell = [None] * (SUB * SUB)
|
||||
self.points[key] = cell
|
||||
sub = sx + sy * SUB
|
||||
buf = cell[sub]
|
||||
if buf is None:
|
||||
buf = bytearray()
|
||||
cell[sub] = buf
|
||||
buf += struct.pack("<HHB", lonq, latq, max(0, min(255, speed)))
|
||||
|
||||
def haversine(lat1, lon1, lat2, lon2):
|
||||
R = 6371000.0
|
||||
p1 = math.radians(lat1); p2 = math.radians(lat2)
|
||||
dp = math.radians(lat2 - lat1); dl = math.radians(lon2 - lon1)
|
||||
a = math.sin(dp / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2
|
||||
return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
|
||||
|
||||
GP_FLAGS = {"byteorder": 0x01, "envelope": 0x02, "empty": 0x10}
|
||||
|
||||
def parse_gp_wkb(raw):
|
||||
"""GeoPackage binary -> liste (lon, lat) pour LineString (ou Multi)."""
|
||||
if not isinstance(raw, (bytes, bytearray)) or len(raw) < 8 or raw[:2] != b"GP":
|
||||
return None
|
||||
flags = raw[3]
|
||||
off = 8
|
||||
if flags & GP_FLAGS["envelope"]:
|
||||
etype = (flags >> 2) & 0x03
|
||||
if etype != 0:
|
||||
return None
|
||||
off += 4 * 8
|
||||
bo = "<" if flags & GP_FLAGS["byteorder"] else ">"
|
||||
if off + 5 > len(raw):
|
||||
return None
|
||||
gtype = struct.unpack(bo + "i", raw[off + 1:off + 5])[0]
|
||||
if gtype == 2: # LineString
|
||||
return parse_wkb_points(raw, off, bo)
|
||||
if gtype == 5: # MultiLineString
|
||||
out = []
|
||||
num = struct.unpack(bo + "i", raw[off + 5:off + 9])[0]
|
||||
p = off + 9
|
||||
for _ in range(num):
|
||||
pts = parse_wkb_points(raw, p, bo)
|
||||
if pts:
|
||||
out.extend(pts)
|
||||
p += 4 + 4 * 8 * (len(pts) if pts else 0)
|
||||
return out if len(out) >= 2 else None
|
||||
return None
|
||||
|
||||
def parse_wkb_points(raw, off, bo):
|
||||
n = struct.unpack(bo + "i", raw[off + 5:off + 9])[0]
|
||||
out = []
|
||||
p = off + 9
|
||||
for _ in range(n):
|
||||
out.append((struct.unpack(bo + "d", raw[p:p + 8])[0],
|
||||
struct.unpack(bo + "d", raw[p + 8:p + 16])[0]))
|
||||
p += 16
|
||||
return out
|
||||
|
||||
MS_RE = re.compile(r'"maxspeed"=>"([^"]*)"')
|
||||
|
||||
def parse_other_tags(s):
|
||||
m = MS_RE.search(s or "")
|
||||
return m.group(1) if m else None
|
||||
|
||||
def load_gpkg(h, gpkg):
|
||||
con = sqlite3.connect(gpkg)
|
||||
cur = con.execute("SELECT geom, highway, other_tags FROM lines")
|
||||
n = 0
|
||||
for raw, hw, tags in cur:
|
||||
if not hw or "maxspeed" not in (tags or ""):
|
||||
continue
|
||||
v = parse_other_tags(tags)
|
||||
speed = parse_speed(v)
|
||||
if speed is None:
|
||||
continue
|
||||
locs = parse_gp_wkb(raw)
|
||||
if not locs or len(locs) < 2:
|
||||
continue
|
||||
h.process_locs(locs, speed)
|
||||
n += 1
|
||||
if n % 500000 == 0:
|
||||
sys.stderr.write("ways maxspeed: %dM\n" % (n // 1000000))
|
||||
con.close()
|
||||
return n
|
||||
|
||||
def build(src_pbf, out_dir, cache_file=None):
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
h = MaxSpeedHandler()
|
||||
if src_pbf.endswith(".gpkg") or src_pbf.endswith(".db"):
|
||||
n = load_gpkg(h, src_pbf)
|
||||
print(f"GPKG : {n} ways traités")
|
||||
else:
|
||||
if cache_file:
|
||||
idx = osmium.index.create_map("sparse_file_array,%s" % cache_file)
|
||||
h.apply_file(src_pbf, locations=osmium.NodeLocationsForWays(idx))
|
||||
else:
|
||||
h.apply_file(src_pbf, locations=True)
|
||||
# écriture : un fichier par cellule (nom: X_Y.bin.gz) + index
|
||||
index = {}
|
||||
total = 0
|
||||
for (cx, cy), cell in h.points.items():
|
||||
count = sum(0 if b is None else len(b) // 5 for b in cell)
|
||||
offsets = [count] * (SUB * SUB)
|
||||
acc = 0
|
||||
for sub, b in enumerate(cell):
|
||||
if b is None:
|
||||
continue
|
||||
offsets[sub] = acc
|
||||
acc += len(b) // 5
|
||||
# remplissage arrière : les sous-cellules vides pointent vers la suivante
|
||||
for s in range(SUB * SUB - 2, -1, -1):
|
||||
if offsets[s] == count:
|
||||
offsets[s] = offsets[s + 1]
|
||||
buf = bytearray()
|
||||
buf += struct.pack("<i", count)
|
||||
buf += struct.pack("<%di" % (SUB * SUB), *offsets)
|
||||
for b in cell:
|
||||
if b is not None:
|
||||
buf += b
|
||||
fname = f"{cx}_{cy}.bin.gz"
|
||||
with open(os.path.join(out_dir, fname), "wb") as f:
|
||||
f.write(gzip.compress(bytes(buf), 6))
|
||||
index[fname] = {"x": cx, "y": cy, "count": count}
|
||||
total += count
|
||||
with open(os.path.join(out_dir, "index.json"), "w") as f:
|
||||
json.dump(index, f)
|
||||
with open(os.path.join(out_dir, "meta.json"), "w") as f:
|
||||
json.dump({"cell": CELL, "sub": SUB, "step_m": STEP_M, "points": total,
|
||||
"files": len(index)}, f)
|
||||
print(f"OK : {total} points dans {len(index)} cellules -> {out_dir}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
cache = None
|
||||
args = sys.argv[1:]
|
||||
extra = {}
|
||||
while args and args[-1].startswith("--"):
|
||||
a = args.pop()
|
||||
if a.startswith("--cache="):
|
||||
cache = a.split("=", 1)[1]
|
||||
elif a == "--no-sample":
|
||||
STEP_M = 0
|
||||
else:
|
||||
print(f"Option inconnue: {a}")
|
||||
sys.exit(1)
|
||||
if len(args) != 2:
|
||||
print("usage: build_maxspeed.py <extrait.osm.pbf|gpkg> <out_dir> [--cache=<file>] [--no-sample]")
|
||||
sys.exit(1)
|
||||
build(args[0], args[1], cache)
|
||||
271
data/maxspeed/build_world.py
Normal file
271
data/maxspeed/build_world.py
Normal file
|
|
@ -0,0 +1,271 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Build world maxspeed grid from planet PBF in 2 passes.
|
||||
|
||||
Pass 1: Stream PBF → collect node IDs from highway+maxspeed ways → bitset
|
||||
Pass 2: Stream PBF → matching nodes → mmap sort → process ways → grid
|
||||
|
||||
Usage: python3 build_world.py planet-latest.osm.pbf grid-planet/
|
||||
"""
|
||||
import sys, os, json, gzip, struct, math, mmap, time
|
||||
import numpy as np
|
||||
import osmium
|
||||
|
||||
CELL = 0.5
|
||||
SUB = 32
|
||||
|
||||
ZONE = {
|
||||
"none": None, "signals": None, "variable": None, "walk": 5, "living_street": 20,
|
||||
"urban": 50, "rural": 80, "trunk": 110, "motorway": 130,
|
||||
}
|
||||
CC = {
|
||||
"FR": (50,80), "BE": (50,90), "DE": (50,100), "NL": (50,100),
|
||||
"GB": (48,96), "ES": (50,90), "IT": (50,90), "CH": (50,80),
|
||||
"AT": (50,100), "LU": (50,90), "PT": (50,90), "PL": (50,90),
|
||||
"CZ": (50,90), "IE": (50,80), "DK": (50,80), "SE": (50,70),
|
||||
"NO": (50,80), "FI": (50,100), "RO": (50,90), "HU": (50,90),
|
||||
"HR": (50,90), "SI": (50,90), "SK": (50,90), "LT": (50,90),
|
||||
"LV": (50,90), "EE": (50,90), "BG": (50,90), "GR": (50,90),
|
||||
"UA": (50,90), "RU": (60,90), "TR": (50,90), "RS": (50,80),
|
||||
"MK": (50,80), "AL": (40,90), "BA": (50,80), "ME": (50,80),
|
||||
"IS": (50,90), "US": (40,105), "CA": (40,100), "MX": (40,90),
|
||||
"BR": (40,80), "AR": (40,110), "CL": (40,120), "CO": (40,90),
|
||||
"PE": (40,90), "JP": (40,60), "CN": (40,100), "IN": (40,80),
|
||||
"AU": (40,100), "NZ": (40,100), "ZA": (40,100), "EG": (50,90),
|
||||
"NG": (50,80), "KE": (50,80), "MA": (50,90), "DZ": (50,90),
|
||||
"TN": (50,90), "LY": (50,90),
|
||||
}
|
||||
|
||||
def parse_speed(v):
|
||||
if not v:
|
||||
return None
|
||||
v = v.strip().lower()
|
||||
if v in ZONE:
|
||||
return ZONE[v]
|
||||
m = __import__("re").match(r"^([0-9]+)(?:\s*(mph|km/h|kmh))?\b", v)
|
||||
if m:
|
||||
n = int(m.group(1))
|
||||
if m.group(2) == "mph":
|
||||
return round(n * 1.609344)
|
||||
return n
|
||||
if ":" in v:
|
||||
cc, kind = v.split(":", 1)
|
||||
cc = cc.upper()
|
||||
if cc in CC and kind in ("urban", "rural"):
|
||||
return CC[cc][0 if kind == "urban" else 1]
|
||||
if cc in CC and kind == "motorway":
|
||||
return None
|
||||
return None
|
||||
|
||||
MAX_NODE_ID = 25_000_000_000 # 25 billion — covers all OSM node IDs with margin
|
||||
|
||||
class Pass1Collector(osmium.SimpleHandler):
|
||||
"""Collect node IDs referenced by highway+maxspeed ways."""
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
bs_size = MAX_NODE_ID // 8 + 1
|
||||
sys.stderr.write(f" Allocating bitset: {bs_size/1073741824:.1f} GB\n")
|
||||
self.bitset = bytearray(bs_size)
|
||||
self.way_count = 0
|
||||
|
||||
def way(self, w):
|
||||
if "highway" not in w.tags:
|
||||
return
|
||||
speed = parse_speed(w.tags.get("maxspeed", ""))
|
||||
if speed is None:
|
||||
return
|
||||
self.way_count += 1
|
||||
for n in w.nodes:
|
||||
nid = n.ref
|
||||
if nid >= MAX_NODE_ID:
|
||||
continue
|
||||
byte_idx = nid >> 3
|
||||
bit_idx = nid & 7
|
||||
self.bitset[byte_idx] |= (1 << bit_idx)
|
||||
if self.way_count % 1_000_000 == 0:
|
||||
sys.stderr.write(f" pass1: {self.way_count//1_000_000}M ways\n")
|
||||
|
||||
def test(self, nid):
|
||||
if nid >= MAX_NODE_ID:
|
||||
return False
|
||||
return (self.bitset[nid >> 3] >> (nid & 7)) & 1
|
||||
|
||||
|
||||
class Pass2Builder(osmium.SimpleHandler):
|
||||
"""Pass 2: collect matching nodes, then process ways."""
|
||||
def __init__(self, bitset_test, outdir):
|
||||
super().__init__()
|
||||
self.bitset_test = bitset_test
|
||||
self.outdir = outdir
|
||||
|
||||
self.node_ids = np.empty(200_000_000, dtype=np.uint64)
|
||||
self.node_lons = np.empty(200_000_000, dtype=np.float32)
|
||||
self.node_lats = np.empty(200_000_000, dtype=np.float32)
|
||||
self.node_count = 0
|
||||
self.node_phase = True
|
||||
|
||||
self.sorted_ids = None
|
||||
self.sorted_lons = None
|
||||
self.sorted_lats = None
|
||||
|
||||
self.points = {}
|
||||
self.way_count = 0
|
||||
self.point_count = 0
|
||||
self.miss_count = 0
|
||||
|
||||
def _ensure_capacity(self):
|
||||
if self.node_count < len(self.node_ids):
|
||||
return
|
||||
new_size = len(self.node_ids) * 2
|
||||
sys.stderr.write(f" Resizing node arrays to {new_size//1_000_000}M...\n")
|
||||
self.node_ids = np.resize(self.node_ids, new_size)
|
||||
self.node_lons = np.resize(self.node_lons, new_size)
|
||||
self.node_lats = np.resize(self.node_lats, new_size)
|
||||
|
||||
def node(self, n):
|
||||
if not self.node_phase:
|
||||
return
|
||||
if self.bitset_test(n.id):
|
||||
self._ensure_capacity()
|
||||
self.node_ids[self.node_count] = n.id
|
||||
self.node_lons[self.node_count] = n.location.lon
|
||||
self.node_lats[self.node_count] = n.location.lat
|
||||
self.node_count += 1
|
||||
if self.node_count % 50_000_000 == 0:
|
||||
sys.stderr.write(f" pass2 nodes: {self.node_count//1_000_000}M\n")
|
||||
|
||||
def _build_index(self):
|
||||
sys.stderr.write(f" Sorting {self.node_count//1_000_000}M nodes...\n")
|
||||
t0 = time.time()
|
||||
ids = self.node_ids[:self.node_count]
|
||||
lons = self.node_lons[:self.node_count]
|
||||
lats = self.node_lats[:self.node_count]
|
||||
self.node_ids = self.node_lons = self.node_lats = None
|
||||
|
||||
order = np.argsort(ids)
|
||||
self.sorted_ids = ids[order]
|
||||
self.sorted_lons = lons[order]
|
||||
self.sorted_lats = lats[order]
|
||||
del ids, lons, lats, order
|
||||
sys.stderr.write(f" Sort done in {time.time()-t0:.1f}s, {len(self.sorted_ids)//1_000_000}M nodes indexed\n")
|
||||
|
||||
def _lookup(self, nid):
|
||||
idx = np.searchsorted(self.sorted_ids, nid)
|
||||
if idx < len(self.sorted_ids) and self.sorted_ids[idx] == nid:
|
||||
return float(self.sorted_lons[idx]), float(self.sorted_lats[idx])
|
||||
return None, None
|
||||
|
||||
def way(self, w):
|
||||
if self.node_phase:
|
||||
self.node_phase = False
|
||||
self._build_index()
|
||||
|
||||
if "highway" not in w.tags:
|
||||
return
|
||||
speed = parse_speed(w.tags.get("maxspeed", ""))
|
||||
if speed is None:
|
||||
return
|
||||
|
||||
self.way_count += 1
|
||||
for n in w.nodes:
|
||||
lon, lat = self._lookup(n.ref)
|
||||
if lon is None:
|
||||
self.miss_count += 1
|
||||
continue
|
||||
self._add_point(lon, lat, speed)
|
||||
|
||||
if self.way_count % 1_000_000 == 0:
|
||||
sys.stderr.write(f" pass2 ways: {self.way_count//1_000_000}M points: {self.point_count//1_000_000}M\n")
|
||||
|
||||
def _add_point(self, lon, lat, speed):
|
||||
cx = int(math.floor(lon / CELL))
|
||||
cy = int(math.floor(lat / CELL))
|
||||
lonq = int(round((lon - cx * CELL) / CELL * 65535))
|
||||
latq = int(round((lat - cy * CELL) / CELL * 65535))
|
||||
lonq = max(0, min(65535, lonq))
|
||||
latq = max(0, min(65535, latq))
|
||||
sx = min(SUB - 1, lonq * SUB // 65536)
|
||||
sy = min(SUB - 1, latq * SUB // 65536)
|
||||
key = (cx, cy)
|
||||
cell = self.points.get(key)
|
||||
if cell is None:
|
||||
cell = [None] * (SUB * SUB)
|
||||
self.points[key] = cell
|
||||
sub = sx + sy * SUB
|
||||
buf = cell[sub]
|
||||
if buf is None:
|
||||
buf = bytearray()
|
||||
cell[sub] = buf
|
||||
buf += struct.pack("<HHB", lonq, latq, max(0, min(255, speed)))
|
||||
|
||||
def write_grid(self):
|
||||
sys.stderr.write(f"\nWriting grid...\n")
|
||||
os.makedirs(self.outdir, exist_ok=True)
|
||||
index = {}
|
||||
total = 0
|
||||
for (cx, cy), cell in self.points.items():
|
||||
count = sum(0 if b is None else len(b) // 5 for b in cell)
|
||||
offsets = [count] * (SUB * SUB)
|
||||
acc = 0
|
||||
for sub, b in enumerate(cell):
|
||||
if b is None:
|
||||
continue
|
||||
offsets[sub] = acc
|
||||
acc += len(b) // 5
|
||||
for s in range(SUB * SUB - 2, -1, -1):
|
||||
if offsets[s] == count:
|
||||
offsets[s] = offsets[s + 1]
|
||||
buf = bytearray()
|
||||
buf += struct.pack("<i", count)
|
||||
buf += struct.pack("<%di" % (SUB * SUB), *offsets)
|
||||
for b in cell:
|
||||
if b is not None:
|
||||
buf += b
|
||||
fname = f"{cx}_{cy}.bin.gz"
|
||||
path = os.path.join(self.outdir, fname)
|
||||
with open(path, "wb") as f:
|
||||
f.write(gzip.compress(bytes(buf), 6))
|
||||
index[fname] = {"x": cx, "y": cy, "count": count}
|
||||
total += count
|
||||
if len(index) % 500 == 0:
|
||||
sys.stderr.write(f" cells: {len(index)} points: {total//1_000_000}M\n")
|
||||
|
||||
with open(os.path.join(self.outdir, "index.json"), "w") as f:
|
||||
json.dump(index, f)
|
||||
with open(os.path.join(self.outdir, "meta.json"), "w") as f:
|
||||
json.dump({"cell": CELL, "sub": SUB, "step_m": 0, "points": total,
|
||||
"files": len(index)}, f)
|
||||
sys.stderr.write(f"OK: {total} points in {len(index)} cells -> {self.outdir}\n")
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 3:
|
||||
print(f"usage: {sys.argv[0]} <planet.pbf> <outdir>")
|
||||
sys.exit(1)
|
||||
pbf_path = sys.argv[1]
|
||||
outdir = sys.argv[2]
|
||||
|
||||
# === PASS 1 ===
|
||||
sys.stderr.write("=== PASS 1: Collect node IDs from highway+maxspeed ways ===\n")
|
||||
t0 = time.time()
|
||||
c = Pass1Collector()
|
||||
c.apply_file(pbf_path, locations=False)
|
||||
sys.stderr.write(f"Pass 1 done: {c.way_count} ways, {time.time()-t0:.0f}s\n")
|
||||
|
||||
# === PASS 2 ===
|
||||
sys.stderr.write("=== PASS 2: Collect nodes + process ways ===\n")
|
||||
t1 = time.time()
|
||||
b = Pass2Builder(c.test, outdir)
|
||||
b.apply_file(pbf_path, locations=False)
|
||||
if not b.node_phase:
|
||||
pass
|
||||
else:
|
||||
sys.stderr.write("WARNING: no ways processed (node phase never ended)\n")
|
||||
sys.stderr.write(f"Pass 2 done: {b.node_count} nodes, {b.way_count} ways, {b.point_count} points, {time.time()-t1:.0f}s\n")
|
||||
|
||||
# === WRITE GRID ===
|
||||
t2 = time.time()
|
||||
b.write_grid()
|
||||
sys.stderr.write(f"Grid written in {time.time()-t0:.0f}s total\n")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
48
data/maxspeed/build_world.sh
Normal file
48
data/maxspeed/build_world.sh
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
cd /home/riricdev/zektyc/data/maxspeed
|
||||
|
||||
echo "=== Étape 1: Filtrer highways du planet ==="
|
||||
if [ ! -f planet-highways.pbf ]; then
|
||||
echo "Extraction des highways..."
|
||||
osmium tags-filter planet-latest.osm.pbf w/highway -o planet-highways.pbf --overwrite
|
||||
ls -lh planet-highways.pbf
|
||||
else
|
||||
echo "planet-highways.pbf existe déjà"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Étape 2: Découper en bandes longitude ==="
|
||||
mkdir -p bands raw
|
||||
|
||||
# 12 bandes de 30° : -180..-150, -150..-120, ..., 150..180
|
||||
BAND_SIZE=30
|
||||
for i in $(seq 0 11); do
|
||||
LON_START=$(( -180 + i * BAND_SIZE ))
|
||||
LON_END=$(( LON_START + BAND_SIZE ))
|
||||
BAND="band_${LON_START}_${LON_END}"
|
||||
|
||||
if [ -f "raw/${BAND}.raw" ]; then
|
||||
echo " ${BAND} déjà fait, skip"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [ ! -f "bands/${BAND}.pbf" ]; then
|
||||
echo " Extraction bande ${LON_START}..${LON_END}..."
|
||||
osmium extract -b ${LON_START},-60,${LON_END},85 planet-highways.pbf -o "bands/${BAND}.pbf" --overwrite
|
||||
ls -lh "bands/${BAND}.pbf"
|
||||
fi
|
||||
|
||||
echo " Build bande ${LON_START}..${LON_END}..."
|
||||
./build_band "bands/${BAND}.pbf" "raw/${BAND}.raw"
|
||||
ls -lh "raw/${BAND}.raw"
|
||||
|
||||
rm -f "bands/${BAND}.pbf"
|
||||
echo ""
|
||||
done
|
||||
|
||||
echo "=== Étape 3: Merge toutes les bandes ==="
|
||||
./merge_grid grid-planet raw/*.raw
|
||||
|
||||
echo "=== Terminé! ==="
|
||||
ls -lh grid-planet/
|
||||
140
data/maxspeed/merge_grid.cpp
Normal file
140
data/maxspeed/merge_grid.cpp
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
#include <cstdio>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <algorithm>
|
||||
#include <filesystem>
|
||||
#include <glob.h>
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
static constexpr double CELL = 0.5;
|
||||
static constexpr int SUB = 32;
|
||||
static constexpr double LON_MIN = -180.0;
|
||||
static constexpr double LAT_MIN = -60.0;
|
||||
static constexpr double LON_MAX = 180.0;
|
||||
static constexpr double LAT_MAX = 85.0;
|
||||
|
||||
struct RawPoint {
|
||||
int32_t col;
|
||||
int32_t row;
|
||||
uint16_t lonq;
|
||||
uint16_t latq;
|
||||
uint8_t speed;
|
||||
};
|
||||
|
||||
struct Point64 {
|
||||
uint16_t lonq;
|
||||
uint16_t latq;
|
||||
uint8_t speed;
|
||||
};
|
||||
|
||||
static bool cmp_cell(const RawPoint& a, const RawPoint& b) {
|
||||
if (a.col != b.col) return a.col < b.col;
|
||||
if (a.row != b.row) return a.row < b.row;
|
||||
return false;
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
if (argc < 3) {
|
||||
fprintf(stderr, "Usage: %s <output_dir> <raw_file1> [raw_file2] ...\n", argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const char* outdir = argv[1];
|
||||
fs::create_directories(outdir);
|
||||
|
||||
std::vector<RawPoint> all_points;
|
||||
|
||||
for (int i = 2; i < argc; i++) {
|
||||
FILE* f = fopen(argv[i], "rb");
|
||||
if (!f) { fprintf(stderr, "Cannot open %s\n", argv[i]); continue; }
|
||||
|
||||
fseek(f, 0, SEEK_END);
|
||||
long fsize = ftell(f);
|
||||
fseek(f, 0, SEEK_SET);
|
||||
|
||||
uint64_t before = all_points.size();
|
||||
uint64_t npoints = fsize / sizeof(RawPoint);
|
||||
all_points.resize(before + npoints);
|
||||
fread(all_points.data() + before, sizeof(RawPoint), npoints, f);
|
||||
fclose(f);
|
||||
|
||||
fprintf(stderr, "Loaded %s: %lu points (total: %lu)\n", argv[i], npoints, all_points.size());
|
||||
}
|
||||
|
||||
fprintf(stderr, "Sorting %lu points...\n", all_points.size());
|
||||
std::sort(all_points.begin(), all_points.end(), cmp_cell);
|
||||
fprintf(stderr, "Sort done. Writing cells...\n");
|
||||
|
||||
std::string index_path = std::string(outdir) + "/index.bin";
|
||||
FILE* index = fopen(index_path.c_str(), "wb");
|
||||
if (!index) { fprintf(stderr, "Cannot write index\n"); return 1; }
|
||||
|
||||
uint32_t magic = 0x5A4D4150;
|
||||
fwrite(&magic, 4, 1, index);
|
||||
uint16_t version = 2;
|
||||
fwrite(&version, 2, 1, index);
|
||||
uint16_t sub = SUB;
|
||||
fwrite(&sub, 2, 1, index);
|
||||
fwrite(&CELL, 8, 1, index);
|
||||
fwrite(&LON_MIN, 8, 1, index);
|
||||
fwrite(&LAT_MIN, 8, 1, index);
|
||||
fwrite(&LON_MAX, 8, 1, index);
|
||||
fwrite(&LAT_MAX, 8, 1, index);
|
||||
|
||||
uint32_t num_cells = 0;
|
||||
fseek(index, 36, SEEK_SET);
|
||||
fwrite(&num_cells, 4, 1, index);
|
||||
fseek(index, 0, SEEK_END);
|
||||
|
||||
int32_t cur_col = -1, cur_row = -1;
|
||||
FILE* cur_file = nullptr;
|
||||
uint32_t cell_points = 0;
|
||||
std::string cur_key;
|
||||
|
||||
for (uint64_t i = 0; i <= all_points.size(); i++) {
|
||||
bool new_cell = (i == all_points.size() ||
|
||||
all_points[i].col != cur_col ||
|
||||
all_points[i].row != cur_row);
|
||||
|
||||
if (new_cell && cur_file) {
|
||||
fclose(cur_file);
|
||||
uint32_t klen = static_cast<uint32_t>(cur_key.size());
|
||||
fwrite(&klen, 4, 1, index);
|
||||
fwrite(cur_key.data(), 1, klen, index);
|
||||
fwrite(&cell_points, 4, 1, index);
|
||||
num_cells++;
|
||||
if (num_cells % 500 == 0) {
|
||||
fprintf(stderr, " %u cells written...\n", num_cells);
|
||||
}
|
||||
}
|
||||
|
||||
if (i == all_points.size()) break;
|
||||
|
||||
if (new_cell) {
|
||||
cur_col = all_points[i].col;
|
||||
cur_row = all_points[i].row;
|
||||
cur_key = std::to_string(cur_col) + "_" + std::to_string(cur_row);
|
||||
std::string cellfile = std::string(outdir) + "/cell_" + cur_key + ".bin";
|
||||
cur_file = fopen(cellfile.c_str(), "wb");
|
||||
cell_points = 0;
|
||||
}
|
||||
|
||||
Point64 p;
|
||||
p.lonq = all_points[i].lonq;
|
||||
p.latq = all_points[i].latq;
|
||||
p.speed = all_points[i].speed;
|
||||
fwrite(&p, sizeof(Point64), 1, cur_file);
|
||||
cell_points++;
|
||||
}
|
||||
|
||||
fseek(index, 36, SEEK_SET);
|
||||
fwrite(&num_cells, 4, 1, index);
|
||||
fclose(index);
|
||||
|
||||
fprintf(stderr, "Total cells: %u\n", num_cells);
|
||||
fprintf(stderr, "Grid written to %s\n", outdir);
|
||||
return 0;
|
||||
}
|
||||
5
data/maxspeed/run_build.sh
Executable file
5
data/maxspeed/run_build.sh
Executable file
|
|
@ -0,0 +1,5 @@
|
|||
#!/bin/bash
|
||||
exec python3 -u /home/riricdev/zektyc/data/maxspeed/build_world.py \
|
||||
/home/riricdev/zektyc/data/maxspeed/planet-latest.osm.pbf \
|
||||
/home/riricdev/zektyc/data/maxspeed/grid-planet/ \
|
||||
>> /home/riricdev/zektyc/data/maxspeed/build-planet.log 2>&1
|
||||
Loading…
Reference in a new issue