Compare commits
13 changed files with 6 additions and 1786 deletions
6
.gitignore
vendored
6
.gitignore
vendored
|
|
@ -48,6 +48,12 @@ data/maxspeed/*.log
|
||||||
# Finder (MacOS) folder config
|
# Finder (MacOS) folder config
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
|
||||||
|
# scripts d'exploitation (gardés sur disque pour les services systemd, hors dépôt)
|
||||||
|
scripts/
|
||||||
|
|
||||||
|
# données de build maxspeed (scripts + artefacts, hors dépôt)
|
||||||
|
data/maxspeed/
|
||||||
|
|
||||||
# sensitive / personal
|
# sensitive / personal
|
||||||
duckdns_token_do_not_open.txt
|
duckdns_token_do_not_open.txt
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,256 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
"""Auto-update OSM planet PBF and rebuild the maxspeed grid.
|
|
||||||
|
|
||||||
Designed to be triggered weekly (Friday 00:00). Checks the RSS feed for a new
|
|
||||||
planet release. If none found yet, retries every 5 minutes until one appears.
|
|
||||||
Then downloads the PBF, verifies MD5, rebuilds the grid, and atomically
|
|
||||||
swaps the old grid directory for the new one.
|
|
||||||
"""
|
|
||||||
import os, sys, time, hashlib, shutil, signal, subprocess, xml.etree.ElementTree as ET
|
|
||||||
from urllib.request import urlopen, Request
|
|
||||||
from pathlib import Path
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
|
|
||||||
DATA_DIR = Path(__file__).resolve().parent
|
|
||||||
PBF_DIR = DATA_DIR
|
|
||||||
RSS_URL = "https://planet.openstreetmap.org/pbf/planet-pbf-rss.xml"
|
|
||||||
BUILD_SCRIPT = DATA_DIR / "build_world.py"
|
|
||||||
CHECK_INTERVAL = 5 * 60 # 5 min between retries when waiting for a new release
|
|
||||||
POLL_TIMEOUT = 6 * 3600 # give up after 6 hours if nothing shows up
|
|
||||||
LOG_PREFIX = "[auto-update]"
|
|
||||||
|
|
||||||
def log(msg):
|
|
||||||
ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
||||||
line = f"{LOG_PREFIX} [{ts}] {msg}"
|
|
||||||
print(line, flush=True)
|
|
||||||
with open(DATA_DIR / "auto-update.log", "a") as f:
|
|
||||||
f.write(line + "\n")
|
|
||||||
|
|
||||||
def fetch_rss():
|
|
||||||
"""Fetch RSS and return the latest planet filename + pubDate."""
|
|
||||||
req = Request(RSS_URL, headers={"User-Agent": "zektyc-auto-updater/1.0"})
|
|
||||||
with urlopen(req, timeout=30) as resp:
|
|
||||||
xml_data = resp.read()
|
|
||||||
root = ET.fromstring(xml_data)
|
|
||||||
items = root.findall(".//item")
|
|
||||||
if not items:
|
|
||||||
return None, None
|
|
||||||
item = items[0]
|
|
||||||
title = (item.findtext("title") or "").strip()
|
|
||||||
pub_date = (item.findtext("pubDate") or "").strip()
|
|
||||||
# title is like "planet-260817.osm.pbf.torrent"
|
|
||||||
filename = title.replace(".torrent", "")
|
|
||||||
return filename, pub_date
|
|
||||||
|
|
||||||
def get_current_pbf():
|
|
||||||
"""Return the basename of the planet PBF currently in use."""
|
|
||||||
# Check which PBF file exists in PBF_DIR
|
|
||||||
for f in sorted(PBF_DIR.glob("planet-*.osm.pbf"), reverse=True):
|
|
||||||
if f.stat().st_size > 10_000_000_000: # > 10 GB = real planet
|
|
||||||
return f.name
|
|
||||||
return None
|
|
||||||
|
|
||||||
def md5_file(path):
|
|
||||||
h = hashlib.md5()
|
|
||||||
with open(path, "rb") as f:
|
|
||||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
|
||||||
h.update(chunk)
|
|
||||||
return h.hexdigest()
|
|
||||||
|
|
||||||
def fetch_md5(filename):
|
|
||||||
"""Download the .md5 file from planet.openstreetmap.org."""
|
|
||||||
url = f"https://planet.openstreetmap.org/pbf/{filename}.md5"
|
|
||||||
try:
|
|
||||||
req = Request(url, headers={"User-Agent": "zektyc-auto-updater/1.0"})
|
|
||||||
with urlopen(req, timeout=30) as resp:
|
|
||||||
text = resp.read().decode().strip()
|
|
||||||
# format: "hash filename"
|
|
||||||
return text.split()[0]
|
|
||||||
except Exception as e:
|
|
||||||
log(f"WARNING: could not fetch MD5: {e}")
|
|
||||||
return None
|
|
||||||
|
|
||||||
def download_pbf(filename):
|
|
||||||
"""Download the planet PBF with resume support."""
|
|
||||||
dest = PBF_DIR / filename
|
|
||||||
url = f"https://planet.openstreetmap.org/pbf/{filename}"
|
|
||||||
existing = dest.stat().st_size if dest.exists() else 0
|
|
||||||
headers = {"User-Agent": "zektyc-auto-updater/1.0"}
|
|
||||||
if existing > 0:
|
|
||||||
headers["Range"] = f"bytes={existing}-"
|
|
||||||
log(f"Resuming download from {existing / 1e9:.1f} GB")
|
|
||||||
req = Request(url, headers=headers)
|
|
||||||
with urlopen(req, timeout=600) as resp:
|
|
||||||
mode = "ab" if existing > 0 and resp.status == 206 else "wb"
|
|
||||||
if mode == "wb":
|
|
||||||
existing = 0
|
|
||||||
total = int(resp.headers.get("Content-Length", 0)) + existing
|
|
||||||
with open(dest, mode) as f:
|
|
||||||
downloaded = existing
|
|
||||||
t0 = time.time()
|
|
||||||
while True:
|
|
||||||
chunk = resp.read(1024 * 1024)
|
|
||||||
if not chunk:
|
|
||||||
break
|
|
||||||
f.write(chunk)
|
|
||||||
downloaded += len(chunk)
|
|
||||||
elapsed = time.time() - t0
|
|
||||||
if elapsed > 0 and downloaded > existing:
|
|
||||||
speed = (downloaded - existing) / elapsed
|
|
||||||
eta = (total - downloaded) / speed if speed > 0 else 0
|
|
||||||
log(f" {downloaded / 1e9:.2f} / {total / 1e9:.2f} GB "
|
|
||||||
f"({speed / 1e6:.1f} MB/s, ETA {int(eta // 60)}m{int(eta % 60)}s)")
|
|
||||||
return dest
|
|
||||||
|
|
||||||
def kill_existing_build():
|
|
||||||
"""Kill any running build_world.py process."""
|
|
||||||
try:
|
|
||||||
result = subprocess.run(
|
|
||||||
["pgrep", "-f", "build_world.py"],
|
|
||||||
capture_output=True, text=True, timeout=5
|
|
||||||
)
|
|
||||||
for pid_str in result.stdout.strip().split("\n"):
|
|
||||||
pid_str = pid_str.strip()
|
|
||||||
if pid_str:
|
|
||||||
pid = int(pid_str)
|
|
||||||
log(f"Killing existing build process PID {pid}")
|
|
||||||
os.kill(pid, signal.SIGTERM)
|
|
||||||
time.sleep(2)
|
|
||||||
try:
|
|
||||||
os.kill(pid, signal.SIGKILL)
|
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
except Exception as e:
|
|
||||||
log(f"WARNING: could not kill existing build: {e}")
|
|
||||||
|
|
||||||
def wait_for_build():
|
|
||||||
"""Wait for any running build_world.py to finish."""
|
|
||||||
while True:
|
|
||||||
result = subprocess.run(
|
|
||||||
["pgrep", "-f", "build_world.py"],
|
|
||||||
capture_output=True, text=True, timeout=5
|
|
||||||
)
|
|
||||||
if not result.stdout.strip():
|
|
||||||
return True
|
|
||||||
log("Waiting for existing build to finish...")
|
|
||||||
time.sleep(30)
|
|
||||||
return False
|
|
||||||
|
|
||||||
def run_build(pbf_path, outdir):
|
|
||||||
"""Run build_world.py."""
|
|
||||||
log(f"Starting build: {pbf_path} -> {outdir}")
|
|
||||||
proc = subprocess.Popen(
|
|
||||||
[sys.executable, "-u", str(BUILD_SCRIPT), str(pbf_path), str(str(outdir))],
|
|
||||||
stdout=subprocess.PIPE,
|
|
||||||
stderr=subprocess.STDOUT,
|
|
||||||
cwd=str(DATA_DIR),
|
|
||||||
)
|
|
||||||
with open(DATA_DIR / "build-planet.log", "a") as logf:
|
|
||||||
for line in iter(proc.stdout.readline, b""):
|
|
||||||
decoded = line.decode(errors="replace")
|
|
||||||
logf.write(decoded)
|
|
||||||
logf.flush()
|
|
||||||
# Also echo last chars for our log
|
|
||||||
proc.wait()
|
|
||||||
if proc.returncode != 0:
|
|
||||||
log(f"ERROR: build_world.py exited with code {proc.returncode}")
|
|
||||||
return False
|
|
||||||
log("Build completed successfully")
|
|
||||||
return True
|
|
||||||
|
|
||||||
def atomic_swap(new_grid, target_grid):
|
|
||||||
"""Atomically replace target_grid with new_grid."""
|
|
||||||
backup = target_grid.parent / (target_grid.name + "-old")
|
|
||||||
if backup.exists():
|
|
||||||
shutil.rmtree(backup)
|
|
||||||
if target_grid.exists():
|
|
||||||
target_grid.rename(backup)
|
|
||||||
new_grid.rename(target_grid)
|
|
||||||
log(f"Swapped {target_grid.name} (backup: {backup.name})")
|
|
||||||
# Clean old backup after a short delay (keep for 1 hour for safety)
|
|
||||||
# Actually let's keep it, it's useful if something goes wrong
|
|
||||||
|
|
||||||
def main():
|
|
||||||
log("=" * 60)
|
|
||||||
log("Auto-update started")
|
|
||||||
|
|
||||||
current_pbf = get_current_pbf()
|
|
||||||
log(f"Current PBF: {current_pbf or 'none'}")
|
|
||||||
|
|
||||||
# Step 1: Check RSS for new release
|
|
||||||
t_start = time.time()
|
|
||||||
new_filename = None
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
filename, pub_date = fetch_rss()
|
|
||||||
if filename and filename != current_pbf:
|
|
||||||
log(f"New release found: {filename} (published: {pub_date})")
|
|
||||||
new_filename = filename
|
|
||||||
break
|
|
||||||
elif filename:
|
|
||||||
log(f"No new release yet (latest: {filename}, current: {current_pbf})")
|
|
||||||
else:
|
|
||||||
log("Could not parse RSS feed")
|
|
||||||
except Exception as e:
|
|
||||||
log(f"RSS check failed: {e}")
|
|
||||||
|
|
||||||
elapsed = time.time() - t_start
|
|
||||||
if elapsed > POLL_TIMEOUT:
|
|
||||||
log(f"No new release after {elapsed/3600:.1f}h, giving up")
|
|
||||||
return
|
|
||||||
log(f"Retrying in {CHECK_INTERVAL // 60} minutes...")
|
|
||||||
time.sleep(CHECK_INTERVAL)
|
|
||||||
|
|
||||||
# Step 2: Kill any existing build
|
|
||||||
kill_existing_build()
|
|
||||||
wait_for_build()
|
|
||||||
|
|
||||||
# Step 3: Download new PBF
|
|
||||||
log(f"Downloading {new_filename}...")
|
|
||||||
pbf_path = download_pbf(new_filename)
|
|
||||||
log(f"Download complete: {pbf_path}")
|
|
||||||
|
|
||||||
# Step 4: Verify MD5
|
|
||||||
expected_md5 = fetch_md5(new_filename)
|
|
||||||
if expected_md5:
|
|
||||||
log(f"Verifying MD5 (expected: {expected_md5})")
|
|
||||||
actual_md5 = md5_file(pbf_path)
|
|
||||||
log(f"Actual MD5: {actual_md5}")
|
|
||||||
if actual_md5 != expected_md5:
|
|
||||||
log("ERROR: MD5 mismatch! Aborting.")
|
|
||||||
return
|
|
||||||
log("MD5 verified OK")
|
|
||||||
else:
|
|
||||||
log("WARNING: skipping MD5 verification")
|
|
||||||
|
|
||||||
# Step 5: Build new grid
|
|
||||||
grid_tmp = PBF_DIR / "grid-planet-new"
|
|
||||||
if grid_tmp.exists():
|
|
||||||
shutil.rmtree(grid_tmp)
|
|
||||||
grid_target = PBF_DIR / "grid-planet"
|
|
||||||
|
|
||||||
if not run_build(pbf_path, grid_tmp):
|
|
||||||
log("ERROR: Build failed, keeping old grid")
|
|
||||||
if grid_tmp.exists():
|
|
||||||
shutil.rmtree(grid_tmp)
|
|
||||||
return
|
|
||||||
|
|
||||||
# Step 6: Atomic swap
|
|
||||||
atomic_swap(grid_tmp, grid_target)
|
|
||||||
|
|
||||||
# Step 7: Update .env symlink (if needed)
|
|
||||||
env_path = Path(__file__).resolve().parent.parent.parent / ".env"
|
|
||||||
if env_path.exists():
|
|
||||||
content = env_path.read_text()
|
|
||||||
if "grid-europe" in content:
|
|
||||||
# Don't change .env — it might be intentionally pointing to grid-europe
|
|
||||||
log("NOTE: .env still points to grid-europe, not auto-updating")
|
|
||||||
elif "grid-planet" in content:
|
|
||||||
log(".env already points to grid-planet")
|
|
||||||
|
|
||||||
log("Auto-update complete!")
|
|
||||||
log("=" * 60)
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
|
|
||||||
main()
|
|
||||||
|
|
@ -1,138 +0,0 @@
|
||||||
#include <osmium/io/any_input.hpp>
|
|
||||||
#include <osmium/visitor.hpp>
|
|
||||||
#include <osmium/handler/node_locations_for_ways.hpp>
|
|
||||||
#include <osmium/index/map/sparse_file_array.hpp>
|
|
||||||
#include <osmium/osm/way.hpp>
|
|
||||||
|
|
||||||
#include <cstdio>
|
|
||||||
#include <cstdint>
|
|
||||||
#include <cstring>
|
|
||||||
#include <cmath>
|
|
||||||
#include <string>
|
|
||||||
#include <vector>
|
|
||||||
#include <algorithm>
|
|
||||||
#include <filesystem>
|
|
||||||
#include <fcntl.h>
|
|
||||||
#include <unistd.h>
|
|
||||||
|
|
||||||
namespace fs = std::filesystem;
|
|
||||||
|
|
||||||
using index_type = osmium::index::map::SparseFileArray<osmium::unsigned_object_id_type, osmium::Location>;
|
|
||||||
using location_handler_type = osmium::handler::NodeLocationsForWays<index_type>;
|
|
||||||
|
|
||||||
static constexpr double CELL = 0.5;
|
|
||||||
static constexpr double LON_MIN = -180.0;
|
|
||||||
static constexpr double LAT_MIN = -60.0;
|
|
||||||
static constexpr double LON_MAX = 180.0;
|
|
||||||
static constexpr double LAT_MAX = 85.0;
|
|
||||||
|
|
||||||
struct RawPoint {
|
|
||||||
int32_t col;
|
|
||||||
int32_t row;
|
|
||||||
uint16_t lonq;
|
|
||||||
uint16_t latq;
|
|
||||||
uint8_t speed;
|
|
||||||
};
|
|
||||||
|
|
||||||
static inline uint16_t quantize_lon(double lon) {
|
|
||||||
return static_cast<uint16_t>((lon - LON_MIN) / (LON_MAX - LON_MIN) * 65535.0);
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline uint16_t quantize_lat(double lat) {
|
|
||||||
return static_cast<uint16_t>((lat - LAT_MIN) / (LAT_MAX - LAT_MIN) * 65535.0);
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline uint8_t parse_speed_val(const char* s) {
|
|
||||||
if (!s || !*s) return 0;
|
|
||||||
char* end = nullptr;
|
|
||||||
double v = strtod(s, &end);
|
|
||||||
if (end != s && v > 0 && v <= 255) return static_cast<uint8_t>(v);
|
|
||||||
if (strstr(s, "mph")) {
|
|
||||||
v = strtod(s, &end);
|
|
||||||
if (end != s) {
|
|
||||||
v *= 1.609;
|
|
||||||
if (v > 0 && v <= 255) return static_cast<uint8_t>(v);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
class StreamHandler : public osmium::handler::Handler {
|
|
||||||
public:
|
|
||||||
FILE* out;
|
|
||||||
uint64_t way_count = 0;
|
|
||||||
uint64_t point_count = 0;
|
|
||||||
|
|
||||||
explicit StreamHandler(FILE* f) : out(f) {}
|
|
||||||
|
|
||||||
void way(const osmium::Way& way) {
|
|
||||||
const char* highway = way.tags().get_value_by_key("highway");
|
|
||||||
if (!highway) return;
|
|
||||||
|
|
||||||
const char* ms = way.tags().get_value_by_key("maxspeed");
|
|
||||||
uint8_t speed = ms ? parse_speed_val(ms) : 0;
|
|
||||||
if (speed == 0) return;
|
|
||||||
|
|
||||||
way_count++;
|
|
||||||
|
|
||||||
for (const auto& wn : way.nodes()) {
|
|
||||||
if (!wn.location().valid()) continue;
|
|
||||||
|
|
||||||
double lon = wn.location().lon();
|
|
||||||
double lat = wn.location().lat();
|
|
||||||
|
|
||||||
if (lon < LON_MIN || lon >= LON_MAX || lat < LAT_MIN || lat >= LAT_MAX) continue;
|
|
||||||
|
|
||||||
RawPoint rp;
|
|
||||||
rp.col = static_cast<int32_t>((lon - LON_MIN) / CELL);
|
|
||||||
rp.row = static_cast<int32_t>((lat - LAT_MIN) / CELL);
|
|
||||||
rp.lonq = quantize_lon(lon);
|
|
||||||
rp.latq = quantize_lat(lat);
|
|
||||||
rp.speed = speed;
|
|
||||||
|
|
||||||
fwrite(&rp, sizeof(RawPoint), 1, out);
|
|
||||||
point_count++;
|
|
||||||
|
|
||||||
if (point_count % 5000000 == 0) {
|
|
||||||
fprintf(stderr, " %lu points (%lu ways)...\n", point_count, way_count);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
int main(int argc, char* argv[]) {
|
|
||||||
if (argc < 3) {
|
|
||||||
fprintf(stderr, "Usage: %s <input.pbf> <output.raw>\n", argv[0]);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
const char* input = argv[1];
|
|
||||||
const char* outpath = argv[2];
|
|
||||||
|
|
||||||
fs::create_directories(fs::path(outpath).parent_path());
|
|
||||||
|
|
||||||
FILE* out = fopen(outpath, "wb");
|
|
||||||
if (!out) { fprintf(stderr, "Cannot open %s\n", outpath); return 1; }
|
|
||||||
|
|
||||||
std::string idx_path = std::string(outpath) + ".idx";
|
|
||||||
int idx_fd = ::open(idx_path.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644);
|
|
||||||
if (idx_fd == -1) { fprintf(stderr, "Cannot create index\n"); fclose(out); return 1; }
|
|
||||||
|
|
||||||
index_type index(idx_fd);
|
|
||||||
location_handler_type location_handler(index);
|
|
||||||
StreamHandler handler(out);
|
|
||||||
|
|
||||||
fprintf(stderr, "Reading %s...\n", input);
|
|
||||||
osmium::io::Reader reader(input);
|
|
||||||
osmium::apply(reader, location_handler, handler);
|
|
||||||
reader.close();
|
|
||||||
|
|
||||||
fclose(out);
|
|
||||||
::close(idx_fd);
|
|
||||||
unlink(idx_path.c_str());
|
|
||||||
|
|
||||||
fprintf(stderr, "Ways with maxspeed: %lu\n", handler.way_count);
|
|
||||||
fprintf(stderr, "Points written: %lu\n", handler.point_count);
|
|
||||||
fprintf(stderr, "Output: %s\n", outpath);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
@ -1,245 +0,0 @@
|
||||||
#include <osmium/io/any_input.hpp>
|
|
||||||
#include <osmium/visitor.hpp>
|
|
||||||
#include <osmium/handler/node_locations_for_ways.hpp>
|
|
||||||
#include <osmium/index/map/sparse_file_array.hpp>
|
|
||||||
#include <osmium/osm/way.hpp>
|
|
||||||
|
|
||||||
#include <cstdio>
|
|
||||||
#include <cstdint>
|
|
||||||
#include <cstring>
|
|
||||||
#include <cmath>
|
|
||||||
#include <string>
|
|
||||||
#include <vector>
|
|
||||||
#include <algorithm>
|
|
||||||
#include <filesystem>
|
|
||||||
#include <fcntl.h>
|
|
||||||
#include <unistd.h>
|
|
||||||
|
|
||||||
namespace fs = std::filesystem;
|
|
||||||
|
|
||||||
using index_type = osmium::index::map::SparseFileArray<osmium::unsigned_object_id_type, osmium::Location>;
|
|
||||||
using location_handler_type = osmium::handler::NodeLocationsForWays<index_type>;
|
|
||||||
|
|
||||||
static constexpr double CELL = 0.5;
|
|
||||||
static constexpr int SUB = 32;
|
|
||||||
static constexpr double LON_MIN = -180.0;
|
|
||||||
static constexpr double LAT_MIN = -60.0;
|
|
||||||
static constexpr double LON_MAX = 180.0;
|
|
||||||
static constexpr double LAT_MAX = 85.0;
|
|
||||||
|
|
||||||
struct RawPoint {
|
|
||||||
int32_t col;
|
|
||||||
int32_t row;
|
|
||||||
uint16_t lonq;
|
|
||||||
uint16_t latq;
|
|
||||||
uint8_t speed;
|
|
||||||
}; // 11 bytes
|
|
||||||
|
|
||||||
static inline uint16_t quantize_lon(double lon) {
|
|
||||||
return static_cast<uint16_t>((lon - LON_MIN) / (LON_MAX - LON_MIN) * 65535.0);
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline uint16_t quantize_lat(double lat) {
|
|
||||||
return static_cast<uint16_t>((lat - LAT_MIN) / (LAT_MAX - LAT_MIN) * 65535.0);
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline uint8_t parse_speed_val(const char* s) {
|
|
||||||
if (!s || !*s) return 0;
|
|
||||||
char* end = nullptr;
|
|
||||||
double v = strtod(s, &end);
|
|
||||||
if (end != s && v > 0 && v <= 255) return static_cast<uint8_t>(v);
|
|
||||||
if (strstr(s, "mph")) {
|
|
||||||
v = strtod(s, &end);
|
|
||||||
if (end != s) {
|
|
||||||
v *= 1.609;
|
|
||||||
if (v > 0 && v <= 255) return static_cast<uint8_t>(v);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
struct Point64 {
|
|
||||||
uint16_t lonq;
|
|
||||||
uint16_t latq;
|
|
||||||
uint8_t speed;
|
|
||||||
};
|
|
||||||
|
|
||||||
class SpeedHandler : public osmium::handler::Handler {
|
|
||||||
public:
|
|
||||||
FILE* tmpfile;
|
|
||||||
uint64_t way_count = 0;
|
|
||||||
uint64_t point_count = 0;
|
|
||||||
|
|
||||||
explicit SpeedHandler(FILE* f) : tmpfile(f) {}
|
|
||||||
|
|
||||||
void way(const osmium::Way& way) {
|
|
||||||
const char* highway = way.tags().get_value_by_key("highway");
|
|
||||||
if (!highway) return;
|
|
||||||
|
|
||||||
const char* ms = way.tags().get_value_by_key("maxspeed");
|
|
||||||
uint8_t speed = ms ? parse_speed_val(ms) : 0;
|
|
||||||
if (speed == 0) return;
|
|
||||||
|
|
||||||
way_count++;
|
|
||||||
|
|
||||||
for (const auto& wn : way.nodes()) {
|
|
||||||
if (!wn.location().valid()) continue;
|
|
||||||
|
|
||||||
double lon = wn.location().lon();
|
|
||||||
double lat = wn.location().lat();
|
|
||||||
|
|
||||||
if (lon < LON_MIN || lon >= LON_MAX || lat < LAT_MIN || lat >= LAT_MAX) continue;
|
|
||||||
|
|
||||||
int32_t col = static_cast<int32_t>((lon - LON_MIN) / CELL);
|
|
||||||
int32_t row = static_cast<int32_t>((lat - LAT_MIN) / CELL);
|
|
||||||
|
|
||||||
RawPoint rp;
|
|
||||||
rp.col = col;
|
|
||||||
rp.row = row;
|
|
||||||
rp.lonq = quantize_lon(lon);
|
|
||||||
rp.latq = quantize_lat(lat);
|
|
||||||
rp.speed = speed;
|
|
||||||
|
|
||||||
fwrite(&rp, sizeof(RawPoint), 1, tmpfile);
|
|
||||||
point_count++;
|
|
||||||
|
|
||||||
if (point_count % 10000000 == 0) {
|
|
||||||
fprintf(stderr, " %lu points (%lu ways)...\n", point_count, way_count);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
static bool compare_by_cell(const RawPoint& a, const RawPoint& b) {
|
|
||||||
if (a.col != b.col) return a.col < b.col;
|
|
||||||
return a.row < b.row;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void split_and_write(const char* tmppath, const std::string& outdir) {
|
|
||||||
fprintf(stderr, "Reading temp file into memory for sort...\n");
|
|
||||||
|
|
||||||
FILE* f = fopen(tmppath, "rb");
|
|
||||||
fseek(f, 0, SEEK_END);
|
|
||||||
long fsize = ftell(f);
|
|
||||||
fseek(f, 0, SEEK_SET);
|
|
||||||
|
|
||||||
uint64_t npoints = fsize / sizeof(RawPoint);
|
|
||||||
fprintf(stderr, "Sorting %lu points (%.1f MB)...\n", npoints, fsize / 1048576.0);
|
|
||||||
|
|
||||||
std::vector<RawPoint> points(npoints);
|
|
||||||
fread(points.data(), sizeof(RawPoint), npoints, f);
|
|
||||||
fclose(f);
|
|
||||||
|
|
||||||
std::sort(points.begin(), points.end(), compare_by_cell);
|
|
||||||
fprintf(stderr, "Sort done. Writing cells...\n");
|
|
||||||
|
|
||||||
fs::create_directories(outdir);
|
|
||||||
|
|
||||||
std::string index_path = outdir + "/index.bin";
|
|
||||||
FILE* index = fopen(index_path.c_str(), "wb");
|
|
||||||
uint32_t magic = 0x5A4D4150;
|
|
||||||
fwrite(&magic, 4, 1, index);
|
|
||||||
uint16_t version = 2;
|
|
||||||
fwrite(&version, 2, 1, index);
|
|
||||||
uint16_t sub = SUB;
|
|
||||||
fwrite(&sub, 2, 1, index);
|
|
||||||
fwrite(&CELL, 8, 1, index);
|
|
||||||
fwrite(&LON_MIN, 8, 1, index);
|
|
||||||
fwrite(&LAT_MIN, 8, 1, index);
|
|
||||||
fwrite(&LON_MAX, 8, 1, index);
|
|
||||||
fwrite(&LAT_MAX, 8, 1, index);
|
|
||||||
|
|
||||||
uint32_t num_cells = 0;
|
|
||||||
fseek(index, 36, SEEK_SET);
|
|
||||||
fwrite(&num_cells, 4, 1, index);
|
|
||||||
fseek(index, 0, SEEK_END);
|
|
||||||
|
|
||||||
int32_t cur_col = -1, cur_row = -1;
|
|
||||||
FILE* cur_file = nullptr;
|
|
||||||
uint32_t cell_points = 0;
|
|
||||||
std::string cur_key;
|
|
||||||
|
|
||||||
for (uint64_t i = 0; i <= npoints; i++) {
|
|
||||||
bool new_cell = (i == npoints || points[i].col != cur_col || points[i].row != cur_row);
|
|
||||||
|
|
||||||
if (new_cell && cur_file) {
|
|
||||||
fclose(cur_file);
|
|
||||||
|
|
||||||
uint32_t klen = static_cast<uint32_t>(cur_key.size());
|
|
||||||
fwrite(&klen, 4, 1, index);
|
|
||||||
fwrite(cur_key.data(), 1, klen, index);
|
|
||||||
fwrite(&cell_points, 4, 1, index);
|
|
||||||
num_cells++;
|
|
||||||
|
|
||||||
if (num_cells % 1000 == 0) {
|
|
||||||
fprintf(stderr, " %u cells written...\n", num_cells);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (i == npoints) break;
|
|
||||||
|
|
||||||
if (new_cell) {
|
|
||||||
cur_col = points[i].col;
|
|
||||||
cur_row = points[i].row;
|
|
||||||
cur_key = std::to_string(cur_col) + "_" + std::to_string(cur_row);
|
|
||||||
std::string cellfile = outdir + "/cell_" + cur_key + ".bin";
|
|
||||||
cur_file = fopen(cellfile.c_str(), "wb");
|
|
||||||
cell_points = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
Point64 p;
|
|
||||||
p.lonq = points[i].lonq;
|
|
||||||
p.latq = points[i].latq;
|
|
||||||
p.speed = points[i].speed;
|
|
||||||
fwrite(&p, sizeof(Point64), 1, cur_file);
|
|
||||||
cell_points++;
|
|
||||||
}
|
|
||||||
|
|
||||||
fseek(index, 36, SEEK_SET);
|
|
||||||
fwrite(&num_cells, 4, 1, index);
|
|
||||||
fclose(index);
|
|
||||||
|
|
||||||
fprintf(stderr, "Total cells: %u\n", num_cells);
|
|
||||||
}
|
|
||||||
|
|
||||||
int main(int argc, char* argv[]) {
|
|
||||||
if (argc < 3) {
|
|
||||||
fprintf(stderr, "Usage: %s <input.pbf> <output_dir>\n", argv[0]);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
const char* input = argv[1];
|
|
||||||
const char* outdir = argv[2];
|
|
||||||
|
|
||||||
std::string tmppath = std::string(outdir) + "/_points.tmp";
|
|
||||||
fs::create_directories(outdir);
|
|
||||||
|
|
||||||
fprintf(stderr, "Step 1: Reading PBF, writing raw points to temp file...\n");
|
|
||||||
FILE* tmpf = fopen(tmppath.c_str(), "w+b");
|
|
||||||
if (!tmpf) { fprintf(stderr, "Cannot create temp file\n"); return 1; }
|
|
||||||
|
|
||||||
std::string idx_path = std::string(outdir) + "/_node_index.tmp";
|
|
||||||
int idx_fd = ::open(idx_path.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644);
|
|
||||||
if (idx_fd == -1) { fprintf(stderr, "Cannot create index temp file\n"); return 1; }
|
|
||||||
|
|
||||||
index_type index(idx_fd);
|
|
||||||
location_handler_type location_handler(index);
|
|
||||||
SpeedHandler handler(tmpf);
|
|
||||||
|
|
||||||
osmium::io::Reader reader(input);
|
|
||||||
osmium::apply(reader, location_handler, handler);
|
|
||||||
reader.close();
|
|
||||||
fclose(tmpf);
|
|
||||||
|
|
||||||
fprintf(stderr, "\nWays with maxspeed: %lu\n", handler.way_count);
|
|
||||||
fprintf(stderr, "Total points: %lu\n", handler.point_count);
|
|
||||||
|
|
||||||
fprintf(stderr, "Step 2: Sort + write grid cells...\n");
|
|
||||||
split_and_write(tmppath.c_str(), outdir);
|
|
||||||
|
|
||||||
::close(idx_fd);
|
|
||||||
unlink(idx_path.c_str());
|
|
||||||
unlink(tmppath.c_str());
|
|
||||||
fprintf(stderr, "Done! Grid written to %s\n", outdir);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
@ -1,273 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
"""Construit une grille binaire des maxspeed OSM (par route, échantillonnée en points).
|
|
||||||
|
|
||||||
Pipeline :
|
|
||||||
osmium tags-filter <extrait>.osm.pbf w/highway -o <extrait>-roads.osm.pbf
|
|
||||||
python3 build_maxspeed.py <extrait>-roads.osm.pbf <sortie> [--cache=<file>]
|
|
||||||
|
|
||||||
Sortie : <sortie>/grid.dat + <sortie>/index.json + <sortie>/meta.json
|
|
||||||
- Chaque cellule = 0.5° x 0.5° (lat, lon).
|
|
||||||
- Blob par cellule (gzip) : int32 count ; 32x32 sous-cellules => int32 offset[1024] ;
|
|
||||||
puis count points de (lonq:int16, latq:int16, speed:int8) rangés par sous-cellule.
|
|
||||||
- lonq/latq : quantifiés sur 0..65535 dans la cellule (precision ~0.8 m).
|
|
||||||
"""
|
|
||||||
import sys, os, json, gzip, math, struct, re
|
|
||||||
import osmium
|
|
||||||
import sqlite3
|
|
||||||
|
|
||||||
CELL = 0.5 # degrés par cellule
|
|
||||||
SUB = 32 # sous-cellules par dimension (grille 32x32)
|
|
||||||
STEP_M_DEFAULT = 40.0 # pas d'échantillonnage le long des routes (m)
|
|
||||||
STEP_M = STEP_M_DEFAULT
|
|
||||||
|
|
||||||
# maxspeed textuel -> km/h (valeurs par défaut OSM)
|
|
||||||
ZONE = {
|
|
||||||
"none": None,
|
|
||||||
"signals": None,
|
|
||||||
"variable": None,
|
|
||||||
"walk": 5,
|
|
||||||
"living_street": 20,
|
|
||||||
"urban": 50, "rural": 80, "trunk": 110, "motorway": 130,
|
|
||||||
}
|
|
||||||
# codes pays -> (urban, rural) km/h ; motorway sans maxspeed = None (ex. DE)
|
|
||||||
CC = {
|
|
||||||
"FR": (50, 80), "BE": (50, 90), "DE": (50, 100), "NL": (50, 100),
|
|
||||||
"GB": (48, 96), "ES": (50, 90), "IT": (50, 90), "CH": (50, 80),
|
|
||||||
"AT": (50, 100), "LU": (50, 90), "PT": (50, 90), "PL": (50, 90),
|
|
||||||
"CZ": (50, 90), "IE": (50, 80), "DK": (50, 80), "SE": (50, 70),
|
|
||||||
"NO": (50, 80), "FI": (50, 100), "RO": (50, 90), "HU": (50, 90),
|
|
||||||
"HR": (50, 90), "SI": (50, 90), "SK": (50, 90), "LT": (50, 90),
|
|
||||||
"LV": (50, 90), "EE": (50, 90), "BG": (50, 90), "GR": (50, 90),
|
|
||||||
"UA": (50, 90), "RU": (60, 90), "TR": (50, 90), "RS": (50, 80),
|
|
||||||
"MK": (50, 80), "AL": (40, 90), "BA": (50, 80), "ME": (50, 80),
|
|
||||||
"IS": (50, 90),
|
|
||||||
}
|
|
||||||
# Les highways de service/privées sans maxspeed sont exclues : on ne garde que les
|
|
||||||
# ways marqués maxspeed (ou avec une valeur par défaut de zone reconnue).
|
|
||||||
|
|
||||||
def parse_speed(v):
|
|
||||||
if not v:
|
|
||||||
return None
|
|
||||||
v = v.strip().lower()
|
|
||||||
if v in ZONE:
|
|
||||||
return ZONE[v]
|
|
||||||
m = __import__("re").match(r"^([0-9]+)(?:\s*(mph|km/h|kmh))?\b", v)
|
|
||||||
if m:
|
|
||||||
n = int(m.group(1))
|
|
||||||
if m.group(2) == "mph":
|
|
||||||
return round(n * 1.609344)
|
|
||||||
return n
|
|
||||||
# code pays type FR:rural / DE:urban
|
|
||||||
if ":" in v:
|
|
||||||
cc, kind = v.split(":", 1)
|
|
||||||
cc = cc.upper()
|
|
||||||
if cc in CC and kind in ("urban", "rural"):
|
|
||||||
return CC[cc][0 if kind == "urban" else 1]
|
|
||||||
if cc in CC and kind == "motorway":
|
|
||||||
return None
|
|
||||||
return None
|
|
||||||
|
|
||||||
class MaxSpeedHandler(osmium.SimpleHandler):
|
|
||||||
def __init__(self):
|
|
||||||
super().__init__()
|
|
||||||
self.points = {} # (cellx, celly) -> dict subcell -> list of (lonq, latq, speed)
|
|
||||||
self.skipped = 0
|
|
||||||
|
|
||||||
def way(self, w):
|
|
||||||
if "highway" not in w.tags:
|
|
||||||
return
|
|
||||||
speed = parse_speed(w.tags.get("maxspeed", ""))
|
|
||||||
if speed is None:
|
|
||||||
return
|
|
||||||
locs = []
|
|
||||||
for n in w.nodes:
|
|
||||||
loc = n.location
|
|
||||||
if loc is not None:
|
|
||||||
locs.append((loc.lon, loc.lat))
|
|
||||||
if len(locs) < 2:
|
|
||||||
return
|
|
||||||
self.process_locs(locs, speed)
|
|
||||||
|
|
||||||
def process_locs(self, locs, speed):
|
|
||||||
if STEP_M <= 0:
|
|
||||||
for (lon, lat) in locs:
|
|
||||||
self.add_point(lon, lat, speed)
|
|
||||||
return
|
|
||||||
prev = locs[0]
|
|
||||||
lon, lat = prev
|
|
||||||
acc = 0.0
|
|
||||||
emit = [(lon, lat)]
|
|
||||||
for node in locs[1:]:
|
|
||||||
a = haversine(prev[1], prev[0], node[1], node[0])
|
|
||||||
acc += a
|
|
||||||
prev = node
|
|
||||||
while acc >= STEP_M:
|
|
||||||
f = 1.0 - (acc - STEP_M) / a if a > 0 else 0.0
|
|
||||||
ilon = lon + (node[0] - lon) * f
|
|
||||||
ilat = lat + (node[1] - lat) * f
|
|
||||||
emit.append((ilon, ilat))
|
|
||||||
acc -= STEP_M
|
|
||||||
lon, lat = node[0], node[1]
|
|
||||||
if (lon, lat) != emit[-1]:
|
|
||||||
emit.append((lon, lat))
|
|
||||||
for (el, ea) in emit:
|
|
||||||
self.add_point(el, ea, speed)
|
|
||||||
|
|
||||||
def add_point(self, lon, lat, speed):
|
|
||||||
cx = int(math.floor(lon / CELL))
|
|
||||||
cy = int(math.floor(lat / CELL))
|
|
||||||
lonq = int(round((lon - cx * CELL) / CELL * 65535))
|
|
||||||
latq = int(round((lat - cy * CELL) / CELL * 65535))
|
|
||||||
lonq = max(0, min(65535, lonq)); latq = max(0, min(65535, latq))
|
|
||||||
sx = min(SUB - 1, lonq * SUB // 65536)
|
|
||||||
sy = min(SUB - 1, latq * SUB // 65536)
|
|
||||||
key = (cx, cy)
|
|
||||||
cell = self.points.get(key)
|
|
||||||
if cell is None:
|
|
||||||
cell = [None] * (SUB * SUB)
|
|
||||||
self.points[key] = cell
|
|
||||||
sub = sx + sy * SUB
|
|
||||||
buf = cell[sub]
|
|
||||||
if buf is None:
|
|
||||||
buf = bytearray()
|
|
||||||
cell[sub] = buf
|
|
||||||
buf += struct.pack("<HHB", lonq, latq, max(0, min(255, speed)))
|
|
||||||
|
|
||||||
def haversine(lat1, lon1, lat2, lon2):
|
|
||||||
R = 6371000.0
|
|
||||||
p1 = math.radians(lat1); p2 = math.radians(lat2)
|
|
||||||
dp = math.radians(lat2 - lat1); dl = math.radians(lon2 - lon1)
|
|
||||||
a = math.sin(dp / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2
|
|
||||||
return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
|
|
||||||
|
|
||||||
GP_FLAGS = {"byteorder": 0x01, "envelope": 0x02, "empty": 0x10}
|
|
||||||
|
|
||||||
def parse_gp_wkb(raw):
|
|
||||||
"""GeoPackage binary -> liste (lon, lat) pour LineString (ou Multi)."""
|
|
||||||
if not isinstance(raw, (bytes, bytearray)) or len(raw) < 8 or raw[:2] != b"GP":
|
|
||||||
return None
|
|
||||||
flags = raw[3]
|
|
||||||
off = 8
|
|
||||||
if flags & GP_FLAGS["envelope"]:
|
|
||||||
etype = (flags >> 2) & 0x03
|
|
||||||
if etype != 0:
|
|
||||||
return None
|
|
||||||
off += 4 * 8
|
|
||||||
bo = "<" if flags & GP_FLAGS["byteorder"] else ">"
|
|
||||||
if off + 5 > len(raw):
|
|
||||||
return None
|
|
||||||
gtype = struct.unpack(bo + "i", raw[off + 1:off + 5])[0]
|
|
||||||
if gtype == 2: # LineString
|
|
||||||
return parse_wkb_points(raw, off, bo)
|
|
||||||
if gtype == 5: # MultiLineString
|
|
||||||
out = []
|
|
||||||
num = struct.unpack(bo + "i", raw[off + 5:off + 9])[0]
|
|
||||||
p = off + 9
|
|
||||||
for _ in range(num):
|
|
||||||
pts = parse_wkb_points(raw, p, bo)
|
|
||||||
if pts:
|
|
||||||
out.extend(pts)
|
|
||||||
p += 4 + 4 * 8 * (len(pts) if pts else 0)
|
|
||||||
return out if len(out) >= 2 else None
|
|
||||||
return None
|
|
||||||
|
|
||||||
def parse_wkb_points(raw, off, bo):
|
|
||||||
n = struct.unpack(bo + "i", raw[off + 5:off + 9])[0]
|
|
||||||
out = []
|
|
||||||
p = off + 9
|
|
||||||
for _ in range(n):
|
|
||||||
out.append((struct.unpack(bo + "d", raw[p:p + 8])[0],
|
|
||||||
struct.unpack(bo + "d", raw[p + 8:p + 16])[0]))
|
|
||||||
p += 16
|
|
||||||
return out
|
|
||||||
|
|
||||||
MS_RE = re.compile(r'"maxspeed"=>"([^"]*)"')
|
|
||||||
|
|
||||||
def parse_other_tags(s):
|
|
||||||
m = MS_RE.search(s or "")
|
|
||||||
return m.group(1) if m else None
|
|
||||||
|
|
||||||
def load_gpkg(h, gpkg):
|
|
||||||
con = sqlite3.connect(gpkg)
|
|
||||||
cur = con.execute("SELECT geom, highway, other_tags FROM lines")
|
|
||||||
n = 0
|
|
||||||
for raw, hw, tags in cur:
|
|
||||||
if not hw or "maxspeed" not in (tags or ""):
|
|
||||||
continue
|
|
||||||
v = parse_other_tags(tags)
|
|
||||||
speed = parse_speed(v)
|
|
||||||
if speed is None:
|
|
||||||
continue
|
|
||||||
locs = parse_gp_wkb(raw)
|
|
||||||
if not locs or len(locs) < 2:
|
|
||||||
continue
|
|
||||||
h.process_locs(locs, speed)
|
|
||||||
n += 1
|
|
||||||
if n % 500000 == 0:
|
|
||||||
sys.stderr.write("ways maxspeed: %dM\n" % (n // 1000000))
|
|
||||||
con.close()
|
|
||||||
return n
|
|
||||||
|
|
||||||
def build(src_pbf, out_dir, cache_file=None):
|
|
||||||
os.makedirs(out_dir, exist_ok=True)
|
|
||||||
h = MaxSpeedHandler()
|
|
||||||
if src_pbf.endswith(".gpkg") or src_pbf.endswith(".db"):
|
|
||||||
n = load_gpkg(h, src_pbf)
|
|
||||||
print(f"GPKG : {n} ways traités")
|
|
||||||
else:
|
|
||||||
if cache_file:
|
|
||||||
idx = osmium.index.create_map("sparse_file_array,%s" % cache_file)
|
|
||||||
h.apply_file(src_pbf, locations=osmium.NodeLocationsForWays(idx))
|
|
||||||
else:
|
|
||||||
h.apply_file(src_pbf, locations=True)
|
|
||||||
# écriture : un fichier par cellule (nom: X_Y.bin.gz) + index
|
|
||||||
index = {}
|
|
||||||
total = 0
|
|
||||||
for (cx, cy), cell in h.points.items():
|
|
||||||
count = sum(0 if b is None else len(b) // 5 for b in cell)
|
|
||||||
offsets = [count] * (SUB * SUB)
|
|
||||||
acc = 0
|
|
||||||
for sub, b in enumerate(cell):
|
|
||||||
if b is None:
|
|
||||||
continue
|
|
||||||
offsets[sub] = acc
|
|
||||||
acc += len(b) // 5
|
|
||||||
# remplissage arrière : les sous-cellules vides pointent vers la suivante
|
|
||||||
for s in range(SUB * SUB - 2, -1, -1):
|
|
||||||
if offsets[s] == count:
|
|
||||||
offsets[s] = offsets[s + 1]
|
|
||||||
buf = bytearray()
|
|
||||||
buf += struct.pack("<i", count)
|
|
||||||
buf += struct.pack("<%di" % (SUB * SUB), *offsets)
|
|
||||||
for b in cell:
|
|
||||||
if b is not None:
|
|
||||||
buf += b
|
|
||||||
fname = f"{cx}_{cy}.bin.gz"
|
|
||||||
with open(os.path.join(out_dir, fname), "wb") as f:
|
|
||||||
f.write(gzip.compress(bytes(buf), 6))
|
|
||||||
index[fname] = {"x": cx, "y": cy, "count": count}
|
|
||||||
total += count
|
|
||||||
with open(os.path.join(out_dir, "index.json"), "w") as f:
|
|
||||||
json.dump(index, f)
|
|
||||||
with open(os.path.join(out_dir, "meta.json"), "w") as f:
|
|
||||||
json.dump({"cell": CELL, "sub": SUB, "step_m": STEP_M, "points": total,
|
|
||||||
"files": len(index)}, f)
|
|
||||||
print(f"OK : {total} points dans {len(index)} cellules -> {out_dir}")
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
cache = None
|
|
||||||
args = sys.argv[1:]
|
|
||||||
extra = {}
|
|
||||||
while args and args[-1].startswith("--"):
|
|
||||||
a = args.pop()
|
|
||||||
if a.startswith("--cache="):
|
|
||||||
cache = a.split("=", 1)[1]
|
|
||||||
elif a == "--no-sample":
|
|
||||||
STEP_M = 0
|
|
||||||
else:
|
|
||||||
print(f"Option inconnue: {a}")
|
|
||||||
sys.exit(1)
|
|
||||||
if len(args) != 2:
|
|
||||||
print("usage: build_maxspeed.py <extrait.osm.pbf|gpkg> <out_dir> [--cache=<file>] [--no-sample]")
|
|
||||||
sys.exit(1)
|
|
||||||
build(args[0], args[1], cache)
|
|
||||||
|
|
@ -1,271 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
"""Build world maxspeed grid from planet PBF in 2 passes.
|
|
||||||
|
|
||||||
Pass 1: Stream PBF → collect node IDs from highway+maxspeed ways → bitset
|
|
||||||
Pass 2: Stream PBF → matching nodes → mmap sort → process ways → grid
|
|
||||||
|
|
||||||
Usage: python3 build_world.py planet-latest.osm.pbf grid-planet/
|
|
||||||
"""
|
|
||||||
import sys, os, json, gzip, struct, math, mmap, time
|
|
||||||
import numpy as np
|
|
||||||
import osmium
|
|
||||||
|
|
||||||
CELL = 0.5
|
|
||||||
SUB = 32
|
|
||||||
|
|
||||||
ZONE = {
|
|
||||||
"none": None, "signals": None, "variable": None, "walk": 5, "living_street": 20,
|
|
||||||
"urban": 50, "rural": 80, "trunk": 110, "motorway": 130,
|
|
||||||
}
|
|
||||||
CC = {
|
|
||||||
"FR": (50,80), "BE": (50,90), "DE": (50,100), "NL": (50,100),
|
|
||||||
"GB": (48,96), "ES": (50,90), "IT": (50,90), "CH": (50,80),
|
|
||||||
"AT": (50,100), "LU": (50,90), "PT": (50,90), "PL": (50,90),
|
|
||||||
"CZ": (50,90), "IE": (50,80), "DK": (50,80), "SE": (50,70),
|
|
||||||
"NO": (50,80), "FI": (50,100), "RO": (50,90), "HU": (50,90),
|
|
||||||
"HR": (50,90), "SI": (50,90), "SK": (50,90), "LT": (50,90),
|
|
||||||
"LV": (50,90), "EE": (50,90), "BG": (50,90), "GR": (50,90),
|
|
||||||
"UA": (50,90), "RU": (60,90), "TR": (50,90), "RS": (50,80),
|
|
||||||
"MK": (50,80), "AL": (40,90), "BA": (50,80), "ME": (50,80),
|
|
||||||
"IS": (50,90), "US": (40,105), "CA": (40,100), "MX": (40,90),
|
|
||||||
"BR": (40,80), "AR": (40,110), "CL": (40,120), "CO": (40,90),
|
|
||||||
"PE": (40,90), "JP": (40,60), "CN": (40,100), "IN": (40,80),
|
|
||||||
"AU": (40,100), "NZ": (40,100), "ZA": (40,100), "EG": (50,90),
|
|
||||||
"NG": (50,80), "KE": (50,80), "MA": (50,90), "DZ": (50,90),
|
|
||||||
"TN": (50,90), "LY": (50,90),
|
|
||||||
}
|
|
||||||
|
|
||||||
def parse_speed(v):
|
|
||||||
if not v:
|
|
||||||
return None
|
|
||||||
v = v.strip().lower()
|
|
||||||
if v in ZONE:
|
|
||||||
return ZONE[v]
|
|
||||||
m = __import__("re").match(r"^([0-9]+)(?:\s*(mph|km/h|kmh))?\b", v)
|
|
||||||
if m:
|
|
||||||
n = int(m.group(1))
|
|
||||||
if m.group(2) == "mph":
|
|
||||||
return round(n * 1.609344)
|
|
||||||
return n
|
|
||||||
if ":" in v:
|
|
||||||
cc, kind = v.split(":", 1)
|
|
||||||
cc = cc.upper()
|
|
||||||
if cc in CC and kind in ("urban", "rural"):
|
|
||||||
return CC[cc][0 if kind == "urban" else 1]
|
|
||||||
if cc in CC and kind == "motorway":
|
|
||||||
return None
|
|
||||||
return None
|
|
||||||
|
|
||||||
MAX_NODE_ID = 25_000_000_000 # 25 billion — covers all OSM node IDs with margin
|
|
||||||
|
|
||||||
class Pass1Collector(osmium.SimpleHandler):
|
|
||||||
"""Collect node IDs referenced by highway+maxspeed ways."""
|
|
||||||
def __init__(self):
|
|
||||||
super().__init__()
|
|
||||||
bs_size = MAX_NODE_ID // 8 + 1
|
|
||||||
sys.stderr.write(f" Allocating bitset: {bs_size/1073741824:.1f} GB\n")
|
|
||||||
self.bitset = bytearray(bs_size)
|
|
||||||
self.way_count = 0
|
|
||||||
|
|
||||||
def way(self, w):
|
|
||||||
if "highway" not in w.tags:
|
|
||||||
return
|
|
||||||
speed = parse_speed(w.tags.get("maxspeed", ""))
|
|
||||||
if speed is None:
|
|
||||||
return
|
|
||||||
self.way_count += 1
|
|
||||||
for n in w.nodes:
|
|
||||||
nid = n.ref
|
|
||||||
if nid >= MAX_NODE_ID:
|
|
||||||
continue
|
|
||||||
byte_idx = nid >> 3
|
|
||||||
bit_idx = nid & 7
|
|
||||||
self.bitset[byte_idx] |= (1 << bit_idx)
|
|
||||||
if self.way_count % 1_000_000 == 0:
|
|
||||||
sys.stderr.write(f" pass1: {self.way_count//1_000_000}M ways\n")
|
|
||||||
|
|
||||||
def test(self, nid):
|
|
||||||
if nid >= MAX_NODE_ID:
|
|
||||||
return False
|
|
||||||
return (self.bitset[nid >> 3] >> (nid & 7)) & 1
|
|
||||||
|
|
||||||
|
|
||||||
class Pass2Builder(osmium.SimpleHandler):
|
|
||||||
"""Pass 2: collect matching nodes, then process ways."""
|
|
||||||
def __init__(self, bitset_test, outdir):
|
|
||||||
super().__init__()
|
|
||||||
self.bitset_test = bitset_test
|
|
||||||
self.outdir = outdir
|
|
||||||
|
|
||||||
self.node_ids = np.empty(200_000_000, dtype=np.uint64)
|
|
||||||
self.node_lons = np.empty(200_000_000, dtype=np.float32)
|
|
||||||
self.node_lats = np.empty(200_000_000, dtype=np.float32)
|
|
||||||
self.node_count = 0
|
|
||||||
self.node_phase = True
|
|
||||||
|
|
||||||
self.sorted_ids = None
|
|
||||||
self.sorted_lons = None
|
|
||||||
self.sorted_lats = None
|
|
||||||
|
|
||||||
self.points = {}
|
|
||||||
self.way_count = 0
|
|
||||||
self.point_count = 0
|
|
||||||
self.miss_count = 0
|
|
||||||
|
|
||||||
def _ensure_capacity(self):
|
|
||||||
if self.node_count < len(self.node_ids):
|
|
||||||
return
|
|
||||||
new_size = len(self.node_ids) * 2
|
|
||||||
sys.stderr.write(f" Resizing node arrays to {new_size//1_000_000}M...\n")
|
|
||||||
self.node_ids = np.resize(self.node_ids, new_size)
|
|
||||||
self.node_lons = np.resize(self.node_lons, new_size)
|
|
||||||
self.node_lats = np.resize(self.node_lats, new_size)
|
|
||||||
|
|
||||||
def node(self, n):
|
|
||||||
if not self.node_phase:
|
|
||||||
return
|
|
||||||
if self.bitset_test(n.id):
|
|
||||||
self._ensure_capacity()
|
|
||||||
self.node_ids[self.node_count] = n.id
|
|
||||||
self.node_lons[self.node_count] = n.location.lon
|
|
||||||
self.node_lats[self.node_count] = n.location.lat
|
|
||||||
self.node_count += 1
|
|
||||||
if self.node_count % 50_000_000 == 0:
|
|
||||||
sys.stderr.write(f" pass2 nodes: {self.node_count//1_000_000}M\n")
|
|
||||||
|
|
||||||
def _build_index(self):
|
|
||||||
sys.stderr.write(f" Sorting {self.node_count//1_000_000}M nodes...\n")
|
|
||||||
t0 = time.time()
|
|
||||||
ids = self.node_ids[:self.node_count]
|
|
||||||
lons = self.node_lons[:self.node_count]
|
|
||||||
lats = self.node_lats[:self.node_count]
|
|
||||||
self.node_ids = self.node_lons = self.node_lats = None
|
|
||||||
|
|
||||||
order = np.argsort(ids)
|
|
||||||
self.sorted_ids = ids[order]
|
|
||||||
self.sorted_lons = lons[order]
|
|
||||||
self.sorted_lats = lats[order]
|
|
||||||
del ids, lons, lats, order
|
|
||||||
sys.stderr.write(f" Sort done in {time.time()-t0:.1f}s, {len(self.sorted_ids)//1_000_000}M nodes indexed\n")
|
|
||||||
|
|
||||||
def _lookup(self, nid):
|
|
||||||
idx = np.searchsorted(self.sorted_ids, nid)
|
|
||||||
if idx < len(self.sorted_ids) and self.sorted_ids[idx] == nid:
|
|
||||||
return float(self.sorted_lons[idx]), float(self.sorted_lats[idx])
|
|
||||||
return None, None
|
|
||||||
|
|
||||||
def way(self, w):
|
|
||||||
if self.node_phase:
|
|
||||||
self.node_phase = False
|
|
||||||
self._build_index()
|
|
||||||
|
|
||||||
if "highway" not in w.tags:
|
|
||||||
return
|
|
||||||
speed = parse_speed(w.tags.get("maxspeed", ""))
|
|
||||||
if speed is None:
|
|
||||||
return
|
|
||||||
|
|
||||||
self.way_count += 1
|
|
||||||
for n in w.nodes:
|
|
||||||
lon, lat = self._lookup(n.ref)
|
|
||||||
if lon is None:
|
|
||||||
self.miss_count += 1
|
|
||||||
continue
|
|
||||||
self._add_point(lon, lat, speed)
|
|
||||||
|
|
||||||
if self.way_count % 1_000_000 == 0:
|
|
||||||
sys.stderr.write(f" pass2 ways: {self.way_count//1_000_000}M points: {self.point_count//1_000_000}M\n")
|
|
||||||
|
|
||||||
def _add_point(self, lon, lat, speed):
|
|
||||||
cx = int(math.floor(lon / CELL))
|
|
||||||
cy = int(math.floor(lat / CELL))
|
|
||||||
lonq = int(round((lon - cx * CELL) / CELL * 65535))
|
|
||||||
latq = int(round((lat - cy * CELL) / CELL * 65535))
|
|
||||||
lonq = max(0, min(65535, lonq))
|
|
||||||
latq = max(0, min(65535, latq))
|
|
||||||
sx = min(SUB - 1, lonq * SUB // 65536)
|
|
||||||
sy = min(SUB - 1, latq * SUB // 65536)
|
|
||||||
key = (cx, cy)
|
|
||||||
cell = self.points.get(key)
|
|
||||||
if cell is None:
|
|
||||||
cell = [None] * (SUB * SUB)
|
|
||||||
self.points[key] = cell
|
|
||||||
sub = sx + sy * SUB
|
|
||||||
buf = cell[sub]
|
|
||||||
if buf is None:
|
|
||||||
buf = bytearray()
|
|
||||||
cell[sub] = buf
|
|
||||||
buf += struct.pack("<HHB", lonq, latq, max(0, min(255, speed)))
|
|
||||||
|
|
||||||
def write_grid(self):
|
|
||||||
sys.stderr.write(f"\nWriting grid...\n")
|
|
||||||
os.makedirs(self.outdir, exist_ok=True)
|
|
||||||
index = {}
|
|
||||||
total = 0
|
|
||||||
for (cx, cy), cell in self.points.items():
|
|
||||||
count = sum(0 if b is None else len(b) // 5 for b in cell)
|
|
||||||
offsets = [count] * (SUB * SUB)
|
|
||||||
acc = 0
|
|
||||||
for sub, b in enumerate(cell):
|
|
||||||
if b is None:
|
|
||||||
continue
|
|
||||||
offsets[sub] = acc
|
|
||||||
acc += len(b) // 5
|
|
||||||
for s in range(SUB * SUB - 2, -1, -1):
|
|
||||||
if offsets[s] == count:
|
|
||||||
offsets[s] = offsets[s + 1]
|
|
||||||
buf = bytearray()
|
|
||||||
buf += struct.pack("<i", count)
|
|
||||||
buf += struct.pack("<%di" % (SUB * SUB), *offsets)
|
|
||||||
for b in cell:
|
|
||||||
if b is not None:
|
|
||||||
buf += b
|
|
||||||
fname = f"{cx}_{cy}.bin.gz"
|
|
||||||
path = os.path.join(self.outdir, fname)
|
|
||||||
with open(path, "wb") as f:
|
|
||||||
f.write(gzip.compress(bytes(buf), 6))
|
|
||||||
index[fname] = {"x": cx, "y": cy, "count": count}
|
|
||||||
total += count
|
|
||||||
if len(index) % 500 == 0:
|
|
||||||
sys.stderr.write(f" cells: {len(index)} points: {total//1_000_000}M\n")
|
|
||||||
|
|
||||||
with open(os.path.join(self.outdir, "index.json"), "w") as f:
|
|
||||||
json.dump(index, f)
|
|
||||||
with open(os.path.join(self.outdir, "meta.json"), "w") as f:
|
|
||||||
json.dump({"cell": CELL, "sub": SUB, "step_m": 0, "points": total,
|
|
||||||
"files": len(index)}, f)
|
|
||||||
sys.stderr.write(f"OK: {total} points in {len(index)} cells -> {self.outdir}\n")
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
if len(sys.argv) != 3:
|
|
||||||
print(f"usage: {sys.argv[0]} <planet.pbf> <outdir>")
|
|
||||||
sys.exit(1)
|
|
||||||
pbf_path = sys.argv[1]
|
|
||||||
outdir = sys.argv[2]
|
|
||||||
|
|
||||||
# === PASS 1 ===
|
|
||||||
sys.stderr.write("=== PASS 1: Collect node IDs from highway+maxspeed ways ===\n")
|
|
||||||
t0 = time.time()
|
|
||||||
c = Pass1Collector()
|
|
||||||
c.apply_file(pbf_path, locations=False)
|
|
||||||
sys.stderr.write(f"Pass 1 done: {c.way_count} ways, {time.time()-t0:.0f}s\n")
|
|
||||||
|
|
||||||
# === PASS 2 ===
|
|
||||||
sys.stderr.write("=== PASS 2: Collect nodes + process ways ===\n")
|
|
||||||
t1 = time.time()
|
|
||||||
b = Pass2Builder(c.test, outdir)
|
|
||||||
b.apply_file(pbf_path, locations=False)
|
|
||||||
if not b.node_phase:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
sys.stderr.write("WARNING: no ways processed (node phase never ended)\n")
|
|
||||||
sys.stderr.write(f"Pass 2 done: {b.node_count} nodes, {b.way_count} ways, {b.point_count} points, {time.time()-t1:.0f}s\n")
|
|
||||||
|
|
||||||
# === WRITE GRID ===
|
|
||||||
t2 = time.time()
|
|
||||||
b.write_grid()
|
|
||||||
sys.stderr.write(f"Grid written in {time.time()-t0:.0f}s total\n")
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
|
|
@ -1,48 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
set -e
|
|
||||||
cd /home/riricdev/zektyc/data/maxspeed
|
|
||||||
|
|
||||||
echo "=== Étape 1: Filtrer highways du planet ==="
|
|
||||||
if [ ! -f planet-highways.pbf ]; then
|
|
||||||
echo "Extraction des highways..."
|
|
||||||
osmium tags-filter planet-latest.osm.pbf w/highway -o planet-highways.pbf --overwrite
|
|
||||||
ls -lh planet-highways.pbf
|
|
||||||
else
|
|
||||||
echo "planet-highways.pbf existe déjà"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "=== Étape 2: Découper en bandes longitude ==="
|
|
||||||
mkdir -p bands raw
|
|
||||||
|
|
||||||
# 12 bandes de 30° : -180..-150, -150..-120, ..., 150..180
|
|
||||||
BAND_SIZE=30
|
|
||||||
for i in $(seq 0 11); do
|
|
||||||
LON_START=$(( -180 + i * BAND_SIZE ))
|
|
||||||
LON_END=$(( LON_START + BAND_SIZE ))
|
|
||||||
BAND="band_${LON_START}_${LON_END}"
|
|
||||||
|
|
||||||
if [ -f "raw/${BAND}.raw" ]; then
|
|
||||||
echo " ${BAND} déjà fait, skip"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ ! -f "bands/${BAND}.pbf" ]; then
|
|
||||||
echo " Extraction bande ${LON_START}..${LON_END}..."
|
|
||||||
osmium extract -b ${LON_START},-60,${LON_END},85 planet-highways.pbf -o "bands/${BAND}.pbf" --overwrite
|
|
||||||
ls -lh "bands/${BAND}.pbf"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo " Build bande ${LON_START}..${LON_END}..."
|
|
||||||
./build_band "bands/${BAND}.pbf" "raw/${BAND}.raw"
|
|
||||||
ls -lh "raw/${BAND}.raw"
|
|
||||||
|
|
||||||
rm -f "bands/${BAND}.pbf"
|
|
||||||
echo ""
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "=== Étape 3: Merge toutes les bandes ==="
|
|
||||||
./merge_grid grid-planet raw/*.raw
|
|
||||||
|
|
||||||
echo "=== Terminé! ==="
|
|
||||||
ls -lh grid-planet/
|
|
||||||
|
|
@ -1,140 +0,0 @@
|
||||||
#include <cstdio>
|
|
||||||
#include <cstdint>
|
|
||||||
#include <cstring>
|
|
||||||
#include <string>
|
|
||||||
#include <vector>
|
|
||||||
#include <algorithm>
|
|
||||||
#include <filesystem>
|
|
||||||
#include <glob.h>
|
|
||||||
|
|
||||||
namespace fs = std::filesystem;
|
|
||||||
|
|
||||||
static constexpr double CELL = 0.5;
|
|
||||||
static constexpr int SUB = 32;
|
|
||||||
static constexpr double LON_MIN = -180.0;
|
|
||||||
static constexpr double LAT_MIN = -60.0;
|
|
||||||
static constexpr double LON_MAX = 180.0;
|
|
||||||
static constexpr double LAT_MAX = 85.0;
|
|
||||||
|
|
||||||
struct RawPoint {
|
|
||||||
int32_t col;
|
|
||||||
int32_t row;
|
|
||||||
uint16_t lonq;
|
|
||||||
uint16_t latq;
|
|
||||||
uint8_t speed;
|
|
||||||
};
|
|
||||||
|
|
||||||
struct Point64 {
|
|
||||||
uint16_t lonq;
|
|
||||||
uint16_t latq;
|
|
||||||
uint8_t speed;
|
|
||||||
};
|
|
||||||
|
|
||||||
static bool cmp_cell(const RawPoint& a, const RawPoint& b) {
|
|
||||||
if (a.col != b.col) return a.col < b.col;
|
|
||||||
if (a.row != b.row) return a.row < b.row;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
int main(int argc, char* argv[]) {
|
|
||||||
if (argc < 3) {
|
|
||||||
fprintf(stderr, "Usage: %s <output_dir> <raw_file1> [raw_file2] ...\n", argv[0]);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
const char* outdir = argv[1];
|
|
||||||
fs::create_directories(outdir);
|
|
||||||
|
|
||||||
std::vector<RawPoint> all_points;
|
|
||||||
|
|
||||||
for (int i = 2; i < argc; i++) {
|
|
||||||
FILE* f = fopen(argv[i], "rb");
|
|
||||||
if (!f) { fprintf(stderr, "Cannot open %s\n", argv[i]); continue; }
|
|
||||||
|
|
||||||
fseek(f, 0, SEEK_END);
|
|
||||||
long fsize = ftell(f);
|
|
||||||
fseek(f, 0, SEEK_SET);
|
|
||||||
|
|
||||||
uint64_t before = all_points.size();
|
|
||||||
uint64_t npoints = fsize / sizeof(RawPoint);
|
|
||||||
all_points.resize(before + npoints);
|
|
||||||
fread(all_points.data() + before, sizeof(RawPoint), npoints, f);
|
|
||||||
fclose(f);
|
|
||||||
|
|
||||||
fprintf(stderr, "Loaded %s: %lu points (total: %lu)\n", argv[i], npoints, all_points.size());
|
|
||||||
}
|
|
||||||
|
|
||||||
fprintf(stderr, "Sorting %lu points...\n", all_points.size());
|
|
||||||
std::sort(all_points.begin(), all_points.end(), cmp_cell);
|
|
||||||
fprintf(stderr, "Sort done. Writing cells...\n");
|
|
||||||
|
|
||||||
std::string index_path = std::string(outdir) + "/index.bin";
|
|
||||||
FILE* index = fopen(index_path.c_str(), "wb");
|
|
||||||
if (!index) { fprintf(stderr, "Cannot write index\n"); return 1; }
|
|
||||||
|
|
||||||
uint32_t magic = 0x5A4D4150;
|
|
||||||
fwrite(&magic, 4, 1, index);
|
|
||||||
uint16_t version = 2;
|
|
||||||
fwrite(&version, 2, 1, index);
|
|
||||||
uint16_t sub = SUB;
|
|
||||||
fwrite(&sub, 2, 1, index);
|
|
||||||
fwrite(&CELL, 8, 1, index);
|
|
||||||
fwrite(&LON_MIN, 8, 1, index);
|
|
||||||
fwrite(&LAT_MIN, 8, 1, index);
|
|
||||||
fwrite(&LON_MAX, 8, 1, index);
|
|
||||||
fwrite(&LAT_MAX, 8, 1, index);
|
|
||||||
|
|
||||||
uint32_t num_cells = 0;
|
|
||||||
fseek(index, 36, SEEK_SET);
|
|
||||||
fwrite(&num_cells, 4, 1, index);
|
|
||||||
fseek(index, 0, SEEK_END);
|
|
||||||
|
|
||||||
int32_t cur_col = -1, cur_row = -1;
|
|
||||||
FILE* cur_file = nullptr;
|
|
||||||
uint32_t cell_points = 0;
|
|
||||||
std::string cur_key;
|
|
||||||
|
|
||||||
for (uint64_t i = 0; i <= all_points.size(); i++) {
|
|
||||||
bool new_cell = (i == all_points.size() ||
|
|
||||||
all_points[i].col != cur_col ||
|
|
||||||
all_points[i].row != cur_row);
|
|
||||||
|
|
||||||
if (new_cell && cur_file) {
|
|
||||||
fclose(cur_file);
|
|
||||||
uint32_t klen = static_cast<uint32_t>(cur_key.size());
|
|
||||||
fwrite(&klen, 4, 1, index);
|
|
||||||
fwrite(cur_key.data(), 1, klen, index);
|
|
||||||
fwrite(&cell_points, 4, 1, index);
|
|
||||||
num_cells++;
|
|
||||||
if (num_cells % 500 == 0) {
|
|
||||||
fprintf(stderr, " %u cells written...\n", num_cells);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (i == all_points.size()) break;
|
|
||||||
|
|
||||||
if (new_cell) {
|
|
||||||
cur_col = all_points[i].col;
|
|
||||||
cur_row = all_points[i].row;
|
|
||||||
cur_key = std::to_string(cur_col) + "_" + std::to_string(cur_row);
|
|
||||||
std::string cellfile = std::string(outdir) + "/cell_" + cur_key + ".bin";
|
|
||||||
cur_file = fopen(cellfile.c_str(), "wb");
|
|
||||||
cell_points = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
Point64 p;
|
|
||||||
p.lonq = all_points[i].lonq;
|
|
||||||
p.latq = all_points[i].latq;
|
|
||||||
p.speed = all_points[i].speed;
|
|
||||||
fwrite(&p, sizeof(Point64), 1, cur_file);
|
|
||||||
cell_points++;
|
|
||||||
}
|
|
||||||
|
|
||||||
fseek(index, 36, SEEK_SET);
|
|
||||||
fwrite(&num_cells, 4, 1, index);
|
|
||||||
fclose(index);
|
|
||||||
|
|
||||||
fprintf(stderr, "Total cells: %u\n", num_cells);
|
|
||||||
fprintf(stderr, "Grid written to %s\n", outdir);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
@ -1,5 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
exec python3 -u /home/riricdev/zektyc/data/maxspeed/build_world.py \
|
|
||||||
/home/riricdev/zektyc/data/maxspeed/planet-latest.osm.pbf \
|
|
||||||
/home/riricdev/zektyc/data/maxspeed/grid-planet/ \
|
|
||||||
>> /home/riricdev/zektyc/data/maxspeed/build-planet.log 2>&1
|
|
||||||
|
|
@ -1,126 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
# Zektyc — protection contre les anomalies de trafic (DoS/DDoS L4, SYN floods)
|
|
||||||
#
|
|
||||||
# Mise au point : NE COUPE QUE lorsque le réseau est réellement saturé, au bord
|
|
||||||
# de tomber. On ne réagit plus à un simple pic de débit (un burst bien absorbé
|
|
||||||
# qui ne sature pas le lien ne doit PAS déclencher). On se base sur le DÉBIT RX
|
|
||||||
# mesuré en octets par rapport au PLAFOND DE BANDE PASSANTE du lien (étalonné) :
|
|
||||||
# - on coupe seulement quand le débit RX soutenu approche le plafond
|
|
||||||
# (le lien est sur le point d'être saturé / de ralentir vraiment),
|
|
||||||
# - pendant plus de PERSIST ticks consécutifs (pas un pic bref).
|
|
||||||
# Le funnel + playit sont réactivés automatiquement quand ça redescend.
|
|
||||||
set -u
|
|
||||||
|
|
||||||
# Webhook Discord chargé depuis un fichier local gitignoré (jamais commité).
|
|
||||||
source /home/riricdev/zektyc/scripts/.webhooks
|
|
||||||
WEBHOOK="${ANTIFLOOD_WEBHOOK:-}"
|
|
||||||
STATE_FILE="/home/riricdev/zektyc/scripts/.synstate"
|
|
||||||
BASE_FILE="/home/riricdev/zektyc/scripts/.synbase"
|
|
||||||
LOG="/var/log/zektyc-syn.log"
|
|
||||||
|
|
||||||
IFACE_ETH="eth0" # interface réseau publique (les SYN floods y arrivent)
|
|
||||||
IFACE_TS="tailscale0" # interface tailscale (légitime)
|
|
||||||
|
|
||||||
INTERVAL=2 # période d'échantillonnage (s)
|
|
||||||
|
|
||||||
# Plafond de bande passante DOWN mesuré via speedtest (~80 Mbit/s ≈ 10 Mo/s).
|
|
||||||
# Tout le trafic entrant (SYN floods compris) passe par eth0 et sature ce plafond.
|
|
||||||
# 10000000 octets/s × INTERVAL(2) = octets par tick au plafond.
|
|
||||||
CAP_BYTES=$(( 10000000 * INTERVAL ))
|
|
||||||
|
|
||||||
# Fraction du plafond au-delà de laquelle on considère le lien "au bord de
|
|
||||||
# tomber" (0.90 = saturé à 90 %). Un burst bref ne suffit pas : il faut tenir
|
|
||||||
# au-dessus pendant PERSIST ticks consécutifs.
|
|
||||||
CAP_RATIO=0.90
|
|
||||||
PERSIST=4 # ticks consécutifs au-dessus du seuil requis avant de couper
|
|
||||||
COOLDOWN_UP=45 # (s) attendre avant de tenter un rétablissement une fois calme
|
|
||||||
|
|
||||||
log() { printf '%s %s\n' "$(date +%F\ %T)" "$1" >>"$LOG"; }
|
|
||||||
notify() { printf '{"content":"%s"}' "$1" | curl -s -m 10 -H "Content-Type: application/json" -X POST "$WEBHOOK" -d @- >/dev/null 2>&1 || true; }
|
|
||||||
|
|
||||||
# Lit le nombre TOTAL d'OCTETS reçus (RX) sur l'interface donnée, depuis
|
|
||||||
# /proc/net/dev (2e champ après le nom d'interface, colonne RX bytes).
|
|
||||||
# La saturation de bande passante se mesure en OCTETS, pas en paquets.
|
|
||||||
read_rx() { awk -v ifx="$1" '$0 ~ ifx {print $2}' /proc/net/dev; }
|
|
||||||
|
|
||||||
# Vrai si le funnel TCP du site (:443) est actif (relais brut + PROXY protocol v2
|
|
||||||
# vers nginx 127.0.0.1:444). Le nom de domaine est suivi de ":443" - on le cherche
|
|
||||||
# explicitement pour ne pas matcher l'entree fantome sur un autre port (:10000).
|
|
||||||
funnel_active() { tailscale funnel status 2>/dev/null | grep -qE 'tcp://riricdev\.tail5ea5cd\.ts\.net:443'; }
|
|
||||||
|
|
||||||
funnel_off() {
|
|
||||||
# coupe le funnel tailscale (site web) — mode TCP brut + PROXY protocol
|
|
||||||
if funnel_active; then
|
|
||||||
tailscale funnel --tcp=443 off >/dev/null 2>&1
|
|
||||||
log "Anomalie de trafic : funnel tailscale coupé"
|
|
||||||
fi
|
|
||||||
# coupe l'entrée playit.gg (serveurs Minecraft publics)
|
|
||||||
if systemctl is-active --quiet playit.service; then
|
|
||||||
systemctl stop playit.service >/dev/null 2>&1
|
|
||||||
log "Anomalie de trafic : entrée playit coupée"
|
|
||||||
fi
|
|
||||||
notify "🔴 **Zektyc** — trafic anormal détecté, accès temporairement suspendu."
|
|
||||||
}
|
|
||||||
|
|
||||||
funnel_on() {
|
|
||||||
# réactive le funnel tailscale (site web) — TCP brut + PPv2 vers nginx :444
|
|
||||||
if ! funnel_active; then
|
|
||||||
tailscale funnel --bg --proxy-protocol=2 --tcp=443 tcp://127.0.0.1:444 >/dev/null 2>&1
|
|
||||||
log "Retour à la normale : funnel tailscale réactivé"
|
|
||||||
fi
|
|
||||||
# réactive l'entrée playit.gg (serveurs Minecraft publics)
|
|
||||||
if ! systemctl is-active --quiet playit.service; then
|
|
||||||
systemctl start playit.service >/dev/null 2>&1
|
|
||||||
log "Retour à la normale : entrée playit réactivée"
|
|
||||||
fi
|
|
||||||
notify "🟢 **Zektyc** — accès rétabli."
|
|
||||||
}
|
|
||||||
|
|
||||||
# --- init ---
|
|
||||||
now=$(read_rx "$IFACE_ETH")
|
|
||||||
prev=$now
|
|
||||||
[[ ! -f "$STATE_FILE" ]] && echo "up" >"$STATE_FILE"
|
|
||||||
state="$(<"$STATE_FILE")"
|
|
||||||
streak=0
|
|
||||||
down_since=0
|
|
||||||
|
|
||||||
# seuil de saturation en octets/tic
|
|
||||||
thresh=$(( CAP_BYTES > 0 ? CAP_BYTES : 1 ))
|
|
||||||
|
|
||||||
while true; do
|
|
||||||
now=$(read_rx "$IFACE_ETH")
|
|
||||||
delta=$(( now - prev ))
|
|
||||||
prev=$now
|
|
||||||
[[ "$delta" -lt 0 ]] && delta=0
|
|
||||||
state="$(<"$STATE_FILE")"
|
|
||||||
|
|
||||||
# --- saturation : débit RX au-dessus de CAP_RATIO × plafond du lien ---
|
|
||||||
# Un SYN flood qui sature le lien fait monter le débit RX vers le plafond.
|
|
||||||
# On ne coupe que si ça tient PERSIST ticks consécutifs (pas un pic bref).
|
|
||||||
sat=0
|
|
||||||
# delta(thresh×CAP_RATIO) ⇔ delta*100 ≥ thresh×(CAP_RATIO×100)
|
|
||||||
if (( delta * 100 >= thresh * 90 )); then
|
|
||||||
sat=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$sat" -eq 1 ]]; then
|
|
||||||
streak=$(( streak + 1 ))
|
|
||||||
else
|
|
||||||
streak=0
|
|
||||||
fi
|
|
||||||
|
|
||||||
# --- actions ---
|
|
||||||
if [[ "$state" == "up" && "$streak" -ge "$PERSIST" ]]; then
|
|
||||||
echo down >"$STATE_FILE"
|
|
||||||
down_since=$(date +%s)
|
|
||||||
funnel_off
|
|
||||||
elif [[ "$state" == "down" ]]; then
|
|
||||||
now_s=$(date +%s)
|
|
||||||
if (( now_s - down_since >= COOLDOWN_UP && streak == 0 )); then
|
|
||||||
echo up >"$STATE_FILE"
|
|
||||||
funnel_on
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
sleep "$INTERVAL"
|
|
||||||
done
|
|
||||||
|
|
@ -1,189 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
# Zektyc — moniteur d'IP suspectes (staff only)
|
|
||||||
#
|
|
||||||
# Toutes les 30 s, analyse deux sources pour détecter des IP anormales qui
|
|
||||||
# tapent sur la stack, et envoie un rapport (embed) sur le webhook Discord staff.
|
|
||||||
#
|
|
||||||
# Source 1 — pare-feu netfilter : log kernel "UFW BLOCK" (vraies IP externes,
|
|
||||||
# probes, scanners, SYN floods résiduels au niveau réseau).
|
|
||||||
#
|
|
||||||
# Source 2 — reverse-proxy nginx : refus HTTP 429/503 émis par les limites
|
|
||||||
# par IP réelle (limit_req / limit_conn). Comme la terminaison TLS a
|
|
||||||
# été déplacée vers nginx (mode funnel tcp brut + PROXY protocol),
|
|
||||||
# c'est ICI qu'apparaissent les attaquants d'applicatif (bursts,
|
|
||||||
# h2load, etc.) que le pare-feu réseau ne voit plus. On relit le log
|
|
||||||
# à partir d'un offset pour ne traiter que les lignes nouvelles.
|
|
||||||
#
|
|
||||||
# Anti-re-spam : une IP déjà signalée n'est renvoyée que si son compteur de
|
|
||||||
# tentatives a augmenté depuis le dernier rapport.
|
|
||||||
set -o pipefail
|
|
||||||
|
|
||||||
# Webhook Discord chargé depuis un fichier local gitignoré (jamais commité).
|
|
||||||
source /home/riricdev/zektyc/scripts/.webhooks
|
|
||||||
WEBHOOK="${IPWATCH_WEBHOOK:-}"
|
|
||||||
INTERVAL=30
|
|
||||||
STATE="/home/riricdev/zektyc/scripts/.ipwatch-state" # IP:compteur déjà signalés
|
|
||||||
NGINX_LOG="/var/log/nginx/access.log"
|
|
||||||
# Taille (octets) du bloc de fin du log relue à chaque cycle puis filtrée au
|
|
||||||
# timestamp : couvre INTERVAL s même au pic (h2load). Rotation-safe, pas d'offset.
|
|
||||||
TL_BACK=16777216 # 16 Mo
|
|
||||||
|
|
||||||
# IP publique ? 0=oui(suspecte possible) 1=non. On exclut aussi le CGNAT
|
|
||||||
# Tailscale (100.64.0.0/10) pour ne pas signaler le VPN légitime / les tests.
|
|
||||||
is_public() {
|
|
||||||
local ip="$1" a b c d
|
|
||||||
IFS='.' read -r a b c d <<<"$ip"
|
|
||||||
[[ "$a" == "127" ]] && return 1
|
|
||||||
[[ "$a" == "10" ]] && return 1
|
|
||||||
[[ "$a" == "192" && "$b" == "168" ]] && return 1
|
|
||||||
[[ "$a" == "172" && "$b" -ge 16 && "$b" -le 31 ]] && return 1
|
|
||||||
[[ "$a" == "100" && "$b" -ge 64 && "$b" -le 127 ]] && return 1
|
|
||||||
[[ "$a" == "169" && "$b" == "254" ]] && return 1
|
|
||||||
[[ "$a" -ge 224 && "$a" -le 239 ]] && return 1
|
|
||||||
[[ "$a" == "0" || "$a" == "255" ]] && return 1
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# Envoie l'embed. Retour 0 si envoyé.
|
|
||||||
send_report() {
|
|
||||||
local report_txt="$1" now
|
|
||||||
now=$(date +"%Y-%m-%d %H:%M:%S")
|
|
||||||
local json
|
|
||||||
json=$(jq -n \
|
|
||||||
--arg title "🚨 IP suspectes détectées" \
|
|
||||||
--arg desc "$report_txt" \
|
|
||||||
--arg time "$now" \
|
|
||||||
--argjson color 15548997 \
|
|
||||||
'{embeds:[{title:$title,description:$desc,color:$color,footer:{text:("Zektyc Sécu — "+$time)}}]}')
|
|
||||||
curl -s -m 10 -H "Content-Type: application/json" -X POST "$WEBHOOK" -d "$json" >/dev/null 2>&1
|
|
||||||
}
|
|
||||||
|
|
||||||
# Charge l'état des IP déjà signalées (format "IP compteur" par ligne)
|
|
||||||
load_state() {
|
|
||||||
declare -gA last=()
|
|
||||||
if [[ -f "$STATE" ]]; then
|
|
||||||
while read -r k v; do last["$k"]="$v"; done <"$STATE"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# Sauvegarde l'état
|
|
||||||
save_state() {
|
|
||||||
: >"$STATE"
|
|
||||||
for k in "${!last[@]}"; do printf '%s %s\n' "$k" "${last[$k]}" >>"$STATE"; done
|
|
||||||
}
|
|
||||||
|
|
||||||
# --- Source 2 : relit la FIN du log nginx et garde les refus des 30 dernières s ---
|
|
||||||
# Pas d'offset : on lit un bloc fixe de la fin du fichier (TL_BACK, dimensionné
|
|
||||||
# pour couvrir INTERVAL s même au pic de trafic) puis on ne garde QUE les lignes
|
|
||||||
# dont l'horodatage est dans la fenêtre [ début, date_actuelle ]. Robuste face
|
|
||||||
# aux rotations/troncatures, aucun offset à resynchroniser manuellement.
|
|
||||||
# Tout le filtrage se fait dans UN SEUL awk (pas de fork par ligne).
|
|
||||||
read_nginx_refus() {
|
|
||||||
local awkprog ip c ts uals from
|
|
||||||
from="$(date -d "@$(( now_s - INTERVAL ))" +%d/%b/%Y:%H:%M:%S)"
|
|
||||||
awkprog='
|
|
||||||
function ispub(ip, a,b,c,d,p,n) {
|
|
||||||
n=split(ip,p,"."); if (n!=4) return 0
|
|
||||||
a=p[1]+0; b=p[2]+0; c=p[3]+0; d=p[4]+0
|
|
||||||
if (a==127||a==10||(a==192&&b==168)||(a==172&&b>=16&&b<=31)\
|
|
||||||
||(a==100&&b>=64&&b<=127)||(a==169&&b==254)) return 0
|
|
||||||
if (a>=224&&a<=239||a==0||a==255) return 0
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
{
|
|
||||||
# Gardes de structure : le buffer tail -c commence en plein milieu d une
|
|
||||||
# ligne -> son 1er enregistrement est un fragment casse. On n accepte que
|
|
||||||
# les lignes completes : IP v4 en tete + timestamp nginx complet en $3.
|
|
||||||
if ($1 !~ /^[0-9]{1,3}(\.[0-9]{1,3}){3}$/) next
|
|
||||||
ts=$3; sub(/^\[/,"",ts)
|
|
||||||
if (ts !~ /^[0-9]{2}\/[A-Za-z]{3}\/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2}/) next
|
|
||||||
# nginx logue en local time (+0200) : on n accepte que les lignes >= debut fenetre
|
|
||||||
if (ts < from) next
|
|
||||||
code=""
|
|
||||||
for (i=1;i<=NF;i++)
|
|
||||||
if ($i ~ /^[0-9]{3}$/ && (i+1)<=NF && $(i+1) ~ /^[0-9]+$/) code=$i
|
|
||||||
if (code!="429" && code!="503") next
|
|
||||||
ip=$1
|
|
||||||
if (!ispub(ip)) next
|
|
||||||
cnt[ip]++
|
|
||||||
n=split($0,q,"\"")
|
|
||||||
ua=(n>=2 ? substr(q[n-1],1,40) : "")
|
|
||||||
if (ua!="" && !((ip SUBSEP ua) in seen)) { seen[ip SUBSEP ua]=1; ual[ip]=(ual[ip]==""?ua:ual[ip]","ua) }
|
|
||||||
last[ip]=ts
|
|
||||||
}
|
|
||||||
END { for (ip in cnt) print ip "|" cnt[ip] "|" last[ip] "|" ual[ip] }
|
|
||||||
'
|
|
||||||
while IFS='|' read -r ip c ts uals; do
|
|
||||||
cnt["$ip"]=$(( ${cnt["$ip"]:-0} + c ))
|
|
||||||
[[ "${protos["$ip"]:-}" != *"http"* ]] && protos["$ip"]="${protos["$ip"]:-}http,"
|
|
||||||
[[ "${dstports["$ip"]:-}" != *"443"* ]] && dstports["$ip"]="${dstports["$ip"]:-}443,"
|
|
||||||
lastts["$ip"]="$ts"
|
|
||||||
[[ -n "$uals" ]] && uas["$ip"]="${uals},"
|
|
||||||
done < <(tail -c "$TL_BACK" "$NGINX_LOG" | awk -v "from=$from" "$awkprog")
|
|
||||||
}
|
|
||||||
|
|
||||||
load_state
|
|
||||||
echo "IP-watch démarré — fenêtre ${INTERVAL}s/signalement (ufw + nginx), webhook staff (actif)."
|
|
||||||
|
|
||||||
while true; do
|
|
||||||
now_s=$(date +%s)
|
|
||||||
since=$(( now_s - INTERVAL ))
|
|
||||||
|
|
||||||
declare -A cnt=()
|
|
||||||
declare -A protos=()
|
|
||||||
declare -A dstports=()
|
|
||||||
declare -A lastts=()
|
|
||||||
declare -A uas=()
|
|
||||||
|
|
||||||
# --- source 1 : blocs netfilter (log kernel UFW BLOCK) ---
|
|
||||||
while IFS= read -r ln; do
|
|
||||||
[[ "$ln" != *"UFW BLOCK"* ]] && continue
|
|
||||||
if [[ "$ln" =~ \ SRC=([0-9.]+)\ ]]; then
|
|
||||||
ip="${BASH_REMATCH[1]}"
|
|
||||||
is_public "$ip" || continue
|
|
||||||
cnt["$ip"]=$(( ${cnt["$ip"]:-0} + 1 ))
|
|
||||||
if [[ "$ln" =~ \ PROTO=([A-Za-z0-9]+)\ ]]; then
|
|
||||||
p="${BASH_REMATCH[1]}"
|
|
||||||
[[ "${protos["$ip"]:-}" != *"$p"* ]] && protos["$ip"]="${protos["$ip"]:-}$p,"
|
|
||||||
fi
|
|
||||||
if [[ "$ln" =~ \ DPT=([0-9]+) ]]; then
|
|
||||||
d="${BASH_REMATCH[1]}"
|
|
||||||
[[ "${dstports["$ip"]:-}" != *"$d"* ]] && dstports["$ip"]="${dstports["$ip"]:-}$d,"
|
|
||||||
fi
|
|
||||||
if [[ "$ln" =~ ^[A-Z][a-z]{2}\ [0-9]{2}\ [0-9]{2}\:[0-9]{2}\:[0-9]{2} ]]; then
|
|
||||||
lastts["$ip"]="${BASH_REMATCH[0]}"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done < <(sudo journalctl -k --no-pager --since "@${since}" 2>/dev/null | grep "UFW BLOCK")
|
|
||||||
|
|
||||||
# --- source 2 : refus HTTP nginx (429/503 par IP réelle) ---
|
|
||||||
read_nginx_refus
|
|
||||||
|
|
||||||
# --- filtrer : ne signaler que les IP nouvelles ou dont le compteur a augmenté ---
|
|
||||||
newfound=0
|
|
||||||
report=""
|
|
||||||
for ip in "${!cnt[@]}"; do
|
|
||||||
c="${cnt[$ip]}"
|
|
||||||
prev="${last["$ip"]:-0}"
|
|
||||||
if (( c > prev )); then
|
|
||||||
proto="${protos[$ip]%,}"
|
|
||||||
dports="${dstports[$ip]%,}"
|
|
||||||
ua="${uas[$ip]%,}"
|
|
||||||
report+=$':shield: **`'"${ip}"$'`** — **'"${c}"$'** tentative(s) bloquée(s)\n'
|
|
||||||
report+=$'└ Sources: `'"${proto:-?}"$'` | Ports cibles: `'"${dports:-?}"$'`\n'
|
|
||||||
if [[ -n "$ua" ]]; then
|
|
||||||
report+=$'└ User-Agent: `'"${ua}"$'`\n'
|
|
||||||
fi
|
|
||||||
report+=$'└ Dernière tentative: `'"${lastts[$ip]:-?}"$'`\n\n'
|
|
||||||
last["$ip"]="$c"
|
|
||||||
((newfound++))
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
if (( newfound > 0 )); then
|
|
||||||
send_report "$report"
|
|
||||||
fi
|
|
||||||
|
|
||||||
save_state
|
|
||||||
sleep "$INTERVAL"
|
|
||||||
done
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
UPNP_URL="http://192.168.1.254:5678/desc/root"
|
|
||||||
LOCAL_IP="192.168.1.50"
|
|
||||||
LOG_TAG="upnp-watch"
|
|
||||||
|
|
||||||
log() {
|
|
||||||
logger -t "$LOG_TAG" "$1"
|
|
||||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"
|
|
||||||
}
|
|
||||||
|
|
||||||
MAPPINGS=$(upnpc -u "$UPNP_URL" -l 2>/dev/null)
|
|
||||||
if [ -z "$MAPPINGS" ]; then
|
|
||||||
log "ERREUR: passerelle UPnP injoignable"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
for PORT in 80 443; do
|
|
||||||
if echo "$MAPPINGS" | grep -q "TCP.*${PORT}->${LOCAL_IP}:${PORT}"; then
|
|
||||||
log "OK: port ${PORT}/TCP déjà ouvert"
|
|
||||||
else
|
|
||||||
log "port ${PORT}/TCP absent -> ouverture"
|
|
||||||
RESULT=$(upnpc -u "$UPNP_URL" -a "$LOCAL_IP" "$PORT" "$PORT" TCP 2>&1 | tail -1)
|
|
||||||
if echo "$RESULT" | grep -q "is redirected"; then
|
|
||||||
log "port ${PORT}/TCP ouvert avec succès"
|
|
||||||
else
|
|
||||||
log "ECHEC ouverture ${PORT}/TCP: $RESULT"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
@ -1,66 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
# Zektyc — moniteur de disponibilité (uptime checker)
|
|
||||||
# Vérifie toutes les 5 secondes le code HTTP de réponse de la homepage.
|
|
||||||
# - Code 2xx/3xx -> online
|
|
||||||
# - Code 000 (échec connexion / funnel coupé), 5xx (erreur serveur) ou autre
|
|
||||||
# code d'échec -> incident technique
|
|
||||||
set -uo pipefail
|
|
||||||
|
|
||||||
URL="https://riricdev.tail5ea5cd.ts.net/"
|
|
||||||
# Webhook Discord chargé depuis un fichier local gitignoré (jamais commité).
|
|
||||||
source /home/riricdev/zektyc/scripts/.webhooks
|
|
||||||
WEBHOOK="${ANTIFLOOD_WEBHOOK:-}"
|
|
||||||
STATE_FILE="/home/riricdev/zektyc/scripts/.uptime-state"
|
|
||||||
INTERVAL=5
|
|
||||||
|
|
||||||
# Un code HTTP est "online" s'il est 2xx/3xx, ou 429 (rate limit = serveur up,
|
|
||||||
# juste throttlé). Tout le reste (000, 4xx hors 429, 5xx) = down.
|
|
||||||
http_is_online() {
|
|
||||||
local code="$1"
|
|
||||||
case "$code" in
|
|
||||||
2*|3*|429) return 0 ;; # 200-399 et 429 : online
|
|
||||||
*) return 1 ;; # 000, 4xx (hors 429), 5xx : down
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
down() {
|
|
||||||
local payload
|
|
||||||
payload=$(printf '{"content":"🔴 **Zektyc** — incident technique."}' )
|
|
||||||
curl -s -m 10 -H "Content-Type: application/json" -X POST "$WEBHOOK" -d "$payload" >/dev/null 2>&1 || true
|
|
||||||
echo down >"$STATE_FILE"
|
|
||||||
}
|
|
||||||
|
|
||||||
up() {
|
|
||||||
local payload
|
|
||||||
payload=$(printf '{"content":"🟢 **Zektyc** — le site est de nouveau en ligne."}' )
|
|
||||||
curl -s -m 10 -H "Content-Type: application/json" -X POST "$WEBHOOK" -d "$payload" >/dev/null 2>&1 || true
|
|
||||||
echo up >"$STATE_FILE"
|
|
||||||
}
|
|
||||||
|
|
||||||
if [[ ! -f "$STATE_FILE" ]]; then
|
|
||||||
echo "unknown" >"$STATE_FILE"
|
|
||||||
fi
|
|
||||||
|
|
||||||
while true; do
|
|
||||||
code="$(curl -s -o /dev/null -w '%{http_code}' -m 10 "$URL" 2>/dev/null || echo 000)"
|
|
||||||
code="${code:-000}"
|
|
||||||
state="$(cat "$STATE_FILE")"
|
|
||||||
|
|
||||||
if http_is_online "$code"; then
|
|
||||||
if [[ "$state" != "up" ]]; then
|
|
||||||
up
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
if [[ "$state" != "down" ]]; then
|
|
||||||
# On n'envoie qu'une fois la notification de panne (quand on était up).
|
|
||||||
if [[ "$state" == "up" ]]; then
|
|
||||||
down
|
|
||||||
else
|
|
||||||
# État initial inconnu : on se contente de mémoriser sans notifier.
|
|
||||||
echo down >"$STATE_FILE"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
sleep "$INTERVAL"
|
|
||||||
done
|
|
||||||
Loading…
Reference in a new issue