diff --git a/.gitignore b/.gitignore index a667d98..fcbd3de 100644 --- a/.gitignore +++ b/.gitignore @@ -48,12 +48,6 @@ data/maxspeed/*.log # Finder (MacOS) folder config .DS_Store -# scripts d'exploitation (gardés sur disque pour les services systemd, hors dépôt) -scripts/ - -# données de build maxspeed (scripts + artefacts, hors dépôt) -data/maxspeed/ - # sensitive / personal duckdns_token_do_not_open.txt diff --git a/data/maxspeed/auto_update.py b/data/maxspeed/auto_update.py new file mode 100755 index 0000000..96e0c04 --- /dev/null +++ b/data/maxspeed/auto_update.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""Auto-update OSM planet PBF and rebuild the maxspeed grid. + +Designed to be triggered weekly (Friday 00:00). Checks the RSS feed for a new +planet release. If none found yet, retries every 5 minutes until one appears. +Then downloads the PBF, verifies MD5, rebuilds the grid, and atomically +swaps the old grid directory for the new one. +""" +import os, sys, time, hashlib, shutil, signal, subprocess, xml.etree.ElementTree as ET +from urllib.request import urlopen, Request +from pathlib import Path +from datetime import datetime, timezone + +DATA_DIR = Path(__file__).resolve().parent +PBF_DIR = DATA_DIR +RSS_URL = "https://planet.openstreetmap.org/pbf/planet-pbf-rss.xml" +BUILD_SCRIPT = DATA_DIR / "build_world.py" +CHECK_INTERVAL = 5 * 60 # 5 min between retries when waiting for a new release +POLL_TIMEOUT = 6 * 3600 # give up after 6 hours if nothing shows up +LOG_PREFIX = "[auto-update]" + +def log(msg): + ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + line = f"{LOG_PREFIX} [{ts}] {msg}" + print(line, flush=True) + with open(DATA_DIR / "auto-update.log", "a") as f: + f.write(line + "\n") + +def fetch_rss(): + """Fetch RSS and return the latest planet filename + pubDate.""" + req = Request(RSS_URL, headers={"User-Agent": "zektyc-auto-updater/1.0"}) + with urlopen(req, timeout=30) as resp: + xml_data = resp.read() + root = ET.fromstring(xml_data) + items = root.findall(".//item") + if not items: + return None, None + item = items[0] + title = (item.findtext("title") or "").strip() + pub_date = (item.findtext("pubDate") or "").strip() + # title is like "planet-260817.osm.pbf.torrent" + filename = title.replace(".torrent", "") + return filename, pub_date + +def get_current_pbf(): + """Return the basename of the planet PBF currently in use.""" + # Check which PBF file exists in PBF_DIR + for f in sorted(PBF_DIR.glob("planet-*.osm.pbf"), reverse=True): + if f.stat().st_size > 10_000_000_000: # > 10 GB = real planet + return f.name + return None + +def md5_file(path): + h = hashlib.md5() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + +def fetch_md5(filename): + """Download the .md5 file from planet.openstreetmap.org.""" + url = f"https://planet.openstreetmap.org/pbf/{filename}.md5" + try: + req = Request(url, headers={"User-Agent": "zektyc-auto-updater/1.0"}) + with urlopen(req, timeout=30) as resp: + text = resp.read().decode().strip() + # format: "hash filename" + return text.split()[0] + except Exception as e: + log(f"WARNING: could not fetch MD5: {e}") + return None + +def download_pbf(filename): + """Download the planet PBF with resume support.""" + dest = PBF_DIR / filename + url = f"https://planet.openstreetmap.org/pbf/{filename}" + existing = dest.stat().st_size if dest.exists() else 0 + headers = {"User-Agent": "zektyc-auto-updater/1.0"} + if existing > 0: + headers["Range"] = f"bytes={existing}-" + log(f"Resuming download from {existing / 1e9:.1f} GB") + req = Request(url, headers=headers) + with urlopen(req, timeout=600) as resp: + mode = "ab" if existing > 0 and resp.status == 206 else "wb" + if mode == "wb": + existing = 0 + total = int(resp.headers.get("Content-Length", 0)) + existing + with open(dest, mode) as f: + downloaded = existing + t0 = time.time() + while True: + chunk = resp.read(1024 * 1024) + if not chunk: + break + f.write(chunk) + downloaded += len(chunk) + elapsed = time.time() - t0 + if elapsed > 0 and downloaded > existing: + speed = (downloaded - existing) / elapsed + eta = (total - downloaded) / speed if speed > 0 else 0 + log(f" {downloaded / 1e9:.2f} / {total / 1e9:.2f} GB " + f"({speed / 1e6:.1f} MB/s, ETA {int(eta // 60)}m{int(eta % 60)}s)") + return dest + +def kill_existing_build(): + """Kill any running build_world.py process.""" + try: + result = subprocess.run( + ["pgrep", "-f", "build_world.py"], + capture_output=True, text=True, timeout=5 + ) + for pid_str in result.stdout.strip().split("\n"): + pid_str = pid_str.strip() + if pid_str: + pid = int(pid_str) + log(f"Killing existing build process PID {pid}") + os.kill(pid, signal.SIGTERM) + time.sleep(2) + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + except Exception as e: + log(f"WARNING: could not kill existing build: {e}") + +def wait_for_build(): + """Wait for any running build_world.py to finish.""" + while True: + result = subprocess.run( + ["pgrep", "-f", "build_world.py"], + capture_output=True, text=True, timeout=5 + ) + if not result.stdout.strip(): + return True + log("Waiting for existing build to finish...") + time.sleep(30) + return False + +def run_build(pbf_path, outdir): + """Run build_world.py.""" + log(f"Starting build: {pbf_path} -> {outdir}") + proc = subprocess.Popen( + [sys.executable, "-u", str(BUILD_SCRIPT), str(pbf_path), str(str(outdir))], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=str(DATA_DIR), + ) + with open(DATA_DIR / "build-planet.log", "a") as logf: + for line in iter(proc.stdout.readline, b""): + decoded = line.decode(errors="replace") + logf.write(decoded) + logf.flush() + # Also echo last chars for our log + proc.wait() + if proc.returncode != 0: + log(f"ERROR: build_world.py exited with code {proc.returncode}") + return False + log("Build completed successfully") + return True + +def atomic_swap(new_grid, target_grid): + """Atomically replace target_grid with new_grid.""" + backup = target_grid.parent / (target_grid.name + "-old") + if backup.exists(): + shutil.rmtree(backup) + if target_grid.exists(): + target_grid.rename(backup) + new_grid.rename(target_grid) + log(f"Swapped {target_grid.name} (backup: {backup.name})") + # Clean old backup after a short delay (keep for 1 hour for safety) + # Actually let's keep it, it's useful if something goes wrong + +def main(): + log("=" * 60) + log("Auto-update started") + + current_pbf = get_current_pbf() + log(f"Current PBF: {current_pbf or 'none'}") + + # Step 1: Check RSS for new release + t_start = time.time() + new_filename = None + while True: + try: + filename, pub_date = fetch_rss() + if filename and filename != current_pbf: + log(f"New release found: {filename} (published: {pub_date})") + new_filename = filename + break + elif filename: + log(f"No new release yet (latest: {filename}, current: {current_pbf})") + else: + log("Could not parse RSS feed") + except Exception as e: + log(f"RSS check failed: {e}") + + elapsed = time.time() - t_start + if elapsed > POLL_TIMEOUT: + log(f"No new release after {elapsed/3600:.1f}h, giving up") + return + log(f"Retrying in {CHECK_INTERVAL // 60} minutes...") + time.sleep(CHECK_INTERVAL) + + # Step 2: Kill any existing build + kill_existing_build() + wait_for_build() + + # Step 3: Download new PBF + log(f"Downloading {new_filename}...") + pbf_path = download_pbf(new_filename) + log(f"Download complete: {pbf_path}") + + # Step 4: Verify MD5 + expected_md5 = fetch_md5(new_filename) + if expected_md5: + log(f"Verifying MD5 (expected: {expected_md5})") + actual_md5 = md5_file(pbf_path) + log(f"Actual MD5: {actual_md5}") + if actual_md5 != expected_md5: + log("ERROR: MD5 mismatch! Aborting.") + return + log("MD5 verified OK") + else: + log("WARNING: skipping MD5 verification") + + # Step 5: Build new grid + grid_tmp = PBF_DIR / "grid-planet-new" + if grid_tmp.exists(): + shutil.rmtree(grid_tmp) + grid_target = PBF_DIR / "grid-planet" + + if not run_build(pbf_path, grid_tmp): + log("ERROR: Build failed, keeping old grid") + if grid_tmp.exists(): + shutil.rmtree(grid_tmp) + return + + # Step 6: Atomic swap + atomic_swap(grid_tmp, grid_target) + + # Step 7: Update .env symlink (if needed) + env_path = Path(__file__).resolve().parent.parent.parent / ".env" + if env_path.exists(): + content = env_path.read_text() + if "grid-europe" in content: + # Don't change .env — it might be intentionally pointing to grid-europe + log("NOTE: .env still points to grid-europe, not auto-updating") + elif "grid-planet" in content: + log(".env already points to grid-planet") + + log("Auto-update complete!") + log("=" * 60) + +if __name__ == "__main__": + signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) + main() diff --git a/data/maxspeed/build_band.cpp b/data/maxspeed/build_band.cpp new file mode 100644 index 0000000..38d0c3c --- /dev/null +++ b/data/maxspeed/build_band.cpp @@ -0,0 +1,138 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +using index_type = osmium::index::map::SparseFileArray; +using location_handler_type = osmium::handler::NodeLocationsForWays; + +static constexpr double CELL = 0.5; +static constexpr double LON_MIN = -180.0; +static constexpr double LAT_MIN = -60.0; +static constexpr double LON_MAX = 180.0; +static constexpr double LAT_MAX = 85.0; + +struct RawPoint { + int32_t col; + int32_t row; + uint16_t lonq; + uint16_t latq; + uint8_t speed; +}; + +static inline uint16_t quantize_lon(double lon) { + return static_cast((lon - LON_MIN) / (LON_MAX - LON_MIN) * 65535.0); +} + +static inline uint16_t quantize_lat(double lat) { + return static_cast((lat - LAT_MIN) / (LAT_MAX - LAT_MIN) * 65535.0); +} + +static inline uint8_t parse_speed_val(const char* s) { + if (!s || !*s) return 0; + char* end = nullptr; + double v = strtod(s, &end); + if (end != s && v > 0 && v <= 255) return static_cast(v); + if (strstr(s, "mph")) { + v = strtod(s, &end); + if (end != s) { + v *= 1.609; + if (v > 0 && v <= 255) return static_cast(v); + } + } + return 0; +} + +class StreamHandler : public osmium::handler::Handler { +public: + FILE* out; + uint64_t way_count = 0; + uint64_t point_count = 0; + + explicit StreamHandler(FILE* f) : out(f) {} + + void way(const osmium::Way& way) { + const char* highway = way.tags().get_value_by_key("highway"); + if (!highway) return; + + const char* ms = way.tags().get_value_by_key("maxspeed"); + uint8_t speed = ms ? parse_speed_val(ms) : 0; + if (speed == 0) return; + + way_count++; + + for (const auto& wn : way.nodes()) { + if (!wn.location().valid()) continue; + + double lon = wn.location().lon(); + double lat = wn.location().lat(); + + if (lon < LON_MIN || lon >= LON_MAX || lat < LAT_MIN || lat >= LAT_MAX) continue; + + RawPoint rp; + rp.col = static_cast((lon - LON_MIN) / CELL); + rp.row = static_cast((lat - LAT_MIN) / CELL); + rp.lonq = quantize_lon(lon); + rp.latq = quantize_lat(lat); + rp.speed = speed; + + fwrite(&rp, sizeof(RawPoint), 1, out); + point_count++; + + if (point_count % 5000000 == 0) { + fprintf(stderr, " %lu points (%lu ways)...\n", point_count, way_count); + } + } + } +}; + +int main(int argc, char* argv[]) { + if (argc < 3) { + fprintf(stderr, "Usage: %s \n", argv[0]); + return 1; + } + + const char* input = argv[1]; + const char* outpath = argv[2]; + + fs::create_directories(fs::path(outpath).parent_path()); + + FILE* out = fopen(outpath, "wb"); + if (!out) { fprintf(stderr, "Cannot open %s\n", outpath); return 1; } + + std::string idx_path = std::string(outpath) + ".idx"; + int idx_fd = ::open(idx_path.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644); + if (idx_fd == -1) { fprintf(stderr, "Cannot create index\n"); fclose(out); return 1; } + + index_type index(idx_fd); + location_handler_type location_handler(index); + StreamHandler handler(out); + + fprintf(stderr, "Reading %s...\n", input); + osmium::io::Reader reader(input); + osmium::apply(reader, location_handler, handler); + reader.close(); + + fclose(out); + ::close(idx_fd); + unlink(idx_path.c_str()); + + fprintf(stderr, "Ways with maxspeed: %lu\n", handler.way_count); + fprintf(stderr, "Points written: %lu\n", handler.point_count); + fprintf(stderr, "Output: %s\n", outpath); + return 0; +} diff --git a/data/maxspeed/build_global.cpp b/data/maxspeed/build_global.cpp new file mode 100644 index 0000000..0ecaf5f --- /dev/null +++ b/data/maxspeed/build_global.cpp @@ -0,0 +1,245 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +using index_type = osmium::index::map::SparseFileArray; +using location_handler_type = osmium::handler::NodeLocationsForWays; + +static constexpr double CELL = 0.5; +static constexpr int SUB = 32; +static constexpr double LON_MIN = -180.0; +static constexpr double LAT_MIN = -60.0; +static constexpr double LON_MAX = 180.0; +static constexpr double LAT_MAX = 85.0; + +struct RawPoint { + int32_t col; + int32_t row; + uint16_t lonq; + uint16_t latq; + uint8_t speed; +}; // 11 bytes + +static inline uint16_t quantize_lon(double lon) { + return static_cast((lon - LON_MIN) / (LON_MAX - LON_MIN) * 65535.0); +} + +static inline uint16_t quantize_lat(double lat) { + return static_cast((lat - LAT_MIN) / (LAT_MAX - LAT_MIN) * 65535.0); +} + +static inline uint8_t parse_speed_val(const char* s) { + if (!s || !*s) return 0; + char* end = nullptr; + double v = strtod(s, &end); + if (end != s && v > 0 && v <= 255) return static_cast(v); + if (strstr(s, "mph")) { + v = strtod(s, &end); + if (end != s) { + v *= 1.609; + if (v > 0 && v <= 255) return static_cast(v); + } + } + return 0; +} + +struct Point64 { + uint16_t lonq; + uint16_t latq; + uint8_t speed; +}; + +class SpeedHandler : public osmium::handler::Handler { +public: + FILE* tmpfile; + uint64_t way_count = 0; + uint64_t point_count = 0; + + explicit SpeedHandler(FILE* f) : tmpfile(f) {} + + void way(const osmium::Way& way) { + const char* highway = way.tags().get_value_by_key("highway"); + if (!highway) return; + + const char* ms = way.tags().get_value_by_key("maxspeed"); + uint8_t speed = ms ? parse_speed_val(ms) : 0; + if (speed == 0) return; + + way_count++; + + for (const auto& wn : way.nodes()) { + if (!wn.location().valid()) continue; + + double lon = wn.location().lon(); + double lat = wn.location().lat(); + + if (lon < LON_MIN || lon >= LON_MAX || lat < LAT_MIN || lat >= LAT_MAX) continue; + + int32_t col = static_cast((lon - LON_MIN) / CELL); + int32_t row = static_cast((lat - LAT_MIN) / CELL); + + RawPoint rp; + rp.col = col; + rp.row = row; + rp.lonq = quantize_lon(lon); + rp.latq = quantize_lat(lat); + rp.speed = speed; + + fwrite(&rp, sizeof(RawPoint), 1, tmpfile); + point_count++; + + if (point_count % 10000000 == 0) { + fprintf(stderr, " %lu points (%lu ways)...\n", point_count, way_count); + } + } + } +}; + +static bool compare_by_cell(const RawPoint& a, const RawPoint& b) { + if (a.col != b.col) return a.col < b.col; + return a.row < b.row; +} + +static void split_and_write(const char* tmppath, const std::string& outdir) { + fprintf(stderr, "Reading temp file into memory for sort...\n"); + + FILE* f = fopen(tmppath, "rb"); + fseek(f, 0, SEEK_END); + long fsize = ftell(f); + fseek(f, 0, SEEK_SET); + + uint64_t npoints = fsize / sizeof(RawPoint); + fprintf(stderr, "Sorting %lu points (%.1f MB)...\n", npoints, fsize / 1048576.0); + + std::vector points(npoints); + fread(points.data(), sizeof(RawPoint), npoints, f); + fclose(f); + + std::sort(points.begin(), points.end(), compare_by_cell); + fprintf(stderr, "Sort done. Writing cells...\n"); + + fs::create_directories(outdir); + + std::string index_path = outdir + "/index.bin"; + FILE* index = fopen(index_path.c_str(), "wb"); + uint32_t magic = 0x5A4D4150; + fwrite(&magic, 4, 1, index); + uint16_t version = 2; + fwrite(&version, 2, 1, index); + uint16_t sub = SUB; + fwrite(&sub, 2, 1, index); + fwrite(&CELL, 8, 1, index); + fwrite(&LON_MIN, 8, 1, index); + fwrite(&LAT_MIN, 8, 1, index); + fwrite(&LON_MAX, 8, 1, index); + fwrite(&LAT_MAX, 8, 1, index); + + uint32_t num_cells = 0; + fseek(index, 36, SEEK_SET); + fwrite(&num_cells, 4, 1, index); + fseek(index, 0, SEEK_END); + + int32_t cur_col = -1, cur_row = -1; + FILE* cur_file = nullptr; + uint32_t cell_points = 0; + std::string cur_key; + + for (uint64_t i = 0; i <= npoints; i++) { + bool new_cell = (i == npoints || points[i].col != cur_col || points[i].row != cur_row); + + if (new_cell && cur_file) { + fclose(cur_file); + + uint32_t klen = static_cast(cur_key.size()); + fwrite(&klen, 4, 1, index); + fwrite(cur_key.data(), 1, klen, index); + fwrite(&cell_points, 4, 1, index); + num_cells++; + + if (num_cells % 1000 == 0) { + fprintf(stderr, " %u cells written...\n", num_cells); + } + } + + if (i == npoints) break; + + if (new_cell) { + cur_col = points[i].col; + cur_row = points[i].row; + cur_key = std::to_string(cur_col) + "_" + std::to_string(cur_row); + std::string cellfile = outdir + "/cell_" + cur_key + ".bin"; + cur_file = fopen(cellfile.c_str(), "wb"); + cell_points = 0; + } + + Point64 p; + p.lonq = points[i].lonq; + p.latq = points[i].latq; + p.speed = points[i].speed; + fwrite(&p, sizeof(Point64), 1, cur_file); + cell_points++; + } + + fseek(index, 36, SEEK_SET); + fwrite(&num_cells, 4, 1, index); + fclose(index); + + fprintf(stderr, "Total cells: %u\n", num_cells); +} + +int main(int argc, char* argv[]) { + if (argc < 3) { + fprintf(stderr, "Usage: %s \n", argv[0]); + return 1; + } + + const char* input = argv[1]; + const char* outdir = argv[2]; + + std::string tmppath = std::string(outdir) + "/_points.tmp"; + fs::create_directories(outdir); + + fprintf(stderr, "Step 1: Reading PBF, writing raw points to temp file...\n"); + FILE* tmpf = fopen(tmppath.c_str(), "w+b"); + if (!tmpf) { fprintf(stderr, "Cannot create temp file\n"); return 1; } + + std::string idx_path = std::string(outdir) + "/_node_index.tmp"; + int idx_fd = ::open(idx_path.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644); + if (idx_fd == -1) { fprintf(stderr, "Cannot create index temp file\n"); return 1; } + + index_type index(idx_fd); + location_handler_type location_handler(index); + SpeedHandler handler(tmpf); + + osmium::io::Reader reader(input); + osmium::apply(reader, location_handler, handler); + reader.close(); + fclose(tmpf); + + fprintf(stderr, "\nWays with maxspeed: %lu\n", handler.way_count); + fprintf(stderr, "Total points: %lu\n", handler.point_count); + + fprintf(stderr, "Step 2: Sort + write grid cells...\n"); + split_and_write(tmppath.c_str(), outdir); + + ::close(idx_fd); + unlink(idx_path.c_str()); + unlink(tmppath.c_str()); + fprintf(stderr, "Done! Grid written to %s\n", outdir); + return 0; +} diff --git a/data/maxspeed/build_maxspeed.py b/data/maxspeed/build_maxspeed.py new file mode 100644 index 0000000..d2642a5 --- /dev/null +++ b/data/maxspeed/build_maxspeed.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""Construit une grille binaire des maxspeed OSM (par route, échantillonnée en points). + +Pipeline : + osmium tags-filter .osm.pbf w/highway -o -roads.osm.pbf + python3 build_maxspeed.py -roads.osm.pbf [--cache=] + +Sortie : /grid.dat + /index.json + /meta.json + - Chaque cellule = 0.5° x 0.5° (lat, lon). + - Blob par cellule (gzip) : int32 count ; 32x32 sous-cellules => int32 offset[1024] ; + puis count points de (lonq:int16, latq:int16, speed:int8) rangés par sous-cellule. + - lonq/latq : quantifiés sur 0..65535 dans la cellule (precision ~0.8 m). +""" +import sys, os, json, gzip, math, struct, re +import osmium +import sqlite3 + +CELL = 0.5 # degrés par cellule +SUB = 32 # sous-cellules par dimension (grille 32x32) +STEP_M_DEFAULT = 40.0 # pas d'échantillonnage le long des routes (m) +STEP_M = STEP_M_DEFAULT + +# maxspeed textuel -> km/h (valeurs par défaut OSM) +ZONE = { + "none": None, + "signals": None, + "variable": None, + "walk": 5, + "living_street": 20, + "urban": 50, "rural": 80, "trunk": 110, "motorway": 130, +} +# codes pays -> (urban, rural) km/h ; motorway sans maxspeed = None (ex. DE) +CC = { + "FR": (50, 80), "BE": (50, 90), "DE": (50, 100), "NL": (50, 100), + "GB": (48, 96), "ES": (50, 90), "IT": (50, 90), "CH": (50, 80), + "AT": (50, 100), "LU": (50, 90), "PT": (50, 90), "PL": (50, 90), + "CZ": (50, 90), "IE": (50, 80), "DK": (50, 80), "SE": (50, 70), + "NO": (50, 80), "FI": (50, 100), "RO": (50, 90), "HU": (50, 90), + "HR": (50, 90), "SI": (50, 90), "SK": (50, 90), "LT": (50, 90), + "LV": (50, 90), "EE": (50, 90), "BG": (50, 90), "GR": (50, 90), + "UA": (50, 90), "RU": (60, 90), "TR": (50, 90), "RS": (50, 80), + "MK": (50, 80), "AL": (40, 90), "BA": (50, 80), "ME": (50, 80), + "IS": (50, 90), +} +# Les highways de service/privées sans maxspeed sont exclues : on ne garde que les +# ways marqués maxspeed (ou avec une valeur par défaut de zone reconnue). + +def parse_speed(v): + if not v: + return None + v = v.strip().lower() + if v in ZONE: + return ZONE[v] + m = __import__("re").match(r"^([0-9]+)(?:\s*(mph|km/h|kmh))?\b", v) + if m: + n = int(m.group(1)) + if m.group(2) == "mph": + return round(n * 1.609344) + return n + # code pays type FR:rural / DE:urban + if ":" in v: + cc, kind = v.split(":", 1) + cc = cc.upper() + if cc in CC and kind in ("urban", "rural"): + return CC[cc][0 if kind == "urban" else 1] + if cc in CC and kind == "motorway": + return None + return None + +class MaxSpeedHandler(osmium.SimpleHandler): + def __init__(self): + super().__init__() + self.points = {} # (cellx, celly) -> dict subcell -> list of (lonq, latq, speed) + self.skipped = 0 + + def way(self, w): + if "highway" not in w.tags: + return + speed = parse_speed(w.tags.get("maxspeed", "")) + if speed is None: + return + locs = [] + for n in w.nodes: + loc = n.location + if loc is not None: + locs.append((loc.lon, loc.lat)) + if len(locs) < 2: + return + self.process_locs(locs, speed) + + def process_locs(self, locs, speed): + if STEP_M <= 0: + for (lon, lat) in locs: + self.add_point(lon, lat, speed) + return + prev = locs[0] + lon, lat = prev + acc = 0.0 + emit = [(lon, lat)] + for node in locs[1:]: + a = haversine(prev[1], prev[0], node[1], node[0]) + acc += a + prev = node + while acc >= STEP_M: + f = 1.0 - (acc - STEP_M) / a if a > 0 else 0.0 + ilon = lon + (node[0] - lon) * f + ilat = lat + (node[1] - lat) * f + emit.append((ilon, ilat)) + acc -= STEP_M + lon, lat = node[0], node[1] + if (lon, lat) != emit[-1]: + emit.append((lon, lat)) + for (el, ea) in emit: + self.add_point(el, ea, speed) + + def add_point(self, lon, lat, speed): + cx = int(math.floor(lon / CELL)) + cy = int(math.floor(lat / CELL)) + lonq = int(round((lon - cx * CELL) / CELL * 65535)) + latq = int(round((lat - cy * CELL) / CELL * 65535)) + lonq = max(0, min(65535, lonq)); latq = max(0, min(65535, latq)) + sx = min(SUB - 1, lonq * SUB // 65536) + sy = min(SUB - 1, latq * SUB // 65536) + key = (cx, cy) + cell = self.points.get(key) + if cell is None: + cell = [None] * (SUB * SUB) + self.points[key] = cell + sub = sx + sy * SUB + buf = cell[sub] + if buf is None: + buf = bytearray() + cell[sub] = buf + buf += struct.pack(" liste (lon, lat) pour LineString (ou Multi).""" + if not isinstance(raw, (bytes, bytearray)) or len(raw) < 8 or raw[:2] != b"GP": + return None + flags = raw[3] + off = 8 + if flags & GP_FLAGS["envelope"]: + etype = (flags >> 2) & 0x03 + if etype != 0: + return None + off += 4 * 8 + bo = "<" if flags & GP_FLAGS["byteorder"] else ">" + if off + 5 > len(raw): + return None + gtype = struct.unpack(bo + "i", raw[off + 1:off + 5])[0] + if gtype == 2: # LineString + return parse_wkb_points(raw, off, bo) + if gtype == 5: # MultiLineString + out = [] + num = struct.unpack(bo + "i", raw[off + 5:off + 9])[0] + p = off + 9 + for _ in range(num): + pts = parse_wkb_points(raw, p, bo) + if pts: + out.extend(pts) + p += 4 + 4 * 8 * (len(pts) if pts else 0) + return out if len(out) >= 2 else None + return None + +def parse_wkb_points(raw, off, bo): + n = struct.unpack(bo + "i", raw[off + 5:off + 9])[0] + out = [] + p = off + 9 + for _ in range(n): + out.append((struct.unpack(bo + "d", raw[p:p + 8])[0], + struct.unpack(bo + "d", raw[p + 8:p + 16])[0])) + p += 16 + return out + +MS_RE = re.compile(r'"maxspeed"=>"([^"]*)"') + +def parse_other_tags(s): + m = MS_RE.search(s or "") + return m.group(1) if m else None + +def load_gpkg(h, gpkg): + con = sqlite3.connect(gpkg) + cur = con.execute("SELECT geom, highway, other_tags FROM lines") + n = 0 + for raw, hw, tags in cur: + if not hw or "maxspeed" not in (tags or ""): + continue + v = parse_other_tags(tags) + speed = parse_speed(v) + if speed is None: + continue + locs = parse_gp_wkb(raw) + if not locs or len(locs) < 2: + continue + h.process_locs(locs, speed) + n += 1 + if n % 500000 == 0: + sys.stderr.write("ways maxspeed: %dM\n" % (n // 1000000)) + con.close() + return n + +def build(src_pbf, out_dir, cache_file=None): + os.makedirs(out_dir, exist_ok=True) + h = MaxSpeedHandler() + if src_pbf.endswith(".gpkg") or src_pbf.endswith(".db"): + n = load_gpkg(h, src_pbf) + print(f"GPKG : {n} ways traités") + else: + if cache_file: + idx = osmium.index.create_map("sparse_file_array,%s" % cache_file) + h.apply_file(src_pbf, locations=osmium.NodeLocationsForWays(idx)) + else: + h.apply_file(src_pbf, locations=True) + # écriture : un fichier par cellule (nom: X_Y.bin.gz) + index + index = {} + total = 0 + for (cx, cy), cell in h.points.items(): + count = sum(0 if b is None else len(b) // 5 for b in cell) + offsets = [count] * (SUB * SUB) + acc = 0 + for sub, b in enumerate(cell): + if b is None: + continue + offsets[sub] = acc + acc += len(b) // 5 + # remplissage arrière : les sous-cellules vides pointent vers la suivante + for s in range(SUB * SUB - 2, -1, -1): + if offsets[s] == count: + offsets[s] = offsets[s + 1] + buf = bytearray() + buf += struct.pack(" {out_dir}") + +if __name__ == "__main__": + cache = None + args = sys.argv[1:] + extra = {} + while args and args[-1].startswith("--"): + a = args.pop() + if a.startswith("--cache="): + cache = a.split("=", 1)[1] + elif a == "--no-sample": + STEP_M = 0 + else: + print(f"Option inconnue: {a}") + sys.exit(1) + if len(args) != 2: + print("usage: build_maxspeed.py [--cache=] [--no-sample]") + sys.exit(1) + build(args[0], args[1], cache) \ No newline at end of file diff --git a/data/maxspeed/build_world.py b/data/maxspeed/build_world.py new file mode 100644 index 0000000..67e1e7b --- /dev/null +++ b/data/maxspeed/build_world.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""Build world maxspeed grid from planet PBF in 2 passes. + +Pass 1: Stream PBF → collect node IDs from highway+maxspeed ways → bitset +Pass 2: Stream PBF → matching nodes → mmap sort → process ways → grid + +Usage: python3 build_world.py planet-latest.osm.pbf grid-planet/ +""" +import sys, os, json, gzip, struct, math, mmap, time +import numpy as np +import osmium + +CELL = 0.5 +SUB = 32 + +ZONE = { + "none": None, "signals": None, "variable": None, "walk": 5, "living_street": 20, + "urban": 50, "rural": 80, "trunk": 110, "motorway": 130, +} +CC = { + "FR": (50,80), "BE": (50,90), "DE": (50,100), "NL": (50,100), + "GB": (48,96), "ES": (50,90), "IT": (50,90), "CH": (50,80), + "AT": (50,100), "LU": (50,90), "PT": (50,90), "PL": (50,90), + "CZ": (50,90), "IE": (50,80), "DK": (50,80), "SE": (50,70), + "NO": (50,80), "FI": (50,100), "RO": (50,90), "HU": (50,90), + "HR": (50,90), "SI": (50,90), "SK": (50,90), "LT": (50,90), + "LV": (50,90), "EE": (50,90), "BG": (50,90), "GR": (50,90), + "UA": (50,90), "RU": (60,90), "TR": (50,90), "RS": (50,80), + "MK": (50,80), "AL": (40,90), "BA": (50,80), "ME": (50,80), + "IS": (50,90), "US": (40,105), "CA": (40,100), "MX": (40,90), + "BR": (40,80), "AR": (40,110), "CL": (40,120), "CO": (40,90), + "PE": (40,90), "JP": (40,60), "CN": (40,100), "IN": (40,80), + "AU": (40,100), "NZ": (40,100), "ZA": (40,100), "EG": (50,90), + "NG": (50,80), "KE": (50,80), "MA": (50,90), "DZ": (50,90), + "TN": (50,90), "LY": (50,90), +} + +def parse_speed(v): + if not v: + return None + v = v.strip().lower() + if v in ZONE: + return ZONE[v] + m = __import__("re").match(r"^([0-9]+)(?:\s*(mph|km/h|kmh))?\b", v) + if m: + n = int(m.group(1)) + if m.group(2) == "mph": + return round(n * 1.609344) + return n + if ":" in v: + cc, kind = v.split(":", 1) + cc = cc.upper() + if cc in CC and kind in ("urban", "rural"): + return CC[cc][0 if kind == "urban" else 1] + if cc in CC and kind == "motorway": + return None + return None + +MAX_NODE_ID = 25_000_000_000 # 25 billion — covers all OSM node IDs with margin + +class Pass1Collector(osmium.SimpleHandler): + """Collect node IDs referenced by highway+maxspeed ways.""" + def __init__(self): + super().__init__() + bs_size = MAX_NODE_ID // 8 + 1 + sys.stderr.write(f" Allocating bitset: {bs_size/1073741824:.1f} GB\n") + self.bitset = bytearray(bs_size) + self.way_count = 0 + + def way(self, w): + if "highway" not in w.tags: + return + speed = parse_speed(w.tags.get("maxspeed", "")) + if speed is None: + return + self.way_count += 1 + for n in w.nodes: + nid = n.ref + if nid >= MAX_NODE_ID: + continue + byte_idx = nid >> 3 + bit_idx = nid & 7 + self.bitset[byte_idx] |= (1 << bit_idx) + if self.way_count % 1_000_000 == 0: + sys.stderr.write(f" pass1: {self.way_count//1_000_000}M ways\n") + + def test(self, nid): + if nid >= MAX_NODE_ID: + return False + return (self.bitset[nid >> 3] >> (nid & 7)) & 1 + + +class Pass2Builder(osmium.SimpleHandler): + """Pass 2: collect matching nodes, then process ways.""" + def __init__(self, bitset_test, outdir): + super().__init__() + self.bitset_test = bitset_test + self.outdir = outdir + + self.node_ids = np.empty(200_000_000, dtype=np.uint64) + self.node_lons = np.empty(200_000_000, dtype=np.float32) + self.node_lats = np.empty(200_000_000, dtype=np.float32) + self.node_count = 0 + self.node_phase = True + + self.sorted_ids = None + self.sorted_lons = None + self.sorted_lats = None + + self.points = {} + self.way_count = 0 + self.point_count = 0 + self.miss_count = 0 + + def _ensure_capacity(self): + if self.node_count < len(self.node_ids): + return + new_size = len(self.node_ids) * 2 + sys.stderr.write(f" Resizing node arrays to {new_size//1_000_000}M...\n") + self.node_ids = np.resize(self.node_ids, new_size) + self.node_lons = np.resize(self.node_lons, new_size) + self.node_lats = np.resize(self.node_lats, new_size) + + def node(self, n): + if not self.node_phase: + return + if self.bitset_test(n.id): + self._ensure_capacity() + self.node_ids[self.node_count] = n.id + self.node_lons[self.node_count] = n.location.lon + self.node_lats[self.node_count] = n.location.lat + self.node_count += 1 + if self.node_count % 50_000_000 == 0: + sys.stderr.write(f" pass2 nodes: {self.node_count//1_000_000}M\n") + + def _build_index(self): + sys.stderr.write(f" Sorting {self.node_count//1_000_000}M nodes...\n") + t0 = time.time() + ids = self.node_ids[:self.node_count] + lons = self.node_lons[:self.node_count] + lats = self.node_lats[:self.node_count] + self.node_ids = self.node_lons = self.node_lats = None + + order = np.argsort(ids) + self.sorted_ids = ids[order] + self.sorted_lons = lons[order] + self.sorted_lats = lats[order] + del ids, lons, lats, order + sys.stderr.write(f" Sort done in {time.time()-t0:.1f}s, {len(self.sorted_ids)//1_000_000}M nodes indexed\n") + + def _lookup(self, nid): + idx = np.searchsorted(self.sorted_ids, nid) + if idx < len(self.sorted_ids) and self.sorted_ids[idx] == nid: + return float(self.sorted_lons[idx]), float(self.sorted_lats[idx]) + return None, None + + def way(self, w): + if self.node_phase: + self.node_phase = False + self._build_index() + + if "highway" not in w.tags: + return + speed = parse_speed(w.tags.get("maxspeed", "")) + if speed is None: + return + + self.way_count += 1 + for n in w.nodes: + lon, lat = self._lookup(n.ref) + if lon is None: + self.miss_count += 1 + continue + self._add_point(lon, lat, speed) + + if self.way_count % 1_000_000 == 0: + sys.stderr.write(f" pass2 ways: {self.way_count//1_000_000}M points: {self.point_count//1_000_000}M\n") + + def _add_point(self, lon, lat, speed): + cx = int(math.floor(lon / CELL)) + cy = int(math.floor(lat / CELL)) + lonq = int(round((lon - cx * CELL) / CELL * 65535)) + latq = int(round((lat - cy * CELL) / CELL * 65535)) + lonq = max(0, min(65535, lonq)) + latq = max(0, min(65535, latq)) + sx = min(SUB - 1, lonq * SUB // 65536) + sy = min(SUB - 1, latq * SUB // 65536) + key = (cx, cy) + cell = self.points.get(key) + if cell is None: + cell = [None] * (SUB * SUB) + self.points[key] = cell + sub = sx + sy * SUB + buf = cell[sub] + if buf is None: + buf = bytearray() + cell[sub] = buf + buf += struct.pack(" {self.outdir}\n") + + +def main(): + if len(sys.argv) != 3: + print(f"usage: {sys.argv[0]} ") + sys.exit(1) + pbf_path = sys.argv[1] + outdir = sys.argv[2] + + # === PASS 1 === + sys.stderr.write("=== PASS 1: Collect node IDs from highway+maxspeed ways ===\n") + t0 = time.time() + c = Pass1Collector() + c.apply_file(pbf_path, locations=False) + sys.stderr.write(f"Pass 1 done: {c.way_count} ways, {time.time()-t0:.0f}s\n") + + # === PASS 2 === + sys.stderr.write("=== PASS 2: Collect nodes + process ways ===\n") + t1 = time.time() + b = Pass2Builder(c.test, outdir) + b.apply_file(pbf_path, locations=False) + if not b.node_phase: + pass + else: + sys.stderr.write("WARNING: no ways processed (node phase never ended)\n") + sys.stderr.write(f"Pass 2 done: {b.node_count} nodes, {b.way_count} ways, {b.point_count} points, {time.time()-t1:.0f}s\n") + + # === WRITE GRID === + t2 = time.time() + b.write_grid() + sys.stderr.write(f"Grid written in {time.time()-t0:.0f}s total\n") + +if __name__ == "__main__": + main() diff --git a/data/maxspeed/build_world.sh b/data/maxspeed/build_world.sh new file mode 100644 index 0000000..9ab1221 --- /dev/null +++ b/data/maxspeed/build_world.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -e +cd /home/riricdev/zektyc/data/maxspeed + +echo "=== Étape 1: Filtrer highways du planet ===" +if [ ! -f planet-highways.pbf ]; then + echo "Extraction des highways..." + osmium tags-filter planet-latest.osm.pbf w/highway -o planet-highways.pbf --overwrite + ls -lh planet-highways.pbf +else + echo "planet-highways.pbf existe déjà" +fi + +echo "" +echo "=== Étape 2: Découper en bandes longitude ===" +mkdir -p bands raw + +# 12 bandes de 30° : -180..-150, -150..-120, ..., 150..180 +BAND_SIZE=30 +for i in $(seq 0 11); do + LON_START=$(( -180 + i * BAND_SIZE )) + LON_END=$(( LON_START + BAND_SIZE )) + BAND="band_${LON_START}_${LON_END}" + + if [ -f "raw/${BAND}.raw" ]; then + echo " ${BAND} déjà fait, skip" + continue + fi + + if [ ! -f "bands/${BAND}.pbf" ]; then + echo " Extraction bande ${LON_START}..${LON_END}..." + osmium extract -b ${LON_START},-60,${LON_END},85 planet-highways.pbf -o "bands/${BAND}.pbf" --overwrite + ls -lh "bands/${BAND}.pbf" + fi + + echo " Build bande ${LON_START}..${LON_END}..." + ./build_band "bands/${BAND}.pbf" "raw/${BAND}.raw" + ls -lh "raw/${BAND}.raw" + + rm -f "bands/${BAND}.pbf" + echo "" +done + +echo "=== Étape 3: Merge toutes les bandes ===" +./merge_grid grid-planet raw/*.raw + +echo "=== Terminé! ===" +ls -lh grid-planet/ diff --git a/data/maxspeed/merge_grid.cpp b/data/maxspeed/merge_grid.cpp new file mode 100644 index 0000000..e4465f7 --- /dev/null +++ b/data/maxspeed/merge_grid.cpp @@ -0,0 +1,140 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +static constexpr double CELL = 0.5; +static constexpr int SUB = 32; +static constexpr double LON_MIN = -180.0; +static constexpr double LAT_MIN = -60.0; +static constexpr double LON_MAX = 180.0; +static constexpr double LAT_MAX = 85.0; + +struct RawPoint { + int32_t col; + int32_t row; + uint16_t lonq; + uint16_t latq; + uint8_t speed; +}; + +struct Point64 { + uint16_t lonq; + uint16_t latq; + uint8_t speed; +}; + +static bool cmp_cell(const RawPoint& a, const RawPoint& b) { + if (a.col != b.col) return a.col < b.col; + if (a.row != b.row) return a.row < b.row; + return false; +} + +int main(int argc, char* argv[]) { + if (argc < 3) { + fprintf(stderr, "Usage: %s [raw_file2] ...\n", argv[0]); + return 1; + } + + const char* outdir = argv[1]; + fs::create_directories(outdir); + + std::vector all_points; + + for (int i = 2; i < argc; i++) { + FILE* f = fopen(argv[i], "rb"); + if (!f) { fprintf(stderr, "Cannot open %s\n", argv[i]); continue; } + + fseek(f, 0, SEEK_END); + long fsize = ftell(f); + fseek(f, 0, SEEK_SET); + + uint64_t before = all_points.size(); + uint64_t npoints = fsize / sizeof(RawPoint); + all_points.resize(before + npoints); + fread(all_points.data() + before, sizeof(RawPoint), npoints, f); + fclose(f); + + fprintf(stderr, "Loaded %s: %lu points (total: %lu)\n", argv[i], npoints, all_points.size()); + } + + fprintf(stderr, "Sorting %lu points...\n", all_points.size()); + std::sort(all_points.begin(), all_points.end(), cmp_cell); + fprintf(stderr, "Sort done. Writing cells...\n"); + + std::string index_path = std::string(outdir) + "/index.bin"; + FILE* index = fopen(index_path.c_str(), "wb"); + if (!index) { fprintf(stderr, "Cannot write index\n"); return 1; } + + uint32_t magic = 0x5A4D4150; + fwrite(&magic, 4, 1, index); + uint16_t version = 2; + fwrite(&version, 2, 1, index); + uint16_t sub = SUB; + fwrite(&sub, 2, 1, index); + fwrite(&CELL, 8, 1, index); + fwrite(&LON_MIN, 8, 1, index); + fwrite(&LAT_MIN, 8, 1, index); + fwrite(&LON_MAX, 8, 1, index); + fwrite(&LAT_MAX, 8, 1, index); + + uint32_t num_cells = 0; + fseek(index, 36, SEEK_SET); + fwrite(&num_cells, 4, 1, index); + fseek(index, 0, SEEK_END); + + int32_t cur_col = -1, cur_row = -1; + FILE* cur_file = nullptr; + uint32_t cell_points = 0; + std::string cur_key; + + for (uint64_t i = 0; i <= all_points.size(); i++) { + bool new_cell = (i == all_points.size() || + all_points[i].col != cur_col || + all_points[i].row != cur_row); + + if (new_cell && cur_file) { + fclose(cur_file); + uint32_t klen = static_cast(cur_key.size()); + fwrite(&klen, 4, 1, index); + fwrite(cur_key.data(), 1, klen, index); + fwrite(&cell_points, 4, 1, index); + num_cells++; + if (num_cells % 500 == 0) { + fprintf(stderr, " %u cells written...\n", num_cells); + } + } + + if (i == all_points.size()) break; + + if (new_cell) { + cur_col = all_points[i].col; + cur_row = all_points[i].row; + cur_key = std::to_string(cur_col) + "_" + std::to_string(cur_row); + std::string cellfile = std::string(outdir) + "/cell_" + cur_key + ".bin"; + cur_file = fopen(cellfile.c_str(), "wb"); + cell_points = 0; + } + + Point64 p; + p.lonq = all_points[i].lonq; + p.latq = all_points[i].latq; + p.speed = all_points[i].speed; + fwrite(&p, sizeof(Point64), 1, cur_file); + cell_points++; + } + + fseek(index, 36, SEEK_SET); + fwrite(&num_cells, 4, 1, index); + fclose(index); + + fprintf(stderr, "Total cells: %u\n", num_cells); + fprintf(stderr, "Grid written to %s\n", outdir); + return 0; +} diff --git a/data/maxspeed/run_build.sh b/data/maxspeed/run_build.sh new file mode 100755 index 0000000..c8c0d72 --- /dev/null +++ b/data/maxspeed/run_build.sh @@ -0,0 +1,5 @@ +#!/bin/bash +exec python3 -u /home/riricdev/zektyc/data/maxspeed/build_world.py \ + /home/riricdev/zektyc/data/maxspeed/planet-latest.osm.pbf \ + /home/riricdev/zektyc/data/maxspeed/grid-planet/ \ + >> /home/riricdev/zektyc/data/maxspeed/build-planet.log 2>&1 diff --git a/scripts/antiflood.sh b/scripts/antiflood.sh new file mode 100755 index 0000000..9e80f0f --- /dev/null +++ b/scripts/antiflood.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Zektyc — protection contre les anomalies de trafic (DoS/DDoS L4, SYN floods) +# +# Mise au point : NE COUPE QUE lorsque le réseau est réellement saturé, au bord +# de tomber. On ne réagit plus à un simple pic de débit (un burst bien absorbé +# qui ne sature pas le lien ne doit PAS déclencher). On se base sur le DÉBIT RX +# mesuré en octets par rapport au PLAFOND DE BANDE PASSANTE du lien (étalonné) : +# - on coupe seulement quand le débit RX soutenu approche le plafond +# (le lien est sur le point d'être saturé / de ralentir vraiment), +# - pendant plus de PERSIST ticks consécutifs (pas un pic bref). +# Le funnel + playit sont réactivés automatiquement quand ça redescend. +set -u + +# Webhook Discord chargé depuis un fichier local gitignoré (jamais commité). +source /home/riricdev/zektyc/scripts/.webhooks +WEBHOOK="${ANTIFLOOD_WEBHOOK:-}" +STATE_FILE="/home/riricdev/zektyc/scripts/.synstate" +BASE_FILE="/home/riricdev/zektyc/scripts/.synbase" +LOG="/var/log/zektyc-syn.log" + +IFACE_ETH="eth0" # interface réseau publique (les SYN floods y arrivent) +IFACE_TS="tailscale0" # interface tailscale (légitime) + +INTERVAL=2 # période d'échantillonnage (s) + +# Plafond de bande passante DOWN mesuré via speedtest (~80 Mbit/s ≈ 10 Mo/s). +# Tout le trafic entrant (SYN floods compris) passe par eth0 et sature ce plafond. +# 10000000 octets/s × INTERVAL(2) = octets par tick au plafond. +CAP_BYTES=$(( 10000000 * INTERVAL )) + +# Fraction du plafond au-delà de laquelle on considère le lien "au bord de +# tomber" (0.90 = saturé à 90 %). Un burst bref ne suffit pas : il faut tenir +# au-dessus pendant PERSIST ticks consécutifs. +CAP_RATIO=0.90 +PERSIST=4 # ticks consécutifs au-dessus du seuil requis avant de couper +COOLDOWN_UP=45 # (s) attendre avant de tenter un rétablissement une fois calme + +log() { printf '%s %s\n' "$(date +%F\ %T)" "$1" >>"$LOG"; } +notify() { printf '{"content":"%s"}' "$1" | curl -s -m 10 -H "Content-Type: application/json" -X POST "$WEBHOOK" -d @- >/dev/null 2>&1 || true; } + +# Lit le nombre TOTAL d'OCTETS reçus (RX) sur l'interface donnée, depuis +# /proc/net/dev (2e champ après le nom d'interface, colonne RX bytes). +# La saturation de bande passante se mesure en OCTETS, pas en paquets. +read_rx() { awk -v ifx="$1" '$0 ~ ifx {print $2}' /proc/net/dev; } + +# Vrai si le funnel TCP du site (:443) est actif (relais brut + PROXY protocol v2 +# vers nginx 127.0.0.1:444). Le nom de domaine est suivi de ":443" - on le cherche +# explicitement pour ne pas matcher l'entree fantome sur un autre port (:10000). +funnel_active() { tailscale funnel status 2>/dev/null | grep -qE 'tcp://riricdev\.tail5ea5cd\.ts\.net:443'; } + +funnel_off() { + # coupe le funnel tailscale (site web) — mode TCP brut + PROXY protocol + if funnel_active; then + tailscale funnel --tcp=443 off >/dev/null 2>&1 + log "Anomalie de trafic : funnel tailscale coupé" + fi + # coupe l'entrée playit.gg (serveurs Minecraft publics) + if systemctl is-active --quiet playit.service; then + systemctl stop playit.service >/dev/null 2>&1 + log "Anomalie de trafic : entrée playit coupée" + fi + notify "🔴 **Zektyc** — trafic anormal détecté, accès temporairement suspendu." +} + +funnel_on() { + # réactive le funnel tailscale (site web) — TCP brut + PPv2 vers nginx :444 + if ! funnel_active; then + tailscale funnel --bg --proxy-protocol=2 --tcp=443 tcp://127.0.0.1:444 >/dev/null 2>&1 + log "Retour à la normale : funnel tailscale réactivé" + fi + # réactive l'entrée playit.gg (serveurs Minecraft publics) + if ! systemctl is-active --quiet playit.service; then + systemctl start playit.service >/dev/null 2>&1 + log "Retour à la normale : entrée playit réactivée" + fi + notify "🟢 **Zektyc** — accès rétabli." +} + +# --- init --- +now=$(read_rx "$IFACE_ETH") +prev=$now +[[ ! -f "$STATE_FILE" ]] && echo "up" >"$STATE_FILE" +state="$(<"$STATE_FILE")" +streak=0 +down_since=0 + +# seuil de saturation en octets/tic +thresh=$(( CAP_BYTES > 0 ? CAP_BYTES : 1 )) + +while true; do + now=$(read_rx "$IFACE_ETH") + delta=$(( now - prev )) + prev=$now + [[ "$delta" -lt 0 ]] && delta=0 + state="$(<"$STATE_FILE")" + + # --- saturation : débit RX au-dessus de CAP_RATIO × plafond du lien --- + # Un SYN flood qui sature le lien fait monter le débit RX vers le plafond. + # On ne coupe que si ça tient PERSIST ticks consécutifs (pas un pic bref). + sat=0 + # delta(thresh×CAP_RATIO) ⇔ delta*100 ≥ thresh×(CAP_RATIO×100) + if (( delta * 100 >= thresh * 90 )); then + sat=1 + fi + + if [[ "$sat" -eq 1 ]]; then + streak=$(( streak + 1 )) + else + streak=0 + fi + + # --- actions --- + if [[ "$state" == "up" && "$streak" -ge "$PERSIST" ]]; then + echo down >"$STATE_FILE" + down_since=$(date +%s) + funnel_off + elif [[ "$state" == "down" ]]; then + now_s=$(date +%s) + if (( now_s - down_since >= COOLDOWN_UP && streak == 0 )); then + echo up >"$STATE_FILE" + funnel_on + fi + fi + + sleep "$INTERVAL" +done diff --git a/scripts/ipwatch.sh b/scripts/ipwatch.sh new file mode 100755 index 0000000..8aad4c9 --- /dev/null +++ b/scripts/ipwatch.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +# Zektyc — moniteur d'IP suspectes (staff only) +# +# Toutes les 30 s, analyse deux sources pour détecter des IP anormales qui +# tapent sur la stack, et envoie un rapport (embed) sur le webhook Discord staff. +# +# Source 1 — pare-feu netfilter : log kernel "UFW BLOCK" (vraies IP externes, +# probes, scanners, SYN floods résiduels au niveau réseau). +# +# Source 2 — reverse-proxy nginx : refus HTTP 429/503 émis par les limites +# par IP réelle (limit_req / limit_conn). Comme la terminaison TLS a +# été déplacée vers nginx (mode funnel tcp brut + PROXY protocol), +# c'est ICI qu'apparaissent les attaquants d'applicatif (bursts, +# h2load, etc.) que le pare-feu réseau ne voit plus. On relit le log +# à partir d'un offset pour ne traiter que les lignes nouvelles. +# +# Anti-re-spam : une IP déjà signalée n'est renvoyée que si son compteur de +# tentatives a augmenté depuis le dernier rapport. +set -o pipefail + +# Webhook Discord chargé depuis un fichier local gitignoré (jamais commité). +source /home/riricdev/zektyc/scripts/.webhooks +WEBHOOK="${IPWATCH_WEBHOOK:-}" +INTERVAL=30 +STATE="/home/riricdev/zektyc/scripts/.ipwatch-state" # IP:compteur déjà signalés +NGINX_LOG="/var/log/nginx/access.log" +# Taille (octets) du bloc de fin du log relue à chaque cycle puis filtrée au +# timestamp : couvre INTERVAL s même au pic (h2load). Rotation-safe, pas d'offset. +TL_BACK=16777216 # 16 Mo + +# IP publique ? 0=oui(suspecte possible) 1=non. On exclut aussi le CGNAT +# Tailscale (100.64.0.0/10) pour ne pas signaler le VPN légitime / les tests. +is_public() { + local ip="$1" a b c d + IFS='.' read -r a b c d <<<"$ip" + [[ "$a" == "127" ]] && return 1 + [[ "$a" == "10" ]] && return 1 + [[ "$a" == "192" && "$b" == "168" ]] && return 1 + [[ "$a" == "172" && "$b" -ge 16 && "$b" -le 31 ]] && return 1 + [[ "$a" == "100" && "$b" -ge 64 && "$b" -le 127 ]] && return 1 + [[ "$a" == "169" && "$b" == "254" ]] && return 1 + [[ "$a" -ge 224 && "$a" -le 239 ]] && return 1 + [[ "$a" == "0" || "$a" == "255" ]] && return 1 + return 0 +} + +# Envoie l'embed. Retour 0 si envoyé. +send_report() { + local report_txt="$1" now + now=$(date +"%Y-%m-%d %H:%M:%S") + local json + json=$(jq -n \ + --arg title "🚨 IP suspectes détectées" \ + --arg desc "$report_txt" \ + --arg time "$now" \ + --argjson color 15548997 \ + '{embeds:[{title:$title,description:$desc,color:$color,footer:{text:("Zektyc Sécu — "+$time)}}]}') + curl -s -m 10 -H "Content-Type: application/json" -X POST "$WEBHOOK" -d "$json" >/dev/null 2>&1 +} + +# Charge l'état des IP déjà signalées (format "IP compteur" par ligne) +load_state() { + declare -gA last=() + if [[ -f "$STATE" ]]; then + while read -r k v; do last["$k"]="$v"; done <"$STATE" + fi +} + +# Sauvegarde l'état +save_state() { + : >"$STATE" + for k in "${!last[@]}"; do printf '%s %s\n' "$k" "${last[$k]}" >>"$STATE"; done +} + +# --- Source 2 : relit la FIN du log nginx et garde les refus des 30 dernières s --- +# Pas d'offset : on lit un bloc fixe de la fin du fichier (TL_BACK, dimensionné +# pour couvrir INTERVAL s même au pic de trafic) puis on ne garde QUE les lignes +# dont l'horodatage est dans la fenêtre [ début, date_actuelle ]. Robuste face +# aux rotations/troncatures, aucun offset à resynchroniser manuellement. +# Tout le filtrage se fait dans UN SEUL awk (pas de fork par ligne). +read_nginx_refus() { + local awkprog ip c ts uals from + from="$(date -d "@$(( now_s - INTERVAL ))" +%d/%b/%Y:%H:%M:%S)" + awkprog=' + function ispub(ip, a,b,c,d,p,n) { + n=split(ip,p,"."); if (n!=4) return 0 + a=p[1]+0; b=p[2]+0; c=p[3]+0; d=p[4]+0 + if (a==127||a==10||(a==192&&b==168)||(a==172&&b>=16&&b<=31)\ + ||(a==100&&b>=64&&b<=127)||(a==169&&b==254)) return 0 + if (a>=224&&a<=239||a==0||a==255) return 0 + return 1 + } + { + # Gardes de structure : le buffer tail -c commence en plein milieu d une + # ligne -> son 1er enregistrement est un fragment casse. On n accepte que + # les lignes completes : IP v4 en tete + timestamp nginx complet en $3. + if ($1 !~ /^[0-9]{1,3}(\.[0-9]{1,3}){3}$/) next + ts=$3; sub(/^\[/,"",ts) + if (ts !~ /^[0-9]{2}\/[A-Za-z]{3}\/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2}/) next + # nginx logue en local time (+0200) : on n accepte que les lignes >= debut fenetre + if (ts < from) next + code="" + for (i=1;i<=NF;i++) + if ($i ~ /^[0-9]{3}$/ && (i+1)<=NF && $(i+1) ~ /^[0-9]+$/) code=$i + if (code!="429" && code!="503") next + ip=$1 + if (!ispub(ip)) next + cnt[ip]++ + n=split($0,q,"\"") + ua=(n>=2 ? substr(q[n-1],1,40) : "") + if (ua!="" && !((ip SUBSEP ua) in seen)) { seen[ip SUBSEP ua]=1; ual[ip]=(ual[ip]==""?ua:ual[ip]","ua) } + last[ip]=ts + } + END { for (ip in cnt) print ip "|" cnt[ip] "|" last[ip] "|" ual[ip] } + ' + while IFS='|' read -r ip c ts uals; do + cnt["$ip"]=$(( ${cnt["$ip"]:-0} + c )) + [[ "${protos["$ip"]:-}" != *"http"* ]] && protos["$ip"]="${protos["$ip"]:-}http," + [[ "${dstports["$ip"]:-}" != *"443"* ]] && dstports["$ip"]="${dstports["$ip"]:-}443," + lastts["$ip"]="$ts" + [[ -n "$uals" ]] && uas["$ip"]="${uals}," + done < <(tail -c "$TL_BACK" "$NGINX_LOG" | awk -v "from=$from" "$awkprog") +} + +load_state +echo "IP-watch démarré — fenêtre ${INTERVAL}s/signalement (ufw + nginx), webhook staff (actif)." + +while true; do + now_s=$(date +%s) + since=$(( now_s - INTERVAL )) + + declare -A cnt=() + declare -A protos=() + declare -A dstports=() + declare -A lastts=() + declare -A uas=() + + # --- source 1 : blocs netfilter (log kernel UFW BLOCK) --- + while IFS= read -r ln; do + [[ "$ln" != *"UFW BLOCK"* ]] && continue + if [[ "$ln" =~ \ SRC=([0-9.]+)\ ]]; then + ip="${BASH_REMATCH[1]}" + is_public "$ip" || continue + cnt["$ip"]=$(( ${cnt["$ip"]:-0} + 1 )) + if [[ "$ln" =~ \ PROTO=([A-Za-z0-9]+)\ ]]; then + p="${BASH_REMATCH[1]}" + [[ "${protos["$ip"]:-}" != *"$p"* ]] && protos["$ip"]="${protos["$ip"]:-}$p," + fi + if [[ "$ln" =~ \ DPT=([0-9]+) ]]; then + d="${BASH_REMATCH[1]}" + [[ "${dstports["$ip"]:-}" != *"$d"* ]] && dstports["$ip"]="${dstports["$ip"]:-}$d," + fi + if [[ "$ln" =~ ^[A-Z][a-z]{2}\ [0-9]{2}\ [0-9]{2}\:[0-9]{2}\:[0-9]{2} ]]; then + lastts["$ip"]="${BASH_REMATCH[0]}" + fi + fi + done < <(sudo journalctl -k --no-pager --since "@${since}" 2>/dev/null | grep "UFW BLOCK") + + # --- source 2 : refus HTTP nginx (429/503 par IP réelle) --- + read_nginx_refus + + # --- filtrer : ne signaler que les IP nouvelles ou dont le compteur a augmenté --- + newfound=0 + report="" + for ip in "${!cnt[@]}"; do + c="${cnt[$ip]}" + prev="${last["$ip"]:-0}" + if (( c > prev )); then + proto="${protos[$ip]%,}" + dports="${dstports[$ip]%,}" + ua="${uas[$ip]%,}" + report+=$':shield: **`'"${ip}"$'`** — **'"${c}"$'** tentative(s) bloquée(s)\n' + report+=$'└ Sources: `'"${proto:-?}"$'` | Ports cibles: `'"${dports:-?}"$'`\n' + if [[ -n "$ua" ]]; then + report+=$'└ User-Agent: `'"${ua}"$'`\n' + fi + report+=$'└ Dernière tentative: `'"${lastts[$ip]:-?}"$'`\n\n' + last["$ip"]="$c" + ((newfound++)) + fi + done + + if (( newfound > 0 )); then + send_report "$report" + fi + + save_state + sleep "$INTERVAL" +done \ No newline at end of file diff --git a/scripts/upnp-watch.sh b/scripts/upnp-watch.sh new file mode 100755 index 0000000..f52e660 --- /dev/null +++ b/scripts/upnp-watch.sh @@ -0,0 +1,29 @@ +#!/bin/bash +UPNP_URL="http://192.168.1.254:5678/desc/root" +LOCAL_IP="192.168.1.50" +LOG_TAG="upnp-watch" + +log() { + logger -t "$LOG_TAG" "$1" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" +} + +MAPPINGS=$(upnpc -u "$UPNP_URL" -l 2>/dev/null) +if [ -z "$MAPPINGS" ]; then + log "ERREUR: passerelle UPnP injoignable" + exit 1 +fi + +for PORT in 80 443; do + if echo "$MAPPINGS" | grep -q "TCP.*${PORT}->${LOCAL_IP}:${PORT}"; then + log "OK: port ${PORT}/TCP déjà ouvert" + else + log "port ${PORT}/TCP absent -> ouverture" + RESULT=$(upnpc -u "$UPNP_URL" -a "$LOCAL_IP" "$PORT" "$PORT" TCP 2>&1 | tail -1) + if echo "$RESULT" | grep -q "is redirected"; then + log "port ${PORT}/TCP ouvert avec succès" + else + log "ECHEC ouverture ${PORT}/TCP: $RESULT" + fi + fi +done diff --git a/scripts/uptime-check.sh b/scripts/uptime-check.sh new file mode 100755 index 0000000..8da6ec4 --- /dev/null +++ b/scripts/uptime-check.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Zektyc — moniteur de disponibilité (uptime checker) +# Vérifie toutes les 5 secondes le code HTTP de réponse de la homepage. +# - Code 2xx/3xx -> online +# - Code 000 (échec connexion / funnel coupé), 5xx (erreur serveur) ou autre +# code d'échec -> incident technique +set -uo pipefail + +URL="https://riricdev.tail5ea5cd.ts.net/" +# Webhook Discord chargé depuis un fichier local gitignoré (jamais commité). +source /home/riricdev/zektyc/scripts/.webhooks +WEBHOOK="${ANTIFLOOD_WEBHOOK:-}" +STATE_FILE="/home/riricdev/zektyc/scripts/.uptime-state" +INTERVAL=5 + +# Un code HTTP est "online" s'il est 2xx/3xx, ou 429 (rate limit = serveur up, +# juste throttlé). Tout le reste (000, 4xx hors 429, 5xx) = down. +http_is_online() { + local code="$1" + case "$code" in + 2*|3*|429) return 0 ;; # 200-399 et 429 : online + *) return 1 ;; # 000, 4xx (hors 429), 5xx : down + esac +} + +down() { + local payload + payload=$(printf '{"content":"🔴 **Zektyc** — incident technique."}' ) + curl -s -m 10 -H "Content-Type: application/json" -X POST "$WEBHOOK" -d "$payload" >/dev/null 2>&1 || true + echo down >"$STATE_FILE" +} + +up() { + local payload + payload=$(printf '{"content":"🟢 **Zektyc** — le site est de nouveau en ligne."}' ) + curl -s -m 10 -H "Content-Type: application/json" -X POST "$WEBHOOK" -d "$payload" >/dev/null 2>&1 || true + echo up >"$STATE_FILE" +} + +if [[ ! -f "$STATE_FILE" ]]; then + echo "unknown" >"$STATE_FILE" +fi + +while true; do + code="$(curl -s -o /dev/null -w '%{http_code}' -m 10 "$URL" 2>/dev/null || echo 000)" + code="${code:-000}" + state="$(cat "$STATE_FILE")" + + if http_is_online "$code"; then + if [[ "$state" != "up" ]]; then + up + fi + else + if [[ "$state" != "down" ]]; then + # On n'envoie qu'une fois la notification de panne (quand on était up). + if [[ "$state" == "up" ]]; then + down + else + # État initial inconnu : on se contente de mémoriser sans notifier. + echo down >"$STATE_FILE" + fi + fi + fi + + sleep "$INTERVAL" +done