diff --git a/.gitignore b/.gitignore index fcbd3de..a667d98 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,12 @@ data/maxspeed/*.log # Finder (MacOS) folder config .DS_Store +# scripts d'exploitation (gardés sur disque pour les services systemd, hors dépôt) +scripts/ + +# données de build maxspeed (scripts + artefacts, hors dépôt) +data/maxspeed/ + # sensitive / personal duckdns_token_do_not_open.txt diff --git a/data/maxspeed/auto_update.py b/data/maxspeed/auto_update.py deleted file mode 100755 index 96e0c04..0000000 --- a/data/maxspeed/auto_update.py +++ /dev/null @@ -1,256 +0,0 @@ -#!/usr/bin/env python3 -"""Auto-update OSM planet PBF and rebuild the maxspeed grid. - -Designed to be triggered weekly (Friday 00:00). Checks the RSS feed for a new -planet release. If none found yet, retries every 5 minutes until one appears. -Then downloads the PBF, verifies MD5, rebuilds the grid, and atomically -swaps the old grid directory for the new one. -""" -import os, sys, time, hashlib, shutil, signal, subprocess, xml.etree.ElementTree as ET -from urllib.request import urlopen, Request -from pathlib import Path -from datetime import datetime, timezone - -DATA_DIR = Path(__file__).resolve().parent -PBF_DIR = DATA_DIR -RSS_URL = "https://planet.openstreetmap.org/pbf/planet-pbf-rss.xml" -BUILD_SCRIPT = DATA_DIR / "build_world.py" -CHECK_INTERVAL = 5 * 60 # 5 min between retries when waiting for a new release -POLL_TIMEOUT = 6 * 3600 # give up after 6 hours if nothing shows up -LOG_PREFIX = "[auto-update]" - -def log(msg): - ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - line = f"{LOG_PREFIX} [{ts}] {msg}" - print(line, flush=True) - with open(DATA_DIR / "auto-update.log", "a") as f: - f.write(line + "\n") - -def fetch_rss(): - """Fetch RSS and return the latest planet filename + pubDate.""" - req = Request(RSS_URL, headers={"User-Agent": "zektyc-auto-updater/1.0"}) - with urlopen(req, timeout=30) as resp: - xml_data = resp.read() - root = ET.fromstring(xml_data) - items = root.findall(".//item") - if not items: - return None, None - item = items[0] - title = (item.findtext("title") or "").strip() - pub_date = (item.findtext("pubDate") or "").strip() - # title is like "planet-260817.osm.pbf.torrent" - filename = title.replace(".torrent", "") - return filename, pub_date - -def get_current_pbf(): - """Return the basename of the planet PBF currently in use.""" - # Check which PBF file exists in PBF_DIR - for f in sorted(PBF_DIR.glob("planet-*.osm.pbf"), reverse=True): - if f.stat().st_size > 10_000_000_000: # > 10 GB = real planet - return f.name - return None - -def md5_file(path): - h = hashlib.md5() - with open(path, "rb") as f: - for chunk in iter(lambda: f.read(1024 * 1024), b""): - h.update(chunk) - return h.hexdigest() - -def fetch_md5(filename): - """Download the .md5 file from planet.openstreetmap.org.""" - url = f"https://planet.openstreetmap.org/pbf/{filename}.md5" - try: - req = Request(url, headers={"User-Agent": "zektyc-auto-updater/1.0"}) - with urlopen(req, timeout=30) as resp: - text = resp.read().decode().strip() - # format: "hash filename" - return text.split()[0] - except Exception as e: - log(f"WARNING: could not fetch MD5: {e}") - return None - -def download_pbf(filename): - """Download the planet PBF with resume support.""" - dest = PBF_DIR / filename - url = f"https://planet.openstreetmap.org/pbf/{filename}" - existing = dest.stat().st_size if dest.exists() else 0 - headers = {"User-Agent": "zektyc-auto-updater/1.0"} - if existing > 0: - headers["Range"] = f"bytes={existing}-" - log(f"Resuming download from {existing / 1e9:.1f} GB") - req = Request(url, headers=headers) - with urlopen(req, timeout=600) as resp: - mode = "ab" if existing > 0 and resp.status == 206 else "wb" - if mode == "wb": - existing = 0 - total = int(resp.headers.get("Content-Length", 0)) + existing - with open(dest, mode) as f: - downloaded = existing - t0 = time.time() - while True: - chunk = resp.read(1024 * 1024) - if not chunk: - break - f.write(chunk) - downloaded += len(chunk) - elapsed = time.time() - t0 - if elapsed > 0 and downloaded > existing: - speed = (downloaded - existing) / elapsed - eta = (total - downloaded) / speed if speed > 0 else 0 - log(f" {downloaded / 1e9:.2f} / {total / 1e9:.2f} GB " - f"({speed / 1e6:.1f} MB/s, ETA {int(eta // 60)}m{int(eta % 60)}s)") - return dest - -def kill_existing_build(): - """Kill any running build_world.py process.""" - try: - result = subprocess.run( - ["pgrep", "-f", "build_world.py"], - capture_output=True, text=True, timeout=5 - ) - for pid_str in result.stdout.strip().split("\n"): - pid_str = pid_str.strip() - if pid_str: - pid = int(pid_str) - log(f"Killing existing build process PID {pid}") - os.kill(pid, signal.SIGTERM) - time.sleep(2) - try: - os.kill(pid, signal.SIGKILL) - except OSError: - pass - except Exception as e: - log(f"WARNING: could not kill existing build: {e}") - -def wait_for_build(): - """Wait for any running build_world.py to finish.""" - while True: - result = subprocess.run( - ["pgrep", "-f", "build_world.py"], - capture_output=True, text=True, timeout=5 - ) - if not result.stdout.strip(): - return True - log("Waiting for existing build to finish...") - time.sleep(30) - return False - -def run_build(pbf_path, outdir): - """Run build_world.py.""" - log(f"Starting build: {pbf_path} -> {outdir}") - proc = subprocess.Popen( - [sys.executable, "-u", str(BUILD_SCRIPT), str(pbf_path), str(str(outdir))], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - cwd=str(DATA_DIR), - ) - with open(DATA_DIR / "build-planet.log", "a") as logf: - for line in iter(proc.stdout.readline, b""): - decoded = line.decode(errors="replace") - logf.write(decoded) - logf.flush() - # Also echo last chars for our log - proc.wait() - if proc.returncode != 0: - log(f"ERROR: build_world.py exited with code {proc.returncode}") - return False - log("Build completed successfully") - return True - -def atomic_swap(new_grid, target_grid): - """Atomically replace target_grid with new_grid.""" - backup = target_grid.parent / (target_grid.name + "-old") - if backup.exists(): - shutil.rmtree(backup) - if target_grid.exists(): - target_grid.rename(backup) - new_grid.rename(target_grid) - log(f"Swapped {target_grid.name} (backup: {backup.name})") - # Clean old backup after a short delay (keep for 1 hour for safety) - # Actually let's keep it, it's useful if something goes wrong - -def main(): - log("=" * 60) - log("Auto-update started") - - current_pbf = get_current_pbf() - log(f"Current PBF: {current_pbf or 'none'}") - - # Step 1: Check RSS for new release - t_start = time.time() - new_filename = None - while True: - try: - filename, pub_date = fetch_rss() - if filename and filename != current_pbf: - log(f"New release found: {filename} (published: {pub_date})") - new_filename = filename - break - elif filename: - log(f"No new release yet (latest: {filename}, current: {current_pbf})") - else: - log("Could not parse RSS feed") - except Exception as e: - log(f"RSS check failed: {e}") - - elapsed = time.time() - t_start - if elapsed > POLL_TIMEOUT: - log(f"No new release after {elapsed/3600:.1f}h, giving up") - return - log(f"Retrying in {CHECK_INTERVAL // 60} minutes...") - time.sleep(CHECK_INTERVAL) - - # Step 2: Kill any existing build - kill_existing_build() - wait_for_build() - - # Step 3: Download new PBF - log(f"Downloading {new_filename}...") - pbf_path = download_pbf(new_filename) - log(f"Download complete: {pbf_path}") - - # Step 4: Verify MD5 - expected_md5 = fetch_md5(new_filename) - if expected_md5: - log(f"Verifying MD5 (expected: {expected_md5})") - actual_md5 = md5_file(pbf_path) - log(f"Actual MD5: {actual_md5}") - if actual_md5 != expected_md5: - log("ERROR: MD5 mismatch! Aborting.") - return - log("MD5 verified OK") - else: - log("WARNING: skipping MD5 verification") - - # Step 5: Build new grid - grid_tmp = PBF_DIR / "grid-planet-new" - if grid_tmp.exists(): - shutil.rmtree(grid_tmp) - grid_target = PBF_DIR / "grid-planet" - - if not run_build(pbf_path, grid_tmp): - log("ERROR: Build failed, keeping old grid") - if grid_tmp.exists(): - shutil.rmtree(grid_tmp) - return - - # Step 6: Atomic swap - atomic_swap(grid_tmp, grid_target) - - # Step 7: Update .env symlink (if needed) - env_path = Path(__file__).resolve().parent.parent.parent / ".env" - if env_path.exists(): - content = env_path.read_text() - if "grid-europe" in content: - # Don't change .env — it might be intentionally pointing to grid-europe - log("NOTE: .env still points to grid-europe, not auto-updating") - elif "grid-planet" in content: - log(".env already points to grid-planet") - - log("Auto-update complete!") - log("=" * 60) - -if __name__ == "__main__": - signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) - main() diff --git a/data/maxspeed/build_band.cpp b/data/maxspeed/build_band.cpp deleted file mode 100644 index 38d0c3c..0000000 --- a/data/maxspeed/build_band.cpp +++ /dev/null @@ -1,138 +0,0 @@ -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; - -using index_type = osmium::index::map::SparseFileArray; -using location_handler_type = osmium::handler::NodeLocationsForWays; - -static constexpr double CELL = 0.5; -static constexpr double LON_MIN = -180.0; -static constexpr double LAT_MIN = -60.0; -static constexpr double LON_MAX = 180.0; -static constexpr double LAT_MAX = 85.0; - -struct RawPoint { - int32_t col; - int32_t row; - uint16_t lonq; - uint16_t latq; - uint8_t speed; -}; - -static inline uint16_t quantize_lon(double lon) { - return static_cast((lon - LON_MIN) / (LON_MAX - LON_MIN) * 65535.0); -} - -static inline uint16_t quantize_lat(double lat) { - return static_cast((lat - LAT_MIN) / (LAT_MAX - LAT_MIN) * 65535.0); -} - -static inline uint8_t parse_speed_val(const char* s) { - if (!s || !*s) return 0; - char* end = nullptr; - double v = strtod(s, &end); - if (end != s && v > 0 && v <= 255) return static_cast(v); - if (strstr(s, "mph")) { - v = strtod(s, &end); - if (end != s) { - v *= 1.609; - if (v > 0 && v <= 255) return static_cast(v); - } - } - return 0; -} - -class StreamHandler : public osmium::handler::Handler { -public: - FILE* out; - uint64_t way_count = 0; - uint64_t point_count = 0; - - explicit StreamHandler(FILE* f) : out(f) {} - - void way(const osmium::Way& way) { - const char* highway = way.tags().get_value_by_key("highway"); - if (!highway) return; - - const char* ms = way.tags().get_value_by_key("maxspeed"); - uint8_t speed = ms ? parse_speed_val(ms) : 0; - if (speed == 0) return; - - way_count++; - - for (const auto& wn : way.nodes()) { - if (!wn.location().valid()) continue; - - double lon = wn.location().lon(); - double lat = wn.location().lat(); - - if (lon < LON_MIN || lon >= LON_MAX || lat < LAT_MIN || lat >= LAT_MAX) continue; - - RawPoint rp; - rp.col = static_cast((lon - LON_MIN) / CELL); - rp.row = static_cast((lat - LAT_MIN) / CELL); - rp.lonq = quantize_lon(lon); - rp.latq = quantize_lat(lat); - rp.speed = speed; - - fwrite(&rp, sizeof(RawPoint), 1, out); - point_count++; - - if (point_count % 5000000 == 0) { - fprintf(stderr, " %lu points (%lu ways)...\n", point_count, way_count); - } - } - } -}; - -int main(int argc, char* argv[]) { - if (argc < 3) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - - const char* input = argv[1]; - const char* outpath = argv[2]; - - fs::create_directories(fs::path(outpath).parent_path()); - - FILE* out = fopen(outpath, "wb"); - if (!out) { fprintf(stderr, "Cannot open %s\n", outpath); return 1; } - - std::string idx_path = std::string(outpath) + ".idx"; - int idx_fd = ::open(idx_path.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644); - if (idx_fd == -1) { fprintf(stderr, "Cannot create index\n"); fclose(out); return 1; } - - index_type index(idx_fd); - location_handler_type location_handler(index); - StreamHandler handler(out); - - fprintf(stderr, "Reading %s...\n", input); - osmium::io::Reader reader(input); - osmium::apply(reader, location_handler, handler); - reader.close(); - - fclose(out); - ::close(idx_fd); - unlink(idx_path.c_str()); - - fprintf(stderr, "Ways with maxspeed: %lu\n", handler.way_count); - fprintf(stderr, "Points written: %lu\n", handler.point_count); - fprintf(stderr, "Output: %s\n", outpath); - return 0; -} diff --git a/data/maxspeed/build_global.cpp b/data/maxspeed/build_global.cpp deleted file mode 100644 index 0ecaf5f..0000000 --- a/data/maxspeed/build_global.cpp +++ /dev/null @@ -1,245 +0,0 @@ -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; - -using index_type = osmium::index::map::SparseFileArray; -using location_handler_type = osmium::handler::NodeLocationsForWays; - -static constexpr double CELL = 0.5; -static constexpr int SUB = 32; -static constexpr double LON_MIN = -180.0; -static constexpr double LAT_MIN = -60.0; -static constexpr double LON_MAX = 180.0; -static constexpr double LAT_MAX = 85.0; - -struct RawPoint { - int32_t col; - int32_t row; - uint16_t lonq; - uint16_t latq; - uint8_t speed; -}; // 11 bytes - -static inline uint16_t quantize_lon(double lon) { - return static_cast((lon - LON_MIN) / (LON_MAX - LON_MIN) * 65535.0); -} - -static inline uint16_t quantize_lat(double lat) { - return static_cast((lat - LAT_MIN) / (LAT_MAX - LAT_MIN) * 65535.0); -} - -static inline uint8_t parse_speed_val(const char* s) { - if (!s || !*s) return 0; - char* end = nullptr; - double v = strtod(s, &end); - if (end != s && v > 0 && v <= 255) return static_cast(v); - if (strstr(s, "mph")) { - v = strtod(s, &end); - if (end != s) { - v *= 1.609; - if (v > 0 && v <= 255) return static_cast(v); - } - } - return 0; -} - -struct Point64 { - uint16_t lonq; - uint16_t latq; - uint8_t speed; -}; - -class SpeedHandler : public osmium::handler::Handler { -public: - FILE* tmpfile; - uint64_t way_count = 0; - uint64_t point_count = 0; - - explicit SpeedHandler(FILE* f) : tmpfile(f) {} - - void way(const osmium::Way& way) { - const char* highway = way.tags().get_value_by_key("highway"); - if (!highway) return; - - const char* ms = way.tags().get_value_by_key("maxspeed"); - uint8_t speed = ms ? parse_speed_val(ms) : 0; - if (speed == 0) return; - - way_count++; - - for (const auto& wn : way.nodes()) { - if (!wn.location().valid()) continue; - - double lon = wn.location().lon(); - double lat = wn.location().lat(); - - if (lon < LON_MIN || lon >= LON_MAX || lat < LAT_MIN || lat >= LAT_MAX) continue; - - int32_t col = static_cast((lon - LON_MIN) / CELL); - int32_t row = static_cast((lat - LAT_MIN) / CELL); - - RawPoint rp; - rp.col = col; - rp.row = row; - rp.lonq = quantize_lon(lon); - rp.latq = quantize_lat(lat); - rp.speed = speed; - - fwrite(&rp, sizeof(RawPoint), 1, tmpfile); - point_count++; - - if (point_count % 10000000 == 0) { - fprintf(stderr, " %lu points (%lu ways)...\n", point_count, way_count); - } - } - } -}; - -static bool compare_by_cell(const RawPoint& a, const RawPoint& b) { - if (a.col != b.col) return a.col < b.col; - return a.row < b.row; -} - -static void split_and_write(const char* tmppath, const std::string& outdir) { - fprintf(stderr, "Reading temp file into memory for sort...\n"); - - FILE* f = fopen(tmppath, "rb"); - fseek(f, 0, SEEK_END); - long fsize = ftell(f); - fseek(f, 0, SEEK_SET); - - uint64_t npoints = fsize / sizeof(RawPoint); - fprintf(stderr, "Sorting %lu points (%.1f MB)...\n", npoints, fsize / 1048576.0); - - std::vector points(npoints); - fread(points.data(), sizeof(RawPoint), npoints, f); - fclose(f); - - std::sort(points.begin(), points.end(), compare_by_cell); - fprintf(stderr, "Sort done. Writing cells...\n"); - - fs::create_directories(outdir); - - std::string index_path = outdir + "/index.bin"; - FILE* index = fopen(index_path.c_str(), "wb"); - uint32_t magic = 0x5A4D4150; - fwrite(&magic, 4, 1, index); - uint16_t version = 2; - fwrite(&version, 2, 1, index); - uint16_t sub = SUB; - fwrite(&sub, 2, 1, index); - fwrite(&CELL, 8, 1, index); - fwrite(&LON_MIN, 8, 1, index); - fwrite(&LAT_MIN, 8, 1, index); - fwrite(&LON_MAX, 8, 1, index); - fwrite(&LAT_MAX, 8, 1, index); - - uint32_t num_cells = 0; - fseek(index, 36, SEEK_SET); - fwrite(&num_cells, 4, 1, index); - fseek(index, 0, SEEK_END); - - int32_t cur_col = -1, cur_row = -1; - FILE* cur_file = nullptr; - uint32_t cell_points = 0; - std::string cur_key; - - for (uint64_t i = 0; i <= npoints; i++) { - bool new_cell = (i == npoints || points[i].col != cur_col || points[i].row != cur_row); - - if (new_cell && cur_file) { - fclose(cur_file); - - uint32_t klen = static_cast(cur_key.size()); - fwrite(&klen, 4, 1, index); - fwrite(cur_key.data(), 1, klen, index); - fwrite(&cell_points, 4, 1, index); - num_cells++; - - if (num_cells % 1000 == 0) { - fprintf(stderr, " %u cells written...\n", num_cells); - } - } - - if (i == npoints) break; - - if (new_cell) { - cur_col = points[i].col; - cur_row = points[i].row; - cur_key = std::to_string(cur_col) + "_" + std::to_string(cur_row); - std::string cellfile = outdir + "/cell_" + cur_key + ".bin"; - cur_file = fopen(cellfile.c_str(), "wb"); - cell_points = 0; - } - - Point64 p; - p.lonq = points[i].lonq; - p.latq = points[i].latq; - p.speed = points[i].speed; - fwrite(&p, sizeof(Point64), 1, cur_file); - cell_points++; - } - - fseek(index, 36, SEEK_SET); - fwrite(&num_cells, 4, 1, index); - fclose(index); - - fprintf(stderr, "Total cells: %u\n", num_cells); -} - -int main(int argc, char* argv[]) { - if (argc < 3) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - - const char* input = argv[1]; - const char* outdir = argv[2]; - - std::string tmppath = std::string(outdir) + "/_points.tmp"; - fs::create_directories(outdir); - - fprintf(stderr, "Step 1: Reading PBF, writing raw points to temp file...\n"); - FILE* tmpf = fopen(tmppath.c_str(), "w+b"); - if (!tmpf) { fprintf(stderr, "Cannot create temp file\n"); return 1; } - - std::string idx_path = std::string(outdir) + "/_node_index.tmp"; - int idx_fd = ::open(idx_path.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644); - if (idx_fd == -1) { fprintf(stderr, "Cannot create index temp file\n"); return 1; } - - index_type index(idx_fd); - location_handler_type location_handler(index); - SpeedHandler handler(tmpf); - - osmium::io::Reader reader(input); - osmium::apply(reader, location_handler, handler); - reader.close(); - fclose(tmpf); - - fprintf(stderr, "\nWays with maxspeed: %lu\n", handler.way_count); - fprintf(stderr, "Total points: %lu\n", handler.point_count); - - fprintf(stderr, "Step 2: Sort + write grid cells...\n"); - split_and_write(tmppath.c_str(), outdir); - - ::close(idx_fd); - unlink(idx_path.c_str()); - unlink(tmppath.c_str()); - fprintf(stderr, "Done! Grid written to %s\n", outdir); - return 0; -} diff --git a/data/maxspeed/build_maxspeed.py b/data/maxspeed/build_maxspeed.py deleted file mode 100644 index d2642a5..0000000 --- a/data/maxspeed/build_maxspeed.py +++ /dev/null @@ -1,273 +0,0 @@ -#!/usr/bin/env python3 -"""Construit une grille binaire des maxspeed OSM (par route, échantillonnée en points). - -Pipeline : - osmium tags-filter .osm.pbf w/highway -o -roads.osm.pbf - python3 build_maxspeed.py -roads.osm.pbf [--cache=] - -Sortie : /grid.dat + /index.json + /meta.json - - Chaque cellule = 0.5° x 0.5° (lat, lon). - - Blob par cellule (gzip) : int32 count ; 32x32 sous-cellules => int32 offset[1024] ; - puis count points de (lonq:int16, latq:int16, speed:int8) rangés par sous-cellule. - - lonq/latq : quantifiés sur 0..65535 dans la cellule (precision ~0.8 m). -""" -import sys, os, json, gzip, math, struct, re -import osmium -import sqlite3 - -CELL = 0.5 # degrés par cellule -SUB = 32 # sous-cellules par dimension (grille 32x32) -STEP_M_DEFAULT = 40.0 # pas d'échantillonnage le long des routes (m) -STEP_M = STEP_M_DEFAULT - -# maxspeed textuel -> km/h (valeurs par défaut OSM) -ZONE = { - "none": None, - "signals": None, - "variable": None, - "walk": 5, - "living_street": 20, - "urban": 50, "rural": 80, "trunk": 110, "motorway": 130, -} -# codes pays -> (urban, rural) km/h ; motorway sans maxspeed = None (ex. DE) -CC = { - "FR": (50, 80), "BE": (50, 90), "DE": (50, 100), "NL": (50, 100), - "GB": (48, 96), "ES": (50, 90), "IT": (50, 90), "CH": (50, 80), - "AT": (50, 100), "LU": (50, 90), "PT": (50, 90), "PL": (50, 90), - "CZ": (50, 90), "IE": (50, 80), "DK": (50, 80), "SE": (50, 70), - "NO": (50, 80), "FI": (50, 100), "RO": (50, 90), "HU": (50, 90), - "HR": (50, 90), "SI": (50, 90), "SK": (50, 90), "LT": (50, 90), - "LV": (50, 90), "EE": (50, 90), "BG": (50, 90), "GR": (50, 90), - "UA": (50, 90), "RU": (60, 90), "TR": (50, 90), "RS": (50, 80), - "MK": (50, 80), "AL": (40, 90), "BA": (50, 80), "ME": (50, 80), - "IS": (50, 90), -} -# Les highways de service/privées sans maxspeed sont exclues : on ne garde que les -# ways marqués maxspeed (ou avec une valeur par défaut de zone reconnue). - -def parse_speed(v): - if not v: - return None - v = v.strip().lower() - if v in ZONE: - return ZONE[v] - m = __import__("re").match(r"^([0-9]+)(?:\s*(mph|km/h|kmh))?\b", v) - if m: - n = int(m.group(1)) - if m.group(2) == "mph": - return round(n * 1.609344) - return n - # code pays type FR:rural / DE:urban - if ":" in v: - cc, kind = v.split(":", 1) - cc = cc.upper() - if cc in CC and kind in ("urban", "rural"): - return CC[cc][0 if kind == "urban" else 1] - if cc in CC and kind == "motorway": - return None - return None - -class MaxSpeedHandler(osmium.SimpleHandler): - def __init__(self): - super().__init__() - self.points = {} # (cellx, celly) -> dict subcell -> list of (lonq, latq, speed) - self.skipped = 0 - - def way(self, w): - if "highway" not in w.tags: - return - speed = parse_speed(w.tags.get("maxspeed", "")) - if speed is None: - return - locs = [] - for n in w.nodes: - loc = n.location - if loc is not None: - locs.append((loc.lon, loc.lat)) - if len(locs) < 2: - return - self.process_locs(locs, speed) - - def process_locs(self, locs, speed): - if STEP_M <= 0: - for (lon, lat) in locs: - self.add_point(lon, lat, speed) - return - prev = locs[0] - lon, lat = prev - acc = 0.0 - emit = [(lon, lat)] - for node in locs[1:]: - a = haversine(prev[1], prev[0], node[1], node[0]) - acc += a - prev = node - while acc >= STEP_M: - f = 1.0 - (acc - STEP_M) / a if a > 0 else 0.0 - ilon = lon + (node[0] - lon) * f - ilat = lat + (node[1] - lat) * f - emit.append((ilon, ilat)) - acc -= STEP_M - lon, lat = node[0], node[1] - if (lon, lat) != emit[-1]: - emit.append((lon, lat)) - for (el, ea) in emit: - self.add_point(el, ea, speed) - - def add_point(self, lon, lat, speed): - cx = int(math.floor(lon / CELL)) - cy = int(math.floor(lat / CELL)) - lonq = int(round((lon - cx * CELL) / CELL * 65535)) - latq = int(round((lat - cy * CELL) / CELL * 65535)) - lonq = max(0, min(65535, lonq)); latq = max(0, min(65535, latq)) - sx = min(SUB - 1, lonq * SUB // 65536) - sy = min(SUB - 1, latq * SUB // 65536) - key = (cx, cy) - cell = self.points.get(key) - if cell is None: - cell = [None] * (SUB * SUB) - self.points[key] = cell - sub = sx + sy * SUB - buf = cell[sub] - if buf is None: - buf = bytearray() - cell[sub] = buf - buf += struct.pack(" liste (lon, lat) pour LineString (ou Multi).""" - if not isinstance(raw, (bytes, bytearray)) or len(raw) < 8 or raw[:2] != b"GP": - return None - flags = raw[3] - off = 8 - if flags & GP_FLAGS["envelope"]: - etype = (flags >> 2) & 0x03 - if etype != 0: - return None - off += 4 * 8 - bo = "<" if flags & GP_FLAGS["byteorder"] else ">" - if off + 5 > len(raw): - return None - gtype = struct.unpack(bo + "i", raw[off + 1:off + 5])[0] - if gtype == 2: # LineString - return parse_wkb_points(raw, off, bo) - if gtype == 5: # MultiLineString - out = [] - num = struct.unpack(bo + "i", raw[off + 5:off + 9])[0] - p = off + 9 - for _ in range(num): - pts = parse_wkb_points(raw, p, bo) - if pts: - out.extend(pts) - p += 4 + 4 * 8 * (len(pts) if pts else 0) - return out if len(out) >= 2 else None - return None - -def parse_wkb_points(raw, off, bo): - n = struct.unpack(bo + "i", raw[off + 5:off + 9])[0] - out = [] - p = off + 9 - for _ in range(n): - out.append((struct.unpack(bo + "d", raw[p:p + 8])[0], - struct.unpack(bo + "d", raw[p + 8:p + 16])[0])) - p += 16 - return out - -MS_RE = re.compile(r'"maxspeed"=>"([^"]*)"') - -def parse_other_tags(s): - m = MS_RE.search(s or "") - return m.group(1) if m else None - -def load_gpkg(h, gpkg): - con = sqlite3.connect(gpkg) - cur = con.execute("SELECT geom, highway, other_tags FROM lines") - n = 0 - for raw, hw, tags in cur: - if not hw or "maxspeed" not in (tags or ""): - continue - v = parse_other_tags(tags) - speed = parse_speed(v) - if speed is None: - continue - locs = parse_gp_wkb(raw) - if not locs or len(locs) < 2: - continue - h.process_locs(locs, speed) - n += 1 - if n % 500000 == 0: - sys.stderr.write("ways maxspeed: %dM\n" % (n // 1000000)) - con.close() - return n - -def build(src_pbf, out_dir, cache_file=None): - os.makedirs(out_dir, exist_ok=True) - h = MaxSpeedHandler() - if src_pbf.endswith(".gpkg") or src_pbf.endswith(".db"): - n = load_gpkg(h, src_pbf) - print(f"GPKG : {n} ways traités") - else: - if cache_file: - idx = osmium.index.create_map("sparse_file_array,%s" % cache_file) - h.apply_file(src_pbf, locations=osmium.NodeLocationsForWays(idx)) - else: - h.apply_file(src_pbf, locations=True) - # écriture : un fichier par cellule (nom: X_Y.bin.gz) + index - index = {} - total = 0 - for (cx, cy), cell in h.points.items(): - count = sum(0 if b is None else len(b) // 5 for b in cell) - offsets = [count] * (SUB * SUB) - acc = 0 - for sub, b in enumerate(cell): - if b is None: - continue - offsets[sub] = acc - acc += len(b) // 5 - # remplissage arrière : les sous-cellules vides pointent vers la suivante - for s in range(SUB * SUB - 2, -1, -1): - if offsets[s] == count: - offsets[s] = offsets[s + 1] - buf = bytearray() - buf += struct.pack(" {out_dir}") - -if __name__ == "__main__": - cache = None - args = sys.argv[1:] - extra = {} - while args and args[-1].startswith("--"): - a = args.pop() - if a.startswith("--cache="): - cache = a.split("=", 1)[1] - elif a == "--no-sample": - STEP_M = 0 - else: - print(f"Option inconnue: {a}") - sys.exit(1) - if len(args) != 2: - print("usage: build_maxspeed.py [--cache=] [--no-sample]") - sys.exit(1) - build(args[0], args[1], cache) \ No newline at end of file diff --git a/data/maxspeed/build_world.py b/data/maxspeed/build_world.py deleted file mode 100644 index 67e1e7b..0000000 --- a/data/maxspeed/build_world.py +++ /dev/null @@ -1,271 +0,0 @@ -#!/usr/bin/env python3 -"""Build world maxspeed grid from planet PBF in 2 passes. - -Pass 1: Stream PBF → collect node IDs from highway+maxspeed ways → bitset -Pass 2: Stream PBF → matching nodes → mmap sort → process ways → grid - -Usage: python3 build_world.py planet-latest.osm.pbf grid-planet/ -""" -import sys, os, json, gzip, struct, math, mmap, time -import numpy as np -import osmium - -CELL = 0.5 -SUB = 32 - -ZONE = { - "none": None, "signals": None, "variable": None, "walk": 5, "living_street": 20, - "urban": 50, "rural": 80, "trunk": 110, "motorway": 130, -} -CC = { - "FR": (50,80), "BE": (50,90), "DE": (50,100), "NL": (50,100), - "GB": (48,96), "ES": (50,90), "IT": (50,90), "CH": (50,80), - "AT": (50,100), "LU": (50,90), "PT": (50,90), "PL": (50,90), - "CZ": (50,90), "IE": (50,80), "DK": (50,80), "SE": (50,70), - "NO": (50,80), "FI": (50,100), "RO": (50,90), "HU": (50,90), - "HR": (50,90), "SI": (50,90), "SK": (50,90), "LT": (50,90), - "LV": (50,90), "EE": (50,90), "BG": (50,90), "GR": (50,90), - "UA": (50,90), "RU": (60,90), "TR": (50,90), "RS": (50,80), - "MK": (50,80), "AL": (40,90), "BA": (50,80), "ME": (50,80), - "IS": (50,90), "US": (40,105), "CA": (40,100), "MX": (40,90), - "BR": (40,80), "AR": (40,110), "CL": (40,120), "CO": (40,90), - "PE": (40,90), "JP": (40,60), "CN": (40,100), "IN": (40,80), - "AU": (40,100), "NZ": (40,100), "ZA": (40,100), "EG": (50,90), - "NG": (50,80), "KE": (50,80), "MA": (50,90), "DZ": (50,90), - "TN": (50,90), "LY": (50,90), -} - -def parse_speed(v): - if not v: - return None - v = v.strip().lower() - if v in ZONE: - return ZONE[v] - m = __import__("re").match(r"^([0-9]+)(?:\s*(mph|km/h|kmh))?\b", v) - if m: - n = int(m.group(1)) - if m.group(2) == "mph": - return round(n * 1.609344) - return n - if ":" in v: - cc, kind = v.split(":", 1) - cc = cc.upper() - if cc in CC and kind in ("urban", "rural"): - return CC[cc][0 if kind == "urban" else 1] - if cc in CC and kind == "motorway": - return None - return None - -MAX_NODE_ID = 25_000_000_000 # 25 billion — covers all OSM node IDs with margin - -class Pass1Collector(osmium.SimpleHandler): - """Collect node IDs referenced by highway+maxspeed ways.""" - def __init__(self): - super().__init__() - bs_size = MAX_NODE_ID // 8 + 1 - sys.stderr.write(f" Allocating bitset: {bs_size/1073741824:.1f} GB\n") - self.bitset = bytearray(bs_size) - self.way_count = 0 - - def way(self, w): - if "highway" not in w.tags: - return - speed = parse_speed(w.tags.get("maxspeed", "")) - if speed is None: - return - self.way_count += 1 - for n in w.nodes: - nid = n.ref - if nid >= MAX_NODE_ID: - continue - byte_idx = nid >> 3 - bit_idx = nid & 7 - self.bitset[byte_idx] |= (1 << bit_idx) - if self.way_count % 1_000_000 == 0: - sys.stderr.write(f" pass1: {self.way_count//1_000_000}M ways\n") - - def test(self, nid): - if nid >= MAX_NODE_ID: - return False - return (self.bitset[nid >> 3] >> (nid & 7)) & 1 - - -class Pass2Builder(osmium.SimpleHandler): - """Pass 2: collect matching nodes, then process ways.""" - def __init__(self, bitset_test, outdir): - super().__init__() - self.bitset_test = bitset_test - self.outdir = outdir - - self.node_ids = np.empty(200_000_000, dtype=np.uint64) - self.node_lons = np.empty(200_000_000, dtype=np.float32) - self.node_lats = np.empty(200_000_000, dtype=np.float32) - self.node_count = 0 - self.node_phase = True - - self.sorted_ids = None - self.sorted_lons = None - self.sorted_lats = None - - self.points = {} - self.way_count = 0 - self.point_count = 0 - self.miss_count = 0 - - def _ensure_capacity(self): - if self.node_count < len(self.node_ids): - return - new_size = len(self.node_ids) * 2 - sys.stderr.write(f" Resizing node arrays to {new_size//1_000_000}M...\n") - self.node_ids = np.resize(self.node_ids, new_size) - self.node_lons = np.resize(self.node_lons, new_size) - self.node_lats = np.resize(self.node_lats, new_size) - - def node(self, n): - if not self.node_phase: - return - if self.bitset_test(n.id): - self._ensure_capacity() - self.node_ids[self.node_count] = n.id - self.node_lons[self.node_count] = n.location.lon - self.node_lats[self.node_count] = n.location.lat - self.node_count += 1 - if self.node_count % 50_000_000 == 0: - sys.stderr.write(f" pass2 nodes: {self.node_count//1_000_000}M\n") - - def _build_index(self): - sys.stderr.write(f" Sorting {self.node_count//1_000_000}M nodes...\n") - t0 = time.time() - ids = self.node_ids[:self.node_count] - lons = self.node_lons[:self.node_count] - lats = self.node_lats[:self.node_count] - self.node_ids = self.node_lons = self.node_lats = None - - order = np.argsort(ids) - self.sorted_ids = ids[order] - self.sorted_lons = lons[order] - self.sorted_lats = lats[order] - del ids, lons, lats, order - sys.stderr.write(f" Sort done in {time.time()-t0:.1f}s, {len(self.sorted_ids)//1_000_000}M nodes indexed\n") - - def _lookup(self, nid): - idx = np.searchsorted(self.sorted_ids, nid) - if idx < len(self.sorted_ids) and self.sorted_ids[idx] == nid: - return float(self.sorted_lons[idx]), float(self.sorted_lats[idx]) - return None, None - - def way(self, w): - if self.node_phase: - self.node_phase = False - self._build_index() - - if "highway" not in w.tags: - return - speed = parse_speed(w.tags.get("maxspeed", "")) - if speed is None: - return - - self.way_count += 1 - for n in w.nodes: - lon, lat = self._lookup(n.ref) - if lon is None: - self.miss_count += 1 - continue - self._add_point(lon, lat, speed) - - if self.way_count % 1_000_000 == 0: - sys.stderr.write(f" pass2 ways: {self.way_count//1_000_000}M points: {self.point_count//1_000_000}M\n") - - def _add_point(self, lon, lat, speed): - cx = int(math.floor(lon / CELL)) - cy = int(math.floor(lat / CELL)) - lonq = int(round((lon - cx * CELL) / CELL * 65535)) - latq = int(round((lat - cy * CELL) / CELL * 65535)) - lonq = max(0, min(65535, lonq)) - latq = max(0, min(65535, latq)) - sx = min(SUB - 1, lonq * SUB // 65536) - sy = min(SUB - 1, latq * SUB // 65536) - key = (cx, cy) - cell = self.points.get(key) - if cell is None: - cell = [None] * (SUB * SUB) - self.points[key] = cell - sub = sx + sy * SUB - buf = cell[sub] - if buf is None: - buf = bytearray() - cell[sub] = buf - buf += struct.pack(" {self.outdir}\n") - - -def main(): - if len(sys.argv) != 3: - print(f"usage: {sys.argv[0]} ") - sys.exit(1) - pbf_path = sys.argv[1] - outdir = sys.argv[2] - - # === PASS 1 === - sys.stderr.write("=== PASS 1: Collect node IDs from highway+maxspeed ways ===\n") - t0 = time.time() - c = Pass1Collector() - c.apply_file(pbf_path, locations=False) - sys.stderr.write(f"Pass 1 done: {c.way_count} ways, {time.time()-t0:.0f}s\n") - - # === PASS 2 === - sys.stderr.write("=== PASS 2: Collect nodes + process ways ===\n") - t1 = time.time() - b = Pass2Builder(c.test, outdir) - b.apply_file(pbf_path, locations=False) - if not b.node_phase: - pass - else: - sys.stderr.write("WARNING: no ways processed (node phase never ended)\n") - sys.stderr.write(f"Pass 2 done: {b.node_count} nodes, {b.way_count} ways, {b.point_count} points, {time.time()-t1:.0f}s\n") - - # === WRITE GRID === - t2 = time.time() - b.write_grid() - sys.stderr.write(f"Grid written in {time.time()-t0:.0f}s total\n") - -if __name__ == "__main__": - main() diff --git a/data/maxspeed/build_world.sh b/data/maxspeed/build_world.sh deleted file mode 100644 index 9ab1221..0000000 --- a/data/maxspeed/build_world.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -set -e -cd /home/riricdev/zektyc/data/maxspeed - -echo "=== Étape 1: Filtrer highways du planet ===" -if [ ! -f planet-highways.pbf ]; then - echo "Extraction des highways..." - osmium tags-filter planet-latest.osm.pbf w/highway -o planet-highways.pbf --overwrite - ls -lh planet-highways.pbf -else - echo "planet-highways.pbf existe déjà" -fi - -echo "" -echo "=== Étape 2: Découper en bandes longitude ===" -mkdir -p bands raw - -# 12 bandes de 30° : -180..-150, -150..-120, ..., 150..180 -BAND_SIZE=30 -for i in $(seq 0 11); do - LON_START=$(( -180 + i * BAND_SIZE )) - LON_END=$(( LON_START + BAND_SIZE )) - BAND="band_${LON_START}_${LON_END}" - - if [ -f "raw/${BAND}.raw" ]; then - echo " ${BAND} déjà fait, skip" - continue - fi - - if [ ! -f "bands/${BAND}.pbf" ]; then - echo " Extraction bande ${LON_START}..${LON_END}..." - osmium extract -b ${LON_START},-60,${LON_END},85 planet-highways.pbf -o "bands/${BAND}.pbf" --overwrite - ls -lh "bands/${BAND}.pbf" - fi - - echo " Build bande ${LON_START}..${LON_END}..." - ./build_band "bands/${BAND}.pbf" "raw/${BAND}.raw" - ls -lh "raw/${BAND}.raw" - - rm -f "bands/${BAND}.pbf" - echo "" -done - -echo "=== Étape 3: Merge toutes les bandes ===" -./merge_grid grid-planet raw/*.raw - -echo "=== Terminé! ===" -ls -lh grid-planet/ diff --git a/data/maxspeed/merge_grid.cpp b/data/maxspeed/merge_grid.cpp deleted file mode 100644 index e4465f7..0000000 --- a/data/maxspeed/merge_grid.cpp +++ /dev/null @@ -1,140 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; - -static constexpr double CELL = 0.5; -static constexpr int SUB = 32; -static constexpr double LON_MIN = -180.0; -static constexpr double LAT_MIN = -60.0; -static constexpr double LON_MAX = 180.0; -static constexpr double LAT_MAX = 85.0; - -struct RawPoint { - int32_t col; - int32_t row; - uint16_t lonq; - uint16_t latq; - uint8_t speed; -}; - -struct Point64 { - uint16_t lonq; - uint16_t latq; - uint8_t speed; -}; - -static bool cmp_cell(const RawPoint& a, const RawPoint& b) { - if (a.col != b.col) return a.col < b.col; - if (a.row != b.row) return a.row < b.row; - return false; -} - -int main(int argc, char* argv[]) { - if (argc < 3) { - fprintf(stderr, "Usage: %s [raw_file2] ...\n", argv[0]); - return 1; - } - - const char* outdir = argv[1]; - fs::create_directories(outdir); - - std::vector all_points; - - for (int i = 2; i < argc; i++) { - FILE* f = fopen(argv[i], "rb"); - if (!f) { fprintf(stderr, "Cannot open %s\n", argv[i]); continue; } - - fseek(f, 0, SEEK_END); - long fsize = ftell(f); - fseek(f, 0, SEEK_SET); - - uint64_t before = all_points.size(); - uint64_t npoints = fsize / sizeof(RawPoint); - all_points.resize(before + npoints); - fread(all_points.data() + before, sizeof(RawPoint), npoints, f); - fclose(f); - - fprintf(stderr, "Loaded %s: %lu points (total: %lu)\n", argv[i], npoints, all_points.size()); - } - - fprintf(stderr, "Sorting %lu points...\n", all_points.size()); - std::sort(all_points.begin(), all_points.end(), cmp_cell); - fprintf(stderr, "Sort done. Writing cells...\n"); - - std::string index_path = std::string(outdir) + "/index.bin"; - FILE* index = fopen(index_path.c_str(), "wb"); - if (!index) { fprintf(stderr, "Cannot write index\n"); return 1; } - - uint32_t magic = 0x5A4D4150; - fwrite(&magic, 4, 1, index); - uint16_t version = 2; - fwrite(&version, 2, 1, index); - uint16_t sub = SUB; - fwrite(&sub, 2, 1, index); - fwrite(&CELL, 8, 1, index); - fwrite(&LON_MIN, 8, 1, index); - fwrite(&LAT_MIN, 8, 1, index); - fwrite(&LON_MAX, 8, 1, index); - fwrite(&LAT_MAX, 8, 1, index); - - uint32_t num_cells = 0; - fseek(index, 36, SEEK_SET); - fwrite(&num_cells, 4, 1, index); - fseek(index, 0, SEEK_END); - - int32_t cur_col = -1, cur_row = -1; - FILE* cur_file = nullptr; - uint32_t cell_points = 0; - std::string cur_key; - - for (uint64_t i = 0; i <= all_points.size(); i++) { - bool new_cell = (i == all_points.size() || - all_points[i].col != cur_col || - all_points[i].row != cur_row); - - if (new_cell && cur_file) { - fclose(cur_file); - uint32_t klen = static_cast(cur_key.size()); - fwrite(&klen, 4, 1, index); - fwrite(cur_key.data(), 1, klen, index); - fwrite(&cell_points, 4, 1, index); - num_cells++; - if (num_cells % 500 == 0) { - fprintf(stderr, " %u cells written...\n", num_cells); - } - } - - if (i == all_points.size()) break; - - if (new_cell) { - cur_col = all_points[i].col; - cur_row = all_points[i].row; - cur_key = std::to_string(cur_col) + "_" + std::to_string(cur_row); - std::string cellfile = std::string(outdir) + "/cell_" + cur_key + ".bin"; - cur_file = fopen(cellfile.c_str(), "wb"); - cell_points = 0; - } - - Point64 p; - p.lonq = all_points[i].lonq; - p.latq = all_points[i].latq; - p.speed = all_points[i].speed; - fwrite(&p, sizeof(Point64), 1, cur_file); - cell_points++; - } - - fseek(index, 36, SEEK_SET); - fwrite(&num_cells, 4, 1, index); - fclose(index); - - fprintf(stderr, "Total cells: %u\n", num_cells); - fprintf(stderr, "Grid written to %s\n", outdir); - return 0; -} diff --git a/data/maxspeed/run_build.sh b/data/maxspeed/run_build.sh deleted file mode 100755 index c8c0d72..0000000 --- a/data/maxspeed/run_build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -exec python3 -u /home/riricdev/zektyc/data/maxspeed/build_world.py \ - /home/riricdev/zektyc/data/maxspeed/planet-latest.osm.pbf \ - /home/riricdev/zektyc/data/maxspeed/grid-planet/ \ - >> /home/riricdev/zektyc/data/maxspeed/build-planet.log 2>&1 diff --git a/scripts/antiflood.sh b/scripts/antiflood.sh deleted file mode 100755 index 9e80f0f..0000000 --- a/scripts/antiflood.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env bash -# Zektyc — protection contre les anomalies de trafic (DoS/DDoS L4, SYN floods) -# -# Mise au point : NE COUPE QUE lorsque le réseau est réellement saturé, au bord -# de tomber. On ne réagit plus à un simple pic de débit (un burst bien absorbé -# qui ne sature pas le lien ne doit PAS déclencher). On se base sur le DÉBIT RX -# mesuré en octets par rapport au PLAFOND DE BANDE PASSANTE du lien (étalonné) : -# - on coupe seulement quand le débit RX soutenu approche le plafond -# (le lien est sur le point d'être saturé / de ralentir vraiment), -# - pendant plus de PERSIST ticks consécutifs (pas un pic bref). -# Le funnel + playit sont réactivés automatiquement quand ça redescend. -set -u - -# Webhook Discord chargé depuis un fichier local gitignoré (jamais commité). -source /home/riricdev/zektyc/scripts/.webhooks -WEBHOOK="${ANTIFLOOD_WEBHOOK:-}" -STATE_FILE="/home/riricdev/zektyc/scripts/.synstate" -BASE_FILE="/home/riricdev/zektyc/scripts/.synbase" -LOG="/var/log/zektyc-syn.log" - -IFACE_ETH="eth0" # interface réseau publique (les SYN floods y arrivent) -IFACE_TS="tailscale0" # interface tailscale (légitime) - -INTERVAL=2 # période d'échantillonnage (s) - -# Plafond de bande passante DOWN mesuré via speedtest (~80 Mbit/s ≈ 10 Mo/s). -# Tout le trafic entrant (SYN floods compris) passe par eth0 et sature ce plafond. -# 10000000 octets/s × INTERVAL(2) = octets par tick au plafond. -CAP_BYTES=$(( 10000000 * INTERVAL )) - -# Fraction du plafond au-delà de laquelle on considère le lien "au bord de -# tomber" (0.90 = saturé à 90 %). Un burst bref ne suffit pas : il faut tenir -# au-dessus pendant PERSIST ticks consécutifs. -CAP_RATIO=0.90 -PERSIST=4 # ticks consécutifs au-dessus du seuil requis avant de couper -COOLDOWN_UP=45 # (s) attendre avant de tenter un rétablissement une fois calme - -log() { printf '%s %s\n' "$(date +%F\ %T)" "$1" >>"$LOG"; } -notify() { printf '{"content":"%s"}' "$1" | curl -s -m 10 -H "Content-Type: application/json" -X POST "$WEBHOOK" -d @- >/dev/null 2>&1 || true; } - -# Lit le nombre TOTAL d'OCTETS reçus (RX) sur l'interface donnée, depuis -# /proc/net/dev (2e champ après le nom d'interface, colonne RX bytes). -# La saturation de bande passante se mesure en OCTETS, pas en paquets. -read_rx() { awk -v ifx="$1" '$0 ~ ifx {print $2}' /proc/net/dev; } - -# Vrai si le funnel TCP du site (:443) est actif (relais brut + PROXY protocol v2 -# vers nginx 127.0.0.1:444). Le nom de domaine est suivi de ":443" - on le cherche -# explicitement pour ne pas matcher l'entree fantome sur un autre port (:10000). -funnel_active() { tailscale funnel status 2>/dev/null | grep -qE 'tcp://riricdev\.tail5ea5cd\.ts\.net:443'; } - -funnel_off() { - # coupe le funnel tailscale (site web) — mode TCP brut + PROXY protocol - if funnel_active; then - tailscale funnel --tcp=443 off >/dev/null 2>&1 - log "Anomalie de trafic : funnel tailscale coupé" - fi - # coupe l'entrée playit.gg (serveurs Minecraft publics) - if systemctl is-active --quiet playit.service; then - systemctl stop playit.service >/dev/null 2>&1 - log "Anomalie de trafic : entrée playit coupée" - fi - notify "🔴 **Zektyc** — trafic anormal détecté, accès temporairement suspendu." -} - -funnel_on() { - # réactive le funnel tailscale (site web) — TCP brut + PPv2 vers nginx :444 - if ! funnel_active; then - tailscale funnel --bg --proxy-protocol=2 --tcp=443 tcp://127.0.0.1:444 >/dev/null 2>&1 - log "Retour à la normale : funnel tailscale réactivé" - fi - # réactive l'entrée playit.gg (serveurs Minecraft publics) - if ! systemctl is-active --quiet playit.service; then - systemctl start playit.service >/dev/null 2>&1 - log "Retour à la normale : entrée playit réactivée" - fi - notify "🟢 **Zektyc** — accès rétabli." -} - -# --- init --- -now=$(read_rx "$IFACE_ETH") -prev=$now -[[ ! -f "$STATE_FILE" ]] && echo "up" >"$STATE_FILE" -state="$(<"$STATE_FILE")" -streak=0 -down_since=0 - -# seuil de saturation en octets/tic -thresh=$(( CAP_BYTES > 0 ? CAP_BYTES : 1 )) - -while true; do - now=$(read_rx "$IFACE_ETH") - delta=$(( now - prev )) - prev=$now - [[ "$delta" -lt 0 ]] && delta=0 - state="$(<"$STATE_FILE")" - - # --- saturation : débit RX au-dessus de CAP_RATIO × plafond du lien --- - # Un SYN flood qui sature le lien fait monter le débit RX vers le plafond. - # On ne coupe que si ça tient PERSIST ticks consécutifs (pas un pic bref). - sat=0 - # delta(thresh×CAP_RATIO) ⇔ delta*100 ≥ thresh×(CAP_RATIO×100) - if (( delta * 100 >= thresh * 90 )); then - sat=1 - fi - - if [[ "$sat" -eq 1 ]]; then - streak=$(( streak + 1 )) - else - streak=0 - fi - - # --- actions --- - if [[ "$state" == "up" && "$streak" -ge "$PERSIST" ]]; then - echo down >"$STATE_FILE" - down_since=$(date +%s) - funnel_off - elif [[ "$state" == "down" ]]; then - now_s=$(date +%s) - if (( now_s - down_since >= COOLDOWN_UP && streak == 0 )); then - echo up >"$STATE_FILE" - funnel_on - fi - fi - - sleep "$INTERVAL" -done diff --git a/scripts/ipwatch.sh b/scripts/ipwatch.sh deleted file mode 100755 index 8aad4c9..0000000 --- a/scripts/ipwatch.sh +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env bash -# Zektyc — moniteur d'IP suspectes (staff only) -# -# Toutes les 30 s, analyse deux sources pour détecter des IP anormales qui -# tapent sur la stack, et envoie un rapport (embed) sur le webhook Discord staff. -# -# Source 1 — pare-feu netfilter : log kernel "UFW BLOCK" (vraies IP externes, -# probes, scanners, SYN floods résiduels au niveau réseau). -# -# Source 2 — reverse-proxy nginx : refus HTTP 429/503 émis par les limites -# par IP réelle (limit_req / limit_conn). Comme la terminaison TLS a -# été déplacée vers nginx (mode funnel tcp brut + PROXY protocol), -# c'est ICI qu'apparaissent les attaquants d'applicatif (bursts, -# h2load, etc.) que le pare-feu réseau ne voit plus. On relit le log -# à partir d'un offset pour ne traiter que les lignes nouvelles. -# -# Anti-re-spam : une IP déjà signalée n'est renvoyée que si son compteur de -# tentatives a augmenté depuis le dernier rapport. -set -o pipefail - -# Webhook Discord chargé depuis un fichier local gitignoré (jamais commité). -source /home/riricdev/zektyc/scripts/.webhooks -WEBHOOK="${IPWATCH_WEBHOOK:-}" -INTERVAL=30 -STATE="/home/riricdev/zektyc/scripts/.ipwatch-state" # IP:compteur déjà signalés -NGINX_LOG="/var/log/nginx/access.log" -# Taille (octets) du bloc de fin du log relue à chaque cycle puis filtrée au -# timestamp : couvre INTERVAL s même au pic (h2load). Rotation-safe, pas d'offset. -TL_BACK=16777216 # 16 Mo - -# IP publique ? 0=oui(suspecte possible) 1=non. On exclut aussi le CGNAT -# Tailscale (100.64.0.0/10) pour ne pas signaler le VPN légitime / les tests. -is_public() { - local ip="$1" a b c d - IFS='.' read -r a b c d <<<"$ip" - [[ "$a" == "127" ]] && return 1 - [[ "$a" == "10" ]] && return 1 - [[ "$a" == "192" && "$b" == "168" ]] && return 1 - [[ "$a" == "172" && "$b" -ge 16 && "$b" -le 31 ]] && return 1 - [[ "$a" == "100" && "$b" -ge 64 && "$b" -le 127 ]] && return 1 - [[ "$a" == "169" && "$b" == "254" ]] && return 1 - [[ "$a" -ge 224 && "$a" -le 239 ]] && return 1 - [[ "$a" == "0" || "$a" == "255" ]] && return 1 - return 0 -} - -# Envoie l'embed. Retour 0 si envoyé. -send_report() { - local report_txt="$1" now - now=$(date +"%Y-%m-%d %H:%M:%S") - local json - json=$(jq -n \ - --arg title "🚨 IP suspectes détectées" \ - --arg desc "$report_txt" \ - --arg time "$now" \ - --argjson color 15548997 \ - '{embeds:[{title:$title,description:$desc,color:$color,footer:{text:("Zektyc Sécu — "+$time)}}]}') - curl -s -m 10 -H "Content-Type: application/json" -X POST "$WEBHOOK" -d "$json" >/dev/null 2>&1 -} - -# Charge l'état des IP déjà signalées (format "IP compteur" par ligne) -load_state() { - declare -gA last=() - if [[ -f "$STATE" ]]; then - while read -r k v; do last["$k"]="$v"; done <"$STATE" - fi -} - -# Sauvegarde l'état -save_state() { - : >"$STATE" - for k in "${!last[@]}"; do printf '%s %s\n' "$k" "${last[$k]}" >>"$STATE"; done -} - -# --- Source 2 : relit la FIN du log nginx et garde les refus des 30 dernières s --- -# Pas d'offset : on lit un bloc fixe de la fin du fichier (TL_BACK, dimensionné -# pour couvrir INTERVAL s même au pic de trafic) puis on ne garde QUE les lignes -# dont l'horodatage est dans la fenêtre [ début, date_actuelle ]. Robuste face -# aux rotations/troncatures, aucun offset à resynchroniser manuellement. -# Tout le filtrage se fait dans UN SEUL awk (pas de fork par ligne). -read_nginx_refus() { - local awkprog ip c ts uals from - from="$(date -d "@$(( now_s - INTERVAL ))" +%d/%b/%Y:%H:%M:%S)" - awkprog=' - function ispub(ip, a,b,c,d,p,n) { - n=split(ip,p,"."); if (n!=4) return 0 - a=p[1]+0; b=p[2]+0; c=p[3]+0; d=p[4]+0 - if (a==127||a==10||(a==192&&b==168)||(a==172&&b>=16&&b<=31)\ - ||(a==100&&b>=64&&b<=127)||(a==169&&b==254)) return 0 - if (a>=224&&a<=239||a==0||a==255) return 0 - return 1 - } - { - # Gardes de structure : le buffer tail -c commence en plein milieu d une - # ligne -> son 1er enregistrement est un fragment casse. On n accepte que - # les lignes completes : IP v4 en tete + timestamp nginx complet en $3. - if ($1 !~ /^[0-9]{1,3}(\.[0-9]{1,3}){3}$/) next - ts=$3; sub(/^\[/,"",ts) - if (ts !~ /^[0-9]{2}\/[A-Za-z]{3}\/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2}/) next - # nginx logue en local time (+0200) : on n accepte que les lignes >= debut fenetre - if (ts < from) next - code="" - for (i=1;i<=NF;i++) - if ($i ~ /^[0-9]{3}$/ && (i+1)<=NF && $(i+1) ~ /^[0-9]+$/) code=$i - if (code!="429" && code!="503") next - ip=$1 - if (!ispub(ip)) next - cnt[ip]++ - n=split($0,q,"\"") - ua=(n>=2 ? substr(q[n-1],1,40) : "") - if (ua!="" && !((ip SUBSEP ua) in seen)) { seen[ip SUBSEP ua]=1; ual[ip]=(ual[ip]==""?ua:ual[ip]","ua) } - last[ip]=ts - } - END { for (ip in cnt) print ip "|" cnt[ip] "|" last[ip] "|" ual[ip] } - ' - while IFS='|' read -r ip c ts uals; do - cnt["$ip"]=$(( ${cnt["$ip"]:-0} + c )) - [[ "${protos["$ip"]:-}" != *"http"* ]] && protos["$ip"]="${protos["$ip"]:-}http," - [[ "${dstports["$ip"]:-}" != *"443"* ]] && dstports["$ip"]="${dstports["$ip"]:-}443," - lastts["$ip"]="$ts" - [[ -n "$uals" ]] && uas["$ip"]="${uals}," - done < <(tail -c "$TL_BACK" "$NGINX_LOG" | awk -v "from=$from" "$awkprog") -} - -load_state -echo "IP-watch démarré — fenêtre ${INTERVAL}s/signalement (ufw + nginx), webhook staff (actif)." - -while true; do - now_s=$(date +%s) - since=$(( now_s - INTERVAL )) - - declare -A cnt=() - declare -A protos=() - declare -A dstports=() - declare -A lastts=() - declare -A uas=() - - # --- source 1 : blocs netfilter (log kernel UFW BLOCK) --- - while IFS= read -r ln; do - [[ "$ln" != *"UFW BLOCK"* ]] && continue - if [[ "$ln" =~ \ SRC=([0-9.]+)\ ]]; then - ip="${BASH_REMATCH[1]}" - is_public "$ip" || continue - cnt["$ip"]=$(( ${cnt["$ip"]:-0} + 1 )) - if [[ "$ln" =~ \ PROTO=([A-Za-z0-9]+)\ ]]; then - p="${BASH_REMATCH[1]}" - [[ "${protos["$ip"]:-}" != *"$p"* ]] && protos["$ip"]="${protos["$ip"]:-}$p," - fi - if [[ "$ln" =~ \ DPT=([0-9]+) ]]; then - d="${BASH_REMATCH[1]}" - [[ "${dstports["$ip"]:-}" != *"$d"* ]] && dstports["$ip"]="${dstports["$ip"]:-}$d," - fi - if [[ "$ln" =~ ^[A-Z][a-z]{2}\ [0-9]{2}\ [0-9]{2}\:[0-9]{2}\:[0-9]{2} ]]; then - lastts["$ip"]="${BASH_REMATCH[0]}" - fi - fi - done < <(sudo journalctl -k --no-pager --since "@${since}" 2>/dev/null | grep "UFW BLOCK") - - # --- source 2 : refus HTTP nginx (429/503 par IP réelle) --- - read_nginx_refus - - # --- filtrer : ne signaler que les IP nouvelles ou dont le compteur a augmenté --- - newfound=0 - report="" - for ip in "${!cnt[@]}"; do - c="${cnt[$ip]}" - prev="${last["$ip"]:-0}" - if (( c > prev )); then - proto="${protos[$ip]%,}" - dports="${dstports[$ip]%,}" - ua="${uas[$ip]%,}" - report+=$':shield: **`'"${ip}"$'`** — **'"${c}"$'** tentative(s) bloquée(s)\n' - report+=$'└ Sources: `'"${proto:-?}"$'` | Ports cibles: `'"${dports:-?}"$'`\n' - if [[ -n "$ua" ]]; then - report+=$'└ User-Agent: `'"${ua}"$'`\n' - fi - report+=$'└ Dernière tentative: `'"${lastts[$ip]:-?}"$'`\n\n' - last["$ip"]="$c" - ((newfound++)) - fi - done - - if (( newfound > 0 )); then - send_report "$report" - fi - - save_state - sleep "$INTERVAL" -done \ No newline at end of file diff --git a/scripts/upnp-watch.sh b/scripts/upnp-watch.sh deleted file mode 100755 index f52e660..0000000 --- a/scripts/upnp-watch.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -UPNP_URL="http://192.168.1.254:5678/desc/root" -LOCAL_IP="192.168.1.50" -LOG_TAG="upnp-watch" - -log() { - logger -t "$LOG_TAG" "$1" - echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" -} - -MAPPINGS=$(upnpc -u "$UPNP_URL" -l 2>/dev/null) -if [ -z "$MAPPINGS" ]; then - log "ERREUR: passerelle UPnP injoignable" - exit 1 -fi - -for PORT in 80 443; do - if echo "$MAPPINGS" | grep -q "TCP.*${PORT}->${LOCAL_IP}:${PORT}"; then - log "OK: port ${PORT}/TCP déjà ouvert" - else - log "port ${PORT}/TCP absent -> ouverture" - RESULT=$(upnpc -u "$UPNP_URL" -a "$LOCAL_IP" "$PORT" "$PORT" TCP 2>&1 | tail -1) - if echo "$RESULT" | grep -q "is redirected"; then - log "port ${PORT}/TCP ouvert avec succès" - else - log "ECHEC ouverture ${PORT}/TCP: $RESULT" - fi - fi -done diff --git a/scripts/uptime-check.sh b/scripts/uptime-check.sh deleted file mode 100755 index 8da6ec4..0000000 --- a/scripts/uptime-check.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -# Zektyc — moniteur de disponibilité (uptime checker) -# Vérifie toutes les 5 secondes le code HTTP de réponse de la homepage. -# - Code 2xx/3xx -> online -# - Code 000 (échec connexion / funnel coupé), 5xx (erreur serveur) ou autre -# code d'échec -> incident technique -set -uo pipefail - -URL="https://riricdev.tail5ea5cd.ts.net/" -# Webhook Discord chargé depuis un fichier local gitignoré (jamais commité). -source /home/riricdev/zektyc/scripts/.webhooks -WEBHOOK="${ANTIFLOOD_WEBHOOK:-}" -STATE_FILE="/home/riricdev/zektyc/scripts/.uptime-state" -INTERVAL=5 - -# Un code HTTP est "online" s'il est 2xx/3xx, ou 429 (rate limit = serveur up, -# juste throttlé). Tout le reste (000, 4xx hors 429, 5xx) = down. -http_is_online() { - local code="$1" - case "$code" in - 2*|3*|429) return 0 ;; # 200-399 et 429 : online - *) return 1 ;; # 000, 4xx (hors 429), 5xx : down - esac -} - -down() { - local payload - payload=$(printf '{"content":"🔴 **Zektyc** — incident technique."}' ) - curl -s -m 10 -H "Content-Type: application/json" -X POST "$WEBHOOK" -d "$payload" >/dev/null 2>&1 || true - echo down >"$STATE_FILE" -} - -up() { - local payload - payload=$(printf '{"content":"🟢 **Zektyc** — le site est de nouveau en ligne."}' ) - curl -s -m 10 -H "Content-Type: application/json" -X POST "$WEBHOOK" -d "$payload" >/dev/null 2>&1 || true - echo up >"$STATE_FILE" -} - -if [[ ! -f "$STATE_FILE" ]]; then - echo "unknown" >"$STATE_FILE" -fi - -while true; do - code="$(curl -s -o /dev/null -w '%{http_code}' -m 10 "$URL" 2>/dev/null || echo 000)" - code="${code:-000}" - state="$(cat "$STATE_FILE")" - - if http_is_online "$code"; then - if [[ "$state" != "up" ]]; then - up - fi - else - if [[ "$state" != "down" ]]; then - # On n'envoie qu'une fois la notification de panne (quand on était up). - if [[ "$state" == "up" ]]; then - down - else - # État initial inconnu : on se contente de mémoriser sans notifier. - echo down >"$STATE_FILE" - fi - fi - fi - - sleep "$INTERVAL" -done