#!/bin/bash set -euo pipefail VERSION="7.2.3" APPDIR="/opt/tklm-backup-agent" CONFDIR="/etc/tklm-backup-agent" STATEDIR="/var/lib/tklm-backup-agent" DBBACKUPDIR="/var/backups/tklm-db" CONFIG="$CONFDIR/config.json" AGENT="$APPDIR/tklm_agent.py" SERVICE="/etc/systemd/system/tklm-backup-agent.service" HELPER="/usr/local/sbin/tklm-agent" WATCHDOG_SERVICE="/etc/systemd/system/tklm-backup-agent-watchdog.service" WATCHDOG_TIMER="/etc/systemd/system/tklm-backup-agent-watchdog.timer" BASE_URL="${TKLM_BASE_URL:-https://crm.tklm.pl/api/v1/backup}" red() { printf '\033[31m%s\033[0m\n' "$*"; } green() { printf '\033[32m%s\033[0m\n' "$*"; } yellow() { printf '\033[33m%s\033[0m\n' "$*"; } cyan() { printf '\033[36m%s\033[0m\n' "$*"; } need_root() { [ "$(id -u)" -eq 0 ] || { red "Uruchom jako root: sudo bash $0"; exit 1; } } normalize_agent_id() { local x x="$(hostname -s 2>/dev/null || hostname)" printf '%s' "$x" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9._-]+/-/g; s/^-+//; s/-+$//' } detect_client() { if [ -s /etc/tklm-client ]; then head -1 /etc/tklm-client | tr -d '\r\n' return fi # Nie wyciągamy klienta z końcówki FQDN (crm.tklm.pl -> "pl" było błędne). # Bez jawnego /etc/tklm-client bezpiecznym domyślnym identyfikatorem jest hostname. hostname -s 2>/dev/null || hostname } pkg_install() { local pkg="$1" if command -v apt-get >/dev/null 2>&1; then DEBIAN_FRONTEND=noninteractive apt-get install -y "$pkg" elif command -v dnf >/dev/null 2>&1; then dnf install -y "$pkg" elif command -v yum >/dev/null 2>&1; then yum install -y "$pkg" elif command -v zypper >/dev/null 2>&1; then zypper --non-interactive install "$pkg" elif command -v pacman >/dev/null 2>&1; then pacman -Sy --noconfirm "$pkg" else return 1 fi } ensure_python() { command -v python3 >/dev/null 2>&1 && return 0 yellow "Brak python3 — próbuję doinstalować..." if command -v apt-get >/dev/null 2>&1; then apt-get update; fi pkg_install python3 || { red "Nie udało się zainstalować python3."; exit 1; } } ensure_optional_tools() { if ! command -v smartctl >/dev/null 2>&1; then yellow "Brak smartctl — próbuję doinstalować smartmontools (zalecane)..." if command -v apt-get >/dev/null 2>&1; then apt-get update >/dev/null 2>&1 || true; fi pkg_install smartmontools >/dev/null 2>&1 || yellow "Nie udało się doinstalować smartmontools. Agent będzie działał, SMART pokaże WARNING." fi local need_mysql=0 need_pg=0 (pgrep -x mysqld >/dev/null 2>&1 || pgrep -x mariadbd >/dev/null 2>&1 || systemctl list-unit-files 2>/dev/null | grep -Eqi '^(mysql|mariadb).*service') && need_mysql=1 || true (pgrep -x postgres >/dev/null 2>&1 || systemctl list-unit-files 2>/dev/null | grep -Eqi '^postgresql.*service') && need_pg=1 || true # Kontenery DB mają własne narzędzia klienckie w środku. Nie instalujemy klienta # na hoście tylko dlatego, że działa kontener PostgreSQL/MySQL. if [ "$need_mysql" -eq 1 ] && ! command -v mysqldump >/dev/null 2>&1 && ! command -v mariadb-dump >/dev/null 2>&1; then yellow "Wykryto MySQL/MariaDB bez klienta dump — próbuję doinstalować klienta..." if command -v apt-get >/dev/null 2>&1; then DEBIAN_FRONTEND=noninteractive apt-get install -y mariadb-client >/dev/null 2>&1 || true; elif command -v dnf >/dev/null 2>&1; then dnf install -y mariadb >/dev/null 2>&1 || true; elif command -v yum >/dev/null 2>&1; then yum install -y mariadb >/dev/null 2>&1 || true; fi fi if [ "$need_pg" -eq 1 ] && ! command -v pg_dump >/dev/null 2>&1; then yellow "Wykryto PostgreSQL bez pg_dump — próbuję doinstalować klienta..." if command -v apt-get >/dev/null 2>&1; then DEBIAN_FRONTEND=noninteractive apt-get install -y postgresql-client >/dev/null 2>&1 || true; elif command -v dnf >/dev/null 2>&1; then dnf install -y postgresql >/dev/null 2>&1 || true; elif command -v yum >/dev/null 2>&1; then yum install -y postgresql >/dev/null 2>&1 || true; fi fi } uninstall_agent() { need_root yellow "Usuwam TKLM Backup Agent Linux..." systemctl disable --now tklm-backup-agent-watchdog.timer >/dev/null 2>&1 || true systemctl disable --now tklm-backup-agent.service >/dev/null 2>&1 || true rm -f "$SERVICE" "$WATCHDOG_SERVICE" "$WATCHDOG_TIMER" "$HELPER" rm -rf "$APPDIR" systemctl daemon-reload || true echo read -r -p "Usunąć też konfigurację i stan z /etc oraz /var/lib? [t/N]: " ans if [[ "$ans" =~ ^[TtYy]$ ]]; then rm -rf "$CONFDIR" "$STATEDIR" yellow "Backupów baz w $DBBACKUPDIR nie usuwam automatycznie." fi green "Odinstalowano." exit 0 } status_agent() { systemctl status tklm-backup-agent.service --no-pager -l || true exit 0 } case "${1:-}" in --uninstall) uninstall_agent ;; --status) status_agent ;; esac need_root ensure_python command -v systemctl >/dev/null 2>&1 || { red "Ten installer wymaga systemd/systemctl."; exit 1; } ensure_optional_tools AGENT_ID="$(normalize_agent_id)" SERVER_NAME="$(hostname -s 2>/dev/null || hostname)" CLIENT_NAME="$(detect_client)" OLD_KEY="" OLD_AGENT_ID="" OLD_CLIENT="" OLD_SERVER="" if [ -f "$CONFIG" ]; then eval "$(python3 - "$CONFIG" <<'PYOLD' import json,shlex,sys try: d=json.load(open(sys.argv[1],encoding='utf-8')) except Exception: d={} a=d.get('agent') or {} api=d.get('api') or {} for k,v in { 'OLD_KEY':api.get('key',''), 'OLD_AGENT_ID':a.get('id',''), 'OLD_CLIENT':a.get('client',''), 'OLD_SERVER':a.get('server',''), }.items(): print(f"{k}={shlex.quote(str(v or ''))}") PYOLD )" fi [ -n "$OLD_AGENT_ID" ] && AGENT_ID="$OLD_AGENT_ID" if [ -n "$OLD_CLIENT" ]; then case "${OLD_CLIENT,,}" in pl|com|net|org|local|lan) yellow "Poprawiam stary błędnie wykryty Client '$OLD_CLIENT' -> '$CLIENT_NAME'." ;; *) CLIENT_NAME="$OLD_CLIENT" ;; esac fi [ -n "$OLD_SERVER" ] && SERVER_NAME="$OLD_SERVER" # === TKLM_CRM_PREFILL_V18 === # Dane z kreatora CRM maja pierwszenstwo nad autodetekcja i stara konfiguracja. [ -n "${TKLM_AGENT_ID:-}" ] && AGENT_ID="$TKLM_AGENT_ID" [ -n "${TKLM_CLIENT:-}" ] && CLIENT_NAME="$TKLM_CLIENT" [ -n "${TKLM_SERVER:-}" ] && SERVER_NAME="$TKLM_SERVER" [ -n "${TKLM_BASE_URL:-}" ] && BASE_URL="$TKLM_BASE_URL" # === /TKLM_CRM_PREFILL_V18 === echo cyan "==============================================================" cyan " TKLM Backup Monitor Linux v$VERSION — monitoring + DB backup" cyan "==============================================================" echo echo "CRM: $BASE_URL" echo "Agent ID: $AGENT_ID" echo "Klient: $CLIENT_NAME" echo "Serwer: $SERVER_NAME" echo if [ -n "${TKLM_API_KEY:-}" ]; then API_KEY="$TKLM_API_KEY" green "API KEY przekazany automatycznie z CRM." elif [ -n "$OLD_KEY" ]; then API_KEY="$OLD_KEY" green "Wykryto istniejącą konfigurację — zachowuję Agent ID i API KEY." else yellow "W CRM utwórz klucz API DOKŁADNIE dla Agent ID: $AGENT_ID" while true; do read -r -s -p "API KEY: " API_KEY echo [ -n "$API_KEY" ] && break red "Klucz API nie może być pusty." done fi umask 077 mkdir -p "$APPDIR" "$CONFDIR" "$STATEDIR" "$DBBACKUPDIR" chmod 700 "$CONFDIR" "$STATEDIR" "$DBBACKUPDIR" cat > "$AGENT" <<'PYAGENT' #!/usr/bin/env python3 # -*- coding: utf-8 -*- from __future__ import annotations import argparse import fnmatch import gzip import hashlib import hmac import json import os import platform import re import shutil import socket import sqlite3 import subprocess import tempfile import threading import time import urllib.error import urllib.request from datetime import datetime, timedelta from pathlib import Path APP_VERSION = "7.2.3" CONFIG_PATH = Path(os.environ.get("TKLM_AGENT_CONFIG", "/etc/tklm-backup-agent/config.json")) STATE_DIR = Path("/var/lib/tklm-backup-agent") DB_PATH = STATE_DIR / "agent.db" DISCOVERY_PATH = STATE_DIR / "discovery.json" DEFAULT_DB_BACKUP_ROOT = "/var/backups/tklm-db" RUNTIME_PATH = STATE_DIR / "agent_runtime.json" STARTED_AT = datetime.now().astimezone().isoformat(timespec="seconds") SCHEMA = """ PRAGMA journal_mode=WAL; CREATE TABLE IF NOT EXISTS api_queue( id INTEGER PRIMARY KEY AUTOINCREMENT, created_at TEXT NOT NULL, endpoint TEXT NOT NULL, payload TEXT NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, last_error TEXT ); CREATE TABLE IF NOT EXISTS checks( id INTEGER PRIMARY KEY AUTOINCREMENT, checked_at TEXT NOT NULL, kind TEXT NOT NULL, resource_id TEXT NOT NULL, status TEXT NOT NULL, message TEXT ); CREATE INDEX IF NOT EXISTS idx_checks_time ON checks(checked_at); CREATE TABLE IF NOT EXISTS metric_state( metric_id TEXT PRIMARY KEY, breach_count INTEGER NOT NULL DEFAULT 0, last_raw_status TEXT NOT NULL DEFAULT 'OK', last_value REAL, updated_at TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS meta( key TEXT PRIMARY KEY, value TEXT ); CREATE TABLE IF NOT EXISTS db_backup_state( engine TEXT NOT NULL, database_name TEXT NOT NULL, checked_at TEXT NOT NULL, status TEXT NOT NULL, file TEXT, size_mb REAL, duration_sec REAL, message TEXT, PRIMARY KEY(engine, database_name) ); CREATE TABLE IF NOT EXISTS db_backup_history( id INTEGER PRIMARY KEY AUTOINCREMENT, engine TEXT NOT NULL, database_name TEXT NOT NULL, checked_at TEXT NOT NULL, status TEXT NOT NULL, file TEXT, size_mb REAL, duration_sec REAL, message TEXT ); CREATE INDEX IF NOT EXISTS idx_dbhist_db_time ON db_backup_history(engine,database_name,checked_at); """ PSEUDO_FS = { "proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "cgroup", "cgroup2", "securityfs", "pstore", "debugfs", "tracefs", "configfs", "fusectl", "mqueue", "hugetlbfs", "ramfs", "autofs", "overlay", "squashfs", "nsfs", "bpf", "binfmt_misc", } IGNORE_MOUNT_PREFIXES = ( "/run/", "/snap/", "/var/lib/docker/", "/var/lib/containers/", "/var/lib/kubelet/", "/proc/", "/sys/", "/dev/", ) SERVICE_KEYWORDS = ( "ssh", "sshd", "cron", "crond", "docker", "containerd", "podman", "mariadb", "mysql", "postgresql", "nginx", "apache2", "httpd", "redis", "memcached", "firebird", "mssql", "samba", "smbd", "nmbd", "winbind", "urbackup", "caddy", "haproxy", "traefik", "vaultwarden", "paperless", "fail2ban", "postfix", "dovecot", "rabbitmq", "mosquitto", "zabbix", "grafana", "prometheus", "influxdb", "telegraf", "elasticsearch", "opensearch", "kibana", "jenkins", "gitlab", "cockpit", "nfs-server", "rpcbind", ) BACKUP_WORDS = ("backup", "backups", "kopia", "kopie", "archiwum", "archive") SEVERITY = {"CRITICAL": 4, "WARNING": 3, "OK": 2, "DISABLED": 1} def now_iso() -> str: return datetime.now().astimezone().isoformat(timespec="seconds") def log(msg: str) -> None: print(f"{datetime.now().astimezone():%Y-%m-%d %H:%M:%S} [TKLM] {msg}", flush=True) def write_runtime(state="RUNNING", detail="") -> None: payload = { "version": APP_VERSION, "engine": "systemd/root", "state": state, "pid": os.getpid(), "started_at": STARTED_AT, "heartbeat_at": now_iso(), "detail": str(detail or "")[:1000], } try: STATE_DIR.mkdir(parents=True, exist_ok=True) tmp = RUNTIME_PATH.with_suffix(".tmp") tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") os.chmod(tmp, 0o600) os.replace(tmp, RUNTIME_PATH) except Exception: pass def runtime_heartbeat_worker() -> None: while True: write_runtime("RUNNING") time.sleep(15) def start_runtime_heartbeat() -> None: t = threading.Thread(target=runtime_heartbeat_worker, name="tklm-runtime-heartbeat", daemon=True) t.start() def db_connect() -> sqlite3.Connection: con = sqlite3.connect(DB_PATH, timeout=30) con.row_factory = sqlite3.Row return con def init_state() -> None: STATE_DIR.mkdir(parents=True, exist_ok=True) os.chmod(STATE_DIR, 0o700) with sqlite3.connect(DB_PATH) as con: con.executescript(SCHEMA) # Migracja kolejki ze starszych wersji: zostaje wyłącznie najnowszy snapshot endpointu. con.execute("DELETE FROM api_queue WHERE id NOT IN (SELECT MAX(id) FROM api_queue GROUP BY endpoint)") cutoff = (datetime.now().astimezone() - timedelta(days=90)).isoformat(timespec="seconds") con.execute("DELETE FROM checks WHERE checked_at < ?", (cutoff,)) hist_cutoff = (datetime.now().astimezone() - timedelta(days=180)).isoformat(timespec="seconds") con.execute("DELETE FROM db_backup_history WHERE checked_at < ?", (hist_cutoff,)) try: os.chmod(DB_PATH, 0o600) except OSError: pass def deep_defaults(dst: dict, defaults: dict) -> dict: for key, value in defaults.items(): if key not in dst: dst[key] = value elif isinstance(value, dict) and isinstance(dst.get(key), dict): deep_defaults(dst[key], value) return dst def config_defaults() -> dict: return { "heartbeat_minutes": 5, "system_check_minutes": 5, "backup_check_minutes": 30, "system_monitor_enabled": True, "disk_warning_percent_free": 15, "disk_critical_percent_free": 8, "inode_warning_percent_free": 15, "inode_critical_percent_free": 5, "mount_check_timeout_seconds": 15, "cpu_warning_percent": 85, "cpu_critical_percent": 95, "ram_warning_percent": 85, "ram_critical_percent": 95, "metric_consecutive_checks": 3, "backup_warning_hours": 26, "backup_critical_hours": 36, "auto_discovery": True, "extra_services": [], "extra_mounts": [], "extra_backup_dirs": [], "auto_backup_arm_on_new_file": True, "docker_monitor_enabled": True, "smart": { "enabled": True, "warning_temperature_c": 55, "critical_temperature_c": 65, "warning_percentage_used": 90, "critical_percentage_used": 100, }, "db_backups": { "enabled": True, "first_backup_on_install": True, "time": "02:30", "root": DEFAULT_DB_BACKUP_ROOT, "retention_days": 14, "min_free_gb": 2, "min_free_percent": 10, "space_guard_enabled": True, "estimate_multiplier": 1.50, "first_backup_source_multiplier": 1.10, "unknown_backup_reserve_gb": 2, "space_check_interval_mb": 32, "retry_after_failure_minutes": 60, "dump_timeout_minutes": 360, "mysql": { "enabled": True, "auto_discover": True, "host": "", "port": 3306, "user": "root", "defaults_file": "", "include": [], "exclude": ["information_schema", "performance_schema", "mysql", "sys"], }, "postgresql": { "enabled": True, "auto_discover": True, "host": "", "port": 5432, "user": "", "run_as_user": "postgres", "pgpass_file": "", "docker_auto_discover": True, "include": [], "exclude": ["template0", "template1"], }, }, } def load_config() -> dict: with CONFIG_PATH.open("r", encoding="utf-8") as f: cfg = json.load(f) if not isinstance(cfg, dict): raise RuntimeError("config.json nie jest obiektem JSON") deep_defaults(cfg, config_defaults()) return cfg def load_discovery() -> dict: try: with DISCOVERY_PATH.open("r", encoding="utf-8") as f: d = json.load(f) if isinstance(d, dict): d.setdefault("mounts", {}) d.setdefault("services", {}) d.setdefault("backups", {}) d.setdefault("containers", {}) return d except Exception: pass return {"mounts": {}, "services": {}, "backups": {}, "containers": {}} def save_discovery(d: dict) -> None: tmp = DISCOVERY_PATH.with_suffix(".tmp") tmp.write_text(json.dumps(d, ensure_ascii=False, indent=2), encoding="utf-8") os.chmod(tmp, 0o600) os.replace(tmp, DISCOVERY_PATH) def run(cmd, timeout=15, env=None, input_text=None): return subprocess.run( cmd, capture_output=True, text=True, timeout=timeout, env=env, input=input_text, ) def hostname_short() -> str: return socket.gethostname().split(".")[0] def os_release() -> dict: data = {} try: for line in Path("/etc/os-release").read_text(encoding="utf-8", errors="replace").splitlines(): if "=" not in line: continue k, v = line.split("=", 1) data[k] = v.strip().strip('"') except Exception: pass return data def agent_base(cfg: dict) -> dict: a = cfg.get("agent", {}) host = hostname_short() rel = os_release() pretty = rel.get("PRETTY_NAME") or platform.platform() return { "agent_id": a.get("id") or host.lower(), "client": a.get("client") or host, "server": a.get("server") or host, "hostname": host, "version": f"{APP_VERSION}-linux", "service": "tklm-backup-agent.service", "os": pretty, "kernel": platform.release(), "platform": "linux", } def api_headers(cfg: dict, body: bytes) -> dict: agent_id = str(cfg["agent"]["id"]) key = str(cfg["api"]["key"]) ts = str(int(time.time())) sig = hmac.new( key.encode("utf-8"), ts.encode("ascii") + b"." + body, hashlib.sha256 ).hexdigest() return { "Content-Type": "application/json", "User-Agent": f"TKLMBackupAgent/{APP_VERSION}-linux", "X-TKLM-Agent": agent_id, "X-TKLM-Timestamp": ts, "X-TKLM-Signature": sig, } def api_post(cfg: dict, endpoint: str, payload: dict, timeout=20): base = str(cfg["api"].get("base_url") or "").rstrip("/") body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") req = urllib.request.Request( base + endpoint, data=body, headers=api_headers(cfg, body), method="POST" ) try: with urllib.request.urlopen(req, timeout=timeout) as resp: text = resp.read(8192).decode("utf-8", "replace") return True, int(resp.status), text except urllib.error.HTTPError as e: try: text = e.read(8192).decode("utf-8", "replace") except Exception: text = str(e) return False, int(e.code), text except Exception as e: return False, 0, str(e) def queue_latest(endpoint: str, payload: dict, error: str) -> None: with db_connect() as con: con.execute("DELETE FROM api_queue WHERE endpoint=?", (endpoint,)) con.execute( "INSERT INTO api_queue(created_at,endpoint,payload,attempts,last_error) VALUES(?,?,?,?,?)", (now_iso(), endpoint, json.dumps(payload, ensure_ascii=False), 1, str(error)[:1000]), ) def send_or_queue(cfg: dict, endpoint: str, payload: dict): ok, code, msg = api_post(cfg, endpoint, payload) if ok: with db_connect() as con: con.execute("DELETE FROM api_queue WHERE endpoint=?", (endpoint,)) else: queue_latest(endpoint, payload, f"{code}: {msg}") return ok, code, msg def flush_queue(cfg: dict, limit=10) -> None: # Najnowsze snapshoty, nie historia. checked_at po stronie CRM dodatkowo chroni przed cofaniem stanu. with db_connect() as con: rows = con.execute("SELECT * FROM api_queue ORDER BY id DESC LIMIT ?", (limit,)).fetchall() for row in reversed(rows): try: payload = json.loads(row["payload"]) except Exception: with db_connect() as con: con.execute("DELETE FROM api_queue WHERE id=?", (row["id"],)) continue ok, code, msg = api_post(cfg, row["endpoint"], payload) with db_connect() as con: if ok: con.execute("DELETE FROM api_queue WHERE id=?", (row["id"],)) else: con.execute( "UPDATE api_queue SET attempts=attempts+1,last_error=? WHERE id=?", (f"{code}: {msg}"[:1000], row["id"]), ) break def meta_get(key: str, default="") -> str: with db_connect() as con: row = con.execute("SELECT value FROM meta WHERE key=?", (key,)).fetchone() return row["value"] if row else default def meta_set(key: str, value: str) -> None: with db_connect() as con: con.execute( "INSERT INTO meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", (key, str(value)), ) def discover_mounts_current() -> dict: mounts = {} try: p = run(["findmnt", "-J", "-o", "TARGET,SOURCE,FSTYPE"], timeout=15) if p.returncode == 0: data = json.loads(p.stdout or "{}") def walk(items): for x in items or []: target = str(x.get("target") or "") source = str(x.get("source") or "") fstype = str(x.get("fstype") or "").lower() if ( target and target.startswith("/") and fstype not in PSEUDO_FS and not any(target.startswith(pref) for pref in IGNORE_MOUNT_PREFIXES) ): mounts[target] = {"source": source, "fstype": fstype} walk(x.get("children")) walk(data.get("filesystems")) mounts.setdefault("/", {"source": "root", "fstype": ""}) return mounts except Exception: pass try: for line in Path("/proc/mounts").read_text(encoding="utf-8", errors="replace").splitlines(): parts = line.split() if len(parts) < 3: continue source, target, fstype = parts[:3] target = target.replace("\\040", " ") if ( target.startswith("/") and fstype.lower() not in PSEUDO_FS and not any(target.startswith(pref) for pref in IGNORE_MOUNT_PREFIXES) ): mounts[target] = {"source": source, "fstype": fstype.lower()} except Exception: pass mounts.setdefault("/", {"source": "root", "fstype": ""}) return mounts def service_unit_is_monitorable(unit: str) -> bool: low = (unit or "").lower().strip() if not low or not low.endswith(".service") or low == "tklm-backup-agent.service": return False if low.endswith("@.service") or low.startswith("sshd@"): return False ignored = ("keygen", "key-generator", "generate", "generator", "-setup.service", "-prepare.service", "-init.service") if any(x in low for x in ignored): return False return any(k in low for k in SERVICE_KEYWORDS) def discover_services_current() -> dict: found = {} if not shutil.which("systemctl"): return found try: p = run([ "systemctl", "list-units", "--type=service", "--all", "--no-legend", "--no-pager", "--plain", ], timeout=25) for line in (p.stdout or "").splitlines(): cols = line.split(None, 4) if len(cols) < 4: continue unit, load_state, active_state, sub_state = cols[:4] if load_state != "loaded" or active_state != "active": continue if service_unit_is_monitorable(unit): found[unit] = {"name": unit, "active": active_state, "sub": sub_state} except Exception: pass return found def discover_docker_containers_current() -> dict: if not shutil.which("docker"): return {} try: p = run(["docker", "ps", "--format", "{{.ID}}\t{{.Names}}\t{{.Status}}"], timeout=15) if p.returncode != 0: return {} out = {} for line in (p.stdout or "").splitlines(): cols = line.split("\t", 2) if len(cols) < 2: continue cid, name = cols[0].strip(), cols[1].strip() status = cols[2].strip() if len(cols) > 2 else "running" if cid and name: out[name] = {"id": cid, "name": name, "status": status} return out except Exception: return {} def candidate_backup_dirs(cfg: dict) -> list[str]: out = set() own = { str(Path("/opt/tklm-backup-agent").resolve()), str(Path("/var/lib/tklm-backup-agent").resolve()), str(Path("/etc/tklm-backup-agent").resolve()), str(Path(cfg.get("db_backups", {}).get("root") or DEFAULT_DB_BACKUP_ROOT).resolve()), } def add_if_backup(path): try: resolved = str(Path(path).resolve()) except Exception: resolved = str(path) if any(resolved == x or resolved.startswith(x.rstrip("/") + "/") for x in own): return out.add(resolved) for x in ("/backup", "/backups", "/srv/backup", "/srv/backups", "/mnt/backup", "/mnt/backups", "/opt/backup", "/opt/backups"): if Path(x).is_dir(): add_if_backup(x) for root in ("/mnt", "/media", "/srv", "/opt"): rp = Path(root) if not rp.is_dir(): continue try: for child in rp.iterdir(): if not child.is_dir(): continue if any(w in child.name.lower() for w in BACKUP_WORDS): add_if_backup(child) try: for sub in child.iterdir(): if sub.is_dir() and any(w in sub.name.lower() for w in BACKUP_WORDS): add_if_backup(sub) except (PermissionError, OSError): pass except (PermissionError, OSError): pass for x in cfg.get("extra_backup_dirs", []) or []: if x: add_if_backup(x) return sorted(out) def update_discovery(cfg: dict): d = load_discovery() now = now_iso() for unit in list(d.get("services", {}).keys()): if not service_unit_is_monitorable(unit) and unit not in (cfg.get("extra_services") or []): d["services"].pop(unit, None) mounts = discover_mounts_current() for path, meta in mounts.items(): d["mounts"].setdefault(path, {}) d["mounts"][path].update(meta) d["mounts"][path]["last_seen"] = now for path in cfg.get("extra_mounts", []) or []: d["mounts"].setdefault(path, {}) d["mounts"][path]["last_seen"] = d["mounts"][path].get("last_seen", now) services = discover_services_current() for unit, meta in services.items(): d["services"].setdefault(unit, {}) d["services"][unit].update(meta) d["services"][unit]["last_seen"] = now for unit in cfg.get("extra_services", []) or []: if unit: d["services"].setdefault(unit, {"name": unit}) manual_backup_dirs = set() for raw in cfg.get("extra_backup_dirs", []) or []: if not raw: continue try: manual_backup_dirs.add(str(Path(raw).resolve())) except Exception: manual_backup_dirs.add(str(raw)) for path in candidate_backup_dirs(cfg): meta = d["backups"].setdefault(path, {}) meta["last_seen"] = now is_manual = path in manual_backup_dirs if is_manual: meta["source"] = "manual" meta["armed"] = True meta.setdefault("first_seen", now) meta.setdefault("armed_at", now) continue meta["source"] = "auto" newest = newest_file(Path(path)) if Path(path).is_dir() else None newest_mtime = 0.0 newest_name = "" if newest is not None: try: newest_mtime = float(newest.stat().st_mtime) newest_name = str(newest) except OSError: pass # Migracja z 7.2.0: stare automatycznie wykryte katalogi nie są już # uznawane od razu za aktywne backupy. Zapamiętujemy stan bazowy i czekamy # na NOWY plik/zmianę po instalacji 7.2.1. if "armed" not in meta: meta["armed"] = False meta["first_seen"] = now meta["baseline_mtime"] = newest_mtime meta["baseline_file"] = newest_name elif not bool(meta.get("armed")) and cfg.get("auto_backup_arm_on_new_file", True): baseline = float(meta.get("baseline_mtime") or 0.0) if newest_mtime > baseline + 0.5: meta["armed"] = True meta["armed_at"] = now meta["first_new_file"] = newest_name meta["baseline_mtime"] = newest_mtime if cfg.get("docker_monitor_enabled", True): containers = discover_docker_containers_current() for name, meta in containers.items(): d["containers"].setdefault(name, {}) d["containers"][name].update(meta) d["containers"][name]["last_seen"] = now save_discovery(d) return d, mounts def metric_gate(metric_id: str, value: float, warn: float, crit: float, needed: int): raw = "CRITICAL" if value >= crit else "WARNING" if value >= warn else "OK" needed = max(1, int(needed or 1)) with db_connect() as con: row = con.execute("SELECT breach_count FROM metric_state WHERE metric_id=?", (metric_id,)).fetchone() prev = int(row["breach_count"] or 0) if row else 0 count = 0 if raw == "OK" else prev + 1 con.execute(""" INSERT INTO metric_state(metric_id,breach_count,last_raw_status,last_value,updated_at) VALUES(?,?,?,?,?) ON CONFLICT(metric_id) DO UPDATE SET breach_count=excluded.breach_count,last_raw_status=excluded.last_raw_status, last_value=excluded.last_value,updated_at=excluded.updated_at """, (metric_id, count, raw, float(value), now_iso())) effective = raw if raw != "OK" and count >= needed else "OK" return effective, raw, count, needed def cpu_percent(sample_seconds=1.0) -> float: def snap(): vals = [int(x) for x in Path("/proc/stat").read_text(encoding="utf-8").splitlines()[0].split()[1:]] while len(vals) < 8: vals.append(0) return vals[3] + vals[4], sum(vals) i1, t1 = snap() time.sleep(max(0.25, float(sample_seconds))) i2, t2 = snap() dt = max(1, t2 - t1) di = max(0, i2 - i1) return max(0.0, min(100.0, (dt - di) * 100.0 / dt)) def memory_stats(): values = {} for line in Path("/proc/meminfo").read_text(encoding="utf-8").splitlines(): if ":" not in line: continue k, rest = line.split(":", 1) try: values[k] = int(rest.strip().split()[0]) * 1024 except Exception: pass total = float(values.get("MemTotal", 0)) avail = float(values.get("MemAvailable", values.get("MemFree", 0))) if total <= 0: raise RuntimeError("Brak MemTotal w /proc/meminfo") used_pct = max(0.0, min(100.0, (total - avail) * 100.0 / total)) return total / 1024**3, avail / 1024**3, used_pct def check_metrics(cfg: dict) -> list[dict]: cpu_warn = float(cfg.get("cpu_warning_percent", 85)) cpu_crit = float(cfg.get("cpu_critical_percent", 95)) ram_warn = float(cfg.get("ram_warning_percent", 85)) ram_crit = float(cfg.get("ram_critical_percent", 95)) needed = max(1, int(cfg.get("metric_consecutive_checks", 3))) resources = [] try: cpu = cpu_percent(1.0) status, raw, count, need = metric_gate("cpu", cpu, cpu_warn, cpu_crit, needed) try: l1, l5, l15 = os.getloadavg() load = f" Load: {l1:.2f} / {l5:.2f} / {l15:.2f}." except Exception: load = "" pending = "" if raw == "OK" or status != "OK" else f" Próg przekroczony {count}/{need} pomiarów." resources.append({ "kind": "metric", "id": "cpu", "name": "CPU", "status": status, "state": f"{cpu:.1f}% USED", "total_gb": None, "free_gb": None, "free_percent": max(0.0, 100.0 - cpu), "message": f"Użycie CPU {cpu:.1f}%.{load} WARNING >= {cpu_warn:.0f}%, CRITICAL >= {cpu_crit:.0f}%.{pending}", }) except Exception as e: resources.append({ "kind": "metric", "id": "cpu", "name": "CPU", "status": "WARNING", "state": "ERROR", "total_gb": None, "free_gb": None, "free_percent": None, "message": f"Nie można odczytać CPU: {e}", }) try: total, avail, used = memory_stats() status, raw, count, need = metric_gate("ram", used, ram_warn, ram_crit, needed) pending = "" if raw == "OK" or status != "OK" else f" Próg przekroczony {count}/{need} pomiarów." resources.append({ "kind": "metric", "id": "ram", "name": "RAM", "status": status, "state": f"{used:.1f}% USED", "total_gb": total, "free_gb": avail, "free_percent": max(0.0, 100.0 - used), "message": f"RAM: użycie {used:.1f}%, dostępne {avail:.1f} GB / {total:.1f} GB. WARNING >= {ram_warn:.0f}%, CRITICAL >= {ram_crit:.0f}%.{pending}", }) except Exception as e: resources.append({ "kind": "metric", "id": "ram", "name": "RAM", "status": "WARNING", "state": "ERROR", "total_gb": None, "free_gb": None, "free_percent": None, "message": f"Nie można odczytać RAM: {e}", }) return resources def _df_one(path: str, inode=False, timeout_sec=15): cmd = ["df", "-Pi" if inode else "-P", path] if not inode: cmd = ["df", "-P", "-B1", path] try: p = subprocess.run(cmd, capture_output=True, text=True, timeout=max(3, int(timeout_sec))) except subprocess.TimeoutExpired: raise RuntimeError(f"timeout df po {timeout_sec}s") if p.returncode != 0: raise RuntimeError((p.stderr or p.stdout or f"df rc={p.returncode}").strip()[:800]) lines = [x for x in (p.stdout or "").splitlines() if x.strip()] if len(lines) < 2: raise RuntimeError("niepełny wynik df") cols = lines[-1].split() if len(cols) < 6: raise RuntimeError("niepoprawny wynik df") if inode: total = int(cols[1]); used = int(cols[2]); free = int(cols[3]) used_pct = float(cols[4].rstrip('%')) return {"total": total, "used": used, "free": free, "free_percent": max(0.0, 100.0-used_pct)} total = int(cols[1]); used = int(cols[2]); free = int(cols[3]) used_pct = float(cols[4].rstrip('%')) return {"total": total, "used": used, "free": free, "free_percent": max(0.0, 100.0-used_pct)} def check_disk(path: str, current_mounts: dict, cfg: dict) -> dict: warn = float(cfg.get("disk_warning_percent_free", 15)) crit = float(cfg.get("disk_critical_percent_free", 8)) iwarn = float(cfg.get("inode_warning_percent_free", 15)) icrit = float(cfg.get("inode_critical_percent_free", 5)) timeout_sec = int(cfg.get("mount_check_timeout_seconds", 15)) name = f"Dysk {path}" if path not in current_mounts: return { "kind":"disk","id":path,"name":name,"status":"CRITICAL","state":"UNMOUNTED", "total_gb":None,"free_gb":None,"free_percent":None,"inode_free_percent":None, "message":"Wcześniej wykryty system plików nie jest obecnie zamontowany.", } try: d = _df_one(path, inode=False, timeout_sec=timeout_sec) total = d["total"] / 1024**3 free = d["free"] / 1024**3 pct = d["free_percent"] status = "CRITICAL" if pct <= crit else "WARNING" if pct <= warn else "OK" inode_pct = None inode_msg = "" try: ino = _df_one(path, inode=True, timeout_sec=timeout_sec) inode_pct = ino["free_percent"] if inode_pct <= icrit: status = "CRITICAL" elif inode_pct <= iwarn and status == "OK": status = "WARNING" inode_msg = f" Inody wolne {inode_pct:.1f}%." except Exception as e: inode_msg = f" Inody: brak odczytu ({e})." if status == "OK": status = "WARNING" meta = current_mounts.get(path, {}) src = meta.get("source") or "" fstype = meta.get("fstype") or "" return { "kind":"disk","id":path,"name":name,"status":status,"state":"MOUNTED", "total_gb":total,"free_gb":free,"free_percent":pct,"inode_free_percent":inode_pct, "source":src,"fstype":fstype, "message":f"Wolne {free:.1f} GB / {total:.1f} GB ({pct:.1f}%).{inode_msg} {fstype} {src}".strip(), } except Exception as e: return { "kind":"disk","id":path,"name":name,"status":"CRITICAL","state":"ERROR", "total_gb":None,"free_gb":None,"free_percent":None,"inode_free_percent":None, "message":f"Błąd/timeout odczytu filesystemu: {e}", } def check_service(unit: str) -> dict: try: p = run(["systemctl", "is-active", unit], timeout=8) state = (p.stdout or p.stderr or "unknown").strip() status = "OK" if state == "active" else "CRITICAL" msg = "Usługa działa." if status == "OK" else f"Usługa nie działa: {state}." except Exception as e: state, status, msg = "error", "CRITICAL", f"Błąd systemctl: {e}" return { "kind": "service", "id": unit, "name": unit, "status": status, "state": state.upper(), "total_gb": None, "free_gb": None, "free_percent": None, "message": msg, } def check_docker_container(name: str) -> dict: if not shutil.which("docker"): return { "kind":"container","id":name,"name":f"Docker {name}","status":"WARNING","state":"UNAVAILABLE", "total_gb":None,"free_gb":None,"free_percent":None, "message":"Docker CLI nie jest dostępny.", } try: fmt = "{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}|{{.RestartCount}}" p = run(["docker", "inspect", "-f", fmt, name], timeout=12) if p.returncode != 0: return { "kind":"container","id":name,"name":f"Docker {name}","status":"CRITICAL","state":"MISSING", "total_gb":None,"free_gb":None,"free_percent":None, "message":"Kontener był wcześniej uruchomiony, ale teraz nie jest dostępny.", } parts = (p.stdout or "").strip().split("|") state = parts[0] if parts else "unknown" health = parts[1] if len(parts) > 1 else "none" restarts = parts[2] if len(parts) > 2 else "0" if state != "running": status = "CRITICAL" elif health == "unhealthy": status = "CRITICAL" elif health not in ("none", "healthy", ""): status = "WARNING" else: status = "OK" return { "kind":"container","id":name,"name":f"Docker {name}","status":status, "state":f"{state.upper()} / {health.upper()}","total_gb":None,"free_gb":None,"free_percent":None, "restart_count":restarts, "message":f"Docker: state={state}, health={health}, restarts={restarts}.", } except Exception as e: return { "kind":"container","id":name,"name":f"Docker {name}","status":"WARNING","state":"ERROR", "total_gb":None,"free_gb":None,"free_percent":None, "message":f"Błąd docker inspect: {e}", } def smart_devices() -> list[tuple[str, list[str]]]: if not shutil.which("smartctl"): return [] try: p = run(["smartctl", "--scan-open"], timeout=20) except Exception: return [] out = [] for line in (p.stdout or "").splitlines(): line = line.split("#", 1)[0].strip() if not line: continue parts = line.split() dev = parts[0] extra = [] if "-d" in parts: i = parts.index("-d") if i + 1 < len(parts): extra = ["-d", parts[i + 1]] out.append((dev, extra)) return out def _smart_attr(table, attr_id): for row in table or []: if int(row.get("id", -1)) == attr_id: raw = row.get("raw", {}) val = raw.get("value") if isinstance(raw, dict) else None try: return int(val) except Exception: return None return None def smart_legacy_resource(dev: str, extra: list[str], cfg: dict, original_error="") -> dict: """Fallback dla smartmontools bez JSON (-j), np. starsze dystrybucje.""" scfg = cfg.get("smart", {}) t_warn = float(scfg.get("warning_temperature_c", 55)) t_crit = float(scfg.get("critical_temperature_c", 65)) try: p = run(["smartctl", "-H", "-A"] + extra + [dev], timeout=40) text = (p.stdout or "") + "\n" + (p.stderr or "") except Exception as e: return { "kind":"smart","id":dev,"name":f"SMART {dev}","status":"WARNING","state":"ERROR", "total_gb":None,"free_gb":None,"free_percent":None, "message":f"Nie można odczytać SMART: {original_error or e}", } model = dev serial = "" for pat in (r"(?im)^Device Model:\s*(.+)$", r"(?im)^Model Number:\s*(.+)$", r"(?im)^Product:\s*(.+)$"): m = re.search(pat, text) if m: model = m.group(1).strip() break m = re.search(r"(?im)^Serial Number:\s*(.+)$", text) if m: serial = m.group(1).strip() passed = None if re.search(r"(?i)(SMART overall-health.*PASSED|SMART Health Status:\s*OK)", text): passed = True elif re.search(r"(?i)(SMART overall-health.*FAILED|SMART Health Status:\s*(?!OK)\S+)", text): passed = False temp = None for pat in ( r"(?im)^Temperature:\s*(\d+)\s*Celsius", r"(?im)^Temperature_Celsius\s+.*?\s(\d+)\s*$", r"(?im)^Current Drive Temperature:\s*(\d+)\s*C", ): m = re.search(pat, text) if m: try: temp = float(m.group(1)) except Exception: temp = None break status = "CRITICAL" if passed is False else "OK" if temp is not None: if temp >= t_crit: status = "CRITICAL" elif temp >= t_warn and status == "OK": status = "WARNING" msg = "SMART odczytany w trybie zgodności" if temp is not None: msg += f", {temp:.0f}°C" if original_error: msg += f" (JSON niedostępny: {original_error[:180]})" return { "kind":"smart","id":dev,"name":model,"status":status, "state":"PASSED" if passed is True else "FAILED" if passed is False else "UNKNOWN", "total_gb":None,"free_gb":None,"free_percent":None, "device":dev,"model":model,"serial":serial,"temperature_c":temp, "power_on_hours":None,"percentage_used":None,"reallocated":None,"pending":None,"uncorrectable":None, "message":msg, } def check_smart(cfg: dict) -> list[dict]: scfg = cfg.get("smart", {}) if not scfg.get("enabled", True): return [] if not shutil.which("smartctl"): return [{ "kind": "smart", "id": "smartctl", "name": "SMART", "status": "WARNING", "state": "UNAVAILABLE", "total_gb": None, "free_gb": None, "free_percent": None, "message": "Brak smartctl/smartmontools — SMART nie jest odczytywany.", }] t_warn = float(scfg.get("warning_temperature_c", 55)) t_crit = float(scfg.get("critical_temperature_c", 65)) u_warn = float(scfg.get("warning_percentage_used", 90)) u_crit = float(scfg.get("critical_percentage_used", 100)) resources = [] devices = smart_devices() if not devices: return [{ "kind": "smart", "id": "smartctl", "name": "SMART", "status": "WARNING", "state": "NO_DEVICES", "total_gb": None, "free_gb": None, "free_percent": None, "message": "smartctl działa, ale nie wykrył urządzeń dostępnych do odczytu.", }] for dev, extra in devices: cmd = ["smartctl", "-a", "-j"] + extra + [dev] try: p = run(cmd, timeout=40) data = json.loads(p.stdout or "{}") if not isinstance(data, dict) or not data: raise ValueError("pusty/niepoprawny JSON smartctl") except Exception as e: resources.append(smart_legacy_resource(dev, extra, cfg, str(e))) continue model = data.get("model_name") or data.get("model_family") or dev serial = data.get("serial_number") or "" passed = (data.get("smart_status") or {}).get("passed") temp = (data.get("temperature") or {}).get("current") poh = (data.get("power_on_time") or {}).get("hours") nvme = data.get("nvme_smart_health_information_log") or {} used = nvme.get("percentage_used") table = ((data.get("ata_smart_attributes") or {}).get("table") or []) realloc = _smart_attr(table, 5) pending = _smart_attr(table, 197) uncorrect = _smart_attr(table, 198) status = "OK" reasons = [] if passed is False: status = "CRITICAL" reasons.append("SMART FAILED") if pending and pending > 0: status = "CRITICAL" reasons.append(f"pending={pending}") if uncorrect and uncorrect > 0: status = "CRITICAL" reasons.append(f"uncorrectable={uncorrect}") if realloc and realloc > 0 and status != "CRITICAL": status = "CRITICAL" if realloc >= 10 else "WARNING" reasons.append(f"reallocated={realloc}") try: if temp is not None: temp = float(temp) if temp >= t_crit: status = "CRITICAL" elif temp >= t_warn and status == "OK": status = "WARNING" if temp >= t_warn: reasons.append(f"temp={temp:.0f}C") except Exception: temp = None try: if used is not None: used = float(used) if used >= u_crit: status = "CRITICAL" elif used >= u_warn and status == "OK": status = "WARNING" if used >= u_warn: reasons.append(f"used={used:.0f}%") except Exception: used = None details = [] if temp is not None: details.append(f"{temp:.0f}°C") if poh is not None: details.append(f"POH {poh} h") if used is not None: details.append(f"zużycie {used:.0f}%") if realloc is not None: details.append(f"realloc {realloc}") if pending is not None: details.append(f"pending {pending}") if uncorrect is not None: details.append(f"uncorr {uncorrect}") message = ", ".join(details) if details else "SMART odczytany." if reasons: message += " | " + ", ".join(reasons) resources.append({ "kind": "smart", "id": dev, "name": str(model), "status": status, "state": "PASSED" if passed is True else "FAILED" if passed is False else "UNKNOWN", "total_gb": None, "free_gb": None, "free_percent": None, "device": dev, "model": model, "serial": serial, "temperature_c": temp, "power_on_hours": poh, "percentage_used": used, "reallocated": realloc, "pending": pending, "uncorrectable": uncorrect, "message": message, }) return resources def newest_file(folder: Path): newest, newest_mtime = None, -1.0 try: for root, dirs, files in os.walk(folder): dirs[:] = [d for d in dirs if not d.startswith(".")] for name in files: low = name.lower() if any(fnmatch.fnmatch(low, pat) for pat in ("*.tmp", "*.lock", "*.log", "*.partial", "*.part")): continue p = Path(root) / name try: mt = p.stat().st_mtime except (PermissionError, OSError): continue if mt > newest_mtime: newest, newest_mtime = p, mt except (PermissionError, OSError): pass return newest def check_backup(path: str, cfg: dict) -> dict: folder = Path(path) rid = re.sub(r"[^a-zA-Z0-9_.-]+", "_", path.strip("/")) or "backup" warn = float(cfg.get("backup_warning_hours", 26)) crit = float(cfg.get("backup_critical_hours", 36)) name = f"Linux: {path}" if not folder.is_dir(): return { "id": rid, "name": name, "status": "CRITICAL", "file": "", "modified_at": None, "age_hours": None, "size_mb": None, "avg7_size_mb": None, "message": "Wcześniej wykryty katalog backupu nie istnieje lub nie jest dostępny.", } p = newest_file(folder) if p is None: return { "id": rid, "name": name, "status": "CRITICAL", "file": "", "modified_at": None, "age_hours": None, "size_mb": None, "avg7_size_mb": None, "message": "Brak plików backupu.", } st = p.stat() modified = datetime.fromtimestamp(st.st_mtime).astimezone() age = (datetime.now().astimezone() - modified).total_seconds() / 3600.0 size_mb = st.st_size / 1024 / 1024 status = "CRITICAL" if age >= crit else "WARNING" if age >= warn else "OK" return { "id": rid, "name": name, "status": status, "file": p.name, "modified_at": modified.isoformat(timespec="seconds"), "age_hours": age, "size_mb": size_mb, "avg7_size_mb": None, "message": f"Ostatni plik ma {age:.1f} h, {size_mb:.1f} MB.", } def save_check(kind: str, rid: str, status: str, message: str) -> None: with db_connect() as con: con.execute( "INSERT INTO checks(checked_at,kind,resource_id,status,message) VALUES(?,?,?,?,?)", (now_iso(), kind, rid, status, message), ) def safe_filename(name: str) -> str: base = re.sub(r"[^A-Za-z0-9._-]+", "_", name).strip("._") or "database" if base != name: base += "_" + hashlib.sha1(name.encode("utf-8", "replace")).hexdigest()[:8] return base[:120] def command_as_user(cmd: list[str], user: str) -> list[str]: user = (user or "").strip() if not user or os.geteuid() != 0: return cmd if shutil.which("runuser"): return ["runuser", "-u", user, "--"] + cmd if shutil.which("sudo"): return ["sudo", "-n", "-u", user, "--"] + cmd return cmd def mysql_tools(): client = shutil.which("mariadb") or shutil.which("mysql") dump = shutil.which("mariadb-dump") or shutil.which("mysqldump") return client, dump def mysql_conn_args(mcfg: dict) -> list[str]: args = [] defaults_file = str(mcfg.get("defaults_file") or "").strip() if defaults_file: args.append(f"--defaults-extra-file={defaults_file}") host = str(mcfg.get("host") or "").strip() if host: args += ["--host", host, "--port", str(int(mcfg.get("port", 3306)))] user = str(mcfg.get("user") or "").strip() if user: args += ["--user", user] return args def discover_mysql_databases(cfg: dict): mcfg = cfg["db_backups"]["mysql"] include = [str(x) for x in (mcfg.get("include") or []) if str(x)] if not mcfg.get("auto_discover", True): return sorted(dict.fromkeys(include)), "" client, _ = mysql_tools() if not client: return [], "Brak klienta mysql/mariadb" cmd = [client] + mysql_conn_args(mcfg) + ["--batch", "--skip-column-names", "-e", "SHOW DATABASES"] try: p = run(cmd, timeout=30) except Exception as e: return [], str(e) if p.returncode != 0: return [], (p.stderr or p.stdout or f"RC={p.returncode}").strip()[:1000] names = [x.strip() for x in (p.stdout or "").splitlines() if x.strip()] exclude = {str(x) for x in (mcfg.get("exclude") or [])} if include: names = [x for x in names if x in include] else: names = [x for x in names if x not in exclude] return sorted(dict.fromkeys(names)), "" def postgres_env(pcfg: dict) -> dict: env = os.environ.copy() pgpass = str(pcfg.get("pgpass_file") or "").strip() if pgpass: env["PGPASSFILE"] = pgpass return env def postgres_conn_args(pcfg: dict, database: str | None = None) -> list[str]: args = [] host = str(pcfg.get("host") or "").strip() user = str(pcfg.get("user") or "").strip() if host: args += ["-h", host, "-p", str(int(pcfg.get("port", 5432)))] if user: args += ["-U", user] if database: args += ["-d", database] return args def docker_postgres_containers() -> list[str]: if not shutil.which("docker"): return [] try: p = run(["docker", "ps", "--format", "{{.Names}}\t{{.Image}}"], timeout=15) if p.returncode != 0: return [] out = [] for line in (p.stdout or "").splitlines(): cols = line.split("\t", 1) name = cols[0].strip() if cols else "" image = cols[1].strip().lower() if len(cols) > 1 else "" low_name = name.lower() if name and ("postgres" in image or "postgres" in low_name or "postgis" in image or "postgis" in low_name): out.append(name) return sorted(dict.fromkeys(out)) except Exception: return [] def _docker_postgres_query(container: str, database: str, query: str): # Hasła nie trafiają do argv hosta. Jeśli oficjalny kontener ma POSTGRES_PASSWORD, # ustawiamy PGPASSWORD dopiero wewnątrz kontenera. script = ( 'U="${POSTGRES_USER:-postgres}"; ' 'export PGPASSWORD="${POSTGRES_PASSWORD:-${PGPASSWORD:-}}"; ' 'exec psql -U "$U" -d "$1" -Atqc "$2"' ) return run(["docker", "exec", container, "sh", "-lc", script, "tklm", database, query], timeout=40) def discover_docker_postgresql_databases(pcfg: dict): if not pcfg.get("docker_auto_discover", True): return [], "" containers = docker_postgres_containers() if not containers: return [], "" query = "SELECT datname FROM pg_database WHERE datallowconn AND NOT datistemplate ORDER BY datname;" include = {str(x) for x in (pcfg.get("include") or []) if str(x)} exclude = {str(x) for x in (pcfg.get("exclude") or [])} found = [] errors = [] for container in containers: p = _docker_postgres_query(container, "postgres", query) if p.returncode != 0: p = _docker_postgres_query(container, "template1", query) if p.returncode != 0: errors.append(f"{container}: {(p.stderr or p.stdout or 'psql error').strip()[:500]}") continue names = [x.strip() for x in (p.stdout or "").splitlines() if x.strip()] if include: names = [x for x in names if x in include] else: names = [x for x in names if x not in exclude] for dbname in names: found.append(f"docker::{container}::{dbname}") if found: return sorted(dict.fromkeys(found)), "" return [], "; ".join(errors)[:1000] def split_postgresql_identifier(value: str): if value.startswith("docker::"): parts = value.split("::", 2) if len(parts) == 3 and parts[1] and parts[2]: return "docker", parts[1], parts[2] return "local", "", value def discover_postgresql_databases(cfg: dict): pcfg = cfg["db_backups"]["postgresql"] include = [str(x) for x in (pcfg.get("include") or []) if str(x)] if not pcfg.get("auto_discover", True): return sorted(dict.fromkeys(include)), "" names = [] errors = [] host = str(pcfg.get("host") or "").strip() run_as = str(pcfg.get("run_as_user") or "postgres").strip() local_postgres = bool(host) if not local_postgres: try: if Path("/var/run/postgresql").exists() or Path("/run/postgresql").exists(): local_postgres = True elif run(["pgrep", "-x", "postgres"], timeout=5).returncode == 0: local_postgres = True except Exception: pass if local_postgres: psql = shutil.which("psql") if not psql: errors.append("Lokalny PostgreSQL wykryty, ale brak psql") else: # Jeżeli łączymy się lokalnie przez konto systemowe, najpierw sprawdzamy, # czy takie konto istnieje. Brak użytkownika postgres na hoście nie jest # błędem, gdy PostgreSQL działa wyłącznie w Dockerze. can_run_local = True if not host and run_as: try: import pwd pwd.getpwnam(run_as) except Exception: can_run_local = False if can_run_local: query = "SELECT datname FROM pg_database WHERE datallowconn AND NOT datistemplate ORDER BY datname;" cmd = [psql] + postgres_conn_args(pcfg, "postgres") + ["-Atqc", query] if not host: cmd = command_as_user(cmd, run_as) try: p = run(cmd, timeout=30, env=postgres_env(pcfg)) if p.returncode != 0: cmd2 = [psql] + postgres_conn_args(pcfg, "template1") + ["-Atqc", query] if not host: cmd2 = command_as_user(cmd2, run_as) p = run(cmd2, timeout=30, env=postgres_env(pcfg)) if p.returncode == 0: local_names = [x.strip() for x in (p.stdout or "").splitlines() if x.strip()] exclude = {str(x) for x in (pcfg.get("exclude") or [])} if include: local_names = [x for x in local_names if x in include] else: local_names = [x for x in local_names if x not in exclude] names.extend(local_names) else: errors.append((p.stderr or p.stdout or f"RC={p.returncode}").strip()[:1000]) except Exception as e: errors.append(str(e)) docker_names, docker_err = discover_docker_postgresql_databases(pcfg) names.extend(docker_names) if docker_err: errors.append(docker_err) names = sorted(dict.fromkeys(names)) if names: return names, "" # Brak jakiegokolwiek PostgreSQL na hoście nie jest awarią. if not local_postgres and not docker_postgres_containers(): return [], "" return [], "; ".join(x for x in errors if x)[:1000] def save_db_state(engine: str, dbname: str, status: str, file: str, size_mb, duration_sec, message: str) -> None: ts = now_iso() msg = message[:2000] with db_connect() as con: con.execute(""" INSERT INTO db_backup_state(engine,database_name,checked_at,status,file,size_mb,duration_sec,message) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(engine,database_name) DO UPDATE SET checked_at=excluded.checked_at,status=excluded.status,file=excluded.file, size_mb=excluded.size_mb,duration_sec=excluded.duration_sec,message=excluded.message """, (engine, dbname, ts, status, file, size_mb, duration_sec, msg)) con.execute(""" INSERT INTO db_backup_history(engine,database_name,checked_at,status,file,size_mb,duration_sec,message) VALUES(?,?,?,?,?,?,?,?) """, (engine, dbname, ts, status, file, size_mb, duration_sec, msg)) class BackupSpaceError(RuntimeError): pass def _gib(value: float) -> int: return int(max(0.0, float(value)) * 1024**3) def backup_space_reserve_bytes(cfg: dict, root: Path) -> int: bcfg = cfg.get("db_backups", {}) usage = shutil.disk_usage(root) fixed = _gib(float(bcfg.get("min_free_gb", 2))) percent = int(usage.total * max(0.0, float(bcfg.get("min_free_percent", 10))) / 100.0) return max(fixed, percent) def backup_space_snapshot(cfg: dict, root: Path) -> dict: usage = shutil.disk_usage(root) reserve = backup_space_reserve_bytes(cfg, root) return { "total": usage.total, "used": usage.used, "free": usage.free, "reserve": reserve, "usable": max(0, usage.free - reserve), } def latest_successful_backup_size_bytes(engine: str, dbname: str) -> int | None: with db_connect() as con: rows = con.execute( "SELECT size_mb FROM db_backup_history " "WHERE engine=? AND database_name=? AND status='OK' AND size_mb IS NOT NULL " "ORDER BY id DESC LIMIT 7", (engine, dbname), ).fetchall() values = [float(r["size_mb"]) * 1024**2 for r in rows if r["size_mb"] is not None] return int(max(values)) if values else None def mysql_database_size_bytes(cfg: dict, dbname: str) -> int | None: client, _ = mysql_tools() if not client: return None quoted = dbname.replace("'", "''") q = ("SELECT COALESCE(SUM(data_length+index_length),0) " "FROM information_schema.tables WHERE table_schema='" + quoted + "';") cmd = [client] + mysql_conn_args(cfg["db_backups"]["mysql"]) + [ "--batch", "--skip-column-names", "-e", q ] try: p = run(cmd, timeout=30) if p.returncode == 0: return max(0, int((p.stdout or "0").strip().splitlines()[-1])) except Exception: pass return None def postgresql_database_size_bytes(cfg: dict, dbname: str) -> int | None: pcfg = cfg["db_backups"]["postgresql"] source, container, real_dbname = split_postgresql_identifier(dbname) quoted = real_dbname.replace("'", "''") q = "SELECT pg_database_size('" + quoted + "');" try: if source == "docker": p = _docker_postgres_query(container, real_dbname, q) else: psql = shutil.which("psql") if not psql: return None cmd = [psql] + postgres_conn_args(pcfg, real_dbname) + ["-Atqc", q] if not str(pcfg.get("host") or "").strip(): cmd = command_as_user(cmd, str(pcfg.get("run_as_user") or "postgres")) p = run(cmd, timeout=30, env=postgres_env(pcfg)) if p.returncode == 0: return max(0, int((p.stdout or "0").strip().splitlines()[-1])) except Exception: pass return None def estimate_backup_bytes(cfg: dict, engine: str, dbname: str) -> tuple[int, str]: bcfg = cfg.get("db_backups", {}) hist = latest_successful_backup_size_bytes(engine, dbname) if hist: factor = max(1.0, float(bcfg.get("estimate_multiplier", 1.50))) return int(hist * factor), f"historia x{factor:.2f}" source = mysql_database_size_bytes(cfg, dbname) if engine == "mysql" else postgresql_database_size_bytes(cfg, dbname) if source is not None and source > 0: factor = max(1.0, float(bcfg.get("first_backup_source_multiplier", 1.10))) return int(source * factor), f"rozmiar bazy x{factor:.2f}" fallback = _gib(float(bcfg.get("unknown_backup_reserve_gb", 2))) return fallback, "fallback dla nieznanego rozmiaru" def preflight_backup_space(cfg: dict, root: Path, engine: str, dbname: str) -> dict: if not cfg.get("db_backups", {}).get("space_guard_enabled", True): return {"enabled": False} snap = backup_space_snapshot(cfg, root) estimate, source = estimate_backup_bytes(cfg, engine, dbname) required = snap["reserve"] + estimate if snap["free"] < required: raise BackupSpaceError( f"Pominięto backup {engine}/{dbname}: za mało miejsca. " f"Wolne {snap['free']/1024**3:.1f} GB, wymagane co najmniej " f"{required/1024**3:.1f} GB (rezerwa systemowa {snap['reserve']/1024**3:.1f} GB + " f"szacunek backupu {estimate/1024**3:.1f} GB, {source})." ) return {**snap, "estimate": estimate, "estimate_source": source, "required": required} def assert_runtime_backup_space(cfg: dict, root: Path, label: str) -> None: if not cfg.get("db_backups", {}).get("space_guard_enabled", True): return snap = backup_space_snapshot(cfg, root) if snap["free"] <= snap["reserve"]: raise BackupSpaceError( f"Przerwano backup {label}: osiągnięto rezerwę wolnego miejsca. " f"Wolne {snap['free']/1024**3:.2f} GB, rezerwa {snap['reserve']/1024**3:.2f} GB." ) def ensure_backup_root(cfg: dict) -> Path: root = Path(str(cfg["db_backups"].get("root") or DEFAULT_DB_BACKUP_ROOT)) root.mkdir(parents=True, exist_ok=True) os.chmod(root, 0o700) # Brak miejsca raportujemy per baza w preflight_backup_space(). Dzięki temu CRM # dostaje konkretny target i powód, zamiast ogólnego błędu schedulera. return root def verify_gzip(path: Path) -> None: with gzip.open(path, "rb") as f: while f.read(1024 * 1024): pass def backup_mysql_database(cfg: dict, dbname: str, root: Path) -> dict: mcfg = cfg["db_backups"]["mysql"] _, dump = mysql_tools() if not dump: raise RuntimeError("Brak mysqldump/mariadb-dump") preflight_backup_space(cfg, root, "mysql", dbname) destdir = root / "mysql" destdir.mkdir(parents=True, exist_ok=True) os.chmod(destdir, 0o700) stamp = datetime.now().astimezone().strftime("%Y%m%d_%H%M%S") final = destdir / f"{safe_filename(dbname)}_{stamp}.sql.gz" partial = final.with_suffix(final.suffix + ".partial") cmd = [dump] + mysql_conn_args(mcfg) + [ "--single-transaction", "--quick", "--routines", "--events", "--triggers", "--hex-blob", "--databases", dbname, ] dump_timeout = max(5, int(cfg.get("db_backups", {}).get("dump_timeout_minutes", 360))) * 60 if shutil.which("timeout"): cmd = [shutil.which("timeout"), "--signal=TERM", "--kill-after=30s", str(dump_timeout)] + cmd started = time.monotonic() with tempfile.TemporaryFile() as errf: proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=errf) try: check_every = max(1, int(cfg.get("db_backups", {}).get("space_check_interval_mb", 32))) * 1024 * 1024 since_check = 0 with gzip.open(partial, "wb", compresslevel=6) as gz: while True: chunk = proc.stdout.read(1024 * 1024) if proc.stdout else b"" if not chunk: break gz.write(chunk) since_check += len(chunk) if since_check >= check_every: assert_runtime_backup_space(cfg, root, f"mysql/{dbname}") since_check = 0 rc = proc.wait(timeout=dump_timeout + 60) except Exception: proc.kill() proc.wait(timeout=5) partial.unlink(missing_ok=True) raise errf.seek(0) err = errf.read().decode("utf-8", "replace")[:4000] if rc != 0: partial.unlink(missing_ok=True) raise RuntimeError(f"dump RC={rc}: {err.strip()}") if partial.stat().st_size <= 0: partial.unlink(missing_ok=True) raise RuntimeError("dump utworzył pusty plik") verify_gzip(partial) os.replace(partial, final) os.chmod(final, 0o600) return { "file": str(final), "size_mb": final.stat().st_size / 1024 / 1024, "duration_sec": time.monotonic() - started, } def backup_postgresql_database(cfg: dict, dbname: str, root: Path) -> dict: pcfg = cfg["db_backups"]["postgresql"] source, container, real_dbname = split_postgresql_identifier(dbname) preflight_backup_space(cfg, root, "postgresql", dbname) destdir = root / "postgresql" destdir.mkdir(parents=True, exist_ok=True) os.chmod(destdir, 0o700) stamp = datetime.now().astimezone().strftime("%Y%m%d_%H%M%S") prefix = f"{container}__{real_dbname}" if source == "docker" else real_dbname final = destdir / f"{safe_filename(prefix)}_{stamp}.dump" partial = final.with_suffix(final.suffix + ".partial") dump_timeout = max(5, int(cfg.get("db_backups", {}).get("dump_timeout_minutes", 360))) * 60 if source == "docker": if not shutil.which("docker"): raise RuntimeError("Brak docker CLI") script = ( 'U="${POSTGRES_USER:-postgres}"; ' 'export PGPASSWORD="${POSTGRES_PASSWORD:-${PGPASSWORD:-}}"; ' 'exec pg_dump -U "$U" -d "$1" -Fc --no-owner --no-acl' ) cmd = ["docker", "exec", container, "sh", "-lc", script, "tklm", real_dbname] env = None else: pg_dump = shutil.which("pg_dump") if not pg_dump: raise RuntimeError("Brak pg_dump") cmd = [pg_dump] + postgres_conn_args(pcfg, real_dbname) + ["-Fc", "--no-owner", "--no-acl"] if not str(pcfg.get("host") or "").strip(): cmd = command_as_user(cmd, str(pcfg.get("run_as_user") or "postgres")) env = postgres_env(pcfg) if shutil.which("timeout"): cmd = [shutil.which("timeout"), "--signal=TERM", "--kill-after=30s", str(dump_timeout)] + cmd started = time.monotonic() check_every = max(1, int(cfg.get("db_backups", {}).get("space_check_interval_mb", 32))) * 1024 * 1024 since_check = 0 with tempfile.TemporaryFile() as errf: proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=errf, env=env) try: with partial.open("wb") as out: while True: chunk = proc.stdout.read(1024 * 1024) if proc.stdout else b"" if not chunk: break out.write(chunk) since_check += len(chunk) if since_check >= check_every: assert_runtime_backup_space(cfg, root, f"postgresql/{dbname}") since_check = 0 rc = proc.wait(timeout=dump_timeout + 60) except Exception: proc.kill() proc.wait(timeout=5) partial.unlink(missing_ok=True) raise errf.seek(0) err = errf.read().decode("utf-8", "replace")[:4000] if rc != 0: partial.unlink(missing_ok=True) raise RuntimeError(f"pg_dump RC={rc}: {err.strip()}") if partial.stat().st_size <= 5: partial.unlink(missing_ok=True) raise RuntimeError("pg_dump utworzył pusty/za mały plik") # Format custom pg_dump zaczyna się od PGDMP. Daje to weryfikację także wtedy, # gdy pg_restore nie jest zainstalowany na hoście (np. DB działa tylko w Dockerze). with partial.open("rb") as f: magic = f.read(5) if magic != b"PGDMP": partial.unlink(missing_ok=True) raise RuntimeError("Nieprawidłowy nagłówek backupu PostgreSQL (brak PGDMP)") pg_restore = shutil.which("pg_restore") if pg_restore: v = run([pg_restore, "-l", str(partial)], timeout=120) if v.returncode != 0: partial.unlink(missing_ok=True) raise RuntimeError(f"pg_restore -l nie zweryfikował backupu: {(v.stderr or v.stdout)[:1000]}") os.replace(partial, final) os.chmod(final, 0o600) return { "file": str(final), "size_mb": final.stat().st_size / 1024 / 1024, "duration_sec": time.monotonic() - started, "source": source, "container": container or None, "database": real_dbname, } def cleanup_retention(cfg: dict, root: Path) -> None: days = max(1, int(cfg["db_backups"].get("retention_days", 14))) cutoff = time.time() - days * 86400 for sub, suffixes in (("mysql", (".sql.gz",)), ("postgresql", (".dump",))): d = root / sub if not d.is_dir(): continue for p in d.iterdir(): try: if p.is_file() and p.name.endswith(suffixes) and p.stat().st_mtime < cutoff: p.unlink() except OSError as e: log(f"retencja: nie mogę usunąć {p}: {e}") def low_space_backup_status(engine: str, dbname: str, cfg: dict) -> str: crit_h = float(cfg.get("backup_critical_hours", 36)) with db_connect() as con: row = con.execute( "SELECT checked_at FROM db_backup_history " "WHERE engine=? AND database_name=? AND status='OK' ORDER BY id DESC LIMIT 1", (engine, dbname), ).fetchone() if not row: return "CRITICAL" try: age_h = (datetime.now().astimezone() - datetime.fromisoformat(row["checked_at"])).total_seconds() / 3600.0 return "CRITICAL" if age_h >= crit_h else "WARNING" except Exception: return "WARNING" def save_db_space_skip_state(engine: str, dbname: str, status: str, message: str) -> None: # Zachowaj nazwę/rozmiar ostatniej dobrej kopii w bieżącym stanie. # Próba pominięta z powodu miejsca nie powinna udawać, że poprzedni backup zniknął. with db_connect() as con: prev = con.execute( "SELECT file,size_mb,duration_sec FROM db_backup_state WHERE engine=? AND database_name=?", (engine, dbname), ).fetchone() save_db_state( engine, dbname, status, prev["file"] if prev and prev["file"] else "", prev["size_mb"] if prev else None, prev["duration_sec"] if prev else None, message, ) def run_database_backups(cfg: dict, force=False) -> dict: bcfg = cfg.get("db_backups", {}) if not bcfg.get("enabled", True): return {"status": "DISABLED", "mysql": [], "postgresql": []} # Najpierw sprzątamy wyłącznie kopie poza retencją. To może odzyskać miejsce # przed preflightem, ale nigdy nie kasujemy świeżych backupów tylko po to, aby zrobić nowy. root = Path(str(bcfg.get("root") or DEFAULT_DB_BACKUP_ROOT)) root.mkdir(parents=True, exist_ok=True) cleanup_retention(cfg, root) root = ensure_backup_root(cfg) results = {"status": "OK", "mysql": [], "postgresql": []} mcfg = bcfg.get("mysql", {}) if mcfg.get("enabled", True): dbs, err = discover_mysql_databases(cfg) if err: client, dump = mysql_tools() # Jeśli narzędzia nie istnieją, traktujemy to jako brak silnika na hoście, nie awarię backupu. if client or dump: save_db_state("mysql", "__engine__", "CRITICAL", "", None, None, f"Nie można wykryć baz MySQL/MariaDB: {err}") results["status"] = "CRITICAL" else: with db_connect() as con: con.execute("DELETE FROM db_backup_state WHERE engine='mysql' AND database_name='__engine__'") if dbs: marks = ",".join("?" for _ in dbs) con.execute(f"DELETE FROM db_backup_state WHERE engine='mysql' AND database_name NOT IN ({marks})", dbs) else: con.execute("DELETE FROM db_backup_state WHERE engine='mysql'") for dbname in dbs: try: r = backup_mysql_database(cfg, dbname, root) msg = f"Backup OK: {Path(r['file']).name}, {r['size_mb']:.1f} MB, {r['duration_sec']:.1f} s. GZIP zweryfikowany." save_db_state("mysql", dbname, "OK", r["file"], r["size_mb"], r["duration_sec"], msg) results["mysql"].append({"database": dbname, "status": "OK", **r}) except BackupSpaceError as e: st = low_space_backup_status("mysql", dbname, cfg) save_db_space_skip_state("mysql", dbname, st, str(e)) results["mysql"].append({"database": dbname, "status": st, "error": str(e), "skipped": True}) if st == "CRITICAL" or results["status"] == "CRITICAL": results["status"] = "CRITICAL" elif results["status"] == "OK": results["status"] = "WARNING" except Exception as e: save_db_state("mysql", dbname, "CRITICAL", "", None, None, str(e)) results["mysql"].append({"database": dbname, "status": "CRITICAL", "error": str(e)}) results["status"] = "CRITICAL" pcfg = bcfg.get("postgresql", {}) if pcfg.get("enabled", True): dbs, err = discover_postgresql_databases(cfg) if err: if shutil.which("psql") or shutil.which("pg_dump"): save_db_state("postgresql", "__engine__", "CRITICAL", "", None, None, f"Nie można wykryć baz PostgreSQL: {err}") results["status"] = "CRITICAL" else: with db_connect() as con: con.execute("DELETE FROM db_backup_state WHERE engine='postgresql' AND database_name='__engine__'") if dbs: marks = ",".join("?" for _ in dbs) con.execute(f"DELETE FROM db_backup_state WHERE engine='postgresql' AND database_name NOT IN ({marks})", dbs) else: con.execute("DELETE FROM db_backup_state WHERE engine='postgresql'") for dbname in dbs: try: r = backup_postgresql_database(cfg, dbname, root) msg = f"Backup OK: {Path(r['file']).name}, {r['size_mb']:.1f} MB, {r['duration_sec']:.1f} s. Format pg_dump zweryfikowany." save_db_state("postgresql", dbname, "OK", r["file"], r["size_mb"], r["duration_sec"], msg) results["postgresql"].append({"database": dbname, "status": "OK", **r}) except BackupSpaceError as e: st = low_space_backup_status("postgresql", dbname, cfg) save_db_space_skip_state("postgresql", dbname, st, str(e)) results["postgresql"].append({"database": dbname, "status": st, "error": str(e), "skipped": True}) if st == "CRITICAL" or results["status"] == "CRITICAL": results["status"] = "CRITICAL" elif results["status"] == "OK": results["status"] = "WARNING" except Exception as e: save_db_state("postgresql", dbname, "CRITICAL", "", None, None, str(e)) results["postgresql"].append({"database": dbname, "status": "CRITICAL", "error": str(e)}) results["status"] = "CRITICAL" cleanup_retention(cfg, root) meta_set("last_db_backup_at", now_iso()) if results["status"] == "OK": meta_set("last_db_backup_date", datetime.now().astimezone().date().isoformat()) return results def maybe_run_database_backups(cfg: dict) -> None: bcfg = cfg.get("db_backups", {}) if not bcfg.get("enabled", True): return value = str(bcfg.get("time") or "02:30") try: hh, mm = [int(x) for x in value.split(":", 1)] if not (0 <= hh <= 23 and 0 <= mm <= 59): raise ValueError except Exception: hh, mm = 2, 30 now = datetime.now().astimezone() if (now.hour, now.minute) < (hh, mm): return today = now.date().isoformat() last_date = meta_get("last_db_backup_date") if last_date == today: return retry_minutes = max(5, int(bcfg.get("retry_after_failure_minutes", 60))) last_attempt = meta_get("last_db_backup_attempt_at") if last_attempt: try: if (now - datetime.fromisoformat(last_attempt)).total_seconds() < retry_minutes * 60: return except Exception: pass meta_set("last_db_backup_attempt_at", now_iso()) # Jeżeli agent został zainstalowany po godzinie harmonogramu i nie ma jeszcze # dzisiejszego backupu, wykonujemy go od razu zamiast czekać do następnej doby. log(f"Start dziennego backupu baz danych ({value})") try: result = run_database_backups(cfg) log(f"Backup baz zakończony: {result['status']}") except Exception as e: meta_set("last_db_backup_at", now_iso()) save_db_state("database", "__scheduler__", "CRITICAL", "", None, None, str(e)) log(f"Backup baz: BŁĄD: {e}") def all_detected_databases_backed_up_today(cfg: dict, dbs: dict | None = None) -> bool: today = datetime.now().astimezone().date().isoformat() dbs = dbs or list_databases(cfg) wanted = [] for engine in ("mysql", "postgresql"): for name in (dbs.get(engine, {}) or {}).get("databases", []) or []: wanted.append((engine, name)) if not wanted: return False with db_connect() as con: for engine, name in wanted: row = con.execute( "SELECT 1 FROM db_backup_history WHERE engine=? AND database_name=? " "AND status='OK' AND substr(checked_at,1,10)=? LIMIT 1", (engine, name, today), ).fetchone() if not row: return False return True def has_successful_db_backup_today() -> bool: # Zachowane dla kompatybilności/support bundle; oznacza dowolny poprawny backup. today = datetime.now().astimezone().date().isoformat() with db_connect() as con: row = con.execute( "SELECT 1 FROM db_backup_history WHERE status='OK' AND substr(checked_at,1,10)=? LIMIT 1", (today,), ).fetchone() return bool(row) def backup_if_needed(cfg: dict) -> dict: dbs = list_databases(cfg) count = sum(len((v or {}).get("databases") or []) for v in dbs.values()) if count == 0: return {"status": "SKIPPED", "reason": "Nie wykryto baz do backupu.", "databases": dbs} if all_detected_databases_backed_up_today(cfg, dbs): return {"status": "SKIPPED", "reason": "Wszystkie wykryte bazy mają poprawny backup z dzisiaj."} return run_database_backups(cfg, force=True) def print_issues(limit: int = 50) -> int: with db_connect() as con: rows = con.execute( "SELECT checked_at,kind,resource_id,status,message FROM checks " "WHERE status NOT IN ('OK','DISABLED') ORDER BY id DESC LIMIT ?", (max(1, int(limit)),), ).fetchall() data = [dict(x) for x in rows] print(json.dumps(data, ensure_ascii=False, indent=2)) return 0 def db_backup_targets(cfg: dict) -> list[dict]: warn = float(cfg.get("backup_warning_hours", 26)) crit = float(cfg.get("backup_critical_hours", 36)) targets = [] with db_connect() as con: rows = con.execute("SELECT * FROM db_backup_state ORDER BY engine,database_name").fetchall() for row in rows: checked = None age = None try: checked = datetime.fromisoformat(row["checked_at"]) age = (datetime.now().astimezone() - checked).total_seconds() / 3600.0 except Exception: pass status = row["status"] dbname = row["database_name"] engine = row["engine"] if status == "OK" and age is not None: status = "CRITICAL" if age >= crit else "WARNING" if age >= warn else "OK" elif status == "WARNING": # Np. backup pominięty przez space guard. Jeśli ostatnia dobra kopia # przekroczy próg CRITICAL, nie zostawiamy targetu wiecznie jako WARNING. with db_connect() as con: good = con.execute( "SELECT checked_at FROM db_backup_history WHERE engine=? AND database_name=? " "AND status='OK' ORDER BY id DESC LIMIT 1", (engine, dbname), ).fetchone() if not good: status = "CRITICAL" else: try: good_age = (datetime.now().astimezone() - datetime.fromisoformat(good["checked_at"])).total_seconds()/3600.0 if good_age >= crit: status = "CRITICAL" except Exception: pass rid = f"{engine}_{re.sub(r'[^A-Za-z0-9_.-]+', '_', dbname)}" if engine == "postgresql" and str(dbname).startswith("docker::"): _src, _container, _realdb = split_postgresql_identifier(str(dbname)) name = f"POSTGRESQL Docker {_container}: {_realdb}" else: name = f"{engine.upper()}: {dbname}" with db_connect() as con: avgrow = con.execute(""" SELECT AVG(size_mb) AS avg_mb FROM ( SELECT size_mb FROM db_backup_history WHERE engine=? AND database_name=? AND status='OK' AND size_mb IS NOT NULL ORDER BY id DESC LIMIT 7 ) """, (engine, dbname)).fetchone() avg7 = float(avgrow["avg_mb"]) if avgrow and avgrow["avg_mb"] is not None else None targets.append({ "id": rid, "name": name, "status": status, "file": Path(row["file"]).name if row["file"] else "", "modified_at": row["checked_at"], "age_hours": age, "size_mb": row["size_mb"], "avg7_size_mb": avg7, "duration_sec": row["duration_sec"], "message": row["message"] or "", "engine": engine, "database": dbname, }) return targets def sync_database_inventory(cfg: dict) -> None: """Dodaje nowo wykryte bazy do raportu zanim wykonany zostanie pierwszy dump.""" bcfg = cfg.get("db_backups", {}) if not bcfg.get("enabled", True): return engines = [] if bcfg.get("mysql", {}).get("enabled", True): engines.append(("mysql", discover_mysql_databases)) if bcfg.get("postgresql", {}).get("enabled", True): engines.append(("postgresql", discover_postgresql_databases)) for engine, discover_fn in engines: dbs, err = discover_fn(cfg) if err: continue with db_connect() as con: existing = {r[0] for r in con.execute( "SELECT database_name FROM db_backup_state WHERE engine=?", (engine,) ).fetchall()} wanted = set(dbs) # Usunięte/wykluczone bazy nie powinny później wisieć jako fałszywie stare backupy. stale = [x for x in existing if not x.startswith("__") and x not in wanted] for name in stale: con.execute("DELETE FROM db_backup_state WHERE engine=? AND database_name=?", (engine, name)) for name in sorted(wanted - existing): con.execute(""" INSERT INTO db_backup_state(engine,database_name,checked_at,status,file,size_mb,duration_sec,message) VALUES(?,?,?,?,?,?,?,?) """, (engine, name, now_iso(), "WARNING", "", None, None, "Baza wykryta — brak pierwszego backupu. Uruchom: tklm-agent backup-now lub poczekaj na harmonogram.")) def list_databases(cfg: dict) -> dict: out = {} if cfg.get("db_backups", {}).get("mysql", {}).get("enabled", True): dbs, err = discover_mysql_databases(cfg) out["mysql"] = {"databases": dbs, "error": err} if cfg.get("db_backups", {}).get("postgresql", {}).get("enabled", True): dbs, err = discover_postgresql_databases(cfg) out["postgresql"] = {"databases": dbs, "error": err} return out def print_backup_space(cfg: dict) -> int: bcfg = cfg.get("db_backups", {}) root = Path(str(bcfg.get("root") or DEFAULT_DB_BACKUP_ROOT)) root.mkdir(parents=True, exist_ok=True) snap = backup_space_snapshot(cfg, root) data = { "root": str(root), "total_gb": round(snap["total"] / 1024**3, 2), "free_gb": round(snap["free"] / 1024**3, 2), "reserve_gb": round(snap["reserve"] / 1024**3, 2), "usable_for_new_backups_gb": round(snap["usable"] / 1024**3, 2), "space_guard_enabled": bool(bcfg.get("space_guard_enabled", True)), "estimates": [], } dbs = list_databases(cfg) for engine in ("mysql", "postgresql"): for dbname in (dbs.get(engine, {}) or {}).get("databases", []) or []: est, src = estimate_backup_bytes(cfg, engine, dbname) data["estimates"].append({ "engine": engine, "database": dbname, "estimated_gb": round(est / 1024**3, 2), "source": src, "would_fit_now": snap["free"] >= snap["reserve"] + est, }) print(json.dumps(data, ensure_ascii=False, indent=2)) return 0 def heartbeat_payload(cfg: dict) -> dict: p = agent_base(cfg) p["timestamp"] = now_iso() return p def system_payload(cfg: dict) -> dict: if not cfg.get("system_monitor_enabled", True): p = agent_base(cfg) p.update({ "checked_at": now_iso(), "status": "DISABLED", "resources": [], "snapshot_complete": False, "message": "Monitoring systemu wyłączony w konfiguracji.", }) return p if cfg.get("auto_discovery", True): discovery, current_mounts = update_discovery(cfg) else: discovery, current_mounts = load_discovery(), discover_mounts_current() resources = check_metrics(cfg) for path in sorted(discovery.get("mounts", {})): resources.append(check_disk(path, current_mounts, cfg)) resources.extend(check_smart(cfg)) for unit in sorted(discovery.get("services", {})): resources.append(check_service(unit)) if cfg.get("docker_monitor_enabled", True): for name in sorted(discovery.get("containers", {})): resources.append(check_docker_container(name)) overall = max(resources, key=lambda x: SEVERITY.get(x.get("status"), 0))["status"] if resources else "OK" p = agent_base(cfg) p.update({ "checked_at": now_iso(), "status": overall, "resources": resources, "snapshot_complete": True, }) for r in resources: save_check(r.get("kind", "resource"), str(r.get("id", "")), r.get("status", "WARNING"), r.get("message", "")) return p def backup_payload(cfg: dict) -> dict: if cfg.get("auto_discovery", True): discovery, _ = update_discovery(cfg) else: discovery = load_discovery() targets = [] for path, meta in sorted(discovery.get("backups", {}).items()): if meta.get("source") == "manual" or bool(meta.get("armed")): targets.append(check_backup(path, cfg)) continue # CRM mógł już zapamiętać ten sam target jako CRITICAL z 7.2.0. # Wysyłamy więc jawne OK/CANDIDATE, żeby zamknąć stary alert, ale nie # uzbrajamy monitoringu do czasu pojawienia się nowego pliku. rid = re.sub(r"[^a-zA-Z0-9_.-]+", "_", path.strip("/")) or "backup" p0 = newest_file(Path(path)) if Path(path).is_dir() else None modified_at = None file_name = "" size_mb = None age_hours = None if p0 is not None: try: st0 = p0.stat() dt0 = datetime.fromtimestamp(st0.st_mtime).astimezone() modified_at = dt0.isoformat(timespec="seconds") file_name = p0.name size_mb = st0.st_size / 1024 / 1024 age_hours = (datetime.now().astimezone() - dt0).total_seconds() / 3600.0 except OSError: pass targets.append({ "id": rid, "name": f"Linux: {path}", "status": "OK", "file": file_name, "modified_at": modified_at, "age_hours": age_hours, "size_mb": size_mb, "avg7_size_mb": None, "monitoring_state": "CANDIDATE", "message": "Kandydat auto-discovery — nie alarmuję od historycznych plików. Monitoring uzbroi się po pojawieniu się NOWEGO pliku backupu po instalacji/upgrade.", }) try: sync_database_inventory(cfg) except Exception as e: log(f"inventory DB: {e}") targets.extend(db_backup_targets(cfg)) overall = max(targets, key=lambda x: SEVERITY.get(x.get("status"), 0))["status"] if targets else "OK" p = agent_base(cfg) p.update({"checked_at": now_iso(), "status": overall, "targets": targets}) for r in targets: save_check("backup", r["id"], r["status"], r.get("message", "")) return p def selftest(cfg: dict) -> int: results = [] def add(name, ok, detail=""): results.append({"name": name, "status": "OK" if ok else "ERROR", "detail": str(detail)[:1000]}) try: with db_connect() as con: con.execute("SELECT 1").fetchone() add("sqlite", True, str(DB_PATH)) except Exception as e: add("sqlite", False, e) try: root = ensure_backup_root(cfg) t = root / ".tklm_write_test" t.write_text("ok", encoding="utf-8") t.unlink() add("backup_root", True, str(root)) except Exception as e: add("backup_root", False, e) try: mounts = discover_mounts_current() add("mount_discovery", bool(mounts), f"mounts={len(mounts)}") except Exception as e: add("mount_discovery", False, e) add("smartctl", bool(shutil.which("smartctl")), shutil.which("smartctl") or "not installed") add("mysql_dump", bool(mysql_tools()[1]), mysql_tools()[1] or "not installed / engine may be absent") add("pg_dump", bool(shutil.which("pg_dump")), shutil.which("pg_dump") or "not installed / engine may be absent") add("systemctl", bool(shutil.which("systemctl")), shutil.which("systemctl") or "missing") write_runtime("SELFTEST", "local selftest") critical = [x for x in results if x["status"] == "ERROR" and x["name"] in ("sqlite", "backup_root", "mount_discovery", "systemctl")] print(json.dumps({"version": APP_VERSION, "results": results, "status": "OK" if not critical else "CRITICAL"}, ensure_ascii=False, indent=2)) return 0 if not critical else 2 def test_auth(cfg: dict) -> int: ok, code, msg = api_post(cfg, "/heartbeat", heartbeat_payload(cfg)) if ok: print(f"OK HTTP {code}") return 0 print(f"ERROR HTTP {code}: {msg}") return 10 if code in (401, 403) else 20 def once(cfg: dict) -> int: flush_queue(cfg) for endpoint, payload in ( ("/heartbeat", heartbeat_payload(cfg)), ("/system", system_payload(cfg)), ("/report", backup_payload(cfg)), ): ok, code, msg = send_or_queue(cfg, endpoint, payload) log(f"{endpoint}: {'OK' if ok else 'ERROR'} HTTP={code} {msg[:180]}") return 0 def daemon(cfg: dict) -> None: log(f"TKLM Backup Agent Linux {APP_VERSION} start | agent={cfg['agent']['id']}") write_runtime("STARTING") start_runtime_heartbeat() last_hb = last_sys = last_bk = last_db_sched_check = 0.0 while True: try: cfg = load_config() now = time.time() hb_i = max(1, float(cfg.get("heartbeat_minutes", 5))) * 60 sy_i = max(1, float(cfg.get("system_check_minutes", 5))) * 60 bk_i = max(1, float(cfg.get("backup_check_minutes", 30))) * 60 if now - last_hb >= hb_i: flush_queue(cfg) ok, code, msg = send_or_queue(cfg, "/heartbeat", heartbeat_payload(cfg)) log(f"heartbeat: {ok} HTTP={code} {msg[:140]}") last_hb = now if now - last_sys >= sy_i: p = system_payload(cfg) ok, code, msg = send_or_queue(cfg, "/system", p) log(f"system: {p['status']} resources={len(p['resources'])} {ok} HTTP={code} {msg[:140]}") last_sys = now if now - last_bk >= bk_i: p = backup_payload(cfg) ok, code, msg = send_or_queue(cfg, "/report", p) log(f"backup: {p['status']} targets={len(p['targets'])} {ok} HTTP={code} {msg[:140]}") last_bk = now if now - last_db_sched_check >= 60: maybe_run_database_backups(cfg) last_db_sched_check = now except Exception as e: write_runtime("DEGRADED", f"{type(e).__name__}: {e}") log(f"BŁĄD pętli: {type(e).__name__}: {e}") time.sleep(2) def print_discovery(cfg: dict) -> None: d, current = update_discovery(cfg) print(json.dumps({ "current_mounts": current, "known_services": d.get("services", {}), "known_backups": d.get("backups", {}), "known_containers": d.get("containers", {}), "databases": list_databases(cfg), "smart_devices": [x[0] for x in smart_devices()], }, ensure_ascii=False, indent=2)) def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--test-auth", action="store_true") parser.add_argument("--once", action="store_true") parser.add_argument("--discover", action="store_true") parser.add_argument("--backup-now", action="store_true") parser.add_argument("--space", action="store_true") parser.add_argument("--list-databases", action="store_true") parser.add_argument("--queue", action="store_true") parser.add_argument("--selftest", action="store_true") parser.add_argument("--backup-if-needed", action="store_true") parser.add_argument("--issues", action="store_true") args = parser.parse_args() init_state() cfg = load_config() if args.selftest: raise SystemExit(selftest(cfg)) if args.test_auth: raise SystemExit(test_auth(cfg)) if args.once: raise SystemExit(once(cfg)) if args.discover: print_discovery(cfg) return if args.list_databases: print(json.dumps(list_databases(cfg), ensure_ascii=False, indent=2)) return if args.backup_now: result = run_database_backups(cfg, force=True) print(json.dumps(result, ensure_ascii=False, indent=2)) raise SystemExit(0 if result.get("status") in ("OK", "DISABLED") else 2) if args.backup_if_needed: result = backup_if_needed(cfg) print(json.dumps(result, ensure_ascii=False, indent=2)) raise SystemExit(0 if result.get("status") in ("OK", "DISABLED", "SKIPPED") else 2) if args.issues: raise SystemExit(print_issues()) if args.queue: with db_connect() as con: rows = con.execute("SELECT id,created_at,endpoint,attempts,last_error FROM api_queue ORDER BY id").fetchall() print(json.dumps([dict(x) for x in rows], ensure_ascii=False, indent=2)) return daemon(cfg) if __name__ == "__main__": main() PYAGENT chmod 755 "$AGENT" # Konfiguracja: przy aktualizacji zachowujemy istniejące ustawienia i tylko dokładamy brakujące sekcje v7. python3 - "$CONFIG" "$AGENT_ID" "$CLIENT_NAME" "$SERVER_NAME" "$BASE_URL" "$API_KEY" <<'PYCFG' import json,sys from pathlib import Path p=Path(sys.argv[1]) try: cfg=json.load(open(p,encoding='utf-8')) if p.exists() else {} if not isinstance(cfg,dict): cfg={} except Exception: cfg={} def merge(dst, src): for k,v in src.items(): if k not in dst: dst[k]=v elif isinstance(v,dict) and isinstance(dst.get(k),dict): merge(dst[k],v) return dst base={ "agent":{"id":sys.argv[2],"client":sys.argv[3],"server":sys.argv[4]}, "api":{"base_url":sys.argv[5],"key":sys.argv[6]}, "heartbeat_minutes":5, "system_check_minutes":5, "backup_check_minutes":30, "system_monitor_enabled":True, "disk_warning_percent_free":15, "disk_critical_percent_free":8, "inode_warning_percent_free":15, "inode_critical_percent_free":5, "mount_check_timeout_seconds":15, "cpu_warning_percent":85, "cpu_critical_percent":95, "ram_warning_percent":85, "ram_critical_percent":95, "metric_consecutive_checks":3, "backup_warning_hours":26, "backup_critical_hours":36, "auto_discovery":True, "extra_services":[], "extra_mounts":[], "extra_backup_dirs":[], "auto_backup_arm_on_new_file":True, "docker_monitor_enabled":True, "smart":{ "enabled":True, "warning_temperature_c":55, "critical_temperature_c":65, "warning_percentage_used":90, "critical_percentage_used":100 }, "db_backups":{ "enabled":True, "first_backup_on_install":True, "time":"02:30", "root":"/var/backups/tklm-db", "retention_days":14, "min_free_gb":2, "min_free_percent":10, "space_guard_enabled":True, "estimate_multiplier":1.50, "first_backup_source_multiplier":1.10, "unknown_backup_reserve_gb":2, "space_check_interval_mb":32, "retry_after_failure_minutes":60, "dump_timeout_minutes":360, "mysql":{ "enabled":True, "auto_discover":True, "host":"", "port":3306, "user":"root", "defaults_file":"", "include":[], "exclude":["information_schema","performance_schema","mysql","sys"] }, "postgresql":{ "enabled":True, "auto_discover":True, "host":"", "port":5432, "user":"", "run_as_user":"postgres", "pgpass_file":"", "docker_auto_discover":True, "include":[], "exclude":["template0","template1"] } } } merge(cfg,base) # To są wartości instalacyjne/połączeniowe, które mają być aktualne po upgrade. cfg.setdefault('agent',{}) cfg['agent']['id']=cfg['agent'].get('id') or sys.argv[2] _old_client=str(cfg['agent'].get('client') or '').strip() if _old_client.lower() in {'pl','com','net','org','local','lan'}: cfg['agent']['client']=sys.argv[3] else: cfg['agent']['client']=_old_client or sys.argv[3] cfg['agent']['server']=cfg['agent'].get('server') or sys.argv[4] cfg.setdefault('api',{}) cfg['api']['base_url']=cfg['api'].get('base_url') or sys.argv[5] cfg['api']['key']=sys.argv[6] p.write_text(json.dumps(cfg,ensure_ascii=False,indent=2),encoding='utf-8') PYCFG chmod 600 "$CONFIG" cat > "$SERVICE" < "$WATCHDOG_SERVICE" < "$WATCHDOG_TIMER" < "$HELPER" <<'EOF' #!/bin/bash set -u AGENT=/opt/tklm-backup-agent/tklm_agent.py CFG=/etc/tklm-backup-agent/config.json STATE=/var/lib/tklm-backup-agent RUNTIME="$STATE/agent_runtime.json" SERVICE=tklm-backup-agent.service WATCHDOG_TIMER=tklm-backup-agent-watchdog.timer need_root() { [ "$(id -u)" -eq 0 ] || { echo "Uruchom jako root: sudo tklm-agent $*"; exit 1; } } case "${1:-status}" in status) systemctl status "$SERVICE" --no-pager -l || true echo [ -f "$RUNTIME" ] && { echo "--- runtime ---"; cat "$RUNTIME"; echo; } echo "--- watchdog ---" systemctl status "$WATCHDOG_TIMER" --no-pager -l || true ;; restart) need_root restart systemctl restart "$SERVICE" sleep 2 systemctl status "$SERVICE" --no-pager -l ;; stop) need_root stop; systemctl stop "$SERVICE" ;; start) need_root start; systemctl start "$SERVICE" ;; logs) journalctl -u "$SERVICE" -f ;; test) /usr/bin/python3 "$AGENT" --once ;; selftest) /usr/bin/python3 "$AGENT" --selftest ;; discover) /usr/bin/python3 "$AGENT" --discover ;; databases|dbs) /usr/bin/python3 "$AGENT" --list-databases ;; backup-now|backup) need_root backup-now; /usr/bin/python3 "$AGENT" --backup-now ;; space) /usr/bin/python3 "$AGENT" --space ;; backup-if-needed) need_root backup-if-needed; /usr/bin/python3 "$AGENT" --backup-if-needed ;; issues) /usr/bin/python3 "$AGENT" --issues ;; queue) /usr/bin/python3 "$AGENT" --queue ;; reset-discovery) need_root reset-discovery rm -f "$STATE/discovery.json" /usr/bin/python3 "$AGENT" --discover systemctl restart "$SERVICE" || true ;; version) /usr/bin/python3 - "$AGENT" <<'PYV' import importlib.util,sys p=sys.argv[1] s=importlib.util.spec_from_file_location('tklm_agent',p) m=importlib.util.module_from_spec(s); s.loader.exec_module(m) print(m.APP_VERSION) PYV ;; watchdog) need_root watchdog bad=0 if ! systemctl is-active --quiet "$SERVICE"; then logger -t tklm-watchdog "agent inactive - restarting" bad=1 elif [ ! -s "$RUNTIME" ]; then logger -t tklm-watchdog "runtime marker missing - restarting agent" bad=1 else now=$(date +%s) mt=$(stat -c %Y "$RUNTIME" 2>/dev/null || echo 0) age=$((now-mt)) if [ "$age" -gt 180 ]; then logger -t tklm-watchdog "runtime marker stale ${age}s - restarting agent" bad=1 fi fi if [ "$bad" -eq 1 ]; then systemctl restart "$SERVICE" || true sleep 10 systemctl is-active --quiet "$SERVICE" || exit 2 fi ;; repair) need_root repair chmod 755 "$AGENT" /usr/local/sbin/tklm-agent chmod 600 "$CFG" 2>/dev/null || true chmod 700 /etc/tklm-backup-agent "$STATE" /var/backups/tklm-db 2>/dev/null || true /usr/bin/python3 -m py_compile "$AGENT" || exit 2 systemctl daemon-reload systemctl enable "$SERVICE" "$WATCHDOG_TIMER" >/dev/null 2>&1 || true systemctl restart "$SERVICE" systemctl restart "$WATCHDOG_TIMER" || true sleep 3 /usr/bin/python3 "$AGENT" --selftest || true systemctl status "$SERVICE" --no-pager -l | head -35 || true ;; support-bundle|support) need_root support-bundle ts=$(date +%Y%m%d_%H%M%S) host=$(hostname -s 2>/dev/null || hostname) dir="/tmp/TKLM_Linux_Support_${host}_${ts}" out="${dir}.tar.gz" mkdir -p "$dir" /usr/bin/python3 - "$CFG" "$dir/config_sanitized.json" <<'PYS' import json,sys src,dst=sys.argv[1:3] try: d=json.load(open(src,encoding='utf-8')) except Exception as e: d={'error':str(e)} if isinstance(d,dict): api=d.get('api') if isinstance(api,dict) and api.get('key'): api['key']='***REDACTED***' open(dst,'w',encoding='utf-8').write(json.dumps(d,ensure_ascii=False,indent=2)) PYS cp -a "$RUNTIME" "$dir/" 2>/dev/null || true cp -a "$STATE/discovery.json" "$dir/" 2>/dev/null || true sha256sum "$AGENT" > "$dir/agent.sha256" 2>&1 || true systemctl status "$SERVICE" --no-pager -l > "$dir/service_status.txt" 2>&1 || true systemctl status "$WATCHDOG_TIMER" --no-pager -l > "$dir/watchdog_status.txt" 2>&1 || true journalctl -u "$SERVICE" -n 1500 --no-pager > "$dir/journal_agent.txt" 2>&1 || true journalctl -u tklm-backup-agent-watchdog.service -n 300 --no-pager > "$dir/journal_watchdog.txt" 2>&1 || true /usr/bin/python3 "$AGENT" --selftest > "$dir/selftest.txt" 2>&1 || true /usr/bin/python3 "$AGENT" --discover > "$dir/discovery_full.txt" 2>&1 || true /usr/bin/python3 "$AGENT" --queue > "$dir/api_queue.txt" 2>&1 || true df -hT > "$dir/df_hT.txt" 2>&1 || true df -ih > "$dir/df_inodes.txt" 2>&1 || true lsblk -a -o NAME,KNAME,TYPE,SIZE,FSTYPE,MOUNTPOINTS,MODEL,SERIAL > "$dir/lsblk.txt" 2>&1 || true smartctl --scan-open > "$dir/smart_scan.txt" 2>&1 || true docker ps -a > "$dir/docker_ps.txt" 2>&1 || true uname -a > "$dir/uname.txt" 2>&1 || true cp -a /etc/os-release "$dir/" 2>/dev/null || true tar -C /tmp -czf "$out" "$(basename "$dir")" rm -rf "$dir" chmod 600 "$out" echo "Support bundle: $out" ;; rekey) need_root rekey TMP=$(mktemp) || exit 1 cp -a "$CFG" "$TMP" || exit 1 printf "Nowy API KEY: " stty -echo IFS= read -r KEY stty echo printf "\n" [ -n "$KEY" ] || { echo "Klucz nie może być pusty."; rm -f "$TMP"; exit 1; } /usr/bin/python3 - "$CFG" "$KEY" <<'PYKEY' import json,sys p=sys.argv[1] d=json.load(open(p,encoding='utf-8')) d['api']['key']=sys.argv[2] open(p,'w',encoding='utf-8').write(json.dumps(d,ensure_ascii=False,indent=2)) PYKEY chmod 600 "$CFG" if /usr/bin/python3 "$AGENT" --test-auth; then rm -f "$TMP" systemctl restart "$SERVICE" echo "Klucz poprawny. Agent zrestartowany." else cp -a "$TMP" "$CFG" rm -f "$TMP" echo "Błędny klucz — przywrócono poprzednią konfigurację." exit 1 fi ;; config) ${EDITOR:-vi} "$CFG" ;; *) echo "Użycie: tklm-agent {status|restart|start|stop|logs|test|selftest|discover|reset-discovery|databases|backup-now|backup-if-needed|space|issues|queue|repair|support-bundle|rekey|config|version}" ;; esac EOF chmod 755 "$HELPER" python3 -m py_compile "$AGENT" systemctl daemon-reload echo cyan "Testuję klucz API / heartbeat..." set +e TEST_OUT="$(python3 "$AGENT" --test-auth 2>&1)" TEST_RC=$? set -e echo "$TEST_OUT" if [ "$TEST_RC" -eq 10 ]; then red "Autoryzacja nie przeszła." yellow "Agent ID w CRM musi być dokładnie: $AGENT_ID" yellow "Po instalacji możesz poprawić klucz poleceniem: tklm-agent rekey" elif [ "$TEST_RC" -ne 0 ]; then yellow "CRM chwilowo nieosiągalny. Agent ma kolejkę offline i spróbuje ponownie." fi systemctl enable tklm-backup-agent.service tklm-backup-agent-watchdog.timer >/dev/null 2>&1 || true systemctl restart tklm-backup-agent.service systemctl restart tklm-backup-agent-watchdog.timer || true sleep 3 echo cyan "Wykrywanie zasobów i baz:" python3 "$AGENT" --discover || true if python3 - "$CONFIG" <<'PYFIRST' import json,sys try: d=json.load(open(sys.argv[1],encoding='utf-8')) raise SystemExit(0 if (d.get('db_backups') or {}).get('first_backup_on_install', True) else 1) except Exception: raise SystemExit(0) PYFIRST then echo cyan "Pierwszy backup baz (tylko jeśli nie ma poprawnej kopii z dzisiaj):" set +e python3 "$AGENT" --backup-if-needed FIRST_BACKUP_RC=$? set -e if [ "$FIRST_BACKUP_RC" -ne 0 ]; then yellow "Pierwszy backup zgłosił błąd. Instalacja pozostaje aktywna; CRM dostanie stan CRITICAL z dokładnym komunikatem." fi echo cyan "Wysyłam świeży stan do CRM po pierwszym backupie:" python3 "$AGENT" --once || true fi echo green "==============================================================" green " GOTOWE — TKLM Backup Monitor Linux v$VERSION" green "==============================================================" echo echo "Agent ID: $AGENT_ID" echo "Serwer: $SERVER_NAME" echo "Klient: $CLIENT_NAME" echo echo "Monitoring:" echo " • heartbeat co 5 min" echo " • CPU / RAM / load" echo " • lokalne i montowane filesystemy (także SMB/NFS) + inode" echo " • SMART przez smartctl" echo " • typowe usługi systemd" echo " • uruchomione kontenery Docker (zapamiętane -> alarm po zatrzymaniu)" echo " • auto-wykryte katalogi backup: kandydat -> monitoring dopiero po NOWYM pliku" echo " • kolejka CRM latest-snapshot (offline)" echo " • watchdog systemd co 5 min + runtime heartbeat co 15 s" echo echo "Backup baz:" echo " • MySQL/MariaDB: mysqldump/mariadb-dump -> .sql.gz" echo " • PostgreSQL: pg_dump -Fc -> .dump" echo " • pierwszy backup automatycznie po instalacji/upgrade (jeśli dziś jeszcze nie było OK)" echo " • następnie codziennie 02:30" echo " • retencja: 14 dni" echo " • SPACE GUARD: rezerwa max(2 GB, 10% filesystemu) + estymacja rozmiaru" echo " • dump jest przerywany i .partial kasowany zanim filesystem dojdzie do rezerwy" echo " • katalog: $DBBACKUPDIR" echo " • MySQL: socket-auth lub db_backups.mysql.defaults_file" echo " • PostgreSQL: lokalnie lub automatycznie wewnątrz kontenera Docker" echo echo "Polecenia:" echo " tklm-agent status" echo " tklm-agent logs" echo " tklm-agent test" echo " tklm-agent selftest" echo " tklm-agent discover" echo " tklm-agent reset-discovery # po świadomym usunięciu starego mountu/usługi/kontenera" echo " tklm-agent databases" echo " tklm-agent backup-now" echo " tklm-agent space # wolne miejsce, rezerwa i estymacja backupów" echo " tklm-agent backup-if-needed" echo " tklm-agent issues # pokaż WARNING/CRITICAL bez sqlite3" echo " tklm-agent queue" echo " tklm-agent repair" echo " tklm-agent support-bundle" echo " tklm-agent config" echo " tklm-agent rekey" echo " tklm-agent restart" echo echo "Konfiguracja: $CONFIG" echo "Backupy DB: $DBBACKUPDIR" echo systemctl status tklm-backup-agent.service --no-pager -l | head -30 || true