v0.5.0: Add RBAC auth, LLM router, audit logging, output sanitization
- New: auth.py – Role-based access (operator/approver/viewer) - New: llm_router.py – Configurable primary/fallback provider chain - New: audit_log.py + audit_query.py – DSGVO-compliant audit trail - New: sanitize_output() – masks secrets in bot responses - Updated: bot.py – Integrated auth guards, router, audit logging - Updated: ollama_client.py – Router-compatible interface - Updated: CLOWNFISCHSERVER.md – Documentation updated - Updated: README.md – Documentation updated
This commit is contained in:
root
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
🐠 Clownfischserver
|
||||
Author: Mehlzoerwer-Claude (https://github.com/mehlzoerwer-claude)
|
||||
License: GPL-3.0 – Keep it open. Always.
|
||||
Repo: https://github.com/mehlzoerwer-claude/clownfischserver
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
INSTALL_DIR = os.getenv("INSTALL_DIR", "/opt/clownfischserver")
|
||||
AIDER_WORKDIR = os.getenv("AIDER_WORKDIR", "/opt/clownfisch-workspace")
|
||||
OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434")
|
||||
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen2.5-coder:7b")
|
||||
TIMEOUT = 600 # 10 minutes for aider tasks on CPU
|
||||
|
||||
|
||||
class AiderWrapper:
|
||||
def __init__(self):
|
||||
self.workdir = Path(AIDER_WORKDIR)
|
||||
self.workdir.mkdir(parents=True, exist_ok=True)
|
||||
self.venv_python = Path(INSTALL_DIR) / "aider-venv" / "bin" / "python3"
|
||||
self.venv_aider = Path(INSTALL_DIR) / "aider-venv" / "bin" / "aider"
|
||||
logger.info(f"AiderWrapper init: workdir={self.workdir}")
|
||||
|
||||
async def run(self, task: str, target_file: str = None) -> str:
|
||||
"""
|
||||
Run aider in script mode for a coding task.
|
||||
Returns the output/result as string.
|
||||
"""
|
||||
logger.info(f"Aider task: {task[:100]}")
|
||||
|
||||
# Determine working directory and files
|
||||
workdir = self.workdir
|
||||
|
||||
# Build aider command
|
||||
cmd = self._build_aider_command(task, target_file)
|
||||
|
||||
try:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=str(workdir),
|
||||
env={
|
||||
**os.environ,
|
||||
"OLLAMA_API_BASE": OLLAMA_BASE_URL,
|
||||
"HOME": f"/home/{os.getenv('MGMT_USER', 'clownfish')}",
|
||||
}
|
||||
)
|
||||
|
||||
try:
|
||||
stdout, stderr = await asyncio.wait_for(
|
||||
process.communicate(),
|
||||
timeout=TIMEOUT
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
process.kill()
|
||||
return f"⏱️ Aider Timeout nach {TIMEOUT}s. Task zu komplex oder Modell überlastet."
|
||||
|
||||
stdout_str = stdout.decode("utf-8", errors="replace").strip()
|
||||
stderr_str = stderr.decode("utf-8", errors="replace").strip()
|
||||
|
||||
# Build response
|
||||
result_parts = []
|
||||
|
||||
if stdout_str:
|
||||
result_parts.append(stdout_str)
|
||||
|
||||
if stderr_str and process.returncode != 0:
|
||||
result_parts.append(f"[STDERR]\n{stderr_str}")
|
||||
|
||||
if process.returncode == 0:
|
||||
result_parts.insert(0, f"✅ Aider Task abgeschlossen\nArbeitsverzeichnis: {workdir}\n")
|
||||
else:
|
||||
result_parts.insert(0, f"⚠️ Aider beendet mit Code {process.returncode}\n")
|
||||
|
||||
return "\n".join(result_parts) or "✓ Task abgeschlossen (keine Ausgabe)"
|
||||
|
||||
except FileNotFoundError:
|
||||
return (
|
||||
"❌ Aider nicht gefunden. Bitte prüfen:\n"
|
||||
f" ls {self.venv_aider}\n"
|
||||
" Ggf. neu installieren mit: pip install aider-chat"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Aider execution error: {e}", exc_info=True)
|
||||
return f"❌ Aider Fehler: {e}"
|
||||
|
||||
def _build_aider_command(self, task: str, target_file: str = None) -> list:
|
||||
"""Build the aider command with appropriate flags"""
|
||||
cmd = [
|
||||
str(self.venv_aider),
|
||||
"--model", f"ollama/{OLLAMA_MODEL}",
|
||||
"--no-git", # Don't require git
|
||||
"--yes", # Auto-confirm all changes
|
||||
"--no-pretty", # Plain output for telegram
|
||||
"--no-show-model-warnings", # Skip env var warnings
|
||||
"--no-analytics", # No analytics prompts
|
||||
"--edit-format", "diff", # Better format for small models
|
||||
"--message", task, # The task as a message
|
||||
]
|
||||
|
||||
if target_file:
|
||||
cmd.append(target_file)
|
||||
|
||||
return cmd
|
||||
|
||||
async def list_workspace_files(self) -> str:
|
||||
"""List files in the aider workspace"""
|
||||
try:
|
||||
files = list(self.workdir.rglob("*"))
|
||||
if not files:
|
||||
return f"📁 Workspace {self.workdir} ist leer."
|
||||
|
||||
file_list = "\n".join([
|
||||
f" {'📁' if f.is_dir() else '📄'} {f.relative_to(self.workdir)}"
|
||||
for f in sorted(files)[:50]
|
||||
])
|
||||
return f"📁 Workspace: {self.workdir}\n\n{file_list}"
|
||||
except Exception as e:
|
||||
return f"❌ Fehler beim Lesen des Workspace: {e}"
|
||||
@@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
🐠 Clownfischserver – Audit Trail (DSGVO Art. 5 + 32)
|
||||
Author: Mehlzoerwer-Claude (https://github.com/mehlzoerwer-claude)
|
||||
License: GPL-3.0
|
||||
|
||||
Writes JSON-line entries for every privileged action.
|
||||
Pseudonymized: stores only numeric Telegram user_ids, never usernames.
|
||||
|
||||
Compliance:
|
||||
- Art. 5(1)(a) lawfulness/transparency: every action is recorded.
|
||||
- Art. 5(1)(c) data minimization: numeric IDs only, command/result trimmed.
|
||||
- Art. 5(1)(e) storage limitation: prune_old_logs() drops entries older
|
||||
than AUDIT_RETENTION_DAYS (default 90).
|
||||
- Art. 32 integrity: append-only file, 0600 permissions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# How much of a command/result we keep – longer payloads are truncated
|
||||
# to honour data minimization. Configurable via env.
|
||||
_MAX_DETAIL_LEN = int(os.getenv("AUDIT_MAX_DETAIL_LEN", "2000"))
|
||||
|
||||
# Lock for append writes – multiple coroutines may log concurrently.
|
||||
_write_lock = threading.Lock()
|
||||
|
||||
|
||||
def _default_audit_path() -> str:
|
||||
install_dir = os.getenv("INSTALL_DIR", "/opt/clownfischserver")
|
||||
return os.getenv("AUDIT_LOG_PATH", os.path.join(install_dir, "audit.jsonl"))
|
||||
|
||||
|
||||
def _retention_days() -> int:
|
||||
try:
|
||||
return max(0, int(os.getenv("AUDIT_RETENTION_DAYS", "90")))
|
||||
except ValueError:
|
||||
return 90
|
||||
|
||||
|
||||
def _trim(value: Any) -> Any:
|
||||
if isinstance(value, str) and len(value) > _MAX_DETAIL_LEN:
|
||||
return value[:_MAX_DETAIL_LEN] + f"... (truncated, {len(value)} chars total)"
|
||||
return value
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def log_action(
|
||||
user_id: str | int | None,
|
||||
action: str,
|
||||
*,
|
||||
role: str | None = None,
|
||||
result: str = "ok",
|
||||
details: dict[str, Any] | None = None,
|
||||
path: str | None = None,
|
||||
) -> None:
|
||||
"""Append a single audit record. Never raises – audit failure must not
|
||||
break the bot."""
|
||||
entry: dict[str, Any] = {
|
||||
"ts": _now_iso(),
|
||||
"user_id": str(user_id) if user_id is not None else None,
|
||||
"action": action,
|
||||
"result": result,
|
||||
}
|
||||
if role:
|
||||
entry["role"] = role
|
||||
if details:
|
||||
entry["details"] = {k: _trim(v) for k, v in details.items()}
|
||||
|
||||
target = path or _default_audit_path()
|
||||
try:
|
||||
Path(target).parent.mkdir(parents=True, exist_ok=True)
|
||||
line = json.dumps(entry, ensure_ascii=False) + "\n"
|
||||
with _write_lock:
|
||||
with open(target, "a", encoding="utf-8") as fh:
|
||||
fh.write(line)
|
||||
try:
|
||||
os.chmod(target, 0o600)
|
||||
except OSError:
|
||||
# chmod can fail on non-POSIX FS (e.g. tests on Windows) –
|
||||
# the JSONL content is still written.
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"Audit log write failed: {e}")
|
||||
|
||||
|
||||
def log_shell(user_id, command: str, *, role: str | None = None,
|
||||
status: str = "executed", returncode: int | None = None,
|
||||
stderr: str | None = None, path: str | None = None) -> None:
|
||||
details: dict[str, Any] = {"command": command}
|
||||
if returncode is not None:
|
||||
details["returncode"] = returncode
|
||||
if stderr:
|
||||
details["stderr"] = stderr
|
||||
log_action(user_id, "shell.execute", role=role, result=status,
|
||||
details=details, path=path)
|
||||
|
||||
|
||||
def log_shell_proposed(user_id, description: str, command: str,
|
||||
role: str | None = None, dangerous: bool = False,
|
||||
path: str | None = None) -> None:
|
||||
log_action(
|
||||
user_id,
|
||||
"shell.proposed",
|
||||
role=role,
|
||||
result="dangerous" if dangerous else "ok",
|
||||
details={"description": description, "command": command},
|
||||
path=path,
|
||||
)
|
||||
|
||||
|
||||
def log_code(user_id, task: str, *, role: str | None = None,
|
||||
status: str = "ok", path: str | None = None) -> None:
|
||||
log_action(user_id, "code.run", role=role, result=status,
|
||||
details={"task": task}, path=path)
|
||||
|
||||
|
||||
def log_approval(user_id, command: str, *, role: str | None = None,
|
||||
approved: bool = True, path: str | None = None) -> None:
|
||||
log_action(
|
||||
user_id,
|
||||
"shell.approval",
|
||||
role=role,
|
||||
result="approved" if approved else "rejected",
|
||||
details={"command": command},
|
||||
path=path,
|
||||
)
|
||||
|
||||
|
||||
def log_denied(user_id, action: str, *, role: str | None = None,
|
||||
reason: str = "insufficient_role",
|
||||
path: str | None = None) -> None:
|
||||
log_action(user_id, action, role=role, result="denied",
|
||||
details={"reason": reason}, path=path)
|
||||
|
||||
|
||||
def prune_old_logs(path: str | None = None,
|
||||
retention_days: int | None = None) -> int:
|
||||
"""Rewrite the audit file, dropping entries older than retention_days.
|
||||
|
||||
Returns the number of dropped lines. retention_days=0 disables pruning.
|
||||
Uses an atomic temp-file + rename so a crash mid-prune cannot corrupt
|
||||
the audit trail.
|
||||
"""
|
||||
target = path or _default_audit_path()
|
||||
days = retention_days if retention_days is not None else _retention_days()
|
||||
if days <= 0 or not os.path.exists(target):
|
||||
return 0
|
||||
|
||||
cutoff = time.time() - days * 86400
|
||||
tmp_path = target + ".prune.tmp"
|
||||
dropped = 0
|
||||
kept = 0
|
||||
|
||||
try:
|
||||
with _write_lock:
|
||||
with open(target, "r", encoding="utf-8") as src, \
|
||||
open(tmp_path, "w", encoding="utf-8") as dst:
|
||||
for line in src:
|
||||
raw = line.strip()
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
record = json.loads(raw)
|
||||
ts = record.get("ts", "")
|
||||
# ISO 8601 – parse with fromisoformat (UTC suffix safe)
|
||||
dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
if dt.timestamp() < cutoff:
|
||||
dropped += 1
|
||||
continue
|
||||
except (ValueError, KeyError):
|
||||
# Malformed line: keep it rather than silently dropping
|
||||
# forensic evidence.
|
||||
pass
|
||||
dst.write(line if line.endswith("\n") else line + "\n")
|
||||
kept += 1
|
||||
os.replace(tmp_path, target)
|
||||
try:
|
||||
os.chmod(target, 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
logger.info(f"Audit prune: kept={kept} dropped={dropped} retention={days}d")
|
||||
except Exception as e:
|
||||
logger.error(f"Audit prune failed: {e}")
|
||||
if os.path.exists(tmp_path):
|
||||
try:
|
||||
os.remove(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
return 0
|
||||
|
||||
return dropped
|
||||
@@ -0,0 +1,189 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
🐠 Clownfischserver – Audit Trail Query (Phase 2b)
|
||||
Author: Mehlzoerwer-Claude (https://github.com/mehlzoerwer-claude)
|
||||
License: GPL-3.0
|
||||
|
||||
Read-side companion to audit_log.py. Filters/sorts JSONL audit records for
|
||||
the `/logs` admin command.
|
||||
|
||||
The query layer is intentionally separate from the writer so that:
|
||||
- Tests can exercise filtering without touching real audit data.
|
||||
- The writer stays append-only and minimal.
|
||||
- Future query backends (SQLite, OpenSearch) can swap in without touching
|
||||
the writer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Any, Iterable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _default_audit_path() -> str:
|
||||
install_dir = os.getenv("INSTALL_DIR", "/opt/clownfischserver")
|
||||
return os.getenv("AUDIT_LOG_PATH", os.path.join(install_dir, "audit.jsonl"))
|
||||
|
||||
|
||||
def _parse_ts(record: dict) -> datetime | None:
|
||||
"""Best-effort parse of the record's ts field. Returns None on
|
||||
malformed timestamps (caller decides how to handle)."""
|
||||
ts = record.get("ts")
|
||||
if not isinstance(ts, str):
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def read_all(path: str | None = None) -> list[dict]:
|
||||
"""Load every audit record from the JSONL file. Malformed lines are
|
||||
skipped with a debug log (the writer keeps them for forensic purposes,
|
||||
but the query layer prefers a clean result set)."""
|
||||
target = path or _default_audit_path()
|
||||
if not os.path.exists(target):
|
||||
return []
|
||||
|
||||
out: list[dict] = []
|
||||
try:
|
||||
with open(target, "r", encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
out.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
logger.debug(f"audit_query: skipping malformed line")
|
||||
continue
|
||||
except OSError as e:
|
||||
logger.error(f"audit_query: cannot read {target}: {e}")
|
||||
return []
|
||||
return out
|
||||
|
||||
|
||||
def filter_records(
|
||||
records: Iterable[dict],
|
||||
*,
|
||||
user_id: str | int | None = None,
|
||||
action: str | None = None,
|
||||
action_prefix: str | None = None,
|
||||
result: str | None = None,
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
) -> list[dict]:
|
||||
"""Apply filters and return matching records, newest first.
|
||||
|
||||
Filters are AND-combined. None values are ignored. action/result match
|
||||
exactly; action_prefix matches if the record's action starts with it.
|
||||
"""
|
||||
target_user = str(user_id) if user_id is not None else None
|
||||
|
||||
matched: list[dict] = []
|
||||
for rec in records:
|
||||
if target_user is not None and rec.get("user_id") != target_user:
|
||||
continue
|
||||
if action is not None and rec.get("action") != action:
|
||||
continue
|
||||
if action_prefix is not None:
|
||||
rec_action = rec.get("action", "")
|
||||
if not isinstance(rec_action, str) or not rec_action.startswith(action_prefix):
|
||||
continue
|
||||
if result is not None and rec.get("result") != result:
|
||||
continue
|
||||
if since is not None or until is not None:
|
||||
dt = _parse_ts(rec)
|
||||
if dt is None:
|
||||
# Malformed ts – exclude when caller asked for a date window.
|
||||
continue
|
||||
if since is not None and dt < since:
|
||||
continue
|
||||
if until is not None and dt > until:
|
||||
continue
|
||||
matched.append(rec)
|
||||
|
||||
# Newest first – stable sort by ts (records without ts go to the bottom).
|
||||
matched.sort(key=lambda r: r.get("ts", ""), reverse=True)
|
||||
return matched
|
||||
|
||||
|
||||
def query(
|
||||
*,
|
||||
user_id: str | int | None = None,
|
||||
action: str | None = None,
|
||||
action_prefix: str | None = None,
|
||||
result: str | None = None,
|
||||
days: int | None = None,
|
||||
limit: int = 20,
|
||||
path: str | None = None,
|
||||
) -> list[dict]:
|
||||
"""Convenience wrapper: read + filter in one call.
|
||||
|
||||
`days` is a shortcut for `since = now - days`. limit clamps the
|
||||
returned record count (use limit=0 for no limit)."""
|
||||
since = None
|
||||
if days is not None and days > 0:
|
||||
since = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
records = read_all(path=path)
|
||||
matched = filter_records(
|
||||
records,
|
||||
user_id=user_id,
|
||||
action=action,
|
||||
action_prefix=action_prefix,
|
||||
result=result,
|
||||
since=since,
|
||||
)
|
||||
if limit and limit > 0:
|
||||
matched = matched[:limit]
|
||||
return matched
|
||||
|
||||
|
||||
def format_record(rec: dict) -> str:
|
||||
"""Render one audit record as a single compact line for Telegram."""
|
||||
ts = rec.get("ts", "?")
|
||||
# Trim ts down to HH:MM:SS or YYYY-MM-DD HH:MM:SS for readability.
|
||||
if isinstance(ts, str) and len(ts) >= 19:
|
||||
ts_short = ts[:19].replace("T", " ")
|
||||
else:
|
||||
ts_short = str(ts)
|
||||
uid = rec.get("user_id") or "—"
|
||||
action = rec.get("action", "?")
|
||||
result = rec.get("result", "?")
|
||||
role = rec.get("role")
|
||||
role_part = f" [{role}]" if role else ""
|
||||
|
||||
details = rec.get("details") or {}
|
||||
detail_str = ""
|
||||
if isinstance(details, dict):
|
||||
# Pick the single most-informative key per common action.
|
||||
for key in ("command", "task", "reason", "snapshot", "method", "tier"):
|
||||
if key in details:
|
||||
val = str(details[key])
|
||||
if len(val) > 80:
|
||||
val = val[:77] + "..."
|
||||
detail_str = f" {val}"
|
||||
break
|
||||
|
||||
return f"{ts_short} {uid}{role_part} {action}={result}{detail_str}"
|
||||
|
||||
|
||||
def summarize(records: list[dict]) -> dict[str, Any]:
|
||||
"""Aggregate counts useful for the /logs header."""
|
||||
by_action: dict[str, int] = {}
|
||||
by_user: dict[str, int] = {}
|
||||
by_result: dict[str, int] = {}
|
||||
for r in records:
|
||||
act = r.get("action", "?")
|
||||
by_action[act] = by_action.get(act, 0) + 1
|
||||
uid = r.get("user_id") or "—"
|
||||
by_user[uid] = by_user.get(uid, 0) + 1
|
||||
res = r.get("result", "?")
|
||||
by_result[res] = by_result.get(res, 0) + 1
|
||||
return {"total": len(records), "by_action": by_action,
|
||||
"by_user": by_user, "by_result": by_result}
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
🐠 Clownfischserver – Role-Based Access Control
|
||||
Author: Mehlzoerwer-Claude (https://github.com/mehlzoerwer-claude)
|
||||
License: GPL-3.0
|
||||
|
||||
Implements DSGVO Art. 32 access control:
|
||||
- operator: full execution rights (/shell, /code, /run, /ja, /ssh, /snapshot)
|
||||
- approver: can approve pending operations (/ja)
|
||||
- viewer: read-only access (/status, /snapshots, /help)
|
||||
|
||||
Backwards compatible: if no role env vars are set but TELEGRAM_CHAT_ID
|
||||
is configured, that user is auto-granted the operator role.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ROLE_OPERATOR = "operator"
|
||||
ROLE_APPROVER = "approver"
|
||||
ROLE_VIEWER = "viewer"
|
||||
|
||||
VALID_ROLES = (ROLE_OPERATOR, ROLE_APPROVER, ROLE_VIEWER)
|
||||
|
||||
# Operators implicitly hold approver + viewer rights.
|
||||
# Approvers implicitly hold viewer rights.
|
||||
_ROLE_INHERITANCE: dict[str, tuple[str, ...]] = {
|
||||
ROLE_OPERATOR: (ROLE_OPERATOR, ROLE_APPROVER, ROLE_VIEWER),
|
||||
ROLE_APPROVER: (ROLE_APPROVER, ROLE_VIEWER),
|
||||
ROLE_VIEWER: (ROLE_VIEWER,),
|
||||
}
|
||||
|
||||
|
||||
def _parse_id_list(raw: str | None) -> set[str]:
|
||||
if not raw:
|
||||
return set()
|
||||
return {part.strip() for part in raw.split(",") if part.strip()}
|
||||
|
||||
|
||||
def _load_role_map() -> dict[str, str]:
|
||||
"""Read role config from env vars.
|
||||
|
||||
Returns {user_id_str: highest_role}. If a user appears in multiple
|
||||
role lists, the strongest role wins (operator > approver > viewer).
|
||||
"""
|
||||
operators = _parse_id_list(os.getenv("CLOWNFISCH_OPERATORS"))
|
||||
approvers = _parse_id_list(os.getenv("CLOWNFISCH_APPROVERS"))
|
||||
viewers = _parse_id_list(os.getenv("CLOWNFISCH_VIEWERS"))
|
||||
|
||||
role_map: dict[str, str] = {}
|
||||
|
||||
for uid in viewers:
|
||||
role_map[uid] = ROLE_VIEWER
|
||||
for uid in approvers:
|
||||
role_map[uid] = ROLE_APPROVER
|
||||
for uid in operators:
|
||||
role_map[uid] = ROLE_OPERATOR
|
||||
|
||||
# Backwards compatibility: legacy TELEGRAM_CHAT_ID becomes operator
|
||||
# when no explicit role lists are defined.
|
||||
if not role_map:
|
||||
legacy = os.getenv("TELEGRAM_CHAT_ID", "").strip()
|
||||
if legacy:
|
||||
role_map[legacy] = ROLE_OPERATOR
|
||||
logger.info(
|
||||
"Auth: no role lists configured – granting operator to legacy "
|
||||
"TELEGRAM_CHAT_ID for backwards compatibility."
|
||||
)
|
||||
|
||||
return role_map
|
||||
|
||||
|
||||
def get_role(user_id: str | int | None) -> str | None:
|
||||
"""Return the role of a user, or None if unauthorized."""
|
||||
if user_id is None:
|
||||
return None
|
||||
return _load_role_map().get(str(user_id))
|
||||
|
||||
|
||||
def has_role(user_id: str | int | None, *required: str) -> bool:
|
||||
"""Check if the user satisfies ANY of the required roles.
|
||||
|
||||
Inheritance applies: an operator passes a check for approver/viewer.
|
||||
"""
|
||||
role = get_role(user_id)
|
||||
if role is None:
|
||||
return False
|
||||
granted = _ROLE_INHERITANCE.get(role, (role,))
|
||||
return any(r in granted for r in required)
|
||||
|
||||
|
||||
def is_authorized(user_id: str | int | None) -> bool:
|
||||
"""True if the user has any role at all."""
|
||||
return get_role(user_id) is not None
|
||||
|
||||
|
||||
def list_users_with_role(role: str) -> list[str]:
|
||||
"""Return all user_ids that hold (or inherit) the given role."""
|
||||
if role not in VALID_ROLES:
|
||||
return []
|
||||
return [uid for uid, r in _load_role_map().items()
|
||||
if role in _ROLE_INHERITANCE.get(r, (r,))]
|
||||
|
||||
|
||||
def describe_roles() -> dict[str, list[str]]:
|
||||
"""Snapshot of current role assignments – useful for /status or audits."""
|
||||
role_map = _load_role_map()
|
||||
result: dict[str, list[str]] = {r: [] for r in VALID_ROLES}
|
||||
for uid, role in role_map.items():
|
||||
result[role].append(uid)
|
||||
return result
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
🐠 Clownfischserver – Boot Notification
|
||||
Sends a Telegram message after server boot with service status.
|
||||
Author: Mehlzoerwer-Claude (https://github.com/mehlzoerwer-claude)
|
||||
License: GPL-3.0 – Keep it open. Always.
|
||||
Repo: https://github.com/mehlzoerwer-claude/clownfischserver
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from dotenv import load_dotenv
|
||||
import requests
|
||||
|
||||
# Load .env
|
||||
INSTALL_DIR = os.getenv("INSTALL_DIR", "/opt/clownfischserver")
|
||||
load_dotenv(os.path.join(INSTALL_DIR, "config", ".env"))
|
||||
|
||||
BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN")
|
||||
CHAT_ID = os.getenv("TELEGRAM_CHAT_ID")
|
||||
|
||||
SERVICES = ["clownfisch", "ollama", "ufw", "ssh", "cron"]
|
||||
|
||||
|
||||
def get_service_status(name: str) -> str:
|
||||
"""Check if a systemd service is active."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["systemctl", "is-active", name],
|
||||
capture_output=True, text=True, timeout=5
|
||||
)
|
||||
status = result.stdout.strip()
|
||||
if status == "active":
|
||||
return "✅"
|
||||
elif status == "inactive":
|
||||
return "⚪"
|
||||
else:
|
||||
return "❌"
|
||||
except Exception:
|
||||
return "❓"
|
||||
|
||||
|
||||
def get_uptime() -> str:
|
||||
"""Get system uptime."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["uptime", "-p"], capture_output=True, text=True, timeout=5
|
||||
)
|
||||
return result.stdout.strip()
|
||||
except Exception:
|
||||
return "unbekannt"
|
||||
|
||||
|
||||
def get_memory() -> str:
|
||||
"""Get memory usage."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["free", "-h", "--si"], capture_output=True, text=True, timeout=5
|
||||
)
|
||||
for line in result.stdout.splitlines():
|
||||
if line.startswith("Mem:"):
|
||||
parts = line.split()
|
||||
return f"{parts[2]} / {parts[1]}"
|
||||
except Exception:
|
||||
pass
|
||||
return "unbekannt"
|
||||
|
||||
|
||||
def send_message(text: str):
|
||||
"""Send Telegram message."""
|
||||
if not BOT_TOKEN or not CHAT_ID:
|
||||
print("ERROR: Token oder Chat-ID fehlt in .env")
|
||||
sys.exit(1)
|
||||
try:
|
||||
requests.post(
|
||||
f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage",
|
||||
json={"chat_id": CHAT_ID, "text": text, "parse_mode": "Markdown"},
|
||||
timeout=10
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Telegram Fehler: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
zeit = datetime.now().strftime("%H:%M:%S")
|
||||
datum = datetime.now().strftime("%d.%m.%Y")
|
||||
|
||||
# Build service status list
|
||||
status_lines = []
|
||||
for svc in SERVICES:
|
||||
icon = get_service_status(svc)
|
||||
status_lines.append(f" {icon} {svc}")
|
||||
|
||||
uptime = get_uptime()
|
||||
memory = get_memory()
|
||||
|
||||
msg = (
|
||||
f"🐠 *Server gestartet!*\n\n"
|
||||
f"Zeit: {datum} {zeit}\n"
|
||||
f"Uptime: {uptime}\n"
|
||||
f"RAM: {memory}\n\n"
|
||||
f"*Dienste:*\n" +
|
||||
"\n".join(status_lines)
|
||||
)
|
||||
|
||||
send_message(msg)
|
||||
print(f"Boot notification sent at {datum} {zeit}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+1081
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
🐠 Clownfischserver
|
||||
Author: Mehlzoerwer-Claude (https://github.com/mehlzoerwer-claude)
|
||||
License: GPL-3.0 – Keep it open. Always.
|
||||
Repo: https://github.com/mehlzoerwer-claude/clownfischserver
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load .env
|
||||
ENV_FILE = "/opt/clownfischserver/config/.env"
|
||||
load_dotenv(ENV_FILE)
|
||||
|
||||
BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN")
|
||||
CHAT_ID = os.getenv("TELEGRAM_CHAT_ID")
|
||||
|
||||
def send_message(text: str):
|
||||
if not BOT_TOKEN or not CHAT_ID:
|
||||
print("ERROR: Token oder Chat-ID fehlt in .env")
|
||||
sys.exit(1)
|
||||
try:
|
||||
requests.post(
|
||||
f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage",
|
||||
json={"chat_id": CHAT_ID, "text": text, "parse_mode": "Markdown"},
|
||||
timeout=10
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Telegram Fehler: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
event = sys.argv[1] if len(sys.argv) > 1 else "unknown"
|
||||
ip = sys.argv[2] if len(sys.argv) > 2 else "unbekannt"
|
||||
|
||||
from datetime import datetime
|
||||
zeit = datetime.now().strftime("%H:%M:%S")
|
||||
datum = datetime.now().strftime("%d.%m.%Y")
|
||||
|
||||
if event == "open":
|
||||
send_message(f"🔓 *SSH geöffnet*\nIP: `{ip}`\nZeit: {datum} {zeit}")
|
||||
elif event == "close":
|
||||
send_message(f"🔒 *SSH geschlossen*\nIP: `{ip}`\nZeit: {datum} {zeit}")
|
||||
elif event == "attempt":
|
||||
send_message(f"⚠️ *Knock-Versuch*\nIP: `{ip}`\nZeit: {datum} {zeit}")
|
||||
@@ -0,0 +1,322 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
🐠 Clownfischserver – LLM Provider Router (Phase 3)
|
||||
Author: Mehlzoerwer-Claude (https://github.com/mehlzoerwer-claude)
|
||||
License: GPL-3.0
|
||||
|
||||
Replaces the previously hard-wired Ollama+OpenRouter fallback logic with a
|
||||
config-driven primary/fallback chain.
|
||||
|
||||
Env vars:
|
||||
LLM_PRIMARY_PROVIDER = "ollama" | "openrouter" (default: ollama)
|
||||
LLM_PRIMARY_MODEL = model id for the primary provider
|
||||
LLM_PRIMARY_MODEL_FAST = optional faster model for chat replies (ollama only)
|
||||
LLM_PRIMARY_NUM_CTX = context window override (ollama)
|
||||
LLM_FALLBACK_PROVIDER = "openrouter" | "ollama" | "" ("" disables fallback)
|
||||
LLM_FALLBACK_MODEL = model id for fallback provider
|
||||
TRACK_LLM_COSTS = "true"|"false" – currently informational only
|
||||
|
||||
Backwards compatibility:
|
||||
- If LLM_*_PROVIDER vars are absent we still default to Ollama primary and,
|
||||
when OPENROUTER_API_KEY exists, OpenRouter as fallback. Existing
|
||||
OLLAMA_MODEL / OLLAMA_MODEL_FAST / OLLAMA_NUM_CTX / OPENROUTER_API_KEY
|
||||
env vars are honoured.
|
||||
|
||||
Routing policy (intentionally explicit, not "self-improving"):
|
||||
- Primary is tried first. Only routable errors (network, timeout, provider
|
||||
unavailable) trigger fallback. Hard errors (invalid JSON, malformed prompt,
|
||||
bad API key) propagate so callers see them.
|
||||
- Every routing decision is written to the audit trail as `llm.route` so
|
||||
admins can see which model produced which output (DSGVO Art. 5(1)(a)
|
||||
transparency).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Awaitable, Callable, Optional
|
||||
|
||||
from ollama_client import OllamaClient
|
||||
from openrouter_client import OpenRouterClient
|
||||
|
||||
try: # audit is optional – router must work even if audit module is missing
|
||||
import audit_log
|
||||
except ImportError: # pragma: no cover
|
||||
audit_log = None # type: ignore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PROVIDER_OLLAMA = "ollama"
|
||||
PROVIDER_OPENROUTER = "openrouter"
|
||||
|
||||
# Error substrings that should trigger a fallback rather than propagate.
|
||||
# Kept narrow on purpose – we don't want to mask programmer errors.
|
||||
_ROUTABLE_ERROR_FRAGMENTS = (
|
||||
"Timeout",
|
||||
"nicht erreichbar",
|
||||
"ConnectionError",
|
||||
"Connection refused",
|
||||
"API Key nicht konfiguriert",
|
||||
)
|
||||
|
||||
|
||||
def _is_routable_error(exc: BaseException) -> bool:
|
||||
msg = str(exc)
|
||||
return any(frag in msg for frag in _ROUTABLE_ERROR_FRAGMENTS)
|
||||
|
||||
|
||||
class LLMProvider:
|
||||
"""Thin interface every provider satisfies."""
|
||||
|
||||
name: str = "base"
|
||||
model: str = ""
|
||||
|
||||
def is_available(self) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
async def chat(self, message: str, history: list | None = None) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
async def generate_shell_command(self, description: str) -> dict:
|
||||
raise NotImplementedError
|
||||
|
||||
async def explain_output(self, command: str, output: str,
|
||||
error: str = "") -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class OllamaProvider(LLMProvider):
|
||||
name = PROVIDER_OLLAMA
|
||||
|
||||
def __init__(self, model: Optional[str] = None,
|
||||
model_fast: Optional[str] = None,
|
||||
num_ctx: Optional[int] = None,
|
||||
base_url: Optional[str] = None):
|
||||
# Pass openrouter_client=None so the underlying client does NOT
|
||||
# perform its own fallback – the router owns that decision now.
|
||||
self.client = OllamaClient(
|
||||
openrouter_client=None,
|
||||
model=model,
|
||||
model_fast=model_fast,
|
||||
num_ctx=num_ctx,
|
||||
base_url=base_url,
|
||||
)
|
||||
self.model = self.client.model
|
||||
|
||||
def is_available(self) -> bool:
|
||||
# Connection health is checked at call time; we always advertise
|
||||
# ourselves as available so the router will attempt and trigger
|
||||
# fallback on actual failure.
|
||||
return True
|
||||
|
||||
async def chat(self, message, history=None):
|
||||
return await self.client.chat(message, history)
|
||||
|
||||
async def generate_shell_command(self, description):
|
||||
return await self.client.generate_shell_command(description)
|
||||
|
||||
async def explain_output(self, command, output, error=""):
|
||||
return await self.client.explain_output(command, output, error)
|
||||
|
||||
|
||||
class OpenRouterProvider(LLMProvider):
|
||||
name = PROVIDER_OPENROUTER
|
||||
|
||||
def __init__(self, model: Optional[str] = None):
|
||||
self.client = OpenRouterClient()
|
||||
# OpenRouterClient reads OPENROUTER_API_KEY at import time. We
|
||||
# only override the model id, leaving the API key handling alone.
|
||||
self.model = model or "openrouter/free"
|
||||
if model:
|
||||
# Patch the FREE_MODELS dict so chat/coder both use the
|
||||
# configured model. Limited blast radius – it only affects
|
||||
# this OpenRouterClient instance, not the module constant.
|
||||
self._patch_model(model)
|
||||
|
||||
def _patch_model(self, model: str) -> None:
|
||||
# The underlying OpenRouterClient hard-codes FREE_MODELS["coder"]/
|
||||
# ["chat"] at call sites. Wrap _chat_sync so every call resolves
|
||||
# to the user-pinned model regardless of what callers pass in.
|
||||
pinned = model
|
||||
original_chat_sync = self.client._chat_sync
|
||||
|
||||
def wrapper(messages, system=None, model=None):
|
||||
return original_chat_sync(messages, system=system, model=pinned)
|
||||
|
||||
self.client._chat_sync = wrapper # type: ignore[assignment]
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return self.client.is_available()
|
||||
|
||||
async def chat(self, message, history=None):
|
||||
return await self.client.chat(message, history)
|
||||
|
||||
async def generate_shell_command(self, description):
|
||||
return await self.client.generate_shell_command(description)
|
||||
|
||||
async def explain_output(self, command, output, error=""):
|
||||
return await self.client.explain_output(command, output, error)
|
||||
|
||||
|
||||
def _build_provider(kind: str) -> Optional[LLMProvider]:
|
||||
"""Build the provider for kind ∈ {"primary", "fallback"}.
|
||||
|
||||
Returns None if the configuration disables this tier (e.g. fallback
|
||||
requested but no key available).
|
||||
"""
|
||||
explicit = os.getenv(f"LLM_{kind.upper()}_PROVIDER", "").strip().lower()
|
||||
|
||||
if kind == "primary":
|
||||
provider_name = explicit or PROVIDER_OLLAMA
|
||||
else:
|
||||
# Fallback default: openrouter only if API key is configured.
|
||||
if explicit:
|
||||
provider_name = explicit
|
||||
elif os.getenv("OPENROUTER_API_KEY"):
|
||||
provider_name = PROVIDER_OPENROUTER
|
||||
else:
|
||||
return None
|
||||
|
||||
if provider_name in ("", "none", "off", "disabled"):
|
||||
return None
|
||||
|
||||
model = os.getenv(f"LLM_{kind.upper()}_MODEL", "").strip() or None
|
||||
|
||||
if provider_name == PROVIDER_OLLAMA:
|
||||
model_fast = os.getenv(f"LLM_{kind.upper()}_MODEL_FAST", "").strip() or None
|
||||
num_ctx_raw = os.getenv(f"LLM_{kind.upper()}_NUM_CTX", "").strip()
|
||||
num_ctx = int(num_ctx_raw) if num_ctx_raw.isdigit() else None
|
||||
return OllamaProvider(model=model, model_fast=model_fast,
|
||||
num_ctx=num_ctx)
|
||||
if provider_name == PROVIDER_OPENROUTER:
|
||||
provider = OpenRouterProvider(model=model)
|
||||
# Skip the tier if OpenRouter is configured but key is missing.
|
||||
if not provider.is_available():
|
||||
logger.warning(
|
||||
f"LLM router: {kind} provider 'openrouter' has no API key – disabled"
|
||||
)
|
||||
return None
|
||||
return provider
|
||||
|
||||
logger.error(f"LLM router: unknown {kind} provider '{provider_name}'")
|
||||
return None
|
||||
|
||||
|
||||
_BUILD_FROM_ENV = object() # sentinel: distinguish "not passed" from "None"
|
||||
|
||||
|
||||
class LLMRouter:
|
||||
"""Drop-in replacement for the old OllamaClient surface.
|
||||
|
||||
Exposes the same async methods (`chat`, `generate_shell_command`,
|
||||
`explain_output`) plus a small status surface (`primary`, `fallback`,
|
||||
`describe()`).
|
||||
|
||||
Pass `primary=None` / `fallback=None` to explicitly disable a tier;
|
||||
omit the kwarg entirely to build the tier from environment.
|
||||
"""
|
||||
|
||||
def __init__(self, primary=_BUILD_FROM_ENV, fallback=_BUILD_FROM_ENV):
|
||||
self.primary = _build_provider("primary") if primary is _BUILD_FROM_ENV else primary
|
||||
self.fallback = _build_provider("fallback") if fallback is _BUILD_FROM_ENV else fallback
|
||||
if self.primary is None and self.fallback is None:
|
||||
logger.error("LLMRouter: neither primary nor fallback configured!")
|
||||
|
||||
# ---- public describe ------------------------------------------------
|
||||
def describe(self) -> dict:
|
||||
def _info(p: LLMProvider | None) -> dict | None:
|
||||
if p is None:
|
||||
return None
|
||||
return {"provider": p.name, "model": p.model,
|
||||
"available": p.is_available()}
|
||||
return {"primary": _info(self.primary), "fallback": _info(self.fallback)}
|
||||
|
||||
# ---- dispatcher -----------------------------------------------------
|
||||
async def _dispatch(self, method_name: str, *args, **kwargs) -> Any:
|
||||
attempts: list[tuple[str, LLMProvider]] = []
|
||||
if self.primary is not None:
|
||||
attempts.append(("primary", self.primary))
|
||||
if self.fallback is not None:
|
||||
attempts.append(("fallback", self.fallback))
|
||||
|
||||
if not attempts:
|
||||
raise RuntimeError(
|
||||
"Kein LLM-Provider konfiguriert (LLM_PRIMARY_PROVIDER fehlt)"
|
||||
)
|
||||
|
||||
last_error: Optional[BaseException] = None
|
||||
for tier, provider in attempts:
|
||||
if not provider.is_available():
|
||||
self._audit("skip", tier, provider, method_name,
|
||||
reason="provider_unavailable")
|
||||
continue
|
||||
try:
|
||||
result = await getattr(provider, method_name)(*args, **kwargs)
|
||||
self._audit("hit", tier, provider, method_name)
|
||||
return result
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
routable = _is_routable_error(e)
|
||||
self._audit("error", tier, provider, method_name,
|
||||
reason=str(e)[:200],
|
||||
routable=routable)
|
||||
if not routable:
|
||||
# Hard error – don't mask it with a fallback.
|
||||
raise
|
||||
# Otherwise loop to next tier.
|
||||
continue
|
||||
|
||||
# All tiers exhausted.
|
||||
raise RuntimeError(
|
||||
f"Alle LLM-Provider fehlgeschlagen für {method_name}: {last_error}"
|
||||
)
|
||||
|
||||
def _audit(self, status: str, tier: str, provider: LLMProvider,
|
||||
method: str, **extra) -> None:
|
||||
if audit_log is None:
|
||||
return
|
||||
try:
|
||||
details = {"tier": tier, "provider": provider.name,
|
||||
"model": provider.model, "method": method}
|
||||
details.update(extra)
|
||||
audit_log.log_action(None, "llm.route", result=status,
|
||||
details=details)
|
||||
except Exception as e: # pragma: no cover – audit must never fail
|
||||
logger.debug(f"audit emit failed: {e}")
|
||||
|
||||
# ---- public surface -------------------------------------------------
|
||||
async def chat(self, message: str, history: list | None = None) -> str:
|
||||
return await self._dispatch("chat", message, history)
|
||||
|
||||
async def generate_shell_command(self, description: str) -> dict:
|
||||
return await self._dispatch("generate_shell_command", description)
|
||||
|
||||
async def explain_output(self, command: str, output: str,
|
||||
error: str = "") -> str:
|
||||
return await self._dispatch("explain_output", command, output, error)
|
||||
|
||||
# ---- convenience for the warmup path in bot.py ----------------------
|
||||
@property
|
||||
def ollama_models(self) -> list[str]:
|
||||
"""Return distinct Ollama model ids configured across tiers –
|
||||
used by the warmup thread in bot.py."""
|
||||
models: list[str] = []
|
||||
for p in (self.primary, self.fallback):
|
||||
if isinstance(p, OllamaProvider):
|
||||
if p.client.model and p.client.model not in models:
|
||||
models.append(p.client.model)
|
||||
if (p.client.model_fast
|
||||
and p.client.model_fast not in models):
|
||||
models.append(p.client.model_fast)
|
||||
return models
|
||||
|
||||
@property
|
||||
def ollama_base_url(self) -> str:
|
||||
for p in (self.primary, self.fallback):
|
||||
if isinstance(p, OllamaProvider):
|
||||
return p.client.base_url
|
||||
return os.getenv("OLLAMA_BASE_URL", "http://localhost:11434")
|
||||
|
||||
def has_fallback(self) -> bool:
|
||||
return self.fallback is not None and self.fallback.is_available()
|
||||
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
🐠 Clownfischserver
|
||||
Author: Mehlzoerwer-Claude (https://github.com/mehlzoerwer-claude)
|
||||
License: GPL-3.0 – Keep it open. Always.
|
||||
Repo: https://github.com/mehlzoerwer-claude/clownfischserver
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import asyncio
|
||||
import requests
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434")
|
||||
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen2.5-coder:7b")
|
||||
OLLAMA_MODEL_FAST = os.getenv("OLLAMA_MODEL_FAST", "")
|
||||
OLLAMA_NUM_CTX = int(os.getenv("OLLAMA_NUM_CTX", "4096"))
|
||||
|
||||
SHELL_PROMPT_TEMPLATE = """You are a Linux bash command generator for an Ubuntu 24.04 server.
|
||||
Convert the user request into a single non-interactive bash command.
|
||||
Respond with ONLY a JSON object.
|
||||
|
||||
CRITICAL RULES:
|
||||
- Commands run WITHOUT a terminal. No interactive commands like nano, vim, crontab -e, less, top.
|
||||
- Use non-interactive alternatives: echo/tee for files, /etc/cron.d/ for cron jobs, sed for edits.
|
||||
- Use sudo when root permissions are needed.
|
||||
- Chain commands with && if needed.
|
||||
|
||||
Example request: "show free ram"
|
||||
Example response: {{"dangerous":false,"reason":"","command":"free -h"}}
|
||||
|
||||
Example request: "installiere nginx"
|
||||
Example response: {{"dangerous":false,"reason":"","command":"sudo apt install -y nginx"}}
|
||||
|
||||
Example request: "erstelle einen cronjob der alle 5 minuten ein script ausfuehrt"
|
||||
Example response: {{"dangerous":false,"reason":"","command":"echo '*/5 * * * * root /path/to/script.sh' | sudo tee /etc/cron.d/myjob"}}
|
||||
|
||||
Example request: "schreibe hello world in eine datei"
|
||||
Example response: {{"dangerous":false,"reason":"","command":"echo 'hello world' | sudo tee /tmp/hello.txt"}}
|
||||
|
||||
Example request: "starte nginx neu"
|
||||
Example response: {{"dangerous":false,"reason":"","command":"sudo systemctl restart nginx"}}
|
||||
|
||||
Example request: "lösche alles in /etc"
|
||||
Example response: {{"dangerous":true,"reason":"Würde Systemkonfiguration zerstören","command":"rm -rf /etc/*"}}
|
||||
|
||||
Now generate JSON for this request: "{description}"
|
||||
|
||||
IMPORTANT: Respond with ONLY the JSON object. No other text. No markdown."""
|
||||
|
||||
CHAT_SYSTEM_PROMPT = """Du bist der Clownfischserver KI-Assistent. Du hilfst beim Verwalten eines Linux-Servers.
|
||||
Antworte kurz und technisch. Antworte IMMER in der Sprache in der der Nutzer schreibt.
|
||||
Wenn du ein Befehlsergebnis erklärst, sei präzise und verständlich."""
|
||||
|
||||
|
||||
def _extract_json(raw: str) -> dict:
|
||||
"""Extract JSON from Ollama response, handling markdown wrappers and stray text."""
|
||||
text = raw.strip()
|
||||
text = re.sub(r"^```(?:json)?\s*", "", text)
|
||||
text = re.sub(r"\s*```$", "", text)
|
||||
text = text.strip()
|
||||
|
||||
try:
|
||||
return json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
match = re.search(r"\{[^{}]*\}", text, re.DOTALL)
|
||||
if match:
|
||||
try:
|
||||
return json.loads(match.group())
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
raise json.JSONDecodeError("No valid JSON found", text, 0)
|
||||
|
||||
|
||||
class OllamaClient:
|
||||
def __init__(self, openrouter_client=None, model: Optional[str] = None,
|
||||
model_fast: Optional[str] = None,
|
||||
num_ctx: Optional[int] = None,
|
||||
base_url: Optional[str] = None):
|
||||
self.base_url = base_url or OLLAMA_BASE_URL
|
||||
self.model = model or OLLAMA_MODEL
|
||||
self.model_fast = model_fast or OLLAMA_MODEL_FAST or self.model
|
||||
self.num_ctx = num_ctx if num_ctx is not None else OLLAMA_NUM_CTX
|
||||
self.openrouter = openrouter_client
|
||||
|
||||
def _chat_sync(self, messages: list, system: Optional[str] = None, think: Optional[bool] = None, use_fast: bool = False) -> str:
|
||||
"""Synchronous Ollama API call"""
|
||||
model = self.model_fast if use_fast else self.model
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"keep_alive": -1,
|
||||
"options": {"num_ctx": self.num_ctx},
|
||||
}
|
||||
if system:
|
||||
payload["system"] = system
|
||||
if think is not None:
|
||||
payload["think"] = think
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
f"{self.base_url}/api/chat",
|
||||
json=payload,
|
||||
timeout=180,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data.get("message", {}).get("content", "")
|
||||
except requests.exceptions.ConnectionError:
|
||||
raise RuntimeError("Ollama nicht erreichbar – läuft der Service? (systemctl status ollama)")
|
||||
except requests.exceptions.Timeout:
|
||||
raise RuntimeError("Ollama Timeout – Modell lädt möglicherweise noch.")
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Ollama Fehler: {e}")
|
||||
|
||||
async def generate_shell_command(self, description: str) -> dict:
|
||||
"""Generate a shell command from natural language description.
|
||||
Tries Ollama first, falls back to OpenRouter on timeout."""
|
||||
loop = asyncio.get_running_loop()
|
||||
raw = ""
|
||||
|
||||
# 1. Try local Ollama
|
||||
try:
|
||||
prompt = SHELL_PROMPT_TEMPLATE.format(description=description)
|
||||
raw = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: self._chat_sync(
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
think=False,
|
||||
use_fast=False,
|
||||
),
|
||||
)
|
||||
result = _extract_json(raw)
|
||||
|
||||
if "command" not in result or not result["command"].strip():
|
||||
logger.warning(f"Shell generate: no command in result: {result}")
|
||||
return {"dangerous": False, "reason": "", "command": ""}
|
||||
if "dangerous" not in result:
|
||||
result["dangerous"] = False
|
||||
|
||||
logger.info(f"Shell generate (Ollama): {result}")
|
||||
return result
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning(f"Shell generate JSON error: {e}, raw: {raw[:500]}")
|
||||
return {"dangerous": False, "reason": "", "command": ""}
|
||||
except RuntimeError as e:
|
||||
if "Timeout" not in str(e) and "nicht erreichbar" not in str(e):
|
||||
raise
|
||||
logger.warning("Ollama nicht verfügbar für /shell – trying OpenRouter fallback")
|
||||
|
||||
# 2. Fallback to OpenRouter
|
||||
if self.openrouter and self.openrouter.is_available():
|
||||
try:
|
||||
logger.info("Using OpenRouter fallback for /shell...")
|
||||
result = await self.openrouter.generate_shell_command(description)
|
||||
logger.info(f"Shell generate (OpenRouter): {result}")
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"OpenRouter fallback failed: {e}")
|
||||
|
||||
return {
|
||||
"dangerous": False, "reason": "",
|
||||
"command": "",
|
||||
"_error": "Ollama Timeout und kein OpenRouter Fallback verfügbar.",
|
||||
}
|
||||
|
||||
async def explain_output(self, command: str, output: str, error: str = "") -> str:
|
||||
"""Ask Ollama to explain/summarize command output"""
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
content = f"Command: {command}\nOutput:\n{output}"
|
||||
if error:
|
||||
content += f"\nErrors:\n{error}"
|
||||
content += "\n\nErkläre kurz was passiert ist und ob es Probleme gibt. Antworte in der gleichen Sprache wie der Nutzer."
|
||||
|
||||
try:
|
||||
return await loop.run_in_executor(
|
||||
None,
|
||||
lambda: self._chat_sync(
|
||||
messages=[{"role": "user", "content": content}],
|
||||
system=CHAT_SYSTEM_PROMPT,
|
||||
think=False,
|
||||
use_fast=True,
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"explain_output failed: {e}")
|
||||
# Try OpenRouter fallback
|
||||
if self.openrouter and self.openrouter.is_available():
|
||||
try:
|
||||
return await self.openrouter.explain_output(command, output, error)
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
async def chat(self, message: str, history: list = None) -> str:
|
||||
"""General chat with context. Tries Ollama, falls back to OpenRouter."""
|
||||
loop = asyncio.get_running_loop()
|
||||
messages = list(history) if history else []
|
||||
messages.append({"role": "user", "content": message})
|
||||
|
||||
# 1. Try local Ollama
|
||||
try:
|
||||
return await loop.run_in_executor(
|
||||
None,
|
||||
lambda: self._chat_sync(
|
||||
messages=messages,
|
||||
system=CHAT_SYSTEM_PROMPT,
|
||||
use_fast=True,
|
||||
),
|
||||
)
|
||||
except RuntimeError as e:
|
||||
if "Timeout" not in str(e) and "nicht erreichbar" not in str(e):
|
||||
return f"Fehler: {e}"
|
||||
logger.warning("Ollama chat nicht verfügbar – trying OpenRouter")
|
||||
|
||||
# 2. Fallback to OpenRouter
|
||||
if self.openrouter and self.openrouter.is_available():
|
||||
try:
|
||||
logger.info("Using OpenRouter for chat...")
|
||||
return await self.openrouter.chat(message, history)
|
||||
except Exception as e:
|
||||
return f"Ollama und OpenRouter nicht verfügbar: {e}"
|
||||
|
||||
return (
|
||||
"Ollama nicht verfügbar und kein OpenRouter Fallback konfiguriert.\n"
|
||||
"Versuche es später oder konfiguriere OpenRouter in .env"
|
||||
)
|
||||
@@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
🐠 Clownfischserver – OpenRouter Free Models Fallback
|
||||
Author: Mehlzoerwer-Claude (https://github.com/mehlzoerwer-claude)
|
||||
License: GPL-3.0 – Keep it open. Always.
|
||||
Repo: https://github.com/mehlzoerwer-claude/clownfischserver
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import asyncio
|
||||
import requests
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY", "")
|
||||
OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
|
||||
|
||||
# Free models on OpenRouter (no credit card needed)
|
||||
# openrouter/free auto-selects the best available free model
|
||||
FREE_MODELS = {
|
||||
"coder": "openrouter/free",
|
||||
"chat": "openrouter/free",
|
||||
}
|
||||
|
||||
|
||||
def _extract_json(raw: str) -> dict:
|
||||
"""Extract JSON from response."""
|
||||
text = raw.strip()
|
||||
text = re.sub(r"^```(?:json)?\s*", "", text)
|
||||
text = re.sub(r"\s*```$", "", text)
|
||||
text = text.strip()
|
||||
|
||||
try:
|
||||
return json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
match = re.search(r"\{[^{}]*\}", text, re.DOTALL)
|
||||
if match:
|
||||
try:
|
||||
return json.loads(match.group())
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
raise json.JSONDecodeError("No valid JSON found", text, 0)
|
||||
|
||||
|
||||
class OpenRouterClient:
|
||||
"""OpenRouter Free Models Client – fallback when Ollama times out."""
|
||||
|
||||
def __init__(self):
|
||||
# Read at construction time, not import time, so env changes between
|
||||
# boot and config reloads (or test monkeypatching) take effect.
|
||||
self.api_key = os.getenv("OPENROUTER_API_KEY", "") or OPENROUTER_API_KEY
|
||||
self.base_url = OPENROUTER_BASE_URL
|
||||
if self.api_key:
|
||||
logger.info("OpenRouter Fallback konfiguriert")
|
||||
else:
|
||||
logger.info("OpenRouter nicht konfiguriert – Fallback deaktiviert")
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return bool(self.api_key)
|
||||
|
||||
def _chat_sync(self, messages: list, system: Optional[str] = None, model: str = None) -> str:
|
||||
"""Synchronous OpenRouter API call."""
|
||||
if not self.api_key:
|
||||
raise RuntimeError("OpenRouter API Key nicht konfiguriert")
|
||||
|
||||
model = model or FREE_MODELS["chat"]
|
||||
|
||||
api_messages = []
|
||||
if system:
|
||||
api_messages.append({"role": "system", "content": system})
|
||||
api_messages.extend(messages)
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
f"{self.base_url}/chat/completions",
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={
|
||||
"model": model,
|
||||
"messages": api_messages,
|
||||
"temperature": 0.7,
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data.get("choices", [{}])[0].get("message", {}).get("content", "")
|
||||
except requests.exceptions.ConnectionError:
|
||||
raise RuntimeError("OpenRouter nicht erreichbar")
|
||||
except requests.exceptions.Timeout:
|
||||
raise RuntimeError("OpenRouter Timeout")
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"OpenRouter Fehler: {e}")
|
||||
|
||||
async def generate_shell_command(self, description: str) -> dict:
|
||||
"""Generate shell command via OpenRouter."""
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
system = "Server: Ubuntu 24.04, Clownfischserver setup."
|
||||
|
||||
prompt = (
|
||||
"You are a Linux bash command generator for Ubuntu 24.04.\n"
|
||||
"Convert the request into a single non-interactive bash command.\n"
|
||||
"Respond ONLY with JSON: {\"dangerous\": bool, \"reason\": string, \"command\": string}\n"
|
||||
"No interactive commands (nano, vim, crontab -e). Use sudo when needed.\n\n"
|
||||
f"Request: {description}\n\nJSON only:"
|
||||
)
|
||||
|
||||
try:
|
||||
raw = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: self._chat_sync(
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
system=system,
|
||||
model=FREE_MODELS["coder"],
|
||||
),
|
||||
)
|
||||
result = _extract_json(raw)
|
||||
if "command" not in result:
|
||||
result["command"] = ""
|
||||
if "dangerous" not in result:
|
||||
result["dangerous"] = False
|
||||
logger.info(f"OpenRouter shell: {result}")
|
||||
return result
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning(f"OpenRouter JSON error: {e}")
|
||||
return {"dangerous": False, "reason": "", "command": ""}
|
||||
except Exception as e:
|
||||
logger.error(f"OpenRouter shell failed: {e}")
|
||||
raise
|
||||
|
||||
async def chat(self, message: str, history: list = None) -> str:
|
||||
"""General chat via OpenRouter."""
|
||||
loop = asyncio.get_running_loop()
|
||||
messages = list(history) if history else []
|
||||
messages.append({"role": "user", "content": message})
|
||||
|
||||
system = (
|
||||
"Du bist der Clownfischserver KI-Assistent. Du hilfst beim Verwalten eines Linux-Servers.\n"
|
||||
"Antworte kurz und technisch. Antworte IMMER in der Sprache in der der Nutzer schreibt."
|
||||
)
|
||||
|
||||
try:
|
||||
return await loop.run_in_executor(
|
||||
None,
|
||||
lambda: self._chat_sync(
|
||||
messages=messages,
|
||||
system=system,
|
||||
model=FREE_MODELS["chat"],
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
return f"OpenRouter Fehler: {e}"
|
||||
|
||||
async def explain_output(self, command: str, output: str, error: str = "") -> str:
|
||||
"""Explain command output via OpenRouter."""
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
content = f"Command: {command}\nOutput:\n{output}"
|
||||
if error:
|
||||
content += f"\nErrors:\n{error}"
|
||||
content += "\n\nErkläre kurz was passiert ist und ob es Probleme gibt. Antworte in der gleichen Sprache wie der Nutzer."
|
||||
|
||||
try:
|
||||
return await loop.run_in_executor(
|
||||
None,
|
||||
lambda: self._chat_sync(
|
||||
messages=[{"role": "user", "content": content}],
|
||||
model=FREE_MODELS["chat"],
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"OpenRouter explain failed: {e}")
|
||||
return ""
|
||||
@@ -0,0 +1,4 @@
|
||||
python-telegram-bot==21.0
|
||||
aider-chat
|
||||
requests
|
||||
python-dotenv
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
🐠 Clownfischserver
|
||||
Author: Mehlzoerwer-Claude (https://github.com/mehlzoerwer-claude)
|
||||
License: GPL-3.0 – Keep it open. Always.
|
||||
Repo: https://github.com/mehlzoerwer-claude/clownfischserver
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MGMT_USER = os.getenv("MGMT_USER", "clownfish")
|
||||
COMMAND_TIMEOUT = 120 # seconds
|
||||
|
||||
|
||||
async def run_shell(command: str) -> tuple[str, str, int]:
|
||||
"""Run a shell command as mgmt user, return (stdout, stderr, returncode)"""
|
||||
try:
|
||||
current_user = os.getenv("USER", "")
|
||||
if current_user != MGMT_USER:
|
||||
full_cmd = f"sudo -u {MGMT_USER} bash -c {shlex.quote(command)}"
|
||||
else:
|
||||
full_cmd = command
|
||||
|
||||
process = await asyncio.create_subprocess_shell(
|
||||
full_cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
env={**os.environ, "HOME": f"/home/{MGMT_USER}"}
|
||||
)
|
||||
|
||||
try:
|
||||
stdout, stderr = await asyncio.wait_for(
|
||||
process.communicate(),
|
||||
timeout=COMMAND_TIMEOUT
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
process.kill()
|
||||
return "", f"Timeout nach {COMMAND_TIMEOUT}s", 1
|
||||
|
||||
return (
|
||||
stdout.decode("utf-8", errors="replace").strip(),
|
||||
stderr.decode("utf-8", errors="replace").strip(),
|
||||
process.returncode
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Shell execution error: {e}")
|
||||
return "", str(e), 1
|
||||
|
||||
|
||||
async def execute_command(shell_cmd: str, ollama_client) -> str:
|
||||
"""
|
||||
Execute a pre-determined shell command and return formatted output.
|
||||
Optionally asks Ollama to explain errors or complex output.
|
||||
"""
|
||||
logger.info(f"Executing: {shell_cmd}")
|
||||
|
||||
stdout, stderr, returncode = await run_shell(shell_cmd)
|
||||
|
||||
# Build response
|
||||
response_parts = [f"$ {shell_cmd}"]
|
||||
|
||||
if stdout:
|
||||
response_parts.append(stdout)
|
||||
|
||||
if stderr and returncode != 0:
|
||||
response_parts.append(f"[STDERR]\n{stderr}")
|
||||
|
||||
if returncode != 0:
|
||||
response_parts.append(f"[Exit code: {returncode}]")
|
||||
|
||||
raw_output = "\n".join(response_parts)
|
||||
|
||||
# If there's an error or output is complex, ask Ollama to explain
|
||||
if returncode != 0 or len(stdout) > 500:
|
||||
explanation = await ollama_client.explain_output(shell_cmd, stdout, stderr)
|
||||
if explanation:
|
||||
raw_output += f"\n\n💡 {explanation}"
|
||||
|
||||
return raw_output or "✓ Ausgeführt (keine Ausgabe)"
|
||||
+290
@@ -0,0 +1,290 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
🐠 Clownfischserver
|
||||
Author: Mehlzoerwer-Claude (https://github.com/mehlzoerwer-claude)
|
||||
License: GPL-3.0 – Keep it open. Always.
|
||||
Repo: https://github.com/mehlzoerwer-claude/clownfischserver
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import shutil
|
||||
import json
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SNAPSHOT_DIR = os.getenv("SNAPSHOT_DIR", "/opt/clownfisch-snapshots")
|
||||
SNAPSHOT_METHOD = os.getenv("SNAPSHOT_METHOD", "tar")
|
||||
MAX_SNAPSHOTS = 20 # Auto-cleanup limit (kept snapshots are exempt)
|
||||
KEPT_FILE = os.path.join(SNAPSHOT_DIR, ".kept.json") # Tracks important snapshots
|
||||
|
||||
# Commands that trigger automatic snapshots
|
||||
AUTO_SNAPSHOT_TRIGGERS = [
|
||||
"apt install", "apt-get install", "apt remove", "apt-get remove",
|
||||
"apt upgrade", "apt-get upgrade", "dnf install", "dnf remove",
|
||||
"pacman -S", "pacman -R",
|
||||
"systemctl enable", "systemctl disable",
|
||||
"rm -", "mv ", "cp -",
|
||||
"chmod ", "chown ",
|
||||
"nano ", "vim ", "vi ", "echo >", "tee ",
|
||||
"dd ", "mkfs", "fdisk",
|
||||
]
|
||||
|
||||
# Commands that are always read-only – never need a snapshot
|
||||
READONLY_PREFIXES = [
|
||||
"ls", "cat ", "grep ", "find ", "df ", "du ", "free",
|
||||
"top", "ps ", "who", "uptime", "journalctl", "tail ",
|
||||
"head ", "wc ", "stat ", "file ", "which ", "type ",
|
||||
"systemctl status", "systemctl is-active", "systemctl show",
|
||||
"ollama list", "docker ps", "docker images",
|
||||
]
|
||||
|
||||
|
||||
def should_snapshot(command: str) -> bool:
|
||||
"""Check if a shell command warrants an automatic snapshot."""
|
||||
cmd_lower = command.lower().strip()
|
||||
|
||||
# Skip read-only commands
|
||||
for ro in READONLY_PREFIXES:
|
||||
if cmd_lower.startswith(ro):
|
||||
return False
|
||||
|
||||
# Check triggers
|
||||
for trigger in AUTO_SNAPSHOT_TRIGGERS:
|
||||
if trigger in cmd_lower:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
class SnapshotManager:
|
||||
def __init__(self):
|
||||
self.snapshot_dir = Path(SNAPSHOT_DIR)
|
||||
self.method = SNAPSHOT_METHOD
|
||||
self.snapshot_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._load_kept()
|
||||
logger.info(f"SnapshotManager init: method={self.method}, dir={self.snapshot_dir}")
|
||||
|
||||
def _timestamp(self) -> str:
|
||||
return datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
|
||||
def _load_kept(self):
|
||||
"""Load list of kept (important) snapshots."""
|
||||
try:
|
||||
if os.path.exists(KEPT_FILE):
|
||||
with open(KEPT_FILE, "r") as f:
|
||||
self.kept = set(json.loads(f.read()))
|
||||
else:
|
||||
self.kept = set()
|
||||
except Exception:
|
||||
self.kept = set()
|
||||
|
||||
def _save_kept(self):
|
||||
"""Save kept snapshots list."""
|
||||
try:
|
||||
with open(KEPT_FILE, "w") as f:
|
||||
json.dump(list(self.kept), f)
|
||||
except Exception as e:
|
||||
logger.error(f"Could not save kept list: {e}")
|
||||
|
||||
def create_snapshot(self, label: str = "") -> Optional[str]:
|
||||
"""Create a snapshot. Returns snapshot name or None on failure."""
|
||||
name = self._timestamp()
|
||||
if label:
|
||||
name += f"_{label}"
|
||||
|
||||
try:
|
||||
result = self._create_tar_snapshot(name)
|
||||
if result:
|
||||
logger.info(f"Snapshot created: {name}")
|
||||
self._cleanup_old_snapshots()
|
||||
return name
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Snapshot creation failed: {e}")
|
||||
return None
|
||||
|
||||
def _create_tar_snapshot(self, name: str) -> bool:
|
||||
"""Create tar.gz snapshot, excluding sensitive files."""
|
||||
snap_path = self.snapshot_dir / f"{name}.tar.gz"
|
||||
|
||||
# Save current package list for rollback comparison
|
||||
pkg_list = self.snapshot_dir / f"{name}.packages.txt"
|
||||
try:
|
||||
with open(pkg_list, "w") as f:
|
||||
subprocess.run(
|
||||
["dpkg", "--get-selections"],
|
||||
stdout=f,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not save package list: {e}")
|
||||
|
||||
workspace = os.getenv("AIDER_WORKDIR", "/opt/clownfisch-workspace")
|
||||
|
||||
dirs_to_backup = [
|
||||
"/etc",
|
||||
"/opt/clownfischserver",
|
||||
"/home",
|
||||
workspace,
|
||||
]
|
||||
existing = [d for d in dirs_to_backup if os.path.exists(d)]
|
||||
|
||||
if not existing:
|
||||
raise Exception("Keine Verzeichnisse zum Sichern gefunden")
|
||||
|
||||
cmd = ["sudo", "tar", "czf", str(snap_path)]
|
||||
cmd += ["--ignore-failed-read"]
|
||||
# Exclude sensitive files – NEVER backup secrets
|
||||
cmd += ["--exclude=/etc/ssh/ssh_host_rsa_key"]
|
||||
cmd += ["--exclude=/etc/ssh/ssh_host_ecdsa_key"]
|
||||
cmd += ["--exclude=/etc/ssh/ssh_host_ed25519_key"]
|
||||
cmd += ["--exclude=/etc/shadow"]
|
||||
cmd += ["--exclude=/etc/gshadow"]
|
||||
cmd += ["--exclude=*/config/.env"] # Don't backup API keys/tokens
|
||||
# Exclude large caches
|
||||
cmd += ["--exclude=__pycache__"]
|
||||
cmd += ["--exclude=*.pyc"]
|
||||
cmd += ["--exclude=node_modules"]
|
||||
cmd += existing
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True)
|
||||
if result.returncode not in (0, 1): # 1 = warnings only
|
||||
raise Exception(result.stderr.decode())
|
||||
return True
|
||||
|
||||
def list_snapshots(self) -> list[dict]:
|
||||
"""List all snapshots with metadata, newest first."""
|
||||
try:
|
||||
snaps = []
|
||||
for f in self.snapshot_dir.iterdir():
|
||||
if f.name.endswith(".tar.gz"):
|
||||
name = f.name.replace(".tar.gz", "")
|
||||
size_mb = f.stat().st_size // (1024 * 1024)
|
||||
kept = name in self.kept
|
||||
snaps.append({
|
||||
"name": name,
|
||||
"size_mb": size_mb,
|
||||
"kept": kept
|
||||
})
|
||||
snaps.sort(key=lambda x: x["name"], reverse=True)
|
||||
return snaps
|
||||
except Exception as e:
|
||||
logger.error(f"list_snapshots failed: {e}")
|
||||
return []
|
||||
|
||||
def keep_snapshot(self, name: str) -> str:
|
||||
"""Mark a snapshot as important (won't be auto-deleted)."""
|
||||
snaps = [s["name"] for s in self.list_snapshots()]
|
||||
if name not in snaps:
|
||||
return f"❌ Snapshot '{name}' nicht gefunden."
|
||||
self.kept.add(name)
|
||||
self._save_kept()
|
||||
return f"⭐ Snapshot '{name}' als wichtig markiert – wird nicht automatisch gelöscht."
|
||||
|
||||
def unkeep_snapshot(self, name: str) -> str:
|
||||
"""Remove important mark from snapshot."""
|
||||
self.kept.discard(name)
|
||||
self._save_kept()
|
||||
return f"✅ Markierung von '{name}' entfernt."
|
||||
|
||||
def delete_snapshot(self, name: str) -> str:
|
||||
"""Delete a specific snapshot."""
|
||||
snap_path = self.snapshot_dir / f"{name}.tar.gz"
|
||||
pkg_path = self.snapshot_dir / f"{name}.packages.txt"
|
||||
if not snap_path.exists():
|
||||
return f"❌ Snapshot '{name}' nicht gefunden."
|
||||
try:
|
||||
snap_path.unlink()
|
||||
if pkg_path.exists():
|
||||
pkg_path.unlink()
|
||||
self.kept.discard(name)
|
||||
self._save_kept()
|
||||
return f"🗑️ Snapshot '{name}' gelöscht."
|
||||
except Exception as e:
|
||||
return f"❌ Fehler beim Löschen: {e}"
|
||||
|
||||
def rollback(self, snapshot_name: str) -> str:
|
||||
"""Roll back to a specific snapshot."""
|
||||
snap_path = self.snapshot_dir / f"{snapshot_name}.tar.gz"
|
||||
if not snap_path.exists():
|
||||
return f"❌ Snapshot '{snapshot_name}' nicht gefunden."
|
||||
try:
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
|
||||
# Extract all content
|
||||
result = subprocess.run(
|
||||
["sudo", "tar", "xzf", str(snap_path), "-C", tmp_dir],
|
||||
capture_output=True
|
||||
)
|
||||
if result.returncode not in (0, 1):
|
||||
raise Exception(f"tar extract failed: {result.stderr.decode()}")
|
||||
|
||||
restored = []
|
||||
|
||||
etc_backup = os.path.join(tmp_dir, "etc")
|
||||
if os.path.exists(etc_backup):
|
||||
subprocess.run(["sudo", "rsync", "-av", "--delete",
|
||||
etc_backup + "/", "/etc/"], capture_output=True)
|
||||
restored.append("/etc")
|
||||
|
||||
cfs_backup = os.path.join(tmp_dir, "opt/clownfischserver")
|
||||
if os.path.exists(cfs_backup):
|
||||
subprocess.run(["sudo", "rsync", "-av", "--delete",
|
||||
cfs_backup + "/", "/opt/clownfischserver/"], capture_output=True)
|
||||
restored.append("/opt/clownfischserver")
|
||||
|
||||
home_backup = os.path.join(tmp_dir, "home")
|
||||
if os.path.exists(home_backup):
|
||||
subprocess.run(["sudo", "rsync", "-av", "--delete",
|
||||
home_backup + "/", "/home/"], capture_output=True)
|
||||
restored.append("/home")
|
||||
|
||||
workspace = os.getenv("AIDER_WORKDIR", "/opt/clownfisch-workspace")
|
||||
ws_backup = os.path.join(tmp_dir, workspace.lstrip("/"))
|
||||
if os.path.exists(ws_backup):
|
||||
subprocess.run(["sudo", "rsync", "-av", "--delete",
|
||||
ws_backup + "/", workspace + "/"], capture_output=True)
|
||||
restored.append(workspace)
|
||||
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
|
||||
restored_str = ", ".join(restored) if restored else "nichts"
|
||||
|
||||
# Check if package list exists for this snapshot
|
||||
pkg_hint = ""
|
||||
pkg_path = self.snapshot_dir / f"{snapshot_name}.packages.txt"
|
||||
if pkg_path.exists():
|
||||
pkg_hint = "\n\n💡 Paketliste verfügbar – prüfe mit:\n/run diff <(dpkg --get-selections) " + str(pkg_path)
|
||||
|
||||
return (
|
||||
f"✅ Rollback zu '{snapshot_name}' abgeschlossen.\n"
|
||||
f"Wiederhergestellt: {restored_str}\n"
|
||||
f"Dienste neu starten: systemctl restart clownfisch ollama"
|
||||
f"{pkg_hint}"
|
||||
)
|
||||
except Exception as e:
|
||||
return f"❌ Rollback Fehler: {e}"
|
||||
|
||||
def _cleanup_old_snapshots(self):
|
||||
"""Remove oldest snapshots if over MAX_SNAPSHOTS, skip kept ones."""
|
||||
snaps = self.list_snapshots()
|
||||
deletable = [s for s in snaps if not s["kept"]]
|
||||
if len(deletable) > MAX_SNAPSHOTS:
|
||||
to_delete = deletable[MAX_SNAPSHOTS:]
|
||||
for snap in to_delete:
|
||||
snap_path = self.snapshot_dir / f"{snap['name']}.tar.gz"
|
||||
pkg_path = self.snapshot_dir / f"{snap['name']}.packages.txt"
|
||||
try:
|
||||
snap_path.unlink()
|
||||
if pkg_path.exists():
|
||||
pkg_path.unlink()
|
||||
logger.info(f"Auto-cleanup: {snap['name']}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not delete {snap['name']}: {e}")
|
||||
Reference in New Issue
Block a user