first commit
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
# sicabe-config-agent — Python stdlib + PyYAML (image ramping).
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY agent.py ./
|
||||
|
||||
# Default; override via env AGENT_PORT.
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=3s --retries=3 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/healthz')" || exit 1
|
||||
|
||||
ENTRYPOINT ["python", "-u", "agent.py"]
|
||||
@@ -0,0 +1,551 @@
|
||||
#!/usr/bin/env python3
|
||||
"""sicabe-config-agent — akses konfigurasi platform SiCABE per host.
|
||||
|
||||
Control-panel TIDAK mengakses filesystem platform langsung; agent ini yang
|
||||
membacanya di host tempat sicabe-doc berada, lalu diekspos lewat HTTP ber-api-key.
|
||||
|
||||
Endpoint (butuh header `X-Api-Key` kecuali `/healthz`):
|
||||
GET /healthz
|
||||
GET /env?key=NAME[&key=NAME2] # read-only, whitelist (minimal 1 key)
|
||||
GET /catalog # daftar katalog Trino (nilai sensitif di-redact)
|
||||
GET /catalog/<name> # detail satu katalog (redact)
|
||||
PUT /catalog/<name> # body {"properties": {...}} — tulis (upsert)
|
||||
DELETE /catalog/<name> # hapus
|
||||
GET /dbt # project dbt terparse (models/sources/DAG)
|
||||
GET /ca # PEM sertifikat TLS host (opsional)
|
||||
GET /opa/data # dokumen data OPA (config/opa/data.json) — mentah
|
||||
PUT /opa/data # body {"content": "<json>"} — validasi + tulis atomic (.bak)
|
||||
POST /services/scan # body {"publicHost","types":[...]} — scan service (panel-driven)
|
||||
|
||||
Env:
|
||||
AGENT_API_KEY (wajib) api key yang harus dikirim panel
|
||||
AGENT_PORT (default 8080)
|
||||
SICABE_ENV_FILE (default /config/.env)
|
||||
TRINO_CATALOG_DIR (default /config/config/trino/catalog)
|
||||
DBT_PROJECT_DIR (default /config/config/airflow/dbt/sicabe)
|
||||
AGENT_CA_FILE (opsional) path PEM untuk GET /ca
|
||||
OPA_DATA_FILE (default /config/config/opa/data.json) dokumen data OPA
|
||||
DOCKER_PROXY_URL (default http://docker-proxy:2375) sumber daftar container
|
||||
"""
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import urllib.request
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
|
||||
import yaml
|
||||
|
||||
AGENT_API_KEY = os.environ.get("AGENT_API_KEY", "")
|
||||
AGENT_PORT = int(os.environ.get("AGENT_PORT", "8080"))
|
||||
ENV_FILE = os.environ.get("SICABE_ENV_FILE", "/config/.env")
|
||||
CATALOG_DIR = os.environ.get("TRINO_CATALOG_DIR", "/config/config/trino/catalog")
|
||||
DBT_DIR = os.environ.get("DBT_PROJECT_DIR", "/config/config/airflow/dbt/sicabe")
|
||||
CA_FILE = os.environ.get("AGENT_CA_FILE", "")
|
||||
OPA_DATA_FILE = os.environ.get("OPA_DATA_FILE", "/config/config/opa/data.json")
|
||||
# docker-proxy internal (satu stack dengan agent) — untuk scan service tanpa
|
||||
# perlu mount docker.sock ke agent.
|
||||
DOCKER_PROXY_URL = os.environ.get("DOCKER_PROXY_URL", "http://docker-proxy:2375")
|
||||
|
||||
VALID_CATALOG = re.compile(r"^[a-z][a-z0-9_-]{0,62}$")
|
||||
SENSITIVE = re.compile(r"(password|secret|credential|access-key|apikey|api-key|token)", re.I)
|
||||
REF_RE = re.compile(r"""ref\(\s*['"]([^'"]+)['"]\s*\)""")
|
||||
SOURCE_RE = re.compile(r"""source\(\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\)""")
|
||||
|
||||
|
||||
# ── helpers: .env ────────────────────────────────────────────────────────────
|
||||
def read_env(keys):
|
||||
"""Baca key tertentu dari ENV_FILE (read-only, whitelist)."""
|
||||
wanted = set(keys)
|
||||
out = {}
|
||||
try:
|
||||
with open(ENV_FILE, "r", encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
k = k.strip()
|
||||
if k not in wanted:
|
||||
continue
|
||||
v = v.strip()
|
||||
if len(v) >= 2 and v[0] == v[-1] and v[0] in "\"'":
|
||||
v = v[1:-1]
|
||||
out[k] = v
|
||||
except OSError:
|
||||
return {}
|
||||
return out
|
||||
|
||||
|
||||
# ── helpers: katalog Trino ───────────────────────────────────────────────────
|
||||
def redact(props):
|
||||
return {k: ("••••••••" if SENSITIVE.search(k) else v) for k, v in props.items()}
|
||||
|
||||
|
||||
def parse_properties(content):
|
||||
props = {}
|
||||
for line in content.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
props[k.strip()] = v.strip()
|
||||
return props
|
||||
|
||||
|
||||
def serialize_properties(props):
|
||||
return "".join(f"{k}={v}\n" for k, v in props.items())
|
||||
|
||||
|
||||
def catalog_path(name):
|
||||
return os.path.join(CATALOG_DIR, f"{name}.properties")
|
||||
|
||||
|
||||
def list_catalogs():
|
||||
out = []
|
||||
try:
|
||||
names = sorted(os.listdir(CATALOG_DIR))
|
||||
except OSError:
|
||||
return out
|
||||
for fname in names:
|
||||
if not fname.endswith(".properties"):
|
||||
continue
|
||||
name = fname[: -len(".properties")]
|
||||
try:
|
||||
with open(os.path.join(CATALOG_DIR, fname), "r", encoding="utf-8") as fh:
|
||||
props = parse_properties(fh.read())
|
||||
except OSError:
|
||||
continue
|
||||
out.append({"name": name, "connector": props.get("connector.name", "unknown"), "properties": redact(props)})
|
||||
return sorted(out, key=lambda c: c["name"])
|
||||
|
||||
|
||||
def get_catalog(name):
|
||||
for c in list_catalogs():
|
||||
if c["name"] == name:
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def put_catalog(name, props):
|
||||
os.makedirs(CATALOG_DIR, exist_ok=True)
|
||||
with open(catalog_path(name), "w", encoding="utf-8") as fh:
|
||||
fh.write(serialize_properties(props))
|
||||
|
||||
|
||||
def write_opa_data(content):
|
||||
"""Tulis dokumen data OPA secara atomic + simpan isi sebelumnya ke .bak.
|
||||
|
||||
Atomic (temp + os.replace) agar crash di tengah tulis tidak meninggalkan file
|
||||
separuh jadi — OPA GAGAL START bila JSON-nya tidak valid, dan itu mematikan
|
||||
authz Trino. Backup `.bak` dipakai untuk rollback (Tahap 4).
|
||||
"""
|
||||
os.makedirs(os.path.dirname(OPA_DATA_FILE) or ".", exist_ok=True)
|
||||
if os.path.exists(OPA_DATA_FILE):
|
||||
with open(OPA_DATA_FILE, "rb") as fh:
|
||||
previous = fh.read()
|
||||
with open(OPA_DATA_FILE + ".bak", "wb") as fh:
|
||||
fh.write(previous)
|
||||
tmp = OPA_DATA_FILE + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as fh:
|
||||
fh.write(content)
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
os.replace(tmp, OPA_DATA_FILE)
|
||||
|
||||
|
||||
def delete_catalog(name):
|
||||
path = catalog_path(name)
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError(name)
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
# ── helpers: project dbt (bentuk = lib/dbt.ts) ───────────────────────────────
|
||||
def layer_from_path(rel):
|
||||
if "bronze_to_silver" in rel:
|
||||
return "silver"
|
||||
if "silver_to_gold" in rel:
|
||||
return "gold"
|
||||
if "incremental" in rel:
|
||||
return "bronze"
|
||||
return "silver"
|
||||
|
||||
|
||||
def load_dbt():
|
||||
name, version, profile = "sicabe", "1.0.0", "trino"
|
||||
try:
|
||||
with open(os.path.join(DBT_DIR, "dbt_project.yml"), "r", encoding="utf-8") as fh:
|
||||
yml = fh.read()
|
||||
m = re.search(r"^name:\s*['\"]?([^'\"\n]+)", yml, re.M)
|
||||
if m:
|
||||
name = m.group(1).strip()
|
||||
m = re.search(r"^version:\s*['\"]?([^'\"\n]+)", yml, re.M)
|
||||
if m:
|
||||
version = m.group(1).strip()
|
||||
m = re.search(r"^profile:\s*['\"]?([^'\"\n]+)", yml, re.M)
|
||||
if m:
|
||||
profile = m.group(1).strip()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
models = []
|
||||
models_dir = os.path.join(DBT_DIR, "models")
|
||||
for root, _dirs, files in os.walk(models_dir):
|
||||
for fname in files:
|
||||
if not fname.endswith(".sql"):
|
||||
continue
|
||||
full = os.path.join(root, fname)
|
||||
rel = os.path.relpath(full, models_dir)
|
||||
try:
|
||||
with open(full, "r", encoding="utf-8") as fh:
|
||||
sql = fh.read()
|
||||
except OSError:
|
||||
continue
|
||||
layer = layer_from_path(rel)
|
||||
models.append({
|
||||
"name": fname[: -len(".sql")],
|
||||
"path": rel,
|
||||
"layer": layer,
|
||||
"materialization": "incremental" if "incremental" in rel else "table",
|
||||
"schema": layer,
|
||||
"dependsOn": REF_RE.findall(sql),
|
||||
"sources": [f"{a}.{b}" for a, b in SOURCE_RE.findall(sql)],
|
||||
"sql": sql,
|
||||
})
|
||||
|
||||
sources = []
|
||||
sources_file = os.path.join(models_dir, "sources.yml")
|
||||
if os.path.exists(sources_file):
|
||||
try:
|
||||
with open(sources_file, "r", encoding="utf-8") as fh:
|
||||
doc = yaml.safe_load(fh.read()) or {}
|
||||
for s in doc.get("sources", []) or []:
|
||||
nm = s.get("name") or ""
|
||||
if not nm:
|
||||
continue
|
||||
sources.append({
|
||||
"name": nm,
|
||||
"database": s.get("database") or "",
|
||||
"schema": s.get("schema") or "",
|
||||
"tables": [{"name": t.get("name") or "", "description": t.get("description")}
|
||||
for t in (s.get("tables") or [])],
|
||||
})
|
||||
except (OSError, yaml.YAMLError):
|
||||
sources = []
|
||||
|
||||
nodes, edges = [], []
|
||||
for src in sources:
|
||||
for tbl in src["tables"]:
|
||||
nodes.append({"id": f"source.{src['name']}.{tbl['name']}", "label": f"{src['name']}.{tbl['name']}",
|
||||
"type": "source", "layer": "bronze"})
|
||||
for m in models:
|
||||
nodes.append({"id": f"model.{m['name']}", "label": m["name"], "type": "model", "layer": m["layer"]})
|
||||
for dep in m["dependsOn"]:
|
||||
edges.append({"id": f"model.{dep}->model.{m['name']}", "source": f"model.{dep}", "target": f"model.{m['name']}"})
|
||||
for s in m["sources"]:
|
||||
db, _, tbl = s.partition(".")
|
||||
edges.append({"id": f"source.{db}.{tbl}->model.{m['name']}", "source": f"source.{db}.{tbl}", "target": f"model.{m['name']}"})
|
||||
|
||||
return {"name": name, "version": version, "profile": profile, "models": models, "sources": sources,
|
||||
"dag": {"nodes": nodes, "edges": edges}}
|
||||
|
||||
|
||||
# ── Scan service (panel-driven) ──────────────────────────────────────────────
|
||||
# Panel mengirim katalog TIPE service (regex + port); agent memindai container
|
||||
# yang SEDANG BERJALAN di host ini (via docker-proxy internal) dan mengembalikan
|
||||
# instance + URL-nya. Logika pengenalan tetap satu tempat di panel (TS) — agent
|
||||
# hanya mesin generik. Hanya container `running` yang dipindai, sehingga job
|
||||
# sekali-jalan (init/migrate) otomatis tidak ikut.
|
||||
|
||||
# Label peta port → skema URL asli (label boleh deskriptif, mis. "console").
|
||||
SCHEME_ALIASES = {
|
||||
"https": "https",
|
||||
"http": "http",
|
||||
"api": "http",
|
||||
"console": "http",
|
||||
"ui": "http",
|
||||
"web": "http",
|
||||
"ldap": "ldap",
|
||||
"mysql": "mysql",
|
||||
"sql": "https",
|
||||
"tcp": "tcp",
|
||||
}
|
||||
def _re_match(pattern, value):
|
||||
if not pattern:
|
||||
return False
|
||||
try:
|
||||
return re.search(pattern, value or "") is not None
|
||||
except re.error:
|
||||
return False
|
||||
|
||||
|
||||
def docker_running_containers():
|
||||
"""Daftar container running dari docker-proxy internal."""
|
||||
req = urllib.request.Request(f"{DOCKER_PROXY_URL}/containers/json?all=false")
|
||||
with urllib.request.urlopen(req, timeout=8) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
out = []
|
||||
for c in data:
|
||||
if (c.get("State") or "") != "running":
|
||||
continue
|
||||
names = c.get("Names") or []
|
||||
name = names[0].lstrip("/") if names else (c.get("Id") or "")[:12]
|
||||
out.append({
|
||||
"name": name,
|
||||
"image": c.get("Image", ""),
|
||||
"labels": c.get("Labels") or {},
|
||||
"ports": c.get("Ports") or [],
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _published_ports(ports_raw):
|
||||
out = []
|
||||
seen = set()
|
||||
for p in ports_raw:
|
||||
if not p.get("PublicPort"):
|
||||
continue
|
||||
key = (p.get("PrivatePort"), p.get("PublicPort"), p.get("Type", "tcp"))
|
||||
if key in seen:
|
||||
continue # docker mengembalikan entri IPv4 & IPv6 → dedupe
|
||||
seen.add(key)
|
||||
out.append({
|
||||
"privatePort": p.get("PrivatePort"),
|
||||
"publicPort": p.get("PublicPort"),
|
||||
"type": p.get("Type", "tcp"),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _urls_for(ports_map, published_raw, public_host):
|
||||
"""Susun URL per label dari peta port container → port publish host.
|
||||
|
||||
Kunci peta = LABEL (mis. "http", "console", "mysql"); skema URL sebenarnya
|
||||
dipetakan lewat SCHEME_ALIASES (label "console" → skema "http").
|
||||
"""
|
||||
urls = {}
|
||||
if not public_host:
|
||||
return urls
|
||||
for label, privates in (ports_map or {}).items():
|
||||
if not isinstance(privates, list):
|
||||
continue
|
||||
scheme = SCHEME_ALIASES.get(str(label).lower(), "http")
|
||||
for priv in privates:
|
||||
for p in published_raw:
|
||||
if int(p.get("PrivatePort") or 0) == int(priv) and p.get("PublicPort"):
|
||||
urls[label] = f"{scheme}://{public_host}:{p.get('PublicPort')}"
|
||||
break
|
||||
if label in urls:
|
||||
break
|
||||
return urls
|
||||
|
||||
|
||||
def scan_services(types, public_host):
|
||||
containers = docker_running_containers()
|
||||
services = []
|
||||
for t in types:
|
||||
if not isinstance(t, dict):
|
||||
continue
|
||||
img_pat = t.get("imagePattern")
|
||||
name_pat = t.get("namePattern")
|
||||
ports_map = t.get("ports") or {}
|
||||
for c in containers:
|
||||
if not (_re_match(img_pat, c["image"]) or _re_match(name_pat, c["name"])):
|
||||
continue
|
||||
services.append({
|
||||
"type": t.get("id"),
|
||||
"container": c["name"],
|
||||
"image": c["image"],
|
||||
"labels": c["labels"],
|
||||
"published": _published_ports(c["ports"]),
|
||||
"urls": _urls_for(ports_map, c["ports"], public_host),
|
||||
})
|
||||
return {
|
||||
"hostname": socket.gethostname(),
|
||||
"containers": len(containers),
|
||||
"config": {
|
||||
"envFile": os.path.exists(ENV_FILE),
|
||||
"catalogDir": os.path.isdir(CATALOG_DIR),
|
||||
"dbtDir": os.path.isdir(DBT_DIR),
|
||||
"caFile": bool(CA_FILE) and os.path.exists(CA_FILE),
|
||||
},
|
||||
"services": services,
|
||||
}
|
||||
|
||||
|
||||
# ── HTTP handler ─────────────────────────────────────────────────────────────
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "sicabe-config-agent/0.1"
|
||||
|
||||
# -- util --
|
||||
def _send(self, code, body, ctype="application/json"):
|
||||
payload = body if isinstance(body, bytes) else json.dumps(body).encode("utf-8")
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", ctype)
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
try:
|
||||
self.wfile.write(payload)
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
pass
|
||||
|
||||
def _auth_ok(self):
|
||||
key = self.headers.get("X-Api-Key", "")
|
||||
return bool(AGENT_API_KEY) and hmac.compare_digest(key, AGENT_API_KEY)
|
||||
|
||||
def _deny(self):
|
||||
self._send(401, {"error": "Unauthorized"})
|
||||
|
||||
def _json_body(self):
|
||||
length = int(self.headers.get("Content-Length") or 0)
|
||||
raw = self.rfile.read(length) if length else b"{}"
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
|
||||
def log_message(self, fmt, *args): # ringkas ke stderr
|
||||
print("[agent] " + (fmt % args), flush=True)
|
||||
|
||||
# -- GET --
|
||||
def do_GET(self):
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
|
||||
if path == "/healthz":
|
||||
return self._send(200, {"status": "ok"})
|
||||
if not self._auth_ok():
|
||||
return self._deny()
|
||||
|
||||
try:
|
||||
if path == "/env":
|
||||
keys = parse_qs(parsed.query).get("key", [])
|
||||
keys = [k for k in keys if k]
|
||||
if not keys:
|
||||
return self._send(400, {"error": "key parameter required (whitelist)"})
|
||||
return self._send(200, read_env(keys))
|
||||
|
||||
if path == "/catalog":
|
||||
return self._send(200, {"catalogs": list_catalogs()})
|
||||
|
||||
if path.startswith("/catalog/"):
|
||||
name = path[len("/catalog/"):]
|
||||
cat = get_catalog(name)
|
||||
if not cat:
|
||||
return self._send(404, {"error": f"catalog '{name}' not found"})
|
||||
return self._send(200, cat)
|
||||
|
||||
if path == "/dbt":
|
||||
return self._send(200, load_dbt())
|
||||
|
||||
if path == "/ca":
|
||||
if not CA_FILE or not os.path.exists(CA_FILE):
|
||||
return self._send(404, {"error": "no CA configured"})
|
||||
# Kirim bytes mentah (jangan di-JSON-encode) agar PEM utuh.
|
||||
with open(CA_FILE, "rb") as fh:
|
||||
return self._send(200, fh.read(), "application/x-pem-file")
|
||||
|
||||
if path == "/opa/data":
|
||||
# Kirim isi file APA ADANYA. Harus bytes: _send() men-JSON-encode
|
||||
# nilai non-bytes (kalau str, isi file jadi string ber-quote ganda
|
||||
# dan editor panel menampilkan JSON yang salah).
|
||||
if not os.path.exists(OPA_DATA_FILE):
|
||||
return self._send(404, {"error": "opa data file not found"})
|
||||
with open(OPA_DATA_FILE, "rb") as fh:
|
||||
return self._send(200, fh.read(), "application/json")
|
||||
|
||||
return self._send(404, {"error": "not found"})
|
||||
except Exception as e: # noqa: BLE001
|
||||
return self._send(500, {"error": "internal error", "details": str(e)})
|
||||
|
||||
# -- PUT --
|
||||
def do_PUT(self):
|
||||
if not self._auth_ok():
|
||||
return self._deny()
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
|
||||
if path == "/opa/data":
|
||||
try:
|
||||
body = self._json_body()
|
||||
content = body.get("content") if isinstance(body, dict) else None
|
||||
if not isinstance(content, str):
|
||||
return self._send(400, {"error": "content (JSON string) required"})
|
||||
# Validasi SEBELUM menulis: JSON invalid membuat OPA gagal start
|
||||
# → authz Trino mati total.
|
||||
try:
|
||||
json.loads(content)
|
||||
except json.JSONDecodeError as e:
|
||||
return self._send(400, {"error": "invalid JSON", "details": e.msg,
|
||||
"line": e.lineno, "col": e.colno})
|
||||
write_opa_data(content)
|
||||
return self._send(200, {"success": True, "bytes": len(content.encode("utf-8"))})
|
||||
except Exception as e: # noqa: BLE001
|
||||
return self._send(500, {"error": "internal error", "details": str(e)})
|
||||
|
||||
if not path.startswith("/catalog/"):
|
||||
return self._send(404, {"error": "not found"})
|
||||
name = path[len("/catalog/"):]
|
||||
if not VALID_CATALOG.match(name):
|
||||
return self._send(400, {"error": "invalid catalog name"})
|
||||
try:
|
||||
body = self._json_body()
|
||||
props = body.get("properties", body) if isinstance(body, dict) else {}
|
||||
if not isinstance(props, dict):
|
||||
return self._send(400, {"error": "properties must be an object"})
|
||||
put_catalog(name, {str(k): str(v) for k, v in props.items()})
|
||||
return self._send(200, {"success": True, "name": name})
|
||||
except Exception as e: # noqa: BLE001
|
||||
return self._send(500, {"error": "internal error", "details": str(e)})
|
||||
|
||||
# -- DELETE --
|
||||
def do_DELETE(self):
|
||||
if not self._auth_ok():
|
||||
return self._deny()
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
if not path.startswith("/catalog/"):
|
||||
return self._send(404, {"error": "not found"})
|
||||
name = path[len("/catalog/"):]
|
||||
if not VALID_CATALOG.match(name):
|
||||
return self._send(400, {"error": "invalid catalog name"})
|
||||
try:
|
||||
delete_catalog(name)
|
||||
return self._send(200, {"success": True, "name": name})
|
||||
except FileNotFoundError:
|
||||
return self._send(404, {"error": f"catalog '{name}' not found"})
|
||||
except Exception as e: # noqa: BLE001
|
||||
return self._send(500, {"error": "internal error", "details": str(e)})
|
||||
|
||||
|
||||
# -- POST --
|
||||
def do_POST(self):
|
||||
if not self._auth_ok():
|
||||
return self._deny()
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
|
||||
if path == "/services/scan":
|
||||
try:
|
||||
body = self._json_body()
|
||||
types = body.get("types") if isinstance(body, dict) else None
|
||||
if not isinstance(types, list) or not types:
|
||||
return self._send(400, {"error": "types (non-empty array) required"})
|
||||
public_host = body.get("publicHost") or ""
|
||||
return self._send(200, scan_services(types, public_host))
|
||||
except Exception as e: # noqa: BLE001
|
||||
return self._send(500, {"error": "scan failed", "details": str(e)})
|
||||
|
||||
return self._send(404, {"error": "not found"})
|
||||
|
||||
|
||||
def main():
|
||||
if not AGENT_API_KEY:
|
||||
print("FATAL: AGENT_API_KEY must be set", flush=True)
|
||||
raise SystemExit(1)
|
||||
print(f"[agent] listening on :{AGENT_PORT} (env={ENV_FILE}, catalog={CATALOG_DIR}, dbt={DBT_DIR})", flush=True)
|
||||
ThreadingHTTPServer(("0.0.0.0", AGENT_PORT), Handler).serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
PyYAML==6.0.2
|
||||
@@ -0,0 +1,128 @@
|
||||
# =============================================================================
|
||||
# sicabe-host-gateway — SATU port untuk docker-proxy + config-agent.
|
||||
#
|
||||
# CATATAN: ini varian LOKAL (build dari ./agent). Untuk DEPLOY dari registry
|
||||
# (termasuk panel, dengan profile) pakai: deploy/stack/docker-compose.yml
|
||||
#
|
||||
# Hanya nginx (gateway) yang membuka port host; docker-socket-proxy dan
|
||||
# config-agent bersifat internal (expose, tanpa port host). Satu API key
|
||||
# (GATEWAY_API_KEY) diperiksa nginx, lalu request diarahkan berdasarkan path:
|
||||
# /_ping /version /info /events /containers/* → docker-proxy
|
||||
# /healthz /env /dbt /ca /catalog(/<name>) → config-agent
|
||||
#
|
||||
# Lihat: docs/desain-host-gateway-terpadu.md
|
||||
#
|
||||
# Cara pakai:
|
||||
# GATEWAY_API_KEY='<secret-kuat>' \
|
||||
# SICABE_DEPLOY_DIR=/path/ke/sicabe-doc/deploy \
|
||||
# AGENT_CA_HOST_PATH=/path/berisi/pem AGENT_CA_FILE=/ca/trino.crt \
|
||||
# docker compose up -d
|
||||
#
|
||||
# Opsional: AGENT_UID/AGENT_GID (default 1000:1000) — UID/GID yang dipakai
|
||||
# config-agent saat menulis ke host, supaya file (katalog Trino, data OPA) tetap
|
||||
# dimiliki user host dan bukan root.
|
||||
#
|
||||
# Panel (satu URL untuk proxy & config):
|
||||
# SICABE_HOSTS=[{"name":"h1","proxyUrl":"http://<host>:22375",
|
||||
# "apiKey":"<GATEWAY_API_KEY>",
|
||||
# "configUrl":"http://<host>:22375",
|
||||
# "configApiKey":"<GATEWAY_API_KEY>","enabled":true}]
|
||||
# =============================================================================
|
||||
|
||||
name: sicabe-host-gateway
|
||||
|
||||
services:
|
||||
# --- Docker Engine API (ACL ketat, internal saja) ------------------------
|
||||
docker-proxy:
|
||||
image: tecnativa/docker-socket-proxy:v0.5.0
|
||||
container_name: sicabe-docker-proxy
|
||||
environment:
|
||||
CONTAINERS: "1"
|
||||
POST: "1"
|
||||
INFO: "1"
|
||||
SYSTEM: "1"
|
||||
EXEC: "0"
|
||||
SWARM: "0"
|
||||
SERVICES: "0"
|
||||
SECRETS: "0"
|
||||
CONFIGS: "0"
|
||||
NETWORKS: "0"
|
||||
VOLUMES: "0"
|
||||
IMAGES: "0"
|
||||
BUILD: "0"
|
||||
GRPC: "0"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
expose:
|
||||
- "2375"
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp
|
||||
- /run
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
# --- Konfigurasi platform per host (internal saja) -----------------------
|
||||
config-agent:
|
||||
build: ./agent
|
||||
image: sicabe/config-agent:0.1.0
|
||||
container_name: sicabe-config-agent
|
||||
# Jalankan sebagai UID/GID host (bukan root) supaya file yang DITULIS agent
|
||||
# (katalog Trino, config/opa/data.json) tetap dimiliki user host — kalau root,
|
||||
# editor/git di host akan kena permission denied. Dir yang di-mount rw memang
|
||||
# milik user host. Bisa di-override via AGENT_UID/AGENT_GID.
|
||||
user: "${AGENT_UID:-1000}:${AGENT_GID:-1000}"
|
||||
environment:
|
||||
# Satu key: nginx memeriksa, agent tetap memvalidasi (defense-in-depth).
|
||||
AGENT_API_KEY: ${GATEWAY_API_KEY:?GATEWAY_API_KEY wajib diisi}
|
||||
AGENT_PORT: "8080"
|
||||
PYTHONDONTWRITEBYTECODE: "1"
|
||||
SICABE_ENV_FILE: /config/.env
|
||||
TRINO_CATALOG_DIR: /config/config/trino/catalog
|
||||
DBT_PROJECT_DIR: /config/config/airflow/dbt/sicabe
|
||||
AGENT_CA_FILE: ${AGENT_CA_FILE:-}
|
||||
# Dokumen data OPA (ACL Trino) — dibaca menu Authorization. Tulis (rw)
|
||||
# menyusul di Tahap 2.
|
||||
OPA_DATA_FILE: /config/config/opa/data.json
|
||||
volumes:
|
||||
- ${SICABE_DEPLOY_DIR:?set SICABE_DEPLOY_DIR}:/config:ro
|
||||
- ${SICABE_DEPLOY_DIR:?set SICABE_DEPLOY_DIR}/config/trino/catalog:/config/config/trino/catalog:rw
|
||||
# Dir OPA (data.json) — rw agar agent bisa menulis di Tahap 2.
|
||||
- ${SICABE_DEPLOY_DIR:?set SICABE_DEPLOY_DIR}/config/opa:/config/config/opa:rw
|
||||
# CA TLS opsional (GET /ca): direktori berisi PEM → /ca.
|
||||
- ${AGENT_CA_HOST_PATH:-${SICABE_DEPLOY_DIR:?set SICABE_DEPLOY_DIR}}:/ca:ro
|
||||
expose:
|
||||
- "8080"
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
# --- Gateway nginx: satu-satunya port ke host ----------------------------
|
||||
gateway:
|
||||
image: nginx:1.30.4-alpine3.24
|
||||
container_name: sicabe-host-gateway
|
||||
depends_on:
|
||||
- docker-proxy
|
||||
- config-agent
|
||||
environment:
|
||||
GATEWAY_API_KEY: ${GATEWAY_API_KEY:?GATEWAY_API_KEY wajib diisi}
|
||||
volumes:
|
||||
- ./nginx/gateway.conf.template:/etc/nginx/templates/default.conf.template:ro
|
||||
ports:
|
||||
- "${GATEWAY_BIND_IP:-127.0.0.1}:${GATEWAY_PORT:-22375}:80"
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
@@ -0,0 +1,66 @@
|
||||
# nginx gateway terpadu — docker-socket-proxy + config-agent dalam SATU port.
|
||||
#
|
||||
# Template: docker-entrypoint nginx men-substitute ${GATEWAY_API_KEY} saat start
|
||||
# (variabel nginx seperti $http_x_api_key TIDAK ikut diganti), lalu menulis hasil
|
||||
# ke /etc/nginx/conf.d/default.conf.
|
||||
#
|
||||
# Kebijakan:
|
||||
# * Semua request WAJIB membawa header `X-Api-Key` == GATEWAY_API_KEY (401 bila
|
||||
# salah/kosong), SEBELUM diproses location mana pun.
|
||||
# * Allowlist path ketat:
|
||||
# - /_ping /version /info /events /containers/* → docker-proxy:2375
|
||||
# - /healthz /env /dbt /ca /catalog(/<name>) /opa/data /services/scan → config-agent:8080
|
||||
# Selain daftar → 403.
|
||||
# * Aksi container restart/start/stop hanya POST (selainnya 405).
|
||||
# * Katalog (cart agent) mengizinkan GET/PUT/DELETE.
|
||||
#
|
||||
# Lihat: docs/desain-host-gateway-terpadu.md
|
||||
|
||||
server {
|
||||
listen 80 default_server;
|
||||
server_name _;
|
||||
|
||||
# ==== API-key check (sebelum location mana pun) ====
|
||||
if ($http_x_api_key != "${GATEWAY_API_KEY}") {
|
||||
return 401;
|
||||
}
|
||||
|
||||
# ==== config-agent (konfigurasi platform) ====
|
||||
location = /healthz { proxy_pass http://config-agent:8080; }
|
||||
location = /env { proxy_pass http://config-agent:8080; }
|
||||
location = /dbt { proxy_pass http://config-agent:8080; }
|
||||
location = /ca { proxy_pass http://config-agent:8080; }
|
||||
location = /catalog { proxy_pass http://config-agent:8080; }
|
||||
# GET/PUT/DELETE satu katalog (tanpa slash tambahan di nama).
|
||||
location ~ ^/catalog/[^/]+$ { proxy_pass http://config-agent:8080; }
|
||||
# Dokumen data OPA (menu Authorization) — baca sekarang, tulis menyusul.
|
||||
location = /opa/data { proxy_pass http://config-agent:8080; }
|
||||
# Scan service (panel-driven) — memindai container host lewat docker-proxy.
|
||||
location = /services/scan { proxy_pass http://config-agent:8080; }
|
||||
|
||||
# ==== docker-socket-proxy (Docker Engine API) ====
|
||||
location = /_ping { proxy_pass http://docker-proxy:2375; }
|
||||
location = /version { proxy_pass http://docker-proxy:2375; }
|
||||
location = /info { proxy_pass http://docker-proxy:2375; }
|
||||
location = /events { proxy_pass http://docker-proxy:2375; }
|
||||
|
||||
location = /containers/json { proxy_pass http://docker-proxy:2375; }
|
||||
|
||||
location ~ ^/containers/[^/]+/json$ { proxy_pass http://docker-proxy:2375; }
|
||||
location ~ ^/containers/[^/]+/logs$ { proxy_pass http://docker-proxy:2375; }
|
||||
location ~ ^/containers/[^/]+/stats$ { proxy_pass http://docker-proxy:2375; }
|
||||
location ~ ^/containers/[^/]+/top$ { proxy_pass http://docker-proxy:2375; }
|
||||
|
||||
# Aksi tulis container — hanya POST.
|
||||
location ~ ^/containers/[^/]+/(restart|start|stop)$ {
|
||||
if ($request_method != POST) {
|
||||
return 405;
|
||||
}
|
||||
proxy_pass http://docker-proxy:2375;
|
||||
}
|
||||
|
||||
# ==== selain allowlist → tolak ====
|
||||
location / {
|
||||
return 403;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user