first commit

This commit is contained in:
2026-10-09 02:48:31 +07:00
commit 8db22bfaa6
17 changed files with 1722 additions and 0 deletions
+20
View File
@@ -0,0 +1,20 @@
node_modules
.next
.git
.gitignore
.env*
npm-debug.log
# Build context (tak perlu untuk build image)
Dockerfile
.dockerignore
deploy
docs
scripts
.opencode
.tmp
test-results
playwright-report
# Runtime state panel — MEMUAT SECRET (api key host). Jangan pernah masuk image.
data
+146
View File
@@ -0,0 +1,146 @@
# ============================================================================
# SiCABE Control Panel — konfigurasi env (salin ke .env lalu sesuaikan)
#
# SEMUA URL service dapat di-override dari environment. Nilai default di bawah
# memakai nama service dalam network compose (sicabe-doc/deploy, `name: sicabe`).
# Bila panel dijalankan DI HOST, pakai port host (12xxx) seperti kolom "host".
#
# Konvensi:
# * var tanpa prefix -> dibaca sisi SERVER (fetch/API, rahasia aman)
# * *_WEB_URL dll -> tautan/UI client; di-fetch dari /api/platform/endpoints
# * .env TIDAK di-commit; contoh ini ikut di-track (!.env.example)
# ============================================================================
# ── Trino ────────────────────────────────────────────────────────────────────
# Endpoint SQL/API (route /api/trino & /api/query). HTTPS saja: Trino memakai
# auth hybrid OAuth2 + Basic dengan allow-insecure-over-http=false (HTTP polos
# tidak melayani apa pun). Login dev pun memakai Bearer, didapat via Keycloak
# ROPC di CredentialsProvider.
# SSO & dev (Bearer/JWT) : HTTPS → https://localhost:12443
# dalam compose : http://trino:8080
TRINO_URL=https://localhost:12443
# CA utk Trino HTTPS self-signed. Pin sertifikat server (PEM) di sini.
# Bisa juga per-host lewat `caPem` di SICABE_HOSTS (lebih diutamakan).
TRINO_CA_FILE=.ca/trino.crt
# Nama host (entri SICABE_HOSTS) yang menjalankan Trino — untuk memilih CA per-host.
#SICABE_TRINO_HOST=host-prod-1
# Trino Web UI.
#TRINO_WEB_URL=https://localhost:12443
# ── Keycloak ─────────────────────────────────────────────────────────────────
# Base URL + realm utk OIDC (login panel & refresh token).
# dalam compose : http://keycloak:8080
# host : http://localhost:12080
KEYCLOAK_URL=http://keycloak:8080
KEYCLOAK_REALM=sicabe
# Client OIDC panel (harus ada di realm tsb).
KEYCLOAK_CLIENT_ID=sicabe-control-panel
# Secret client. WAJIB di-production (tanpa nilai -> fail saat login).
#KEYCLOAK_CLIENT_SECRET=
# Admin Console Keycloak (tautan dashboard utk admin).
#KEYCLOAK_WEB_URL=http://localhost:12080
# ── Iceberg table explorer backend (Nessie) ──────────────────────────────────
# Halaman /tables memakai catalog Iceberg REST Nessie.
# base ops = {NESSIE_URL}/iceberg/v1/{NESSIE_BRANCH}.
# dalam compose : http://nessie:19120
# host : http://localhost:12120
#NESSIE_URL=http://nessie:19120
#NESSIE_BRANCH=main
# ── Dremio (REST v3 — sources & SQL) ────────────────────────────────────────
# Endpoint REST Dremio OSS + kredensial lokal (login /apiv2 -> token _dremio).
# dalam compose : http://dremio:9047
# host : http://localhost:12947
DREMIO_URL=http://dremio:9047
DREMIO_USER=
DREMIO_PASSWORD=
# ── MinIO ────────────────────────────────────────────────────────────────────
# MinIO Console (tautan dashboard & sidebar).
# host : http://localhost:12901
#MINIO_CONSOLE_URL=http://localhost:12901
# ── StarRocks ────────────────────────────────────────────────────────────────
# FE MySQL protocol (route /api/starrocks).
# dalam compose : STARROCKS_HOST=starrocks-fe, PORT=9030
# host : STARROCKS_HOST=localhost, PORT=12930
STARROCKS_HOST=localhost
STARROCKS_PORT=12930
STARROCKS_USER=root
#STARROCKS_PASSWORD=
# StarRocks UI HTTP (tautan halaman StarRocks).
#STARROCKS_WEB_URL=http://localhost:12030
# ── Apache Airflow ───────────────────────────────────────────────────────────
# Base URL — dipakai utk REST API (lib/airflow) DAN tautan UI (sidebar/dashboard).
# dalam compose : http://airflow-webserver:8080
# host : http://localhost:12082
AIRFLOW_URL=http://localhost:12082
AIRFLOW_USER=admin
# ── OpenMetadata & Grafana (governance) ──────────────────────────────────────
# URL API/UI — single source (halaman governance + tautan).
# host : OpenMetadata http://localhost:12085, Grafana http://localhost:12300
OPENMETADATA_URL=http://localhost:12085
GRAFANA_URL=http://localhost:12300
# ── Auth (NextAuth) ──────────────────────────────────────────────────────────
NEXTAUTH_URL=http://localhost:3000
# WAJIB di-production.
# ── Login lokal (hybrid dengan SSO) ──────────────────────────────────────────
# Akun admin LOKAL dari env — bisa login tanpa browser SSO, berlaku juga di
# produksi. Password = BCRYPT HASH (bukan plaintext).
# Buat hash: node -e "console.log(require('bcryptjs').hashSync('PASSWORD',10))"
# Untuk Docker/Compose pakai varian _B64 (bcrypt memuat '$' yang bisa di-
# interpolasi Compose):
# node -e "console.log(Buffer.from(require('bcryptjs').hashSync('PASSWORD',10)).toString('base64'))"
#PANEL_LOCAL_USER=admin
#PANEL_LOCAL_PASSWORD_HASH=$2b$10$...
#PANEL_LOCAL_PASSWORD_HASH_B64=JDJiJDEwJC4uLg==
#PANEL_LOCAL_ROLE=data-admin
# Identitas Trino untuk sesi login lokal (tanpa JWT Keycloak): service account.
# Password dibaca dari .env platform via config-agent → TRINO_SVC_<ACC>_PASSWORD.
#TRINO_SERVICE_ACCOUNT=svc_dbt
#NEXTAUTH_SECRET=
# ── Multi-host Docker control ────────────────────────────────────────────────
# Registry host Docker yang dikelola panel (via docker-socket-proxy per host).
# Nilai JSON array. Bila kosong → default single host http://127.0.0.1:2375
# (kompatibel perilaku lama).
# Field per host: name (unik), proxyUrl, apiKey (opsional), enabled,
# sshTunnel (opsional: {host, port, user}).
# Multi-host (opsional, untuk Tahap 2): configUrl/configApiKey menunjuk ke
# `sicabe-config-agent` host tsb; caPem = CA TLS host (PEM) bila perlu.
# ⚠️ apiKey/configApiKey adalah SECRET — jangan di-commit; isi via env/deploy.
# Development lokal (single host, tanpa proxy):
# SICABE_HOSTS='[{"name":"local","proxyUrl":"unix:///var/run/docker.sock","enabled":true}]'
#SICABE_HOSTS='[{"name":"host-prod-1","proxyUrl":"http://127.0.0.1:22375","apiKey":"","configUrl":"http://127.0.0.1:22375","configApiKey":"","enabled":true}]'
# Config panel yang PERSISTEN (hosts/modules/services), diedit dari UI Settings.
# Bila file ini ada, ia MENGGANTIKAN SICABE_HOSTS di atas (env = seed saja).
# Memuat api key → gitignored, perms 0600.
#SICABE_CONFIG_FILE=data/config.json
# Lokasi file state (monitored toggle per container). Default data/state.json.
# Key = `${host}/${container}` (kompatibel dgn key lama `${container}`).
#SICABE_STATE_FILE=data/state.json
# Daftar project Docker Compose yang dimonitor (label com.docker.compose.project).
# Comma-separated; default "sicabe".
#SICABE_COMPOSE_PROJECTS=sicabe
# Override host per-service (opsional). Bila kosong, host di-resolve otomatis
# dari daftar host/komponen (dipakai untuk membaca kredensial dari host yang tepat).
#SICABE_AIRFLOW_HOST=host-prod-1
#SICABE_STARROCKS_HOST=host-prod-1
# Host yang menjalankan OPA (menu Authorization; baca/tulis config/opa/data.json).
#SICABE_OPA_HOST=host-prod-1
# ── Lain-lain ────────────────────────────────────────────────────────────────
# Direktori catalog Trino & file .env platform (bila dijalankan non-standar).
# .env platform dibaca READ-ONLY oleh panel (tidak pernah ditimpa).
#TRINO_CATALOG_DIR=deploy/config/trino/catalog
#SICABE_ENV_FILE=deploy/.env
# Dokumen data OPA (ACL authz Trino) — dibaca menu Authorization lewat ConfigStore.
#OPA_DATA_FILE=/path/ke/sicabe-doc/deploy/config/opa/data.json
# Project dbt SiCABE (dibaca /api/dbt untuk models, sources, lineage DAG).
# Panel di host: arahkan ke folder platform (mis. sicabe-doc/deploy/config/airflow/dbt/sicabe).
#DBT_PROJECT_DIR=/path/ke/sicabe-doc/deploy/config/airflow/dbt/sicabe
# Password admin Airflow dibaca dari SICABE_ENV_FILE (AIRFLOW_ADMIN_PASSWORD).
+61
View File
@@ -0,0 +1,61 @@
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
# dependencies
/node_modules
/.pnp
.pnp.*
.yarn/*
!.yarn/patches
!.yarn/plugins
!.yarn/releases
!.yarn/versions
# testing
/coverage
# next.js
/.next/
/out/
# production
/build
# misc
.DS_Store
*.pem
# debug
npm-debug.log*
yarn-debug.log*
yarn-error.log*
.pnpm-debug.log*
# env files (can opt-in for committing if needed)
.env*
!.env.example
# stack env — nama `env` (tanpa titik) tidak tertangkap `.env*`
/deploy/stack/env
/deploy/panel/env
# vercel
.vercel
# typescript
*.tsbuildinfo
next-env.d.ts
# AetherLake root CA exported by install.sh (trusted by local dev TLS calls)
.ca/
# playwright
/test-results/
/playwright-report/
# task/session artifacts (agent)
/.tmp/
# runtime panel state & config — data/config.json memuat api key host (SECRET)
/data/
# local agent tooling & install helper
/.opencode/
/install.sh
View File
@@ -0,0 +1,104 @@
# =============================================================================
# docker-socket-proxy + gateway nginx (api-key) — per host SiCABE.
#
# Control-panel (sicabe-control-panel) TIDAK pernah menyentuh /var/run/docker.sock
# langsung. Di setiap host yang dikelola dipasang stack ini; panel berbicara ke
# endpoint HTTP gateway (lihat desain:
# docs/desain-kontrol-multi-host-docker-socket-proxy.md).
#
# Arsitektur:
# panel ──HTTP + X-Api-Key──► gateway (nginx, allowlist path)
# └──► docker-proxy (docker-socket-proxy, ACL)
# └──► /var/run/docker.sock (:ro)
#
# Keamanan (jangan longgarkan tanpa kebutuhan):
# * Socket daemon di-mount READ-ONLY (:ro).
# * docker-socket-proxy TIDAK punya auth — container tsb HANYA di jaringan
# internal compose (expose, tanpa port host). Yang diekspos ke host hanyalah
# gateway nginx.
# * nginx memeriksa header `X-Api-Key` (wajib sama dengan PROXY_API_KEY) dan
# allowlist path (hanya route yang dibutuhkan kontrol, sisanya 403).
# * ACL docker-socket-proxy hanya mengizinkan yang dibutuhkan:
# CONTAINERS (list/inspect/restart/stop/start/logs/top), POST (restart),
# INFO (version/ping), SYSTEM (/events).
# Kategori lain (EXEC, BUILD, IMAGES, NETWORKS, VOLUMES, ...) DIMATIKAN.
# * Versi image DIPATOK (bukan :latest) untuk produksi.
#
# Mode alternatif SSH-tunnel / single-host (tanpa api-key):
# Hapus/komentari service `gateway`, lalu pada service `docker-proxy` ganti
# `expose` dengan port publik:
# ports:
# - "127.0.0.1:${PROXY_PORT:-22375}:2375"
#
# Cara pakai (HTTP + api-key):
# PROXY_API_KEY='<secret-acak-kuat>' PROXY_BIND_IP=127.0.0.1 PROXY_PORT=22375 \
# docker compose up -d
# - PROXY_API_KEY WAJIB diisi (compose akan error bila kosong). Ini secret —
# jangan di-commit; isi via env/deploy.
# - PROXY_BIND_IP: 127.0.0.1 utk single-host/SSH-tunnel; IP internal host bila
# panel di host lain dalam network tepercaya. JANGAN expose ke internet.
# - Panel: SICABE_HOSTS=[{"name":"...","proxyUrl":"http://<bind-ip>:<port>",
# "apiKey":"<PROXY_API_KEY>","enabled":true}]
# =============================================================================
name: sicabe-docker-proxy
services:
# --- docker-socket-proxy: ACL, hanya di jaringan internal compose ---------
docker-proxy:
image: tecnativa/docker-socket-proxy:v0.5.0
container_name: sicabe-docker-proxy
environment:
# --- ACL: hanya yang dibutuhkan kontrol ---
CONTAINERS: "1" # list/inspect/restart/stop/start/logs/top
POST: "1" # dibutuhkan utk restart/start/stop
INFO: "1" # info daemon (version, /_ping, healthcheck)
SYSTEM: "1" # /events (feed Observability). Set "0" bila tak dipakai
# --- matikan semua yang tidak perlu ---
EXEC: "0"
SWARM: "0"
SERVICES: "0"
SECRETS: "0"
CONFIGS: "0"
NETWORKS: "0"
VOLUMES: "0"
IMAGES: "0"
BUILD: "0"
GRPC: "0"
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
expose:
- "2375" # hanya reachable dari gateway (network compose)
read_only: true
tmpfs:
- /tmp
- /run # HAProxy menulis pidfile /run/haproxy.pid
restart: unless-stopped
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
# --- gateway nginx: api-key + allowlist path (satu-satunya port ke host) --
gateway:
image: nginx:1.30.4-alpine3.24
container_name: sicabe-docker-proxy-gateway
depends_on:
- docker-proxy
environment:
# WAJIB: compose error bila PROXY_API_KEY tidak di-set.
PROXY_API_KEY: ${PROXY_API_KEY:?PROXY_API_KEY wajib diisi (secret acak kuat)}
ports:
- "${PROXY_BIND_IP:-127.0.0.1}:${PROXY_PORT:-22375}:80"
volumes:
# Template di-substitute oleh envsubst docker-entrypoint nginx saat start
# (${PROXY_API_KEY} diganti nilai env; $http_* tetap utuh) → output ke
# /etc/nginx/conf.d/default.conf (menggantikan default bawaan image).
- ./nginx/gateway.conf.template:/etc/nginx/templates/default.conf.template:ro
restart: unless-stopped
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
@@ -0,0 +1,54 @@
# nginx gateway untuk docker-socket-proxy.
#
# File ini adalah TEMPLATE: pada container start, docker-entrypoint nginx
# menjalankan envsubst dan mengganti ${PROXY_API_KEY} dengan nilai env
# PROXY_API_KEY (variabel nginx seperti $http_x_api_key TIDAK ikut diganti),
# lalu menulis hasilnya ke /etc/nginx/conf.d/default.conf.
#
# Kebijakan:
# * Semua request WAJIB membawa header `X-Api-Key` == PROXY_API_KEY (401 bila
# salah/kosong). nginx membalas 401 TANPA meneruskan ke proxy.
# * Allowlist path (desain §2): hanya route yang dibutuhkan KONTROL oleh
# control-panel. Selain itu di-block 403 — termasuk route berbahaya
# (exec, create, images, networks, volumes, dll) sekalipun ACL proxy longgar.
# * Aksi tulis restart/start/stop hanya via POST (selainnya 405).
server {
listen 80 default_server;
server_name _;
# ==== API-key check (sebelum diproses location mana pun) ====
if ($http_x_api_key != "${PROXY_API_KEY}") {
return 401;
}
# ==== Route yang diizinkan (allowlist) ====
# Health/info daemon + event stream
location = /_ping { proxy_pass http://docker-proxy:2375; }
location = /version { proxy_pass http://docker-proxy:2375; }
location = /info { proxy_pass http://docker-proxy:2375; }
location = /events { proxy_pass http://docker-proxy:2375; }
# List container (tanpa id) — GET /containers/json
location = /containers/json { proxy_pass http://docker-proxy:2375; }
# Operasi per-container yang dibutuhkan kontrol/observability
location ~ ^/containers/[^/]+/json$ { proxy_pass http://docker-proxy:2375; }
location ~ ^/containers/[^/]+/logs$ { proxy_pass http://docker-proxy:2375; }
location ~ ^/containers/[^/]+/stats$ { proxy_pass http://docker-proxy:2375; }
location ~ ^/containers/[^/]+/top$ { proxy_pass http://docker-proxy:2375; }
# Aksi tulis — hanya POST (restart/start/stop)
location ~ ^/containers/[^/]+/(restart|start|stop)$ {
if ($request_method != POST) {
return 405;
}
proxy_pass http://docker-proxy:2375;
}
# ==== Semua path lain di-block ====
location / {
return 403;
}
}
+17
View File
@@ -0,0 +1,17 @@
# sicabe-config-agent — Python stdlib + PyYAML (image ramping).
FROM python:3.12-slim
WORKDIR /app
COPY requirements.txt ./
RUN pip install --no-cache-dir -r requirements.txt
COPY agent.py ./
# Default; override via env AGENT_PORT.
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=3s --retries=3 \
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/healthz')" || exit 1
ENTRYPOINT ["python", "-u", "agent.py"]
+551
View File
@@ -0,0 +1,551 @@
#!/usr/bin/env python3
"""sicabe-config-agent — akses konfigurasi platform SiCABE per host.
Control-panel TIDAK mengakses filesystem platform langsung; agent ini yang
membacanya di host tempat sicabe-doc berada, lalu diekspos lewat HTTP ber-api-key.
Endpoint (butuh header `X-Api-Key` kecuali `/healthz`):
GET /healthz
GET /env?key=NAME[&key=NAME2] # read-only, whitelist (minimal 1 key)
GET /catalog # daftar katalog Trino (nilai sensitif di-redact)
GET /catalog/<name> # detail satu katalog (redact)
PUT /catalog/<name> # body {"properties": {...}} — tulis (upsert)
DELETE /catalog/<name> # hapus
GET /dbt # project dbt terparse (models/sources/DAG)
GET /ca # PEM sertifikat TLS host (opsional)
GET /opa/data # dokumen data OPA (config/opa/data.json) — mentah
PUT /opa/data # body {"content": "<json>"} — validasi + tulis atomic (.bak)
POST /services/scan # body {"publicHost","types":[...]} — scan service (panel-driven)
Env:
AGENT_API_KEY (wajib) api key yang harus dikirim panel
AGENT_PORT (default 8080)
SICABE_ENV_FILE (default /config/.env)
TRINO_CATALOG_DIR (default /config/config/trino/catalog)
DBT_PROJECT_DIR (default /config/config/airflow/dbt/sicabe)
AGENT_CA_FILE (opsional) path PEM untuk GET /ca
OPA_DATA_FILE (default /config/config/opa/data.json) dokumen data OPA
DOCKER_PROXY_URL (default http://docker-proxy:2375) sumber daftar container
"""
import hmac
import json
import os
import re
import socket
import urllib.request
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import urlparse, parse_qs
import yaml
AGENT_API_KEY = os.environ.get("AGENT_API_KEY", "")
AGENT_PORT = int(os.environ.get("AGENT_PORT", "8080"))
ENV_FILE = os.environ.get("SICABE_ENV_FILE", "/config/.env")
CATALOG_DIR = os.environ.get("TRINO_CATALOG_DIR", "/config/config/trino/catalog")
DBT_DIR = os.environ.get("DBT_PROJECT_DIR", "/config/config/airflow/dbt/sicabe")
CA_FILE = os.environ.get("AGENT_CA_FILE", "")
OPA_DATA_FILE = os.environ.get("OPA_DATA_FILE", "/config/config/opa/data.json")
# docker-proxy internal (satu stack dengan agent) — untuk scan service tanpa
# perlu mount docker.sock ke agent.
DOCKER_PROXY_URL = os.environ.get("DOCKER_PROXY_URL", "http://docker-proxy:2375")
VALID_CATALOG = re.compile(r"^[a-z][a-z0-9_-]{0,62}$")
SENSITIVE = re.compile(r"(password|secret|credential|access-key|apikey|api-key|token)", re.I)
REF_RE = re.compile(r"""ref\(\s*['"]([^'"]+)['"]\s*\)""")
SOURCE_RE = re.compile(r"""source\(\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\)""")
# ── helpers: .env ────────────────────────────────────────────────────────────
def read_env(keys):
"""Baca key tertentu dari ENV_FILE (read-only, whitelist)."""
wanted = set(keys)
out = {}
try:
with open(ENV_FILE, "r", encoding="utf-8") as fh:
for line in fh:
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
k, v = line.split("=", 1)
k = k.strip()
if k not in wanted:
continue
v = v.strip()
if len(v) >= 2 and v[0] == v[-1] and v[0] in "\"'":
v = v[1:-1]
out[k] = v
except OSError:
return {}
return out
# ── helpers: katalog Trino ───────────────────────────────────────────────────
def redact(props):
return {k: ("••••••••" if SENSITIVE.search(k) else v) for k, v in props.items()}
def parse_properties(content):
props = {}
for line in content.splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
k, v = line.split("=", 1)
props[k.strip()] = v.strip()
return props
def serialize_properties(props):
return "".join(f"{k}={v}\n" for k, v in props.items())
def catalog_path(name):
return os.path.join(CATALOG_DIR, f"{name}.properties")
def list_catalogs():
out = []
try:
names = sorted(os.listdir(CATALOG_DIR))
except OSError:
return out
for fname in names:
if not fname.endswith(".properties"):
continue
name = fname[: -len(".properties")]
try:
with open(os.path.join(CATALOG_DIR, fname), "r", encoding="utf-8") as fh:
props = parse_properties(fh.read())
except OSError:
continue
out.append({"name": name, "connector": props.get("connector.name", "unknown"), "properties": redact(props)})
return sorted(out, key=lambda c: c["name"])
def get_catalog(name):
for c in list_catalogs():
if c["name"] == name:
return c
return None
def put_catalog(name, props):
os.makedirs(CATALOG_DIR, exist_ok=True)
with open(catalog_path(name), "w", encoding="utf-8") as fh:
fh.write(serialize_properties(props))
def write_opa_data(content):
"""Tulis dokumen data OPA secara atomic + simpan isi sebelumnya ke .bak.
Atomic (temp + os.replace) agar crash di tengah tulis tidak meninggalkan file
separuh jadi — OPA GAGAL START bila JSON-nya tidak valid, dan itu mematikan
authz Trino. Backup `.bak` dipakai untuk rollback (Tahap 4).
"""
os.makedirs(os.path.dirname(OPA_DATA_FILE) or ".", exist_ok=True)
if os.path.exists(OPA_DATA_FILE):
with open(OPA_DATA_FILE, "rb") as fh:
previous = fh.read()
with open(OPA_DATA_FILE + ".bak", "wb") as fh:
fh.write(previous)
tmp = OPA_DATA_FILE + ".tmp"
with open(tmp, "w", encoding="utf-8") as fh:
fh.write(content)
fh.flush()
os.fsync(fh.fileno())
os.replace(tmp, OPA_DATA_FILE)
def delete_catalog(name):
path = catalog_path(name)
if not os.path.exists(path):
raise FileNotFoundError(name)
os.unlink(path)
# ── helpers: project dbt (bentuk = lib/dbt.ts) ───────────────────────────────
def layer_from_path(rel):
if "bronze_to_silver" in rel:
return "silver"
if "silver_to_gold" in rel:
return "gold"
if "incremental" in rel:
return "bronze"
return "silver"
def load_dbt():
name, version, profile = "sicabe", "1.0.0", "trino"
try:
with open(os.path.join(DBT_DIR, "dbt_project.yml"), "r", encoding="utf-8") as fh:
yml = fh.read()
m = re.search(r"^name:\s*['\"]?([^'\"\n]+)", yml, re.M)
if m:
name = m.group(1).strip()
m = re.search(r"^version:\s*['\"]?([^'\"\n]+)", yml, re.M)
if m:
version = m.group(1).strip()
m = re.search(r"^profile:\s*['\"]?([^'\"\n]+)", yml, re.M)
if m:
profile = m.group(1).strip()
except OSError:
pass
models = []
models_dir = os.path.join(DBT_DIR, "models")
for root, _dirs, files in os.walk(models_dir):
for fname in files:
if not fname.endswith(".sql"):
continue
full = os.path.join(root, fname)
rel = os.path.relpath(full, models_dir)
try:
with open(full, "r", encoding="utf-8") as fh:
sql = fh.read()
except OSError:
continue
layer = layer_from_path(rel)
models.append({
"name": fname[: -len(".sql")],
"path": rel,
"layer": layer,
"materialization": "incremental" if "incremental" in rel else "table",
"schema": layer,
"dependsOn": REF_RE.findall(sql),
"sources": [f"{a}.{b}" for a, b in SOURCE_RE.findall(sql)],
"sql": sql,
})
sources = []
sources_file = os.path.join(models_dir, "sources.yml")
if os.path.exists(sources_file):
try:
with open(sources_file, "r", encoding="utf-8") as fh:
doc = yaml.safe_load(fh.read()) or {}
for s in doc.get("sources", []) or []:
nm = s.get("name") or ""
if not nm:
continue
sources.append({
"name": nm,
"database": s.get("database") or "",
"schema": s.get("schema") or "",
"tables": [{"name": t.get("name") or "", "description": t.get("description")}
for t in (s.get("tables") or [])],
})
except (OSError, yaml.YAMLError):
sources = []
nodes, edges = [], []
for src in sources:
for tbl in src["tables"]:
nodes.append({"id": f"source.{src['name']}.{tbl['name']}", "label": f"{src['name']}.{tbl['name']}",
"type": "source", "layer": "bronze"})
for m in models:
nodes.append({"id": f"model.{m['name']}", "label": m["name"], "type": "model", "layer": m["layer"]})
for dep in m["dependsOn"]:
edges.append({"id": f"model.{dep}->model.{m['name']}", "source": f"model.{dep}", "target": f"model.{m['name']}"})
for s in m["sources"]:
db, _, tbl = s.partition(".")
edges.append({"id": f"source.{db}.{tbl}->model.{m['name']}", "source": f"source.{db}.{tbl}", "target": f"model.{m['name']}"})
return {"name": name, "version": version, "profile": profile, "models": models, "sources": sources,
"dag": {"nodes": nodes, "edges": edges}}
# ── Scan service (panel-driven) ──────────────────────────────────────────────
# Panel mengirim katalog TIPE service (regex + port); agent memindai container
# yang SEDANG BERJALAN di host ini (via docker-proxy internal) dan mengembalikan
# instance + URL-nya. Logika pengenalan tetap satu tempat di panel (TS) — agent
# hanya mesin generik. Hanya container `running` yang dipindai, sehingga job
# sekali-jalan (init/migrate) otomatis tidak ikut.
# Label peta port → skema URL asli (label boleh deskriptif, mis. "console").
SCHEME_ALIASES = {
"https": "https",
"http": "http",
"api": "http",
"console": "http",
"ui": "http",
"web": "http",
"ldap": "ldap",
"mysql": "mysql",
"sql": "https",
"tcp": "tcp",
}
def _re_match(pattern, value):
if not pattern:
return False
try:
return re.search(pattern, value or "") is not None
except re.error:
return False
def docker_running_containers():
"""Daftar container running dari docker-proxy internal."""
req = urllib.request.Request(f"{DOCKER_PROXY_URL}/containers/json?all=false")
with urllib.request.urlopen(req, timeout=8) as resp:
data = json.loads(resp.read().decode("utf-8"))
out = []
for c in data:
if (c.get("State") or "") != "running":
continue
names = c.get("Names") or []
name = names[0].lstrip("/") if names else (c.get("Id") or "")[:12]
out.append({
"name": name,
"image": c.get("Image", ""),
"labels": c.get("Labels") or {},
"ports": c.get("Ports") or [],
})
return out
def _published_ports(ports_raw):
out = []
seen = set()
for p in ports_raw:
if not p.get("PublicPort"):
continue
key = (p.get("PrivatePort"), p.get("PublicPort"), p.get("Type", "tcp"))
if key in seen:
continue # docker mengembalikan entri IPv4 & IPv6 → dedupe
seen.add(key)
out.append({
"privatePort": p.get("PrivatePort"),
"publicPort": p.get("PublicPort"),
"type": p.get("Type", "tcp"),
})
return out
def _urls_for(ports_map, published_raw, public_host):
"""Susun URL per label dari peta port container → port publish host.
Kunci peta = LABEL (mis. "http", "console", "mysql"); skema URL sebenarnya
dipetakan lewat SCHEME_ALIASES (label "console" → skema "http").
"""
urls = {}
if not public_host:
return urls
for label, privates in (ports_map or {}).items():
if not isinstance(privates, list):
continue
scheme = SCHEME_ALIASES.get(str(label).lower(), "http")
for priv in privates:
for p in published_raw:
if int(p.get("PrivatePort") or 0) == int(priv) and p.get("PublicPort"):
urls[label] = f"{scheme}://{public_host}:{p.get('PublicPort')}"
break
if label in urls:
break
return urls
def scan_services(types, public_host):
containers = docker_running_containers()
services = []
for t in types:
if not isinstance(t, dict):
continue
img_pat = t.get("imagePattern")
name_pat = t.get("namePattern")
ports_map = t.get("ports") or {}
for c in containers:
if not (_re_match(img_pat, c["image"]) or _re_match(name_pat, c["name"])):
continue
services.append({
"type": t.get("id"),
"container": c["name"],
"image": c["image"],
"labels": c["labels"],
"published": _published_ports(c["ports"]),
"urls": _urls_for(ports_map, c["ports"], public_host),
})
return {
"hostname": socket.gethostname(),
"containers": len(containers),
"config": {
"envFile": os.path.exists(ENV_FILE),
"catalogDir": os.path.isdir(CATALOG_DIR),
"dbtDir": os.path.isdir(DBT_DIR),
"caFile": bool(CA_FILE) and os.path.exists(CA_FILE),
},
"services": services,
}
# ── HTTP handler ─────────────────────────────────────────────────────────────
class Handler(BaseHTTPRequestHandler):
server_version = "sicabe-config-agent/0.1"
# -- util --
def _send(self, code, body, ctype="application/json"):
payload = body if isinstance(body, bytes) else json.dumps(body).encode("utf-8")
self.send_response(code)
self.send_header("Content-Type", ctype)
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
try:
self.wfile.write(payload)
except (BrokenPipeError, ConnectionResetError):
pass
def _auth_ok(self):
key = self.headers.get("X-Api-Key", "")
return bool(AGENT_API_KEY) and hmac.compare_digest(key, AGENT_API_KEY)
def _deny(self):
self._send(401, {"error": "Unauthorized"})
def _json_body(self):
length = int(self.headers.get("Content-Length") or 0)
raw = self.rfile.read(length) if length else b"{}"
return json.loads(raw.decode("utf-8"))
def log_message(self, fmt, *args): # ringkas ke stderr
print("[agent] " + (fmt % args), flush=True)
# -- GET --
def do_GET(self):
parsed = urlparse(self.path)
path = parsed.path.rstrip("/") or "/"
if path == "/healthz":
return self._send(200, {"status": "ok"})
if not self._auth_ok():
return self._deny()
try:
if path == "/env":
keys = parse_qs(parsed.query).get("key", [])
keys = [k for k in keys if k]
if not keys:
return self._send(400, {"error": "key parameter required (whitelist)"})
return self._send(200, read_env(keys))
if path == "/catalog":
return self._send(200, {"catalogs": list_catalogs()})
if path.startswith("/catalog/"):
name = path[len("/catalog/"):]
cat = get_catalog(name)
if not cat:
return self._send(404, {"error": f"catalog '{name}' not found"})
return self._send(200, cat)
if path == "/dbt":
return self._send(200, load_dbt())
if path == "/ca":
if not CA_FILE or not os.path.exists(CA_FILE):
return self._send(404, {"error": "no CA configured"})
# Kirim bytes mentah (jangan di-JSON-encode) agar PEM utuh.
with open(CA_FILE, "rb") as fh:
return self._send(200, fh.read(), "application/x-pem-file")
if path == "/opa/data":
# Kirim isi file APA ADANYA. Harus bytes: _send() men-JSON-encode
# nilai non-bytes (kalau str, isi file jadi string ber-quote ganda
# dan editor panel menampilkan JSON yang salah).
if not os.path.exists(OPA_DATA_FILE):
return self._send(404, {"error": "opa data file not found"})
with open(OPA_DATA_FILE, "rb") as fh:
return self._send(200, fh.read(), "application/json")
return self._send(404, {"error": "not found"})
except Exception as e: # noqa: BLE001
return self._send(500, {"error": "internal error", "details": str(e)})
# -- PUT --
def do_PUT(self):
if not self._auth_ok():
return self._deny()
parsed = urlparse(self.path)
path = parsed.path.rstrip("/") or "/"
if path == "/opa/data":
try:
body = self._json_body()
content = body.get("content") if isinstance(body, dict) else None
if not isinstance(content, str):
return self._send(400, {"error": "content (JSON string) required"})
# Validasi SEBELUM menulis: JSON invalid membuat OPA gagal start
# → authz Trino mati total.
try:
json.loads(content)
except json.JSONDecodeError as e:
return self._send(400, {"error": "invalid JSON", "details": e.msg,
"line": e.lineno, "col": e.colno})
write_opa_data(content)
return self._send(200, {"success": True, "bytes": len(content.encode("utf-8"))})
except Exception as e: # noqa: BLE001
return self._send(500, {"error": "internal error", "details": str(e)})
if not path.startswith("/catalog/"):
return self._send(404, {"error": "not found"})
name = path[len("/catalog/"):]
if not VALID_CATALOG.match(name):
return self._send(400, {"error": "invalid catalog name"})
try:
body = self._json_body()
props = body.get("properties", body) if isinstance(body, dict) else {}
if not isinstance(props, dict):
return self._send(400, {"error": "properties must be an object"})
put_catalog(name, {str(k): str(v) for k, v in props.items()})
return self._send(200, {"success": True, "name": name})
except Exception as e: # noqa: BLE001
return self._send(500, {"error": "internal error", "details": str(e)})
# -- DELETE --
def do_DELETE(self):
if not self._auth_ok():
return self._deny()
parsed = urlparse(self.path)
path = parsed.path.rstrip("/") or "/"
if not path.startswith("/catalog/"):
return self._send(404, {"error": "not found"})
name = path[len("/catalog/"):]
if not VALID_CATALOG.match(name):
return self._send(400, {"error": "invalid catalog name"})
try:
delete_catalog(name)
return self._send(200, {"success": True, "name": name})
except FileNotFoundError:
return self._send(404, {"error": f"catalog '{name}' not found"})
except Exception as e: # noqa: BLE001
return self._send(500, {"error": "internal error", "details": str(e)})
# -- POST --
def do_POST(self):
if not self._auth_ok():
return self._deny()
parsed = urlparse(self.path)
path = parsed.path.rstrip("/") or "/"
if path == "/services/scan":
try:
body = self._json_body()
types = body.get("types") if isinstance(body, dict) else None
if not isinstance(types, list) or not types:
return self._send(400, {"error": "types (non-empty array) required"})
public_host = body.get("publicHost") or ""
return self._send(200, scan_services(types, public_host))
except Exception as e: # noqa: BLE001
return self._send(500, {"error": "scan failed", "details": str(e)})
return self._send(404, {"error": "not found"})
def main():
if not AGENT_API_KEY:
print("FATAL: AGENT_API_KEY must be set", flush=True)
raise SystemExit(1)
print(f"[agent] listening on :{AGENT_PORT} (env={ENV_FILE}, catalog={CATALOG_DIR}, dbt={DBT_DIR})", flush=True)
ThreadingHTTPServer(("0.0.0.0", AGENT_PORT), Handler).serve_forever()
if __name__ == "__main__":
main()
@@ -0,0 +1 @@
PyYAML==6.0.2
+128
View File
@@ -0,0 +1,128 @@
# =============================================================================
# sicabe-host-gateway — SATU port untuk docker-proxy + config-agent.
#
# CATATAN: ini varian LOKAL (build dari ./agent). Untuk DEPLOY dari registry
# (termasuk panel, dengan profile) pakai: deploy/stack/docker-compose.yml
#
# Hanya nginx (gateway) yang membuka port host; docker-socket-proxy dan
# config-agent bersifat internal (expose, tanpa port host). Satu API key
# (GATEWAY_API_KEY) diperiksa nginx, lalu request diarahkan berdasarkan path:
# /_ping /version /info /events /containers/* → docker-proxy
# /healthz /env /dbt /ca /catalog(/<name>) → config-agent
#
# Lihat: docs/desain-host-gateway-terpadu.md
#
# Cara pakai:
# GATEWAY_API_KEY='<secret-kuat>' \
# SICABE_DEPLOY_DIR=/path/ke/sicabe-doc/deploy \
# AGENT_CA_HOST_PATH=/path/berisi/pem AGENT_CA_FILE=/ca/trino.crt \
# docker compose up -d
#
# Opsional: AGENT_UID/AGENT_GID (default 1000:1000) — UID/GID yang dipakai
# config-agent saat menulis ke host, supaya file (katalog Trino, data OPA) tetap
# dimiliki user host dan bukan root.
#
# Panel (satu URL untuk proxy & config):
# SICABE_HOSTS=[{"name":"h1","proxyUrl":"http://<host>:22375",
# "apiKey":"<GATEWAY_API_KEY>",
# "configUrl":"http://<host>:22375",
# "configApiKey":"<GATEWAY_API_KEY>","enabled":true}]
# =============================================================================
name: sicabe-host-gateway
services:
# --- Docker Engine API (ACL ketat, internal saja) ------------------------
docker-proxy:
image: tecnativa/docker-socket-proxy:v0.5.0
container_name: sicabe-docker-proxy
environment:
CONTAINERS: "1"
POST: "1"
INFO: "1"
SYSTEM: "1"
EXEC: "0"
SWARM: "0"
SERVICES: "0"
SECRETS: "0"
CONFIGS: "0"
NETWORKS: "0"
VOLUMES: "0"
IMAGES: "0"
BUILD: "0"
GRPC: "0"
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
expose:
- "2375"
read_only: true
tmpfs:
- /tmp
- /run
restart: unless-stopped
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
# --- Konfigurasi platform per host (internal saja) -----------------------
config-agent:
build: ./agent
image: sicabe/config-agent:0.1.0
container_name: sicabe-config-agent
# Jalankan sebagai UID/GID host (bukan root) supaya file yang DITULIS agent
# (katalog Trino, config/opa/data.json) tetap dimiliki user host — kalau root,
# editor/git di host akan kena permission denied. Dir yang di-mount rw memang
# milik user host. Bisa di-override via AGENT_UID/AGENT_GID.
user: "${AGENT_UID:-1000}:${AGENT_GID:-1000}"
environment:
# Satu key: nginx memeriksa, agent tetap memvalidasi (defense-in-depth).
AGENT_API_KEY: ${GATEWAY_API_KEY:?GATEWAY_API_KEY wajib diisi}
AGENT_PORT: "8080"
PYTHONDONTWRITEBYTECODE: "1"
SICABE_ENV_FILE: /config/.env
TRINO_CATALOG_DIR: /config/config/trino/catalog
DBT_PROJECT_DIR: /config/config/airflow/dbt/sicabe
AGENT_CA_FILE: ${AGENT_CA_FILE:-}
# Dokumen data OPA (ACL Trino) — dibaca menu Authorization. Tulis (rw)
# menyusul di Tahap 2.
OPA_DATA_FILE: /config/config/opa/data.json
volumes:
- ${SICABE_DEPLOY_DIR:?set SICABE_DEPLOY_DIR}:/config:ro
- ${SICABE_DEPLOY_DIR:?set SICABE_DEPLOY_DIR}/config/trino/catalog:/config/config/trino/catalog:rw
# Dir OPA (data.json) — rw agar agent bisa menulis di Tahap 2.
- ${SICABE_DEPLOY_DIR:?set SICABE_DEPLOY_DIR}/config/opa:/config/config/opa:rw
# CA TLS opsional (GET /ca): direktori berisi PEM → /ca.
- ${AGENT_CA_HOST_PATH:-${SICABE_DEPLOY_DIR:?set SICABE_DEPLOY_DIR}}:/ca:ro
expose:
- "8080"
read_only: true
tmpfs:
- /tmp
restart: unless-stopped
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
# --- Gateway nginx: satu-satunya port ke host ----------------------------
gateway:
image: nginx:1.30.4-alpine3.24
container_name: sicabe-host-gateway
depends_on:
- docker-proxy
- config-agent
environment:
GATEWAY_API_KEY: ${GATEWAY_API_KEY:?GATEWAY_API_KEY wajib diisi}
volumes:
- ./nginx/gateway.conf.template:/etc/nginx/templates/default.conf.template:ro
ports:
- "${GATEWAY_BIND_IP:-127.0.0.1}:${GATEWAY_PORT:-22375}:80"
restart: unless-stopped
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
@@ -0,0 +1,66 @@
# nginx gateway terpadu — docker-socket-proxy + config-agent dalam SATU port.
#
# Template: docker-entrypoint nginx men-substitute ${GATEWAY_API_KEY} saat start
# (variabel nginx seperti $http_x_api_key TIDAK ikut diganti), lalu menulis hasil
# ke /etc/nginx/conf.d/default.conf.
#
# Kebijakan:
# * Semua request WAJIB membawa header `X-Api-Key` == GATEWAY_API_KEY (401 bila
# salah/kosong), SEBELUM diproses location mana pun.
# * Allowlist path ketat:
# - /_ping /version /info /events /containers/* → docker-proxy:2375
# - /healthz /env /dbt /ca /catalog(/<name>) /opa/data /services/scan → config-agent:8080
# Selain daftar → 403.
# * Aksi container restart/start/stop hanya POST (selainnya 405).
# * Katalog (cart agent) mengizinkan GET/PUT/DELETE.
#
# Lihat: docs/desain-host-gateway-terpadu.md
server {
listen 80 default_server;
server_name _;
# ==== API-key check (sebelum location mana pun) ====
if ($http_x_api_key != "${GATEWAY_API_KEY}") {
return 401;
}
# ==== config-agent (konfigurasi platform) ====
location = /healthz { proxy_pass http://config-agent:8080; }
location = /env { proxy_pass http://config-agent:8080; }
location = /dbt { proxy_pass http://config-agent:8080; }
location = /ca { proxy_pass http://config-agent:8080; }
location = /catalog { proxy_pass http://config-agent:8080; }
# GET/PUT/DELETE satu katalog (tanpa slash tambahan di nama).
location ~ ^/catalog/[^/]+$ { proxy_pass http://config-agent:8080; }
# Dokumen data OPA (menu Authorization) — baca sekarang, tulis menyusul.
location = /opa/data { proxy_pass http://config-agent:8080; }
# Scan service (panel-driven) — memindai container host lewat docker-proxy.
location = /services/scan { proxy_pass http://config-agent:8080; }
# ==== docker-socket-proxy (Docker Engine API) ====
location = /_ping { proxy_pass http://docker-proxy:2375; }
location = /version { proxy_pass http://docker-proxy:2375; }
location = /info { proxy_pass http://docker-proxy:2375; }
location = /events { proxy_pass http://docker-proxy:2375; }
location = /containers/json { proxy_pass http://docker-proxy:2375; }
location ~ ^/containers/[^/]+/json$ { proxy_pass http://docker-proxy:2375; }
location ~ ^/containers/[^/]+/logs$ { proxy_pass http://docker-proxy:2375; }
location ~ ^/containers/[^/]+/stats$ { proxy_pass http://docker-proxy:2375; }
location ~ ^/containers/[^/]+/top$ { proxy_pass http://docker-proxy:2375; }
# Aksi tulis container — hanya POST.
location ~ ^/containers/[^/]+/(restart|start|stop)$ {
if ($request_method != POST) {
return 405;
}
proxy_pass http://docker-proxy:2375;
}
# ==== selain allowlist → tolak ====
location / {
return 403;
}
}
+104
View File
@@ -0,0 +1,104 @@
# Deploy (build/lokal): SiCABE Control Panel
> Untuk deploy PRODUKSI dari image registry (menyatu dengan gateway: docker-proxy +
> config-agent + nginx, dengan profile), pakai **`deploy/stack/`**. Compose di sini
> untuk **build lokal** dan menjalankan panel sendirian.
Panel (Next.js) dijalankan sebagai container. Image produksi memakai
**Next.js standalone** + user non-root (`nextjs`, uid 1001).
> Gateway **per host** (`deploy/host-gateway/`) adalah stack terpisah — sudah punya
> compose sendiri. Panel hanya perlu menunjuk ke gateway itu lewat `SICABE_HOSTS`.
## 1. Build image
```bash
./scripts/build-image.sh # tag default: sicabe/control-panel:<git-sha> + :latest
./scripts/build-image.sh --tag sicabe/control-panel:1.0.0
./scripts/build-image.sh --push --registry registry.example.com/sicabe
./scripts/build-image.sh --no-cache --platform linux/amd64
```
Script memverifikasi Dockerfile/`public/`, menandai `:latest` + `<sha>`, dan
mencetak ukuran image.
## 2. Siapkan env
```bash
cd deploy/panel
cp env.example .env # lalu isi NEXTAUTH_SECRET, api key gateway, dll.
# NEXTAUTH_SECRET: openssl rand -base64 32
```
**Penting:** panel di container **tidak bisa membaca filesystem platform** → tiap
host di `SICABE_HOSTS` **wajib** punya `configUrl` (config-agent). Karena itu
**tidak perlu** mount folder platform: katalog Trino, project dbt, dokumen OPA,
`.env` platform, dan CA Trino semuanya dibaca lewat agent (HTTP). Env
`TRINO_CATALOG_DIR` / `DBT_PROJECT_DIR` / `OPA_DATA_FILE` / `SICABE_ENV_FILE` /
`TRINO_CA_FILE` hanya dipakai `LocalConfigStore` (dev/single-host tanpa agent) —
jangan diandalkan di container.
## 3. Jalankan
```bash
cd deploy/panel
docker compose up -d # pakai image hasil build
# docker compose up -d --build # sekaligus build dari source
docker compose logs -f panel
```
Buka `http://localhost:3000` (atau `PANEL_PORT`).
## Yang persisten
| Data | Lokasi |
| --- | --- |
| Config panel (hosts/modules/services/service types) | volume `panel-data` → `/app/data/config.json` |
| Monitored state | volume `panel-data` → `/app/data/state.json` |
Backup/pindah config lewat **Settings → Backup** (Export/Import) — tanpa
menyentuh volume. Untuk migrasi dari deployment host lama: Export di yang lama →
Import di yang baru.
## Catatan jaringan & TLS
- **Default compose = `network_mode: host`.** Gateway host
(`deploy/host-gateway`) hanya membuka `127.0.0.1:22375` (aman), sehingga panel
di **bridge network tidak bisa** menjangkaunya lewat `host.docker.internal`.
Terverifikasi: dari bridge, Keycloak (`0.0.0.0:12080`) **bisa**, gateway
(`127.0.0.1:22375`) **gagal**; dari host network keduanya **bisa**. Host network
juga membuat `127.0.0.1` & SAN sertifikat Trino (`localhost`) cocok.
Port ditentukan `PORT` (default 3000), bukan `ports:`.
- **Varian bridge** (bila gateway dibuka ke alamat non-loopback): lihat komentar
di `docker-compose.yml` — hapus `network_mode: host`, tambah `ports:` +
`extra_hosts: host.docker.internal`.
- **TLS Trino**: sertifikat self-signed Trino hanya mencakup SAN
`localhost / trino / sicabe-trino / trino-coordinator / 127.0.0.1`.
`host.docker.internal` **tidak** termasuk → verifikasi TLS gagal di bridge.
- CA Trino **tidak perlu** file lokal bila host punya `configUrl` (agent `/ca`).
## Login di produksi = SSO (Keycloak)
Container berjalan `NODE_ENV=production`, dan **login kredensial dev dinonaktifkan
di produksi** (hanya provider `keycloak` yang aktif — cek
`GET /api/auth/providers`). Artinya:
- Login lewat tombol **Sign in with Keycloak** di browser (bukan form dev).
- **`NEXTAUTH_URL` harus sama dengan URL publik panel**, dan panel Keycloak
(`scripts/create-keycloak-client.sh`, `APP_URL`) harus punya redirect URI
`${NEXTAUTH_URL}/api/auth/callback/keycloak`. Kalau panel disajikan di port
selain 3000, jalankan ulang script dengan `APP_URL=http://<host>:<port>`.
- Image **gagal-cepat** bila `NEXTAUTH_SECRET` atau `KEYCLOAK_CLIENT_SECRET`
kosong (memang disengaja) — panel menolak start dengan pesan jelas.
## Update
```bash
./scripts/build-image.sh --tag sicabe/control-panel:1.1.0
PANEL_TAG=1.1.0 docker compose up -d # dari deploy/panel
```
## Healthcheck
Image menyertakan `HEALTHCHECK` (`GET /api/auth/session` → 200 walau belum login),
jadi `docker compose ps` menampilkan status `healthy`.
+61
View File
@@ -0,0 +1,61 @@
# =============================================================================
# sicabe-control-panel — deploy panel (Next.js) via Docker Compose.
#
# Build image dulu: ../../scripts/build-image.sh
# Lalu jalankan : docker compose up -d
#
# Persisten: volume bernama `panel-data` → `data/config.json` (hosts/modules/
# services/service types) + `data/state.json`. Backup/restore dari UI
# (Settings → Backup) tanpa menyentuh volume.
#
# PENTING: panel di container TIDAK bisa membaca filesystem platform → setiap
# host di SICABE_HOSTS WAJIB punya `configUrl` (config-agent).
#
# NETWORK: default `network_mode: host` karena gateway host
# (deploy/host-gateway) membuka `127.0.0.1:22375` — container di bridge TIDAK
# bisa menjangkaunya via host.docker.internal. Dengan host network, `127.0.0.1`
# berfungsi seperti di host (dan SAN sertifikat Trino `localhost` cocok).
# Lihat README.md untuk varian bridge.
# =============================================================================
name: sicabe-control-panel
services:
panel:
image: sicabe/control-panel:${PANEL_TAG:-latest}
# Biarkan `build` aktif bila ingin `docker compose up --build`.
build:
context: ../..
dockerfile: Dockerfile
container_name: sicabe-control-panel
network_mode: host
# `.env` dibuat dari env.example (cp env.example .env). `required: false`
# agar `docker compose config` tetap valid sebelum file dibuat.
env_file:
- path: .env
required: false
environment:
NODE_ENV: production
HOSTNAME: "0.0.0.0"
# Host network: port binding ditentukan PORT (bukan `ports:`).
PORT: "${PANEL_PORT:-3000}"
volumes:
- panel-data:/app/data
restart: unless-stopped
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
# ── Varian BRIDGE (bila gateway dibuka ke alamat non-loopback) ────────────
# Hapus `network_mode: host` di atas, lalu tambahkan:
# ports:
# - "${PANEL_PORT:-3000}:3000"
# extra_hosts:
# - "host.docker.internal:host-gateway"
# dan set SICABE_HOSTS/URL memakai host.docker.internal (perhatikan SAN TLS
# Trino — host.docker.internal TIDAK ada di SAN).
volumes:
panel-data:
+70
View File
@@ -0,0 +1,70 @@
# =============================================================================
# Env untuk panel yang berjalan sebagai CONTAINER.
# Salin ke .env LALU ISI NILAINYA (jangan commit .env):
# cp deploy/panel/env.example deploy/panel/.env
#
# Beda penting dari .env dev (panel di host):
# - Panel di container TIDAK bisa membaca filesystem platform → WAJIB pakai
# config-agent (`configUrl`) di SICABE_HOSTS.
# - Alamat ke service di HOST memakai `host.docker.internal` (atau
# `network_mode: host`, lihat docker-compose.yml).
# =============================================================================
# ── Wajib ────────────────────────────────────────────────────────────────────
# URL publik panel — HARUS sama dengan yang diakses browser.
NEXTAUTH_URL=http://localhost:3000
# Rahasia NextAuth (WAJIB di produksi). Buat: openssl rand -base64 32
NEXTAUTH_SECRET=
# ── Login lokal (hybrid dengan SSO) — OPSIONAL ───────────────────────────────
#PANEL_LOCAL_USER=admin
#PANEL_LOCAL_PASSWORD_HASH=$2b$10$...
#PANEL_LOCAL_PASSWORD_HASH_B64=
#PANEL_LOCAL_ROLE=data-admin
#TRINO_SERVICE_ACCOUNT=svc_dbt
# ── Keycloak (SSO & login dev ROPC) ──────────────────────────────────────────
# Dari container, Keycloak di host diakses via host.docker.internal.
KEYCLOAK_URL=http://host.docker.internal:12080
KEYCLOAK_CLIENT_ID=sicabe-control-panel
KEYCLOAK_CLIENT_SECRET=dev-keycloak-secret
# URL yang dilihat browser (tautan Keycloak).
KEYCLOAK_WEB_URL=http://localhost:12080
#KEYCLOAK_REALM=sicabe
# ── Host/agent yang dikelola (WAJIB configUrl) ───────────────────────────────
# Gateway per host (nginx: docker-proxy + config-agent) jalan di HOST:22375.
# Dari container → host.docker.internal:22375. `publicHost` = alamat yang
# dilihat BROWSER (untuk tautan UI), mis. IP LAN host.
SICABE_HOSTS='[{"name":"local-gw","proxyUrl":"http://host.docker.internal:22375","apiKey":"<gateway-key>","configUrl":"http://host.docker.internal:22375","configApiKey":"<gateway-key>","publicHost":"localhost","enabled":true}]'
# ── Service (fallback; registry hasil scan diutamakan) ───────────────────────
# CATATAN TLS Trino: sertifikat Trino (self-signed) punya SAN
# `localhost / trino / sicabe-trino / trino-coordinator / 127.0.0.1`. Dari
# container, `host.docker.internal` TIDAK ada di SAN → verifikasi TLS gagal.
# Pilihan:
# (a) pakai `network_mode: host` (lihat docker-compose.yml) → 127.0.0.1 cocok;
# (b) tambah alias di extra_hosts yang ADA di SAN (mis. `sicabe-trino`), lalu
# set TRINO_URL=https://sicabe-trino:12443
TRINO_URL=https://host.docker.internal:12443
TRINO_WEB_URL=http://localhost:12443
# CA diambil otomatis via config-agent (`/ca`) → tak perlu file lokal.
NESSIE_URL=http://host.docker.internal:12120
AIRFLOW_URL=http://host.docker.internal:12082
STARROCKS_HOST=host.docker.internal
STARROCKS_PORT=12930
STARROCKS_WEB_URL=http://localhost:12030
OPENMETADATA_URL=http://host.docker.internal:12085
GRAFANA_URL=http://host.docker.internal:12300
MINIO_CONSOLE_URL=http://localhost:12901
# Dremio (bila ada)
#DREMIO_URL=http://host.docker.internal:12947
# ── Runtime state (di dalam volume panel-data) ───────────────────────────────
SICABE_CONFIG_FILE=data/config.json
SICABE_STATE_FILE=data/state.json
SICABE_COMPOSE_PROJECTS=sicabe
+84
View File
@@ -0,0 +1,84 @@
# Deploy: stack `sicabe-host` (tanpa build)
Satu compose untuk satu host SiCABE, memuat empat service:
| Service | Image | Profile | Di host mana |
| --- | --- | --- | --- |
| `docker-socket-proxy` | `tecnativa/docker-socket-proxy` | – | **semua** host |
| `config-agent` | `<registry>/config-agent` | – | **semua** host |
| `gateway` (nginx) | `nginx:alpine` | – | **semua** host |
| `sicabe-control-panel` | `<registry>/control-panel` | `panel` | **satu** host (panel) |
Compose di sini **tidak membangun image** — semua diambil dari registry. Untuk
build & push image ke registry privat, pakai `deploy/panel/` + `scripts/build-image.sh`.
## 1. Build & push image (sekali, dari mesin dev)
```bash
./scripts/build-image.sh --all --push --registry registry.example.com/sicabe
# → registry.example.com/sicabe/control-panel:<git-sha> + :latest
# → registry.example.com/sicabe/config-agent:<git-sha> + :latest
```
`--all` membangun **panel + config-agent** dengan tag yang sama (agar satu
`SICABE_TAG` cukup di stack). Login registry dulu bila perlu: `docker login <registry>`.
## 2. Siapkan env (per host)
```bash
cd deploy/stack
cp env.example .env # isi GATEWAY_API_KEY, SICABE_REGISTRY, SICABE_DEPLOY_DIR
```
Di host **panel**, isi juga bagian bawah (`PANEL_URL`, `NEXTAUTH_SECRET`,
`KEYCLOAK_CLIENT_SECRET`, `SICABE_HOSTS`, dst).
## 3. Jalankan
```bash
# SEMUA host SiCABE (docker-proxy + config-agent + gateway):
docker compose up -d
# HANYA di host panel — sekaligus menjalankan panelnya:
docker compose --profile panel up -d
```
Cek: `docker compose ps` dan `docker compose logs -f gateway`.
## Update image
```bash
# setelah build+push tag baru:
SICABE_TAG=<tag-baru> docker compose pull # (di host panel: + --profile panel)
docker compose up -d # recreate
```
## Menghentikan / menghapus
`docker compose down` hanya menghapus service **pada profile yang aktif**. Karena
itu, di host panel jalankan dengan profile yang sama:
```bash
# host panel:
docker compose --profile panel down # hapus SEMUA (termasuk panel)
# host non-panel:
docker compose down
```
> Kalau `down` dijalankan tanpa `--profile panel` di host panel, container
> `sicabe-control-panel` akan tertinggal sebagai *orphan*. Compose akan
> memperingatkan; hapus dengan `--profile panel down` atau `docker rm -f`.
## Catatan
- **Panel memakai `network_mode: host`** karena gateway bind `127.0.0.1` (tak
terjangkau dari bridge Docker; sekaligus SAN sertifikat Trino `localhost`
cocok). Port panel = `PANEL_PORT` (default 3000).
- **Template nginx dipakai bersama** dari `../host-gateway/nginx/` (tanpa
duplikasi). Setelah mengubah template, jalankan `docker compose restart gateway`
(nginx men-substitute template saat start).
- **Menambah host baru**: jalankan `docker compose up -d` (tanpa profile) di host
itu, lalu daftarkan di panel (**Settings → Hosts & Agents**) dan **Scan**.
- Compose ini **menggantikan** `deploy/host-gateway/` + `deploy/panel/` sebagai
artefak deploy. Keduanya tetap ada untuk **build lokal**/dev.
+178
View File
@@ -0,0 +1,178 @@
# =============================================================================
# sicabe-host — SATU compose untuk satu host SiCABE.
#
# Berisi: docker-socket-proxy + config-agent + gateway(nginx) + control-panel.
#
# PROFILE:
# - tanpa profile → docker-socket-proxy + config-agent + gateway
# (WAJIB di SEMUA host SiCABE)
# - profile panel → + sicabe-control-panel
# (hanya di SATU host: host panel)
#
# Pemakaian:
# # di SEMUA host SiCABE:
# docker compose up -d
#
# # di HOST PANEL (selain di atas):
# docker compose --profile panel up -d
#
# Env: salin `env.example` → `.env` (compose membacanya otomatis).
# Image TIDAK di-build di sini — pakai image dari registry:
# SICABE_REGISTRY=<registry> SICABE_TAG=<tag> (lihat scripts/build-image.sh --all --push)
#
# Lihat README.md.
# =============================================================================
name: sicabe-host
services:
# ── Docker Engine API (ACL ketat, internal saja) ──────────────────────────
docker-proxy:
image: tecnativa/docker-socket-proxy:v0.5.0
container_name: sicabe-docker-proxy
environment:
CONTAINERS: "1"
POST: "1"
INFO: "1"
SYSTEM: "1"
EXEC: "0"
SWARM: "0"
SERVICES: "0"
SECRETS: "0"
CONFIGS: "0"
NETWORKS: "0"
VOLUMES: "0"
IMAGES: "0"
BUILD: "0"
GRPC: "0"
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
expose:
- "2375"
read_only: true
tmpfs:
- /tmp
- /run
restart: unless-stopped
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
# ── Konfigurasi platform per host (internal saja) ─────────────────────────
config-agent:
image: ${SICABE_REGISTRY:-sicabe}/config-agent:${SICABE_TAG:-latest}
container_name: sicabe-config-agent
# Jalan sebagai UID/GID host agar file yang ditulis (katalog Trino, data OPA)
# tetap milik user host, bukan root.
user: "${AGENT_UID:-1000}:${AGENT_GID:-1000}"
environment:
AGENT_API_KEY: ${GATEWAY_API_KEY:?GATEWAY_API_KEY wajib diisi (env)}
AGENT_PORT: "8080"
PYTHONDONTWRITEBYTECODE: "1"
SICABE_ENV_FILE: /config/.env
TRINO_CATALOG_DIR: /config/config/trino/catalog
DBT_PROJECT_DIR: /config/config/airflow/dbt/sicabe
OPA_DATA_FILE: /config/config/opa/data.json
# Nama file PEM di dalam /ca (lihat AGENT_CA_HOST_PATH). Diperlukan agar
# panel bisa memverifikasi TLS Trino (self-signed).
AGENT_CA_FILE: ${AGENT_CA_FILE:-/ca/trino.crt}
# Sumber daftar container untuk scan (docker-proxy internal).
DOCKER_PROXY_URL: http://docker-proxy:2375
volumes:
- ${SICABE_DEPLOY_DIR:?set SICABE_DEPLOY_DIR (env)}:/config:ro
- ${SICABE_DEPLOY_DIR:?}/config/trino/catalog:/config/config/trino/catalog:rw
- ${SICABE_DEPLOY_DIR:?}/config/opa:/config/config/opa:rw
# CA TLS (GET /ca): direktori berisi PEM → /ca. WAJIB diisi di host
# panel agar TLS Trino (self-signed) bisa diverifikasi.
- ${AGENT_CA_HOST_PATH:-${SICABE_DEPLOY_DIR:?}}:/ca:ro
expose:
- "8080"
read_only: true
tmpfs:
- /tmp
restart: unless-stopped
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
# ── Gateway nginx: satu-satunya port ke host ──────────────────────────────
gateway:
image: nginx:1.30.4-alpine3.24
container_name: sicabe-host-gateway
depends_on:
- docker-proxy
- config-agent
environment:
GATEWAY_API_KEY: ${GATEWAY_API_KEY:?GATEWAY_API_KEY wajib diisi (env)}
volumes:
# Template nginx dipakai bersama dari deploy/host-gateway (tanpa duplikasi).
- ../host-gateway/nginx/gateway.conf.template:/etc/nginx/templates/default.conf.template:ro
ports:
- "${GATEWAY_BIND_IP:-127.0.0.1}:${GATEWAY_PORT:-22375}:80"
restart: unless-stopped
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
# ── Control panel — HANYA di host panel (profile "panel") ────────────────
panel:
profiles: ["panel"]
image: ${SICABE_REGISTRY:-sicabe}/control-panel:${SICABE_TAG:-latest}
container_name: sicabe-control-panel
depends_on:
- gateway
# Host network: gateway host bind 127.0.0.1 → tak terjangkau dari bridge,
# dan SAN sertifikat Trino (localhost) cocok. Port via PORT.
network_mode: host
environment:
NODE_ENV: production
HOSTNAME: "0.0.0.0"
PORT: ${PANEL_PORT:-3000}
# ── Wajib (panel gagal-cepat bila kosong) ──
NEXTAUTH_URL: ${PANEL_URL:-}
NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:-}
KEYCLOAK_CLIENT_SECRET: ${KEYCLOAK_CLIENT_SECRET:-}
SICABE_HOSTS: ${SICABE_HOSTS:-}
# ── Login lokal (hybrid, opsional) ──
# Akun lokal berlaku juga di produksi. Pakai varian _B64 (bcrypt memuat
# '$' yang bisa di-interpolasi Compose). Kosong = hanya SSO.
PANEL_LOCAL_USER: ${PANEL_LOCAL_USER:-}
PANEL_LOCAL_PASSWORD_HASH: ${PANEL_LOCAL_PASSWORD_HASH:-}
PANEL_LOCAL_PASSWORD_HASH_B64: ${PANEL_LOCAL_PASSWORD_HASH_B64:-}
PANEL_LOCAL_ROLE: ${PANEL_LOCAL_ROLE:-}
# Identitas Trino untuk sesi lokal (tanpa JWT) — service account Basic.
TRINO_SERVICE_ACCOUNT: ${TRINO_SERVICE_ACCOUNT:-}
# ── Opsional (ada default di kode) ──
KEYCLOAK_URL: ${KEYCLOAK_URL:-}
KEYCLOAK_CLIENT_ID: ${KEYCLOAK_CLIENT_ID:-sicabe-control-panel}
KEYCLOAK_WEB_URL: ${KEYCLOAK_WEB_URL:-}
TRINO_URL: ${TRINO_URL:-}
TRINO_WEB_URL: ${TRINO_WEB_URL:-}
NESSIE_URL: ${NESSIE_URL:-}
AIRFLOW_URL: ${AIRFLOW_URL:-}
STARROCKS_HOST: ${STARROCKS_HOST:-}
STARROCKS_PORT: ${STARROCKS_PORT:-}
STARROCKS_WEB_URL: ${STARROCKS_WEB_URL:-}
OPENMETADATA_URL: ${OPENMETADATA_URL:-}
GRAFANA_URL: ${GRAFANA_URL:-}
MINIO_CONSOLE_URL: ${MINIO_CONSOLE_URL:-}
SICABE_COMPOSE_PROJECTS: ${SICABE_COMPOSE_PROJECTS:-sicabe}
SICABE_CONFIG_FILE: data/config.json
SICABE_STATE_FILE: data/state.json
volumes:
- panel-data:/app/data
restart: unless-stopped
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
volumes:
panel-data:
+77
View File
@@ -0,0 +1,77 @@
# =============================================================================
# Env untuk stack sicabe-host (satu file untuk SEMUA host).
# Salin ke .env lalu isi nilainya: cp env.example .env
#
# Dipakai oleh: docker-socket-proxy + config-agent + gateway (semua host),
# dan control-panel (hanya host panel / profile "panel").
# =============================================================================
# ── Image dari registry (WAJIB) ──────────────────────────────────────────────
# Hasil: ./scripts/build-image.sh --all --push --registry <registry>
# Kosongkan SICABE_REGISTRY bila memakai image lokal (mis. saat uji).
#SICABE_REGISTRY=registry.example.com/sicabe
SICABE_TAG=latest
# ── Gateway (WAJIB, semua host) ──────────────────────────────────────────────
# Satu API key untuk docker-proxy + config-agent (diperiksa nginx).
GATEWAY_API_KEY=
# Bind ke 127.0.0.1 (default, aman) atau IP internal. JANGAN ke internet.
GATEWAY_BIND_IP=127.0.0.1
GATEWAY_PORT=22375
# ── Config-agent (WAJIB, semua host) ─────────────────────────────────────────
# Direktori deploy sicabe-doc di host ini (berisi .env, config/trino/catalog, ...).
SICABE_DEPLOY_DIR=/path/ke/sicabe-doc/deploy
# UID/GID host yang menulis file (katalog/OPA) — agar tidak root-owned.
AGENT_UID=1000
AGENT_GID=1000
# CA TLS Trino — WAJIB di host panel agar panel memverifikasi sertifikat Trino
# (self-signed). Isi direktori yang memuat PEM-nya (dihasilkan
# ./scripts/export-trino-ca.sh → <panel>/.ca/trino.crt).
# Di host NON-panel boleh dikosongkan (panel-lah yang butuh CA).
AGENT_CA_HOST_PATH=/path/ke/sicabe-control-panel/.ca
AGENT_CA_FILE=/ca/trino.crt
# =============================================================================
# Di bawah ini HANYA untuk host panel (profile "panel").
# Boleh dibiarkan kosong di host non-panel (compose tetap valid).
# =============================================================================
# ── Panel: wajib ─────────────────────────────────────────────────────────────
# URL publik panel (harus sama dengan yang diakses browser & redirect URI Keycloak).
PANEL_URL=http://localhost:3000
PANEL_PORT=3000
# openssl rand -base64 32
NEXTAUTH_SECRET=
# ── Login lokal (hybrid dengan SSO) — OPSIONAL ───────────────────────────────
# Akun lokal tanpa Keycloak; password = bcrypt hash. Pakai varian _B64 di sini
# karena Compose meng-interpolasi '$' (bcrypt selalu memuat '$').
# node -e "console.log(Buffer.from(require('bcryptjs').hashSync('PASSWORD',10)).toString('base64'))"
#PANEL_LOCAL_USER=admin
#PANEL_LOCAL_PASSWORD_HASH_B64=
#PANEL_LOCAL_ROLE=data-admin
# Identitas Trino utk sesi lokal (tanpa JWT) — service account (Basic).
#TRINO_SERVICE_ACCOUNT=svc_dbt
# Client Keycloak panel (scripts/create-keycloak-client.sh).
KEYCLOAK_CLIENT_SECRET=dev-keycloak-secret
# Host/agent yang dikelola panel. WAJIB ada `configUrl` (config-agent).
# Di host panel (network host) gateway lokal = 127.0.0.1:${GATEWAY_PORT}.
# Format JSON tanpa kutip pembungkus (hindari masalah parsing .env).
SICABE_HOSTS=[{"name":"local-gw","proxyUrl":"http://127.0.0.1:22375","apiKey":"CHANGE_ME","configUrl":"http://127.0.0.1:22375","configApiKey":"CHANGE_ME","publicHost":"localhost","enabled":true}]
# ── Panel: opsional (ada default di kode) ────────────────────────────────────
KEYCLOAK_URL=http://127.0.0.1:12080
KEYCLOAK_CLIENT_ID=sicabe-control-panel
KEYCLOAK_WEB_URL=http://localhost:12080
TRINO_URL=https://localhost:12443
TRINO_WEB_URL=https://localhost:12443
NESSIE_URL=http://127.0.0.1:12120
AIRFLOW_URL=http://127.0.0.1:12082
STARROCKS_HOST=127.0.0.1
STARROCKS_PORT=12930
STARROCKS_WEB_URL=http://localhost:12030
OPENMETADATA_URL=http://127.0.0.1:12085
GRAFANA_URL=http://127.0.0.1:12300
MINIO_CONSOLE_URL=http://localhost:12901
SICABE_COMPOSE_PROJECTS=sicabe