first commit
This commit is contained in:
@@ -0,0 +1,20 @@
|
||||
node_modules
|
||||
.next
|
||||
.git
|
||||
.gitignore
|
||||
.env*
|
||||
npm-debug.log
|
||||
|
||||
# Build context (tak perlu untuk build image)
|
||||
Dockerfile
|
||||
.dockerignore
|
||||
deploy
|
||||
docs
|
||||
scripts
|
||||
.opencode
|
||||
.tmp
|
||||
test-results
|
||||
playwright-report
|
||||
|
||||
# Runtime state panel — MEMUAT SECRET (api key host). Jangan pernah masuk image.
|
||||
data
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
# ============================================================================
|
||||
# SiCABE Control Panel — konfigurasi env (salin ke .env lalu sesuaikan)
|
||||
#
|
||||
# SEMUA URL service dapat di-override dari environment. Nilai default di bawah
|
||||
# memakai nama service dalam network compose (sicabe-doc/deploy, `name: sicabe`).
|
||||
# Bila panel dijalankan DI HOST, pakai port host (12xxx) seperti kolom "host".
|
||||
#
|
||||
# Konvensi:
|
||||
# * var tanpa prefix -> dibaca sisi SERVER (fetch/API, rahasia aman)
|
||||
# * *_WEB_URL dll -> tautan/UI client; di-fetch dari /api/platform/endpoints
|
||||
# * .env TIDAK di-commit; contoh ini ikut di-track (!.env.example)
|
||||
# ============================================================================
|
||||
|
||||
# ── Trino ────────────────────────────────────────────────────────────────────
|
||||
# Endpoint SQL/API (route /api/trino & /api/query). HTTPS saja: Trino memakai
|
||||
# auth hybrid OAuth2 + Basic dengan allow-insecure-over-http=false (HTTP polos
|
||||
# tidak melayani apa pun). Login dev pun memakai Bearer, didapat via Keycloak
|
||||
# ROPC di CredentialsProvider.
|
||||
# SSO & dev (Bearer/JWT) : HTTPS → https://localhost:12443
|
||||
# dalam compose : http://trino:8080
|
||||
TRINO_URL=https://localhost:12443
|
||||
# CA utk Trino HTTPS self-signed. Pin sertifikat server (PEM) di sini.
|
||||
# Bisa juga per-host lewat `caPem` di SICABE_HOSTS (lebih diutamakan).
|
||||
TRINO_CA_FILE=.ca/trino.crt
|
||||
# Nama host (entri SICABE_HOSTS) yang menjalankan Trino — untuk memilih CA per-host.
|
||||
#SICABE_TRINO_HOST=host-prod-1
|
||||
# Trino Web UI.
|
||||
#TRINO_WEB_URL=https://localhost:12443
|
||||
|
||||
# ── Keycloak ─────────────────────────────────────────────────────────────────
|
||||
# Base URL + realm utk OIDC (login panel & refresh token).
|
||||
# dalam compose : http://keycloak:8080
|
||||
# host : http://localhost:12080
|
||||
KEYCLOAK_URL=http://keycloak:8080
|
||||
KEYCLOAK_REALM=sicabe
|
||||
# Client OIDC panel (harus ada di realm tsb).
|
||||
KEYCLOAK_CLIENT_ID=sicabe-control-panel
|
||||
# Secret client. WAJIB di-production (tanpa nilai -> fail saat login).
|
||||
#KEYCLOAK_CLIENT_SECRET=
|
||||
# Admin Console Keycloak (tautan dashboard utk admin).
|
||||
#KEYCLOAK_WEB_URL=http://localhost:12080
|
||||
|
||||
# ── Iceberg table explorer backend (Nessie) ──────────────────────────────────
|
||||
# Halaman /tables memakai catalog Iceberg REST Nessie.
|
||||
# base ops = {NESSIE_URL}/iceberg/v1/{NESSIE_BRANCH}.
|
||||
# dalam compose : http://nessie:19120
|
||||
# host : http://localhost:12120
|
||||
#NESSIE_URL=http://nessie:19120
|
||||
#NESSIE_BRANCH=main
|
||||
|
||||
# ── Dremio (REST v3 — sources & SQL) ────────────────────────────────────────
|
||||
# Endpoint REST Dremio OSS + kredensial lokal (login /apiv2 -> token _dremio).
|
||||
# dalam compose : http://dremio:9047
|
||||
# host : http://localhost:12947
|
||||
DREMIO_URL=http://dremio:9047
|
||||
DREMIO_USER=
|
||||
DREMIO_PASSWORD=
|
||||
|
||||
# ── MinIO ────────────────────────────────────────────────────────────────────
|
||||
# MinIO Console (tautan dashboard & sidebar).
|
||||
# host : http://localhost:12901
|
||||
#MINIO_CONSOLE_URL=http://localhost:12901
|
||||
|
||||
# ── StarRocks ────────────────────────────────────────────────────────────────
|
||||
# FE MySQL protocol (route /api/starrocks).
|
||||
# dalam compose : STARROCKS_HOST=starrocks-fe, PORT=9030
|
||||
# host : STARROCKS_HOST=localhost, PORT=12930
|
||||
STARROCKS_HOST=localhost
|
||||
STARROCKS_PORT=12930
|
||||
STARROCKS_USER=root
|
||||
#STARROCKS_PASSWORD=
|
||||
# StarRocks UI HTTP (tautan halaman StarRocks).
|
||||
#STARROCKS_WEB_URL=http://localhost:12030
|
||||
|
||||
# ── Apache Airflow ───────────────────────────────────────────────────────────
|
||||
# Base URL — dipakai utk REST API (lib/airflow) DAN tautan UI (sidebar/dashboard).
|
||||
# dalam compose : http://airflow-webserver:8080
|
||||
# host : http://localhost:12082
|
||||
AIRFLOW_URL=http://localhost:12082
|
||||
AIRFLOW_USER=admin
|
||||
|
||||
# ── OpenMetadata & Grafana (governance) ──────────────────────────────────────
|
||||
# URL API/UI — single source (halaman governance + tautan).
|
||||
# host : OpenMetadata http://localhost:12085, Grafana http://localhost:12300
|
||||
OPENMETADATA_URL=http://localhost:12085
|
||||
GRAFANA_URL=http://localhost:12300
|
||||
|
||||
# ── Auth (NextAuth) ──────────────────────────────────────────────────────────
|
||||
NEXTAUTH_URL=http://localhost:3000
|
||||
# WAJIB di-production.
|
||||
# ── Login lokal (hybrid dengan SSO) ──────────────────────────────────────────
|
||||
# Akun admin LOKAL dari env — bisa login tanpa browser SSO, berlaku juga di
|
||||
# produksi. Password = BCRYPT HASH (bukan plaintext).
|
||||
# Buat hash: node -e "console.log(require('bcryptjs').hashSync('PASSWORD',10))"
|
||||
# Untuk Docker/Compose pakai varian _B64 (bcrypt memuat '$' yang bisa di-
|
||||
# interpolasi Compose):
|
||||
# node -e "console.log(Buffer.from(require('bcryptjs').hashSync('PASSWORD',10)).toString('base64'))"
|
||||
#PANEL_LOCAL_USER=admin
|
||||
#PANEL_LOCAL_PASSWORD_HASH=$2b$10$...
|
||||
#PANEL_LOCAL_PASSWORD_HASH_B64=JDJiJDEwJC4uLg==
|
||||
#PANEL_LOCAL_ROLE=data-admin
|
||||
# Identitas Trino untuk sesi login lokal (tanpa JWT Keycloak): service account.
|
||||
# Password dibaca dari .env platform via config-agent → TRINO_SVC_<ACC>_PASSWORD.
|
||||
#TRINO_SERVICE_ACCOUNT=svc_dbt
|
||||
#NEXTAUTH_SECRET=
|
||||
|
||||
# ── Multi-host Docker control ────────────────────────────────────────────────
|
||||
# Registry host Docker yang dikelola panel (via docker-socket-proxy per host).
|
||||
# Nilai JSON array. Bila kosong → default single host http://127.0.0.1:2375
|
||||
# (kompatibel perilaku lama).
|
||||
# Field per host: name (unik), proxyUrl, apiKey (opsional), enabled,
|
||||
# sshTunnel (opsional: {host, port, user}).
|
||||
# Multi-host (opsional, untuk Tahap 2): configUrl/configApiKey menunjuk ke
|
||||
# `sicabe-config-agent` host tsb; caPem = CA TLS host (PEM) bila perlu.
|
||||
# ⚠️ apiKey/configApiKey adalah SECRET — jangan di-commit; isi via env/deploy.
|
||||
# Development lokal (single host, tanpa proxy):
|
||||
# SICABE_HOSTS='[{"name":"local","proxyUrl":"unix:///var/run/docker.sock","enabled":true}]'
|
||||
#SICABE_HOSTS='[{"name":"host-prod-1","proxyUrl":"http://127.0.0.1:22375","apiKey":"","configUrl":"http://127.0.0.1:22375","configApiKey":"","enabled":true}]'
|
||||
# Config panel yang PERSISTEN (hosts/modules/services), diedit dari UI Settings.
|
||||
# Bila file ini ada, ia MENGGANTIKAN SICABE_HOSTS di atas (env = seed saja).
|
||||
# Memuat api key → gitignored, perms 0600.
|
||||
#SICABE_CONFIG_FILE=data/config.json
|
||||
# Lokasi file state (monitored toggle per container). Default data/state.json.
|
||||
# Key = `${host}/${container}` (kompatibel dgn key lama `${container}`).
|
||||
#SICABE_STATE_FILE=data/state.json
|
||||
# Daftar project Docker Compose yang dimonitor (label com.docker.compose.project).
|
||||
# Comma-separated; default "sicabe".
|
||||
#SICABE_COMPOSE_PROJECTS=sicabe
|
||||
# Override host per-service (opsional). Bila kosong, host di-resolve otomatis
|
||||
# dari daftar host/komponen (dipakai untuk membaca kredensial dari host yang tepat).
|
||||
#SICABE_AIRFLOW_HOST=host-prod-1
|
||||
#SICABE_STARROCKS_HOST=host-prod-1
|
||||
# Host yang menjalankan OPA (menu Authorization; baca/tulis config/opa/data.json).
|
||||
#SICABE_OPA_HOST=host-prod-1
|
||||
|
||||
# ── Lain-lain ────────────────────────────────────────────────────────────────
|
||||
# Direktori catalog Trino & file .env platform (bila dijalankan non-standar).
|
||||
# .env platform dibaca READ-ONLY oleh panel (tidak pernah ditimpa).
|
||||
#TRINO_CATALOG_DIR=deploy/config/trino/catalog
|
||||
#SICABE_ENV_FILE=deploy/.env
|
||||
# Dokumen data OPA (ACL authz Trino) — dibaca menu Authorization lewat ConfigStore.
|
||||
#OPA_DATA_FILE=/path/ke/sicabe-doc/deploy/config/opa/data.json
|
||||
# Project dbt SiCABE (dibaca /api/dbt untuk models, sources, lineage DAG).
|
||||
# Panel di host: arahkan ke folder platform (mis. sicabe-doc/deploy/config/airflow/dbt/sicabe).
|
||||
#DBT_PROJECT_DIR=/path/ke/sicabe-doc/deploy/config/airflow/dbt/sicabe
|
||||
# Password admin Airflow dibaca dari SICABE_ENV_FILE (AIRFLOW_ADMIN_PASSWORD).
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
/node_modules
|
||||
/.pnp
|
||||
.pnp.*
|
||||
.yarn/*
|
||||
!.yarn/patches
|
||||
!.yarn/plugins
|
||||
!.yarn/releases
|
||||
!.yarn/versions
|
||||
|
||||
# testing
|
||||
/coverage
|
||||
|
||||
# next.js
|
||||
/.next/
|
||||
/out/
|
||||
|
||||
# production
|
||||
/build
|
||||
|
||||
# misc
|
||||
.DS_Store
|
||||
*.pem
|
||||
|
||||
# debug
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
.pnpm-debug.log*
|
||||
|
||||
# env files (can opt-in for committing if needed)
|
||||
.env*
|
||||
!.env.example
|
||||
# stack env — nama `env` (tanpa titik) tidak tertangkap `.env*`
|
||||
/deploy/stack/env
|
||||
/deploy/panel/env
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
# typescript
|
||||
*.tsbuildinfo
|
||||
next-env.d.ts
|
||||
# AetherLake root CA exported by install.sh (trusted by local dev TLS calls)
|
||||
.ca/
|
||||
|
||||
# playwright
|
||||
/test-results/
|
||||
/playwright-report/
|
||||
|
||||
# task/session artifacts (agent)
|
||||
/.tmp/
|
||||
|
||||
# runtime panel state & config — data/config.json memuat api key host (SECRET)
|
||||
/data/
|
||||
|
||||
# local agent tooling & install helper
|
||||
/.opencode/
|
||||
/install.sh
|
||||
@@ -0,0 +1,104 @@
|
||||
# =============================================================================
|
||||
# docker-socket-proxy + gateway nginx (api-key) — per host SiCABE.
|
||||
#
|
||||
# Control-panel (sicabe-control-panel) TIDAK pernah menyentuh /var/run/docker.sock
|
||||
# langsung. Di setiap host yang dikelola dipasang stack ini; panel berbicara ke
|
||||
# endpoint HTTP gateway (lihat desain:
|
||||
# docs/desain-kontrol-multi-host-docker-socket-proxy.md).
|
||||
#
|
||||
# Arsitektur:
|
||||
# panel ──HTTP + X-Api-Key──► gateway (nginx, allowlist path)
|
||||
# └──► docker-proxy (docker-socket-proxy, ACL)
|
||||
# └──► /var/run/docker.sock (:ro)
|
||||
#
|
||||
# Keamanan (jangan longgarkan tanpa kebutuhan):
|
||||
# * Socket daemon di-mount READ-ONLY (:ro).
|
||||
# * docker-socket-proxy TIDAK punya auth — container tsb HANYA di jaringan
|
||||
# internal compose (expose, tanpa port host). Yang diekspos ke host hanyalah
|
||||
# gateway nginx.
|
||||
# * nginx memeriksa header `X-Api-Key` (wajib sama dengan PROXY_API_KEY) dan
|
||||
# allowlist path (hanya route yang dibutuhkan kontrol, sisanya 403).
|
||||
# * ACL docker-socket-proxy hanya mengizinkan yang dibutuhkan:
|
||||
# CONTAINERS (list/inspect/restart/stop/start/logs/top), POST (restart),
|
||||
# INFO (version/ping), SYSTEM (/events).
|
||||
# Kategori lain (EXEC, BUILD, IMAGES, NETWORKS, VOLUMES, ...) DIMATIKAN.
|
||||
# * Versi image DIPATOK (bukan :latest) untuk produksi.
|
||||
#
|
||||
# Mode alternatif SSH-tunnel / single-host (tanpa api-key):
|
||||
# Hapus/komentari service `gateway`, lalu pada service `docker-proxy` ganti
|
||||
# `expose` dengan port publik:
|
||||
# ports:
|
||||
# - "127.0.0.1:${PROXY_PORT:-22375}:2375"
|
||||
#
|
||||
# Cara pakai (HTTP + api-key):
|
||||
# PROXY_API_KEY='<secret-acak-kuat>' PROXY_BIND_IP=127.0.0.1 PROXY_PORT=22375 \
|
||||
# docker compose up -d
|
||||
# - PROXY_API_KEY WAJIB diisi (compose akan error bila kosong). Ini secret —
|
||||
# jangan di-commit; isi via env/deploy.
|
||||
# - PROXY_BIND_IP: 127.0.0.1 utk single-host/SSH-tunnel; IP internal host bila
|
||||
# panel di host lain dalam network tepercaya. JANGAN expose ke internet.
|
||||
# - Panel: SICABE_HOSTS=[{"name":"...","proxyUrl":"http://<bind-ip>:<port>",
|
||||
# "apiKey":"<PROXY_API_KEY>","enabled":true}]
|
||||
# =============================================================================
|
||||
|
||||
name: sicabe-docker-proxy
|
||||
|
||||
services:
|
||||
# --- docker-socket-proxy: ACL, hanya di jaringan internal compose ---------
|
||||
docker-proxy:
|
||||
image: tecnativa/docker-socket-proxy:v0.5.0
|
||||
container_name: sicabe-docker-proxy
|
||||
environment:
|
||||
# --- ACL: hanya yang dibutuhkan kontrol ---
|
||||
CONTAINERS: "1" # list/inspect/restart/stop/start/logs/top
|
||||
POST: "1" # dibutuhkan utk restart/start/stop
|
||||
INFO: "1" # info daemon (version, /_ping, healthcheck)
|
||||
SYSTEM: "1" # /events (feed Observability). Set "0" bila tak dipakai
|
||||
# --- matikan semua yang tidak perlu ---
|
||||
EXEC: "0"
|
||||
SWARM: "0"
|
||||
SERVICES: "0"
|
||||
SECRETS: "0"
|
||||
CONFIGS: "0"
|
||||
NETWORKS: "0"
|
||||
VOLUMES: "0"
|
||||
IMAGES: "0"
|
||||
BUILD: "0"
|
||||
GRPC: "0"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
expose:
|
||||
- "2375" # hanya reachable dari gateway (network compose)
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp
|
||||
- /run # HAProxy menulis pidfile /run/haproxy.pid
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
# --- gateway nginx: api-key + allowlist path (satu-satunya port ke host) --
|
||||
gateway:
|
||||
image: nginx:1.30.4-alpine3.24
|
||||
container_name: sicabe-docker-proxy-gateway
|
||||
depends_on:
|
||||
- docker-proxy
|
||||
environment:
|
||||
# WAJIB: compose error bila PROXY_API_KEY tidak di-set.
|
||||
PROXY_API_KEY: ${PROXY_API_KEY:?PROXY_API_KEY wajib diisi (secret acak kuat)}
|
||||
ports:
|
||||
- "${PROXY_BIND_IP:-127.0.0.1}:${PROXY_PORT:-22375}:80"
|
||||
volumes:
|
||||
# Template di-substitute oleh envsubst docker-entrypoint nginx saat start
|
||||
# (${PROXY_API_KEY} diganti nilai env; $http_* tetap utuh) → output ke
|
||||
# /etc/nginx/conf.d/default.conf (menggantikan default bawaan image).
|
||||
- ./nginx/gateway.conf.template:/etc/nginx/templates/default.conf.template:ro
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
@@ -0,0 +1,54 @@
|
||||
# nginx gateway untuk docker-socket-proxy.
|
||||
#
|
||||
# File ini adalah TEMPLATE: pada container start, docker-entrypoint nginx
|
||||
# menjalankan envsubst dan mengganti ${PROXY_API_KEY} dengan nilai env
|
||||
# PROXY_API_KEY (variabel nginx seperti $http_x_api_key TIDAK ikut diganti),
|
||||
# lalu menulis hasilnya ke /etc/nginx/conf.d/default.conf.
|
||||
#
|
||||
# Kebijakan:
|
||||
# * Semua request WAJIB membawa header `X-Api-Key` == PROXY_API_KEY (401 bila
|
||||
# salah/kosong). nginx membalas 401 TANPA meneruskan ke proxy.
|
||||
# * Allowlist path (desain §2): hanya route yang dibutuhkan KONTROL oleh
|
||||
# control-panel. Selain itu di-block 403 — termasuk route berbahaya
|
||||
# (exec, create, images, networks, volumes, dll) sekalipun ACL proxy longgar.
|
||||
# * Aksi tulis restart/start/stop hanya via POST (selainnya 405).
|
||||
|
||||
server {
|
||||
listen 80 default_server;
|
||||
server_name _;
|
||||
|
||||
# ==== API-key check (sebelum diproses location mana pun) ====
|
||||
if ($http_x_api_key != "${PROXY_API_KEY}") {
|
||||
return 401;
|
||||
}
|
||||
|
||||
# ==== Route yang diizinkan (allowlist) ====
|
||||
|
||||
# Health/info daemon + event stream
|
||||
location = /_ping { proxy_pass http://docker-proxy:2375; }
|
||||
location = /version { proxy_pass http://docker-proxy:2375; }
|
||||
location = /info { proxy_pass http://docker-proxy:2375; }
|
||||
location = /events { proxy_pass http://docker-proxy:2375; }
|
||||
|
||||
# List container (tanpa id) — GET /containers/json
|
||||
location = /containers/json { proxy_pass http://docker-proxy:2375; }
|
||||
|
||||
# Operasi per-container yang dibutuhkan kontrol/observability
|
||||
location ~ ^/containers/[^/]+/json$ { proxy_pass http://docker-proxy:2375; }
|
||||
location ~ ^/containers/[^/]+/logs$ { proxy_pass http://docker-proxy:2375; }
|
||||
location ~ ^/containers/[^/]+/stats$ { proxy_pass http://docker-proxy:2375; }
|
||||
location ~ ^/containers/[^/]+/top$ { proxy_pass http://docker-proxy:2375; }
|
||||
|
||||
# Aksi tulis — hanya POST (restart/start/stop)
|
||||
location ~ ^/containers/[^/]+/(restart|start|stop)$ {
|
||||
if ($request_method != POST) {
|
||||
return 405;
|
||||
}
|
||||
proxy_pass http://docker-proxy:2375;
|
||||
}
|
||||
|
||||
# ==== Semua path lain di-block ====
|
||||
location / {
|
||||
return 403;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
# sicabe-config-agent — Python stdlib + PyYAML (image ramping).
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY agent.py ./
|
||||
|
||||
# Default; override via env AGENT_PORT.
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=3s --retries=3 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/healthz')" || exit 1
|
||||
|
||||
ENTRYPOINT ["python", "-u", "agent.py"]
|
||||
@@ -0,0 +1,551 @@
|
||||
#!/usr/bin/env python3
|
||||
"""sicabe-config-agent — akses konfigurasi platform SiCABE per host.
|
||||
|
||||
Control-panel TIDAK mengakses filesystem platform langsung; agent ini yang
|
||||
membacanya di host tempat sicabe-doc berada, lalu diekspos lewat HTTP ber-api-key.
|
||||
|
||||
Endpoint (butuh header `X-Api-Key` kecuali `/healthz`):
|
||||
GET /healthz
|
||||
GET /env?key=NAME[&key=NAME2] # read-only, whitelist (minimal 1 key)
|
||||
GET /catalog # daftar katalog Trino (nilai sensitif di-redact)
|
||||
GET /catalog/<name> # detail satu katalog (redact)
|
||||
PUT /catalog/<name> # body {"properties": {...}} — tulis (upsert)
|
||||
DELETE /catalog/<name> # hapus
|
||||
GET /dbt # project dbt terparse (models/sources/DAG)
|
||||
GET /ca # PEM sertifikat TLS host (opsional)
|
||||
GET /opa/data # dokumen data OPA (config/opa/data.json) — mentah
|
||||
PUT /opa/data # body {"content": "<json>"} — validasi + tulis atomic (.bak)
|
||||
POST /services/scan # body {"publicHost","types":[...]} — scan service (panel-driven)
|
||||
|
||||
Env:
|
||||
AGENT_API_KEY (wajib) api key yang harus dikirim panel
|
||||
AGENT_PORT (default 8080)
|
||||
SICABE_ENV_FILE (default /config/.env)
|
||||
TRINO_CATALOG_DIR (default /config/config/trino/catalog)
|
||||
DBT_PROJECT_DIR (default /config/config/airflow/dbt/sicabe)
|
||||
AGENT_CA_FILE (opsional) path PEM untuk GET /ca
|
||||
OPA_DATA_FILE (default /config/config/opa/data.json) dokumen data OPA
|
||||
DOCKER_PROXY_URL (default http://docker-proxy:2375) sumber daftar container
|
||||
"""
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import urllib.request
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
|
||||
import yaml
|
||||
|
||||
AGENT_API_KEY = os.environ.get("AGENT_API_KEY", "")
|
||||
AGENT_PORT = int(os.environ.get("AGENT_PORT", "8080"))
|
||||
ENV_FILE = os.environ.get("SICABE_ENV_FILE", "/config/.env")
|
||||
CATALOG_DIR = os.environ.get("TRINO_CATALOG_DIR", "/config/config/trino/catalog")
|
||||
DBT_DIR = os.environ.get("DBT_PROJECT_DIR", "/config/config/airflow/dbt/sicabe")
|
||||
CA_FILE = os.environ.get("AGENT_CA_FILE", "")
|
||||
OPA_DATA_FILE = os.environ.get("OPA_DATA_FILE", "/config/config/opa/data.json")
|
||||
# docker-proxy internal (satu stack dengan agent) — untuk scan service tanpa
|
||||
# perlu mount docker.sock ke agent.
|
||||
DOCKER_PROXY_URL = os.environ.get("DOCKER_PROXY_URL", "http://docker-proxy:2375")
|
||||
|
||||
VALID_CATALOG = re.compile(r"^[a-z][a-z0-9_-]{0,62}$")
|
||||
SENSITIVE = re.compile(r"(password|secret|credential|access-key|apikey|api-key|token)", re.I)
|
||||
REF_RE = re.compile(r"""ref\(\s*['"]([^'"]+)['"]\s*\)""")
|
||||
SOURCE_RE = re.compile(r"""source\(\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\)""")
|
||||
|
||||
|
||||
# ── helpers: .env ────────────────────────────────────────────────────────────
|
||||
def read_env(keys):
|
||||
"""Baca key tertentu dari ENV_FILE (read-only, whitelist)."""
|
||||
wanted = set(keys)
|
||||
out = {}
|
||||
try:
|
||||
with open(ENV_FILE, "r", encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
k = k.strip()
|
||||
if k not in wanted:
|
||||
continue
|
||||
v = v.strip()
|
||||
if len(v) >= 2 and v[0] == v[-1] and v[0] in "\"'":
|
||||
v = v[1:-1]
|
||||
out[k] = v
|
||||
except OSError:
|
||||
return {}
|
||||
return out
|
||||
|
||||
|
||||
# ── helpers: katalog Trino ───────────────────────────────────────────────────
|
||||
def redact(props):
|
||||
return {k: ("••••••••" if SENSITIVE.search(k) else v) for k, v in props.items()}
|
||||
|
||||
|
||||
def parse_properties(content):
|
||||
props = {}
|
||||
for line in content.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
props[k.strip()] = v.strip()
|
||||
return props
|
||||
|
||||
|
||||
def serialize_properties(props):
|
||||
return "".join(f"{k}={v}\n" for k, v in props.items())
|
||||
|
||||
|
||||
def catalog_path(name):
|
||||
return os.path.join(CATALOG_DIR, f"{name}.properties")
|
||||
|
||||
|
||||
def list_catalogs():
|
||||
out = []
|
||||
try:
|
||||
names = sorted(os.listdir(CATALOG_DIR))
|
||||
except OSError:
|
||||
return out
|
||||
for fname in names:
|
||||
if not fname.endswith(".properties"):
|
||||
continue
|
||||
name = fname[: -len(".properties")]
|
||||
try:
|
||||
with open(os.path.join(CATALOG_DIR, fname), "r", encoding="utf-8") as fh:
|
||||
props = parse_properties(fh.read())
|
||||
except OSError:
|
||||
continue
|
||||
out.append({"name": name, "connector": props.get("connector.name", "unknown"), "properties": redact(props)})
|
||||
return sorted(out, key=lambda c: c["name"])
|
||||
|
||||
|
||||
def get_catalog(name):
|
||||
for c in list_catalogs():
|
||||
if c["name"] == name:
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def put_catalog(name, props):
|
||||
os.makedirs(CATALOG_DIR, exist_ok=True)
|
||||
with open(catalog_path(name), "w", encoding="utf-8") as fh:
|
||||
fh.write(serialize_properties(props))
|
||||
|
||||
|
||||
def write_opa_data(content):
|
||||
"""Tulis dokumen data OPA secara atomic + simpan isi sebelumnya ke .bak.
|
||||
|
||||
Atomic (temp + os.replace) agar crash di tengah tulis tidak meninggalkan file
|
||||
separuh jadi — OPA GAGAL START bila JSON-nya tidak valid, dan itu mematikan
|
||||
authz Trino. Backup `.bak` dipakai untuk rollback (Tahap 4).
|
||||
"""
|
||||
os.makedirs(os.path.dirname(OPA_DATA_FILE) or ".", exist_ok=True)
|
||||
if os.path.exists(OPA_DATA_FILE):
|
||||
with open(OPA_DATA_FILE, "rb") as fh:
|
||||
previous = fh.read()
|
||||
with open(OPA_DATA_FILE + ".bak", "wb") as fh:
|
||||
fh.write(previous)
|
||||
tmp = OPA_DATA_FILE + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as fh:
|
||||
fh.write(content)
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
os.replace(tmp, OPA_DATA_FILE)
|
||||
|
||||
|
||||
def delete_catalog(name):
|
||||
path = catalog_path(name)
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError(name)
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
# ── helpers: project dbt (bentuk = lib/dbt.ts) ───────────────────────────────
|
||||
def layer_from_path(rel):
|
||||
if "bronze_to_silver" in rel:
|
||||
return "silver"
|
||||
if "silver_to_gold" in rel:
|
||||
return "gold"
|
||||
if "incremental" in rel:
|
||||
return "bronze"
|
||||
return "silver"
|
||||
|
||||
|
||||
def load_dbt():
|
||||
name, version, profile = "sicabe", "1.0.0", "trino"
|
||||
try:
|
||||
with open(os.path.join(DBT_DIR, "dbt_project.yml"), "r", encoding="utf-8") as fh:
|
||||
yml = fh.read()
|
||||
m = re.search(r"^name:\s*['\"]?([^'\"\n]+)", yml, re.M)
|
||||
if m:
|
||||
name = m.group(1).strip()
|
||||
m = re.search(r"^version:\s*['\"]?([^'\"\n]+)", yml, re.M)
|
||||
if m:
|
||||
version = m.group(1).strip()
|
||||
m = re.search(r"^profile:\s*['\"]?([^'\"\n]+)", yml, re.M)
|
||||
if m:
|
||||
profile = m.group(1).strip()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
models = []
|
||||
models_dir = os.path.join(DBT_DIR, "models")
|
||||
for root, _dirs, files in os.walk(models_dir):
|
||||
for fname in files:
|
||||
if not fname.endswith(".sql"):
|
||||
continue
|
||||
full = os.path.join(root, fname)
|
||||
rel = os.path.relpath(full, models_dir)
|
||||
try:
|
||||
with open(full, "r", encoding="utf-8") as fh:
|
||||
sql = fh.read()
|
||||
except OSError:
|
||||
continue
|
||||
layer = layer_from_path(rel)
|
||||
models.append({
|
||||
"name": fname[: -len(".sql")],
|
||||
"path": rel,
|
||||
"layer": layer,
|
||||
"materialization": "incremental" if "incremental" in rel else "table",
|
||||
"schema": layer,
|
||||
"dependsOn": REF_RE.findall(sql),
|
||||
"sources": [f"{a}.{b}" for a, b in SOURCE_RE.findall(sql)],
|
||||
"sql": sql,
|
||||
})
|
||||
|
||||
sources = []
|
||||
sources_file = os.path.join(models_dir, "sources.yml")
|
||||
if os.path.exists(sources_file):
|
||||
try:
|
||||
with open(sources_file, "r", encoding="utf-8") as fh:
|
||||
doc = yaml.safe_load(fh.read()) or {}
|
||||
for s in doc.get("sources", []) or []:
|
||||
nm = s.get("name") or ""
|
||||
if not nm:
|
||||
continue
|
||||
sources.append({
|
||||
"name": nm,
|
||||
"database": s.get("database") or "",
|
||||
"schema": s.get("schema") or "",
|
||||
"tables": [{"name": t.get("name") or "", "description": t.get("description")}
|
||||
for t in (s.get("tables") or [])],
|
||||
})
|
||||
except (OSError, yaml.YAMLError):
|
||||
sources = []
|
||||
|
||||
nodes, edges = [], []
|
||||
for src in sources:
|
||||
for tbl in src["tables"]:
|
||||
nodes.append({"id": f"source.{src['name']}.{tbl['name']}", "label": f"{src['name']}.{tbl['name']}",
|
||||
"type": "source", "layer": "bronze"})
|
||||
for m in models:
|
||||
nodes.append({"id": f"model.{m['name']}", "label": m["name"], "type": "model", "layer": m["layer"]})
|
||||
for dep in m["dependsOn"]:
|
||||
edges.append({"id": f"model.{dep}->model.{m['name']}", "source": f"model.{dep}", "target": f"model.{m['name']}"})
|
||||
for s in m["sources"]:
|
||||
db, _, tbl = s.partition(".")
|
||||
edges.append({"id": f"source.{db}.{tbl}->model.{m['name']}", "source": f"source.{db}.{tbl}", "target": f"model.{m['name']}"})
|
||||
|
||||
return {"name": name, "version": version, "profile": profile, "models": models, "sources": sources,
|
||||
"dag": {"nodes": nodes, "edges": edges}}
|
||||
|
||||
|
||||
# ── Scan service (panel-driven) ──────────────────────────────────────────────
|
||||
# Panel mengirim katalog TIPE service (regex + port); agent memindai container
|
||||
# yang SEDANG BERJALAN di host ini (via docker-proxy internal) dan mengembalikan
|
||||
# instance + URL-nya. Logika pengenalan tetap satu tempat di panel (TS) — agent
|
||||
# hanya mesin generik. Hanya container `running` yang dipindai, sehingga job
|
||||
# sekali-jalan (init/migrate) otomatis tidak ikut.
|
||||
|
||||
# Label peta port → skema URL asli (label boleh deskriptif, mis. "console").
|
||||
SCHEME_ALIASES = {
|
||||
"https": "https",
|
||||
"http": "http",
|
||||
"api": "http",
|
||||
"console": "http",
|
||||
"ui": "http",
|
||||
"web": "http",
|
||||
"ldap": "ldap",
|
||||
"mysql": "mysql",
|
||||
"sql": "https",
|
||||
"tcp": "tcp",
|
||||
}
|
||||
def _re_match(pattern, value):
|
||||
if not pattern:
|
||||
return False
|
||||
try:
|
||||
return re.search(pattern, value or "") is not None
|
||||
except re.error:
|
||||
return False
|
||||
|
||||
|
||||
def docker_running_containers():
|
||||
"""Daftar container running dari docker-proxy internal."""
|
||||
req = urllib.request.Request(f"{DOCKER_PROXY_URL}/containers/json?all=false")
|
||||
with urllib.request.urlopen(req, timeout=8) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
out = []
|
||||
for c in data:
|
||||
if (c.get("State") or "") != "running":
|
||||
continue
|
||||
names = c.get("Names") or []
|
||||
name = names[0].lstrip("/") if names else (c.get("Id") or "")[:12]
|
||||
out.append({
|
||||
"name": name,
|
||||
"image": c.get("Image", ""),
|
||||
"labels": c.get("Labels") or {},
|
||||
"ports": c.get("Ports") or [],
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _published_ports(ports_raw):
|
||||
out = []
|
||||
seen = set()
|
||||
for p in ports_raw:
|
||||
if not p.get("PublicPort"):
|
||||
continue
|
||||
key = (p.get("PrivatePort"), p.get("PublicPort"), p.get("Type", "tcp"))
|
||||
if key in seen:
|
||||
continue # docker mengembalikan entri IPv4 & IPv6 → dedupe
|
||||
seen.add(key)
|
||||
out.append({
|
||||
"privatePort": p.get("PrivatePort"),
|
||||
"publicPort": p.get("PublicPort"),
|
||||
"type": p.get("Type", "tcp"),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _urls_for(ports_map, published_raw, public_host):
|
||||
"""Susun URL per label dari peta port container → port publish host.
|
||||
|
||||
Kunci peta = LABEL (mis. "http", "console", "mysql"); skema URL sebenarnya
|
||||
dipetakan lewat SCHEME_ALIASES (label "console" → skema "http").
|
||||
"""
|
||||
urls = {}
|
||||
if not public_host:
|
||||
return urls
|
||||
for label, privates in (ports_map or {}).items():
|
||||
if not isinstance(privates, list):
|
||||
continue
|
||||
scheme = SCHEME_ALIASES.get(str(label).lower(), "http")
|
||||
for priv in privates:
|
||||
for p in published_raw:
|
||||
if int(p.get("PrivatePort") or 0) == int(priv) and p.get("PublicPort"):
|
||||
urls[label] = f"{scheme}://{public_host}:{p.get('PublicPort')}"
|
||||
break
|
||||
if label in urls:
|
||||
break
|
||||
return urls
|
||||
|
||||
|
||||
def scan_services(types, public_host):
|
||||
containers = docker_running_containers()
|
||||
services = []
|
||||
for t in types:
|
||||
if not isinstance(t, dict):
|
||||
continue
|
||||
img_pat = t.get("imagePattern")
|
||||
name_pat = t.get("namePattern")
|
||||
ports_map = t.get("ports") or {}
|
||||
for c in containers:
|
||||
if not (_re_match(img_pat, c["image"]) or _re_match(name_pat, c["name"])):
|
||||
continue
|
||||
services.append({
|
||||
"type": t.get("id"),
|
||||
"container": c["name"],
|
||||
"image": c["image"],
|
||||
"labels": c["labels"],
|
||||
"published": _published_ports(c["ports"]),
|
||||
"urls": _urls_for(ports_map, c["ports"], public_host),
|
||||
})
|
||||
return {
|
||||
"hostname": socket.gethostname(),
|
||||
"containers": len(containers),
|
||||
"config": {
|
||||
"envFile": os.path.exists(ENV_FILE),
|
||||
"catalogDir": os.path.isdir(CATALOG_DIR),
|
||||
"dbtDir": os.path.isdir(DBT_DIR),
|
||||
"caFile": bool(CA_FILE) and os.path.exists(CA_FILE),
|
||||
},
|
||||
"services": services,
|
||||
}
|
||||
|
||||
|
||||
# ── HTTP handler ─────────────────────────────────────────────────────────────
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "sicabe-config-agent/0.1"
|
||||
|
||||
# -- util --
|
||||
def _send(self, code, body, ctype="application/json"):
|
||||
payload = body if isinstance(body, bytes) else json.dumps(body).encode("utf-8")
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", ctype)
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
try:
|
||||
self.wfile.write(payload)
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
pass
|
||||
|
||||
def _auth_ok(self):
|
||||
key = self.headers.get("X-Api-Key", "")
|
||||
return bool(AGENT_API_KEY) and hmac.compare_digest(key, AGENT_API_KEY)
|
||||
|
||||
def _deny(self):
|
||||
self._send(401, {"error": "Unauthorized"})
|
||||
|
||||
def _json_body(self):
|
||||
length = int(self.headers.get("Content-Length") or 0)
|
||||
raw = self.rfile.read(length) if length else b"{}"
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
|
||||
def log_message(self, fmt, *args): # ringkas ke stderr
|
||||
print("[agent] " + (fmt % args), flush=True)
|
||||
|
||||
# -- GET --
|
||||
def do_GET(self):
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
|
||||
if path == "/healthz":
|
||||
return self._send(200, {"status": "ok"})
|
||||
if not self._auth_ok():
|
||||
return self._deny()
|
||||
|
||||
try:
|
||||
if path == "/env":
|
||||
keys = parse_qs(parsed.query).get("key", [])
|
||||
keys = [k for k in keys if k]
|
||||
if not keys:
|
||||
return self._send(400, {"error": "key parameter required (whitelist)"})
|
||||
return self._send(200, read_env(keys))
|
||||
|
||||
if path == "/catalog":
|
||||
return self._send(200, {"catalogs": list_catalogs()})
|
||||
|
||||
if path.startswith("/catalog/"):
|
||||
name = path[len("/catalog/"):]
|
||||
cat = get_catalog(name)
|
||||
if not cat:
|
||||
return self._send(404, {"error": f"catalog '{name}' not found"})
|
||||
return self._send(200, cat)
|
||||
|
||||
if path == "/dbt":
|
||||
return self._send(200, load_dbt())
|
||||
|
||||
if path == "/ca":
|
||||
if not CA_FILE or not os.path.exists(CA_FILE):
|
||||
return self._send(404, {"error": "no CA configured"})
|
||||
# Kirim bytes mentah (jangan di-JSON-encode) agar PEM utuh.
|
||||
with open(CA_FILE, "rb") as fh:
|
||||
return self._send(200, fh.read(), "application/x-pem-file")
|
||||
|
||||
if path == "/opa/data":
|
||||
# Kirim isi file APA ADANYA. Harus bytes: _send() men-JSON-encode
|
||||
# nilai non-bytes (kalau str, isi file jadi string ber-quote ganda
|
||||
# dan editor panel menampilkan JSON yang salah).
|
||||
if not os.path.exists(OPA_DATA_FILE):
|
||||
return self._send(404, {"error": "opa data file not found"})
|
||||
with open(OPA_DATA_FILE, "rb") as fh:
|
||||
return self._send(200, fh.read(), "application/json")
|
||||
|
||||
return self._send(404, {"error": "not found"})
|
||||
except Exception as e: # noqa: BLE001
|
||||
return self._send(500, {"error": "internal error", "details": str(e)})
|
||||
|
||||
# -- PUT --
|
||||
def do_PUT(self):
|
||||
if not self._auth_ok():
|
||||
return self._deny()
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
|
||||
if path == "/opa/data":
|
||||
try:
|
||||
body = self._json_body()
|
||||
content = body.get("content") if isinstance(body, dict) else None
|
||||
if not isinstance(content, str):
|
||||
return self._send(400, {"error": "content (JSON string) required"})
|
||||
# Validasi SEBELUM menulis: JSON invalid membuat OPA gagal start
|
||||
# → authz Trino mati total.
|
||||
try:
|
||||
json.loads(content)
|
||||
except json.JSONDecodeError as e:
|
||||
return self._send(400, {"error": "invalid JSON", "details": e.msg,
|
||||
"line": e.lineno, "col": e.colno})
|
||||
write_opa_data(content)
|
||||
return self._send(200, {"success": True, "bytes": len(content.encode("utf-8"))})
|
||||
except Exception as e: # noqa: BLE001
|
||||
return self._send(500, {"error": "internal error", "details": str(e)})
|
||||
|
||||
if not path.startswith("/catalog/"):
|
||||
return self._send(404, {"error": "not found"})
|
||||
name = path[len("/catalog/"):]
|
||||
if not VALID_CATALOG.match(name):
|
||||
return self._send(400, {"error": "invalid catalog name"})
|
||||
try:
|
||||
body = self._json_body()
|
||||
props = body.get("properties", body) if isinstance(body, dict) else {}
|
||||
if not isinstance(props, dict):
|
||||
return self._send(400, {"error": "properties must be an object"})
|
||||
put_catalog(name, {str(k): str(v) for k, v in props.items()})
|
||||
return self._send(200, {"success": True, "name": name})
|
||||
except Exception as e: # noqa: BLE001
|
||||
return self._send(500, {"error": "internal error", "details": str(e)})
|
||||
|
||||
# -- DELETE --
|
||||
def do_DELETE(self):
|
||||
if not self._auth_ok():
|
||||
return self._deny()
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
if not path.startswith("/catalog/"):
|
||||
return self._send(404, {"error": "not found"})
|
||||
name = path[len("/catalog/"):]
|
||||
if not VALID_CATALOG.match(name):
|
||||
return self._send(400, {"error": "invalid catalog name"})
|
||||
try:
|
||||
delete_catalog(name)
|
||||
return self._send(200, {"success": True, "name": name})
|
||||
except FileNotFoundError:
|
||||
return self._send(404, {"error": f"catalog '{name}' not found"})
|
||||
except Exception as e: # noqa: BLE001
|
||||
return self._send(500, {"error": "internal error", "details": str(e)})
|
||||
|
||||
|
||||
# -- POST --
|
||||
def do_POST(self):
|
||||
if not self._auth_ok():
|
||||
return self._deny()
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
|
||||
if path == "/services/scan":
|
||||
try:
|
||||
body = self._json_body()
|
||||
types = body.get("types") if isinstance(body, dict) else None
|
||||
if not isinstance(types, list) or not types:
|
||||
return self._send(400, {"error": "types (non-empty array) required"})
|
||||
public_host = body.get("publicHost") or ""
|
||||
return self._send(200, scan_services(types, public_host))
|
||||
except Exception as e: # noqa: BLE001
|
||||
return self._send(500, {"error": "scan failed", "details": str(e)})
|
||||
|
||||
return self._send(404, {"error": "not found"})
|
||||
|
||||
|
||||
def main():
|
||||
if not AGENT_API_KEY:
|
||||
print("FATAL: AGENT_API_KEY must be set", flush=True)
|
||||
raise SystemExit(1)
|
||||
print(f"[agent] listening on :{AGENT_PORT} (env={ENV_FILE}, catalog={CATALOG_DIR}, dbt={DBT_DIR})", flush=True)
|
||||
ThreadingHTTPServer(("0.0.0.0", AGENT_PORT), Handler).serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
PyYAML==6.0.2
|
||||
@@ -0,0 +1,128 @@
|
||||
# =============================================================================
|
||||
# sicabe-host-gateway — SATU port untuk docker-proxy + config-agent.
|
||||
#
|
||||
# CATATAN: ini varian LOKAL (build dari ./agent). Untuk DEPLOY dari registry
|
||||
# (termasuk panel, dengan profile) pakai: deploy/stack/docker-compose.yml
|
||||
#
|
||||
# Hanya nginx (gateway) yang membuka port host; docker-socket-proxy dan
|
||||
# config-agent bersifat internal (expose, tanpa port host). Satu API key
|
||||
# (GATEWAY_API_KEY) diperiksa nginx, lalu request diarahkan berdasarkan path:
|
||||
# /_ping /version /info /events /containers/* → docker-proxy
|
||||
# /healthz /env /dbt /ca /catalog(/<name>) → config-agent
|
||||
#
|
||||
# Lihat: docs/desain-host-gateway-terpadu.md
|
||||
#
|
||||
# Cara pakai:
|
||||
# GATEWAY_API_KEY='<secret-kuat>' \
|
||||
# SICABE_DEPLOY_DIR=/path/ke/sicabe-doc/deploy \
|
||||
# AGENT_CA_HOST_PATH=/path/berisi/pem AGENT_CA_FILE=/ca/trino.crt \
|
||||
# docker compose up -d
|
||||
#
|
||||
# Opsional: AGENT_UID/AGENT_GID (default 1000:1000) — UID/GID yang dipakai
|
||||
# config-agent saat menulis ke host, supaya file (katalog Trino, data OPA) tetap
|
||||
# dimiliki user host dan bukan root.
|
||||
#
|
||||
# Panel (satu URL untuk proxy & config):
|
||||
# SICABE_HOSTS=[{"name":"h1","proxyUrl":"http://<host>:22375",
|
||||
# "apiKey":"<GATEWAY_API_KEY>",
|
||||
# "configUrl":"http://<host>:22375",
|
||||
# "configApiKey":"<GATEWAY_API_KEY>","enabled":true}]
|
||||
# =============================================================================
|
||||
|
||||
name: sicabe-host-gateway
|
||||
|
||||
services:
|
||||
# --- Docker Engine API (ACL ketat, internal saja) ------------------------
|
||||
docker-proxy:
|
||||
image: tecnativa/docker-socket-proxy:v0.5.0
|
||||
container_name: sicabe-docker-proxy
|
||||
environment:
|
||||
CONTAINERS: "1"
|
||||
POST: "1"
|
||||
INFO: "1"
|
||||
SYSTEM: "1"
|
||||
EXEC: "0"
|
||||
SWARM: "0"
|
||||
SERVICES: "0"
|
||||
SECRETS: "0"
|
||||
CONFIGS: "0"
|
||||
NETWORKS: "0"
|
||||
VOLUMES: "0"
|
||||
IMAGES: "0"
|
||||
BUILD: "0"
|
||||
GRPC: "0"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
expose:
|
||||
- "2375"
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp
|
||||
- /run
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
# --- Konfigurasi platform per host (internal saja) -----------------------
|
||||
config-agent:
|
||||
build: ./agent
|
||||
image: sicabe/config-agent:0.1.0
|
||||
container_name: sicabe-config-agent
|
||||
# Jalankan sebagai UID/GID host (bukan root) supaya file yang DITULIS agent
|
||||
# (katalog Trino, config/opa/data.json) tetap dimiliki user host — kalau root,
|
||||
# editor/git di host akan kena permission denied. Dir yang di-mount rw memang
|
||||
# milik user host. Bisa di-override via AGENT_UID/AGENT_GID.
|
||||
user: "${AGENT_UID:-1000}:${AGENT_GID:-1000}"
|
||||
environment:
|
||||
# Satu key: nginx memeriksa, agent tetap memvalidasi (defense-in-depth).
|
||||
AGENT_API_KEY: ${GATEWAY_API_KEY:?GATEWAY_API_KEY wajib diisi}
|
||||
AGENT_PORT: "8080"
|
||||
PYTHONDONTWRITEBYTECODE: "1"
|
||||
SICABE_ENV_FILE: /config/.env
|
||||
TRINO_CATALOG_DIR: /config/config/trino/catalog
|
||||
DBT_PROJECT_DIR: /config/config/airflow/dbt/sicabe
|
||||
AGENT_CA_FILE: ${AGENT_CA_FILE:-}
|
||||
# Dokumen data OPA (ACL Trino) — dibaca menu Authorization. Tulis (rw)
|
||||
# menyusul di Tahap 2.
|
||||
OPA_DATA_FILE: /config/config/opa/data.json
|
||||
volumes:
|
||||
- ${SICABE_DEPLOY_DIR:?set SICABE_DEPLOY_DIR}:/config:ro
|
||||
- ${SICABE_DEPLOY_DIR:?set SICABE_DEPLOY_DIR}/config/trino/catalog:/config/config/trino/catalog:rw
|
||||
# Dir OPA (data.json) — rw agar agent bisa menulis di Tahap 2.
|
||||
- ${SICABE_DEPLOY_DIR:?set SICABE_DEPLOY_DIR}/config/opa:/config/config/opa:rw
|
||||
# CA TLS opsional (GET /ca): direktori berisi PEM → /ca.
|
||||
- ${AGENT_CA_HOST_PATH:-${SICABE_DEPLOY_DIR:?set SICABE_DEPLOY_DIR}}:/ca:ro
|
||||
expose:
|
||||
- "8080"
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
# --- Gateway nginx: satu-satunya port ke host ----------------------------
|
||||
gateway:
|
||||
image: nginx:1.30.4-alpine3.24
|
||||
container_name: sicabe-host-gateway
|
||||
depends_on:
|
||||
- docker-proxy
|
||||
- config-agent
|
||||
environment:
|
||||
GATEWAY_API_KEY: ${GATEWAY_API_KEY:?GATEWAY_API_KEY wajib diisi}
|
||||
volumes:
|
||||
- ./nginx/gateway.conf.template:/etc/nginx/templates/default.conf.template:ro
|
||||
ports:
|
||||
- "${GATEWAY_BIND_IP:-127.0.0.1}:${GATEWAY_PORT:-22375}:80"
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
@@ -0,0 +1,66 @@
|
||||
# nginx gateway terpadu — docker-socket-proxy + config-agent dalam SATU port.
|
||||
#
|
||||
# Template: docker-entrypoint nginx men-substitute ${GATEWAY_API_KEY} saat start
|
||||
# (variabel nginx seperti $http_x_api_key TIDAK ikut diganti), lalu menulis hasil
|
||||
# ke /etc/nginx/conf.d/default.conf.
|
||||
#
|
||||
# Kebijakan:
|
||||
# * Semua request WAJIB membawa header `X-Api-Key` == GATEWAY_API_KEY (401 bila
|
||||
# salah/kosong), SEBELUM diproses location mana pun.
|
||||
# * Allowlist path ketat:
|
||||
# - /_ping /version /info /events /containers/* → docker-proxy:2375
|
||||
# - /healthz /env /dbt /ca /catalog(/<name>) /opa/data /services/scan → config-agent:8080
|
||||
# Selain daftar → 403.
|
||||
# * Aksi container restart/start/stop hanya POST (selainnya 405).
|
||||
# * Katalog (cart agent) mengizinkan GET/PUT/DELETE.
|
||||
#
|
||||
# Lihat: docs/desain-host-gateway-terpadu.md
|
||||
|
||||
server {
|
||||
listen 80 default_server;
|
||||
server_name _;
|
||||
|
||||
# ==== API-key check (sebelum location mana pun) ====
|
||||
if ($http_x_api_key != "${GATEWAY_API_KEY}") {
|
||||
return 401;
|
||||
}
|
||||
|
||||
# ==== config-agent (konfigurasi platform) ====
|
||||
location = /healthz { proxy_pass http://config-agent:8080; }
|
||||
location = /env { proxy_pass http://config-agent:8080; }
|
||||
location = /dbt { proxy_pass http://config-agent:8080; }
|
||||
location = /ca { proxy_pass http://config-agent:8080; }
|
||||
location = /catalog { proxy_pass http://config-agent:8080; }
|
||||
# GET/PUT/DELETE satu katalog (tanpa slash tambahan di nama).
|
||||
location ~ ^/catalog/[^/]+$ { proxy_pass http://config-agent:8080; }
|
||||
# Dokumen data OPA (menu Authorization) — baca sekarang, tulis menyusul.
|
||||
location = /opa/data { proxy_pass http://config-agent:8080; }
|
||||
# Scan service (panel-driven) — memindai container host lewat docker-proxy.
|
||||
location = /services/scan { proxy_pass http://config-agent:8080; }
|
||||
|
||||
# ==== docker-socket-proxy (Docker Engine API) ====
|
||||
location = /_ping { proxy_pass http://docker-proxy:2375; }
|
||||
location = /version { proxy_pass http://docker-proxy:2375; }
|
||||
location = /info { proxy_pass http://docker-proxy:2375; }
|
||||
location = /events { proxy_pass http://docker-proxy:2375; }
|
||||
|
||||
location = /containers/json { proxy_pass http://docker-proxy:2375; }
|
||||
|
||||
location ~ ^/containers/[^/]+/json$ { proxy_pass http://docker-proxy:2375; }
|
||||
location ~ ^/containers/[^/]+/logs$ { proxy_pass http://docker-proxy:2375; }
|
||||
location ~ ^/containers/[^/]+/stats$ { proxy_pass http://docker-proxy:2375; }
|
||||
location ~ ^/containers/[^/]+/top$ { proxy_pass http://docker-proxy:2375; }
|
||||
|
||||
# Aksi tulis container — hanya POST.
|
||||
location ~ ^/containers/[^/]+/(restart|start|stop)$ {
|
||||
if ($request_method != POST) {
|
||||
return 405;
|
||||
}
|
||||
proxy_pass http://docker-proxy:2375;
|
||||
}
|
||||
|
||||
# ==== selain allowlist → tolak ====
|
||||
location / {
|
||||
return 403;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
# Deploy (build/lokal): SiCABE Control Panel
|
||||
|
||||
> Untuk deploy PRODUKSI dari image registry (menyatu dengan gateway: docker-proxy +
|
||||
> config-agent + nginx, dengan profile), pakai **`deploy/stack/`**. Compose di sini
|
||||
> untuk **build lokal** dan menjalankan panel sendirian.
|
||||
|
||||
Panel (Next.js) dijalankan sebagai container. Image produksi memakai
|
||||
**Next.js standalone** + user non-root (`nextjs`, uid 1001).
|
||||
|
||||
> Gateway **per host** (`deploy/host-gateway/`) adalah stack terpisah — sudah punya
|
||||
> compose sendiri. Panel hanya perlu menunjuk ke gateway itu lewat `SICABE_HOSTS`.
|
||||
|
||||
## 1. Build image
|
||||
|
||||
```bash
|
||||
./scripts/build-image.sh # tag default: sicabe/control-panel:<git-sha> + :latest
|
||||
./scripts/build-image.sh --tag sicabe/control-panel:1.0.0
|
||||
./scripts/build-image.sh --push --registry registry.example.com/sicabe
|
||||
./scripts/build-image.sh --no-cache --platform linux/amd64
|
||||
```
|
||||
|
||||
Script memverifikasi Dockerfile/`public/`, menandai `:latest` + `<sha>`, dan
|
||||
mencetak ukuran image.
|
||||
|
||||
## 2. Siapkan env
|
||||
|
||||
```bash
|
||||
cd deploy/panel
|
||||
cp env.example .env # lalu isi NEXTAUTH_SECRET, api key gateway, dll.
|
||||
# NEXTAUTH_SECRET: openssl rand -base64 32
|
||||
```
|
||||
|
||||
**Penting:** panel di container **tidak bisa membaca filesystem platform** → tiap
|
||||
host di `SICABE_HOSTS` **wajib** punya `configUrl` (config-agent). Karena itu
|
||||
**tidak perlu** mount folder platform: katalog Trino, project dbt, dokumen OPA,
|
||||
`.env` platform, dan CA Trino semuanya dibaca lewat agent (HTTP). Env
|
||||
`TRINO_CATALOG_DIR` / `DBT_PROJECT_DIR` / `OPA_DATA_FILE` / `SICABE_ENV_FILE` /
|
||||
`TRINO_CA_FILE` hanya dipakai `LocalConfigStore` (dev/single-host tanpa agent) —
|
||||
jangan diandalkan di container.
|
||||
|
||||
## 3. Jalankan
|
||||
|
||||
```bash
|
||||
cd deploy/panel
|
||||
docker compose up -d # pakai image hasil build
|
||||
# docker compose up -d --build # sekaligus build dari source
|
||||
docker compose logs -f panel
|
||||
```
|
||||
|
||||
Buka `http://localhost:3000` (atau `PANEL_PORT`).
|
||||
|
||||
## Yang persisten
|
||||
|
||||
| Data | Lokasi |
|
||||
| --- | --- |
|
||||
| Config panel (hosts/modules/services/service types) | volume `panel-data` → `/app/data/config.json` |
|
||||
| Monitored state | volume `panel-data` → `/app/data/state.json` |
|
||||
|
||||
Backup/pindah config lewat **Settings → Backup** (Export/Import) — tanpa
|
||||
menyentuh volume. Untuk migrasi dari deployment host lama: Export di yang lama →
|
||||
Import di yang baru.
|
||||
|
||||
## Catatan jaringan & TLS
|
||||
|
||||
- **Default compose = `network_mode: host`.** Gateway host
|
||||
(`deploy/host-gateway`) hanya membuka `127.0.0.1:22375` (aman), sehingga panel
|
||||
di **bridge network tidak bisa** menjangkaunya lewat `host.docker.internal`.
|
||||
Terverifikasi: dari bridge, Keycloak (`0.0.0.0:12080`) **bisa**, gateway
|
||||
(`127.0.0.1:22375`) **gagal**; dari host network keduanya **bisa**. Host network
|
||||
juga membuat `127.0.0.1` & SAN sertifikat Trino (`localhost`) cocok.
|
||||
Port ditentukan `PORT` (default 3000), bukan `ports:`.
|
||||
- **Varian bridge** (bila gateway dibuka ke alamat non-loopback): lihat komentar
|
||||
di `docker-compose.yml` — hapus `network_mode: host`, tambah `ports:` +
|
||||
`extra_hosts: host.docker.internal`.
|
||||
- **TLS Trino**: sertifikat self-signed Trino hanya mencakup SAN
|
||||
`localhost / trino / sicabe-trino / trino-coordinator / 127.0.0.1`.
|
||||
`host.docker.internal` **tidak** termasuk → verifikasi TLS gagal di bridge.
|
||||
- CA Trino **tidak perlu** file lokal bila host punya `configUrl` (agent `/ca`).
|
||||
|
||||
## Login di produksi = SSO (Keycloak)
|
||||
|
||||
Container berjalan `NODE_ENV=production`, dan **login kredensial dev dinonaktifkan
|
||||
di produksi** (hanya provider `keycloak` yang aktif — cek
|
||||
`GET /api/auth/providers`). Artinya:
|
||||
|
||||
- Login lewat tombol **Sign in with Keycloak** di browser (bukan form dev).
|
||||
- **`NEXTAUTH_URL` harus sama dengan URL publik panel**, dan panel Keycloak
|
||||
(`scripts/create-keycloak-client.sh`, `APP_URL`) harus punya redirect URI
|
||||
`${NEXTAUTH_URL}/api/auth/callback/keycloak`. Kalau panel disajikan di port
|
||||
selain 3000, jalankan ulang script dengan `APP_URL=http://<host>:<port>`.
|
||||
- Image **gagal-cepat** bila `NEXTAUTH_SECRET` atau `KEYCLOAK_CLIENT_SECRET`
|
||||
kosong (memang disengaja) — panel menolak start dengan pesan jelas.
|
||||
|
||||
## Update
|
||||
|
||||
```bash
|
||||
./scripts/build-image.sh --tag sicabe/control-panel:1.1.0
|
||||
PANEL_TAG=1.1.0 docker compose up -d # dari deploy/panel
|
||||
```
|
||||
|
||||
## Healthcheck
|
||||
|
||||
Image menyertakan `HEALTHCHECK` (`GET /api/auth/session` → 200 walau belum login),
|
||||
jadi `docker compose ps` menampilkan status `healthy`.
|
||||
@@ -0,0 +1,61 @@
|
||||
# =============================================================================
|
||||
# sicabe-control-panel — deploy panel (Next.js) via Docker Compose.
|
||||
#
|
||||
# Build image dulu: ../../scripts/build-image.sh
|
||||
# Lalu jalankan : docker compose up -d
|
||||
#
|
||||
# Persisten: volume bernama `panel-data` → `data/config.json` (hosts/modules/
|
||||
# services/service types) + `data/state.json`. Backup/restore dari UI
|
||||
# (Settings → Backup) tanpa menyentuh volume.
|
||||
#
|
||||
# PENTING: panel di container TIDAK bisa membaca filesystem platform → setiap
|
||||
# host di SICABE_HOSTS WAJIB punya `configUrl` (config-agent).
|
||||
#
|
||||
# NETWORK: default `network_mode: host` karena gateway host
|
||||
# (deploy/host-gateway) membuka `127.0.0.1:22375` — container di bridge TIDAK
|
||||
# bisa menjangkaunya via host.docker.internal. Dengan host network, `127.0.0.1`
|
||||
# berfungsi seperti di host (dan SAN sertifikat Trino `localhost` cocok).
|
||||
# Lihat README.md untuk varian bridge.
|
||||
# =============================================================================
|
||||
|
||||
name: sicabe-control-panel
|
||||
|
||||
services:
|
||||
panel:
|
||||
image: sicabe/control-panel:${PANEL_TAG:-latest}
|
||||
# Biarkan `build` aktif bila ingin `docker compose up --build`.
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: Dockerfile
|
||||
container_name: sicabe-control-panel
|
||||
network_mode: host
|
||||
# `.env` dibuat dari env.example (cp env.example .env). `required: false`
|
||||
# agar `docker compose config` tetap valid sebelum file dibuat.
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
environment:
|
||||
NODE_ENV: production
|
||||
HOSTNAME: "0.0.0.0"
|
||||
# Host network: port binding ditentukan PORT (bukan `ports:`).
|
||||
PORT: "${PANEL_PORT:-3000}"
|
||||
volumes:
|
||||
- panel-data:/app/data
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
# ── Varian BRIDGE (bila gateway dibuka ke alamat non-loopback) ────────────
|
||||
# Hapus `network_mode: host` di atas, lalu tambahkan:
|
||||
# ports:
|
||||
# - "${PANEL_PORT:-3000}:3000"
|
||||
# extra_hosts:
|
||||
# - "host.docker.internal:host-gateway"
|
||||
# dan set SICABE_HOSTS/URL memakai host.docker.internal (perhatikan SAN TLS
|
||||
# Trino — host.docker.internal TIDAK ada di SAN).
|
||||
|
||||
volumes:
|
||||
panel-data:
|
||||
@@ -0,0 +1,70 @@
|
||||
# =============================================================================
|
||||
# Env untuk panel yang berjalan sebagai CONTAINER.
|
||||
# Salin ke .env LALU ISI NILAINYA (jangan commit .env):
|
||||
# cp deploy/panel/env.example deploy/panel/.env
|
||||
#
|
||||
# Beda penting dari .env dev (panel di host):
|
||||
# - Panel di container TIDAK bisa membaca filesystem platform → WAJIB pakai
|
||||
# config-agent (`configUrl`) di SICABE_HOSTS.
|
||||
# - Alamat ke service di HOST memakai `host.docker.internal` (atau
|
||||
# `network_mode: host`, lihat docker-compose.yml).
|
||||
# =============================================================================
|
||||
|
||||
# ── Wajib ────────────────────────────────────────────────────────────────────
|
||||
# URL publik panel — HARUS sama dengan yang diakses browser.
|
||||
NEXTAUTH_URL=http://localhost:3000
|
||||
# Rahasia NextAuth (WAJIB di produksi). Buat: openssl rand -base64 32
|
||||
NEXTAUTH_SECRET=
|
||||
|
||||
# ── Login lokal (hybrid dengan SSO) — OPSIONAL ───────────────────────────────
|
||||
#PANEL_LOCAL_USER=admin
|
||||
#PANEL_LOCAL_PASSWORD_HASH=$2b$10$...
|
||||
#PANEL_LOCAL_PASSWORD_HASH_B64=
|
||||
#PANEL_LOCAL_ROLE=data-admin
|
||||
#TRINO_SERVICE_ACCOUNT=svc_dbt
|
||||
|
||||
# ── Keycloak (SSO & login dev ROPC) ──────────────────────────────────────────
|
||||
# Dari container, Keycloak di host diakses via host.docker.internal.
|
||||
KEYCLOAK_URL=http://host.docker.internal:12080
|
||||
KEYCLOAK_CLIENT_ID=sicabe-control-panel
|
||||
KEYCLOAK_CLIENT_SECRET=dev-keycloak-secret
|
||||
# URL yang dilihat browser (tautan Keycloak).
|
||||
KEYCLOAK_WEB_URL=http://localhost:12080
|
||||
#KEYCLOAK_REALM=sicabe
|
||||
|
||||
# ── Host/agent yang dikelola (WAJIB configUrl) ───────────────────────────────
|
||||
# Gateway per host (nginx: docker-proxy + config-agent) jalan di HOST:22375.
|
||||
# Dari container → host.docker.internal:22375. `publicHost` = alamat yang
|
||||
# dilihat BROWSER (untuk tautan UI), mis. IP LAN host.
|
||||
SICABE_HOSTS='[{"name":"local-gw","proxyUrl":"http://host.docker.internal:22375","apiKey":"<gateway-key>","configUrl":"http://host.docker.internal:22375","configApiKey":"<gateway-key>","publicHost":"localhost","enabled":true}]'
|
||||
|
||||
# ── Service (fallback; registry hasil scan diutamakan) ───────────────────────
|
||||
# CATATAN TLS Trino: sertifikat Trino (self-signed) punya SAN
|
||||
# `localhost / trino / sicabe-trino / trino-coordinator / 127.0.0.1`. Dari
|
||||
# container, `host.docker.internal` TIDAK ada di SAN → verifikasi TLS gagal.
|
||||
# Pilihan:
|
||||
# (a) pakai `network_mode: host` (lihat docker-compose.yml) → 127.0.0.1 cocok;
|
||||
# (b) tambah alias di extra_hosts yang ADA di SAN (mis. `sicabe-trino`), lalu
|
||||
# set TRINO_URL=https://sicabe-trino:12443
|
||||
TRINO_URL=https://host.docker.internal:12443
|
||||
TRINO_WEB_URL=http://localhost:12443
|
||||
# CA diambil otomatis via config-agent (`/ca`) → tak perlu file lokal.
|
||||
|
||||
NESSIE_URL=http://host.docker.internal:12120
|
||||
|
||||
AIRFLOW_URL=http://host.docker.internal:12082
|
||||
STARROCKS_HOST=host.docker.internal
|
||||
STARROCKS_PORT=12930
|
||||
STARROCKS_WEB_URL=http://localhost:12030
|
||||
|
||||
OPENMETADATA_URL=http://host.docker.internal:12085
|
||||
GRAFANA_URL=http://host.docker.internal:12300
|
||||
MINIO_CONSOLE_URL=http://localhost:12901
|
||||
|
||||
# Dremio (bila ada)
|
||||
#DREMIO_URL=http://host.docker.internal:12947
|
||||
|
||||
# ── Runtime state (di dalam volume panel-data) ───────────────────────────────
|
||||
SICABE_CONFIG_FILE=data/config.json
|
||||
SICABE_STATE_FILE=data/state.json
|
||||
SICABE_COMPOSE_PROJECTS=sicabe
|
||||
@@ -0,0 +1,84 @@
|
||||
# Deploy: stack `sicabe-host` (tanpa build)
|
||||
|
||||
Satu compose untuk satu host SiCABE, memuat empat service:
|
||||
|
||||
| Service | Image | Profile | Di host mana |
|
||||
| --- | --- | --- | --- |
|
||||
| `docker-socket-proxy` | `tecnativa/docker-socket-proxy` | – | **semua** host |
|
||||
| `config-agent` | `<registry>/config-agent` | – | **semua** host |
|
||||
| `gateway` (nginx) | `nginx:alpine` | – | **semua** host |
|
||||
| `sicabe-control-panel` | `<registry>/control-panel` | `panel` | **satu** host (panel) |
|
||||
|
||||
Compose di sini **tidak membangun image** — semua diambil dari registry. Untuk
|
||||
build & push image ke registry privat, pakai `deploy/panel/` + `scripts/build-image.sh`.
|
||||
|
||||
## 1. Build & push image (sekali, dari mesin dev)
|
||||
|
||||
```bash
|
||||
./scripts/build-image.sh --all --push --registry registry.example.com/sicabe
|
||||
# → registry.example.com/sicabe/control-panel:<git-sha> + :latest
|
||||
# → registry.example.com/sicabe/config-agent:<git-sha> + :latest
|
||||
```
|
||||
|
||||
`--all` membangun **panel + config-agent** dengan tag yang sama (agar satu
|
||||
`SICABE_TAG` cukup di stack). Login registry dulu bila perlu: `docker login <registry>`.
|
||||
|
||||
## 2. Siapkan env (per host)
|
||||
|
||||
```bash
|
||||
cd deploy/stack
|
||||
cp env.example .env # isi GATEWAY_API_KEY, SICABE_REGISTRY, SICABE_DEPLOY_DIR
|
||||
```
|
||||
|
||||
Di host **panel**, isi juga bagian bawah (`PANEL_URL`, `NEXTAUTH_SECRET`,
|
||||
`KEYCLOAK_CLIENT_SECRET`, `SICABE_HOSTS`, dst).
|
||||
|
||||
## 3. Jalankan
|
||||
|
||||
```bash
|
||||
# SEMUA host SiCABE (docker-proxy + config-agent + gateway):
|
||||
docker compose up -d
|
||||
|
||||
# HANYA di host panel — sekaligus menjalankan panelnya:
|
||||
docker compose --profile panel up -d
|
||||
```
|
||||
|
||||
Cek: `docker compose ps` dan `docker compose logs -f gateway`.
|
||||
|
||||
## Update image
|
||||
|
||||
```bash
|
||||
# setelah build+push tag baru:
|
||||
SICABE_TAG=<tag-baru> docker compose pull # (di host panel: + --profile panel)
|
||||
docker compose up -d # recreate
|
||||
```
|
||||
|
||||
## Menghentikan / menghapus
|
||||
|
||||
`docker compose down` hanya menghapus service **pada profile yang aktif**. Karena
|
||||
itu, di host panel jalankan dengan profile yang sama:
|
||||
|
||||
```bash
|
||||
# host panel:
|
||||
docker compose --profile panel down # hapus SEMUA (termasuk panel)
|
||||
|
||||
# host non-panel:
|
||||
docker compose down
|
||||
```
|
||||
|
||||
> Kalau `down` dijalankan tanpa `--profile panel` di host panel, container
|
||||
> `sicabe-control-panel` akan tertinggal sebagai *orphan*. Compose akan
|
||||
> memperingatkan; hapus dengan `--profile panel down` atau `docker rm -f`.
|
||||
|
||||
## Catatan
|
||||
|
||||
- **Panel memakai `network_mode: host`** karena gateway bind `127.0.0.1` (tak
|
||||
terjangkau dari bridge Docker; sekaligus SAN sertifikat Trino `localhost`
|
||||
cocok). Port panel = `PANEL_PORT` (default 3000).
|
||||
- **Template nginx dipakai bersama** dari `../host-gateway/nginx/` (tanpa
|
||||
duplikasi). Setelah mengubah template, jalankan `docker compose restart gateway`
|
||||
(nginx men-substitute template saat start).
|
||||
- **Menambah host baru**: jalankan `docker compose up -d` (tanpa profile) di host
|
||||
itu, lalu daftarkan di panel (**Settings → Hosts & Agents**) dan **Scan**.
|
||||
- Compose ini **menggantikan** `deploy/host-gateway/` + `deploy/panel/` sebagai
|
||||
artefak deploy. Keduanya tetap ada untuk **build lokal**/dev.
|
||||
@@ -0,0 +1,178 @@
|
||||
# =============================================================================
|
||||
# sicabe-host — SATU compose untuk satu host SiCABE.
|
||||
#
|
||||
# Berisi: docker-socket-proxy + config-agent + gateway(nginx) + control-panel.
|
||||
#
|
||||
# PROFILE:
|
||||
# - tanpa profile → docker-socket-proxy + config-agent + gateway
|
||||
# (WAJIB di SEMUA host SiCABE)
|
||||
# - profile panel → + sicabe-control-panel
|
||||
# (hanya di SATU host: host panel)
|
||||
#
|
||||
# Pemakaian:
|
||||
# # di SEMUA host SiCABE:
|
||||
# docker compose up -d
|
||||
#
|
||||
# # di HOST PANEL (selain di atas):
|
||||
# docker compose --profile panel up -d
|
||||
#
|
||||
# Env: salin `env.example` → `.env` (compose membacanya otomatis).
|
||||
# Image TIDAK di-build di sini — pakai image dari registry:
|
||||
# SICABE_REGISTRY=<registry> SICABE_TAG=<tag> (lihat scripts/build-image.sh --all --push)
|
||||
#
|
||||
# Lihat README.md.
|
||||
# =============================================================================
|
||||
|
||||
name: sicabe-host
|
||||
|
||||
services:
|
||||
# ── Docker Engine API (ACL ketat, internal saja) ──────────────────────────
|
||||
docker-proxy:
|
||||
image: tecnativa/docker-socket-proxy:v0.5.0
|
||||
container_name: sicabe-docker-proxy
|
||||
environment:
|
||||
CONTAINERS: "1"
|
||||
POST: "1"
|
||||
INFO: "1"
|
||||
SYSTEM: "1"
|
||||
EXEC: "0"
|
||||
SWARM: "0"
|
||||
SERVICES: "0"
|
||||
SECRETS: "0"
|
||||
CONFIGS: "0"
|
||||
NETWORKS: "0"
|
||||
VOLUMES: "0"
|
||||
IMAGES: "0"
|
||||
BUILD: "0"
|
||||
GRPC: "0"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
expose:
|
||||
- "2375"
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp
|
||||
- /run
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
# ── Konfigurasi platform per host (internal saja) ─────────────────────────
|
||||
config-agent:
|
||||
image: ${SICABE_REGISTRY:-sicabe}/config-agent:${SICABE_TAG:-latest}
|
||||
container_name: sicabe-config-agent
|
||||
# Jalan sebagai UID/GID host agar file yang ditulis (katalog Trino, data OPA)
|
||||
# tetap milik user host, bukan root.
|
||||
user: "${AGENT_UID:-1000}:${AGENT_GID:-1000}"
|
||||
environment:
|
||||
AGENT_API_KEY: ${GATEWAY_API_KEY:?GATEWAY_API_KEY wajib diisi (env)}
|
||||
AGENT_PORT: "8080"
|
||||
PYTHONDONTWRITEBYTECODE: "1"
|
||||
SICABE_ENV_FILE: /config/.env
|
||||
TRINO_CATALOG_DIR: /config/config/trino/catalog
|
||||
DBT_PROJECT_DIR: /config/config/airflow/dbt/sicabe
|
||||
OPA_DATA_FILE: /config/config/opa/data.json
|
||||
# Nama file PEM di dalam /ca (lihat AGENT_CA_HOST_PATH). Diperlukan agar
|
||||
# panel bisa memverifikasi TLS Trino (self-signed).
|
||||
AGENT_CA_FILE: ${AGENT_CA_FILE:-/ca/trino.crt}
|
||||
# Sumber daftar container untuk scan (docker-proxy internal).
|
||||
DOCKER_PROXY_URL: http://docker-proxy:2375
|
||||
volumes:
|
||||
- ${SICABE_DEPLOY_DIR:?set SICABE_DEPLOY_DIR (env)}:/config:ro
|
||||
- ${SICABE_DEPLOY_DIR:?}/config/trino/catalog:/config/config/trino/catalog:rw
|
||||
- ${SICABE_DEPLOY_DIR:?}/config/opa:/config/config/opa:rw
|
||||
# CA TLS (GET /ca): direktori berisi PEM → /ca. WAJIB diisi di host
|
||||
# panel agar TLS Trino (self-signed) bisa diverifikasi.
|
||||
- ${AGENT_CA_HOST_PATH:-${SICABE_DEPLOY_DIR:?}}:/ca:ro
|
||||
expose:
|
||||
- "8080"
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
# ── Gateway nginx: satu-satunya port ke host ──────────────────────────────
|
||||
gateway:
|
||||
image: nginx:1.30.4-alpine3.24
|
||||
container_name: sicabe-host-gateway
|
||||
depends_on:
|
||||
- docker-proxy
|
||||
- config-agent
|
||||
environment:
|
||||
GATEWAY_API_KEY: ${GATEWAY_API_KEY:?GATEWAY_API_KEY wajib diisi (env)}
|
||||
volumes:
|
||||
# Template nginx dipakai bersama dari deploy/host-gateway (tanpa duplikasi).
|
||||
- ../host-gateway/nginx/gateway.conf.template:/etc/nginx/templates/default.conf.template:ro
|
||||
ports:
|
||||
- "${GATEWAY_BIND_IP:-127.0.0.1}:${GATEWAY_PORT:-22375}:80"
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
# ── Control panel — HANYA di host panel (profile "panel") ────────────────
|
||||
panel:
|
||||
profiles: ["panel"]
|
||||
image: ${SICABE_REGISTRY:-sicabe}/control-panel:${SICABE_TAG:-latest}
|
||||
container_name: sicabe-control-panel
|
||||
depends_on:
|
||||
- gateway
|
||||
# Host network: gateway host bind 127.0.0.1 → tak terjangkau dari bridge,
|
||||
# dan SAN sertifikat Trino (localhost) cocok. Port via PORT.
|
||||
network_mode: host
|
||||
environment:
|
||||
NODE_ENV: production
|
||||
HOSTNAME: "0.0.0.0"
|
||||
PORT: ${PANEL_PORT:-3000}
|
||||
# ── Wajib (panel gagal-cepat bila kosong) ──
|
||||
NEXTAUTH_URL: ${PANEL_URL:-}
|
||||
NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:-}
|
||||
KEYCLOAK_CLIENT_SECRET: ${KEYCLOAK_CLIENT_SECRET:-}
|
||||
SICABE_HOSTS: ${SICABE_HOSTS:-}
|
||||
# ── Login lokal (hybrid, opsional) ──
|
||||
# Akun lokal berlaku juga di produksi. Pakai varian _B64 (bcrypt memuat
|
||||
# '$' yang bisa di-interpolasi Compose). Kosong = hanya SSO.
|
||||
PANEL_LOCAL_USER: ${PANEL_LOCAL_USER:-}
|
||||
PANEL_LOCAL_PASSWORD_HASH: ${PANEL_LOCAL_PASSWORD_HASH:-}
|
||||
PANEL_LOCAL_PASSWORD_HASH_B64: ${PANEL_LOCAL_PASSWORD_HASH_B64:-}
|
||||
PANEL_LOCAL_ROLE: ${PANEL_LOCAL_ROLE:-}
|
||||
# Identitas Trino untuk sesi lokal (tanpa JWT) — service account Basic.
|
||||
TRINO_SERVICE_ACCOUNT: ${TRINO_SERVICE_ACCOUNT:-}
|
||||
# ── Opsional (ada default di kode) ──
|
||||
KEYCLOAK_URL: ${KEYCLOAK_URL:-}
|
||||
KEYCLOAK_CLIENT_ID: ${KEYCLOAK_CLIENT_ID:-sicabe-control-panel}
|
||||
KEYCLOAK_WEB_URL: ${KEYCLOAK_WEB_URL:-}
|
||||
TRINO_URL: ${TRINO_URL:-}
|
||||
TRINO_WEB_URL: ${TRINO_WEB_URL:-}
|
||||
NESSIE_URL: ${NESSIE_URL:-}
|
||||
AIRFLOW_URL: ${AIRFLOW_URL:-}
|
||||
STARROCKS_HOST: ${STARROCKS_HOST:-}
|
||||
STARROCKS_PORT: ${STARROCKS_PORT:-}
|
||||
STARROCKS_WEB_URL: ${STARROCKS_WEB_URL:-}
|
||||
OPENMETADATA_URL: ${OPENMETADATA_URL:-}
|
||||
GRAFANA_URL: ${GRAFANA_URL:-}
|
||||
MINIO_CONSOLE_URL: ${MINIO_CONSOLE_URL:-}
|
||||
SICABE_COMPOSE_PROJECTS: ${SICABE_COMPOSE_PROJECTS:-sicabe}
|
||||
SICABE_CONFIG_FILE: data/config.json
|
||||
SICABE_STATE_FILE: data/state.json
|
||||
volumes:
|
||||
- panel-data:/app/data
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
volumes:
|
||||
panel-data:
|
||||
@@ -0,0 +1,77 @@
|
||||
# =============================================================================
|
||||
# Env untuk stack sicabe-host (satu file untuk SEMUA host).
|
||||
# Salin ke .env lalu isi nilainya: cp env.example .env
|
||||
#
|
||||
# Dipakai oleh: docker-socket-proxy + config-agent + gateway (semua host),
|
||||
# dan control-panel (hanya host panel / profile "panel").
|
||||
# =============================================================================
|
||||
|
||||
# ── Image dari registry (WAJIB) ──────────────────────────────────────────────
|
||||
# Hasil: ./scripts/build-image.sh --all --push --registry <registry>
|
||||
# Kosongkan SICABE_REGISTRY bila memakai image lokal (mis. saat uji).
|
||||
#SICABE_REGISTRY=registry.example.com/sicabe
|
||||
SICABE_TAG=latest
|
||||
|
||||
# ── Gateway (WAJIB, semua host) ──────────────────────────────────────────────
|
||||
# Satu API key untuk docker-proxy + config-agent (diperiksa nginx).
|
||||
GATEWAY_API_KEY=
|
||||
# Bind ke 127.0.0.1 (default, aman) atau IP internal. JANGAN ke internet.
|
||||
GATEWAY_BIND_IP=127.0.0.1
|
||||
GATEWAY_PORT=22375
|
||||
|
||||
# ── Config-agent (WAJIB, semua host) ─────────────────────────────────────────
|
||||
# Direktori deploy sicabe-doc di host ini (berisi .env, config/trino/catalog, ...).
|
||||
SICABE_DEPLOY_DIR=/path/ke/sicabe-doc/deploy
|
||||
# UID/GID host yang menulis file (katalog/OPA) — agar tidak root-owned.
|
||||
AGENT_UID=1000
|
||||
AGENT_GID=1000
|
||||
# CA TLS Trino — WAJIB di host panel agar panel memverifikasi sertifikat Trino
|
||||
# (self-signed). Isi direktori yang memuat PEM-nya (dihasilkan
|
||||
# ./scripts/export-trino-ca.sh → <panel>/.ca/trino.crt).
|
||||
# Di host NON-panel boleh dikosongkan (panel-lah yang butuh CA).
|
||||
AGENT_CA_HOST_PATH=/path/ke/sicabe-control-panel/.ca
|
||||
AGENT_CA_FILE=/ca/trino.crt
|
||||
|
||||
# =============================================================================
|
||||
# Di bawah ini HANYA untuk host panel (profile "panel").
|
||||
# Boleh dibiarkan kosong di host non-panel (compose tetap valid).
|
||||
# =============================================================================
|
||||
|
||||
# ── Panel: wajib ─────────────────────────────────────────────────────────────
|
||||
# URL publik panel (harus sama dengan yang diakses browser & redirect URI Keycloak).
|
||||
PANEL_URL=http://localhost:3000
|
||||
PANEL_PORT=3000
|
||||
# openssl rand -base64 32
|
||||
NEXTAUTH_SECRET=
|
||||
|
||||
# ── Login lokal (hybrid dengan SSO) — OPSIONAL ───────────────────────────────
|
||||
# Akun lokal tanpa Keycloak; password = bcrypt hash. Pakai varian _B64 di sini
|
||||
# karena Compose meng-interpolasi '$' (bcrypt selalu memuat '$').
|
||||
# node -e "console.log(Buffer.from(require('bcryptjs').hashSync('PASSWORD',10)).toString('base64'))"
|
||||
#PANEL_LOCAL_USER=admin
|
||||
#PANEL_LOCAL_PASSWORD_HASH_B64=
|
||||
#PANEL_LOCAL_ROLE=data-admin
|
||||
# Identitas Trino utk sesi lokal (tanpa JWT) — service account (Basic).
|
||||
#TRINO_SERVICE_ACCOUNT=svc_dbt
|
||||
# Client Keycloak panel (scripts/create-keycloak-client.sh).
|
||||
KEYCLOAK_CLIENT_SECRET=dev-keycloak-secret
|
||||
# Host/agent yang dikelola panel. WAJIB ada `configUrl` (config-agent).
|
||||
# Di host panel (network host) gateway lokal = 127.0.0.1:${GATEWAY_PORT}.
|
||||
# Format JSON tanpa kutip pembungkus (hindari masalah parsing .env).
|
||||
SICABE_HOSTS=[{"name":"local-gw","proxyUrl":"http://127.0.0.1:22375","apiKey":"CHANGE_ME","configUrl":"http://127.0.0.1:22375","configApiKey":"CHANGE_ME","publicHost":"localhost","enabled":true}]
|
||||
|
||||
# ── Panel: opsional (ada default di kode) ────────────────────────────────────
|
||||
KEYCLOAK_URL=http://127.0.0.1:12080
|
||||
KEYCLOAK_CLIENT_ID=sicabe-control-panel
|
||||
KEYCLOAK_WEB_URL=http://localhost:12080
|
||||
TRINO_URL=https://localhost:12443
|
||||
TRINO_WEB_URL=https://localhost:12443
|
||||
NESSIE_URL=http://127.0.0.1:12120
|
||||
AIRFLOW_URL=http://127.0.0.1:12082
|
||||
STARROCKS_HOST=127.0.0.1
|
||||
STARROCKS_PORT=12930
|
||||
STARROCKS_WEB_URL=http://localhost:12030
|
||||
OPENMETADATA_URL=http://127.0.0.1:12085
|
||||
GRAFANA_URL=http://127.0.0.1:12300
|
||||
MINIO_CONSOLE_URL=http://localhost:12901
|
||||
SICABE_COMPOSE_PROJECTS=sicabe
|
||||
Reference in New Issue
Block a user