first commit
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
# =============================================================================
|
||||
# Apache Airflow — Configuration (Airflow 3.3.x)
|
||||
# Referensi arsitektur: bab 4.5.1 (Apache Airflow — Orchestrator)
|
||||
# =============================================================================
|
||||
# ⚠️ Airflow 3.3 memakai SimpleAuthManager (bukan FabAuthManager).
|
||||
# Config [api] auth_backends dan [webserver] auth_backend dari Airflow 2.x
|
||||
# TIDAK berlaku lagi — dihapus agar task runner bisa autentikasi internal.
|
||||
# =============================================================================
|
||||
|
||||
[core]
|
||||
executor = LocalExecutor
|
||||
simple_auth_manager_users = admin:Admin
|
||||
simple_auth_manager_passwords_file = /opt/airflow/simple_auth_manager_passwords.json
|
||||
parallelism = 32
|
||||
max_active_tasks_per_dag = 16
|
||||
dagbag_import_timeout = 30
|
||||
load_examples = False
|
||||
default_timezone = Asia/Jakarta
|
||||
|
||||
[database]
|
||||
# ⚠️ ConfigParser does NOT expand ${VAR} syntax — this is a placeholder only.
|
||||
# The actual connection string is set via AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
|
||||
# environment variable in docker-compose.yml.
|
||||
sql_alchemy_conn = postgresql+psycopg2://airflow:PLACEHOLDER@postgres:5432/airflow
|
||||
sql_alchemy_pool_enabled = True
|
||||
sql_alchemy_pool_size = 10
|
||||
sql_alchemy_max_overflow = 20
|
||||
|
||||
[webserver]
|
||||
# base_url internal (container) — task runner connect ke sini
|
||||
base_url = http://airflow-webserver:8080
|
||||
default_ui_timezone = Asia/Jakarta
|
||||
expose_config = True
|
||||
|
||||
[scheduler]
|
||||
parsing_processes = 4
|
||||
max_threads = 4
|
||||
scheduler_heartbeat_sec = 5
|
||||
|
||||
[dag_processor]
|
||||
# Airflow 3.x: dag_dir_list_interval dipindah dari [scheduler] ke sini
|
||||
# (menghindari DeprecationWarning saat start).
|
||||
refresh_interval = 30
|
||||
|
||||
[logging]
|
||||
remote_logging = False
|
||||
log_level = INFO
|
||||
|
||||
[email]
|
||||
email_backend = airflow.utils.email.send_email_smtp
|
||||
|
||||
[smtp]
|
||||
smtp_host = localhost
|
||||
smtp_port = 25
|
||||
smtp_timeout = 30
|
||||
smtp_mail_from = airflow@sicabe.local
|
||||
@@ -0,0 +1,96 @@
|
||||
"""sitomat — DAG-3 (gold build) — TAHAP A: CONNECTIVITY SMOKE DAG (Airflow 3.x).
|
||||
|
||||
HONESTY (baca dulu):
|
||||
DAG ini HANYA memvalidasi konektivitas sitomat <-> Airflow (validate DAG,
|
||||
trigger run, poll status). Ia TIDAK membangun gold dan **TIDAK** memanggil
|
||||
callback `register_gold` — jadi **tidak ada artefak gold yang diklaim**.
|
||||
|
||||
Tahap B akan menggantinya dengan transform nyata: baca silver (via Trino),
|
||||
tulis gold, lalu callback ke sitomat. Lihat README.md di folder yang sama.
|
||||
|
||||
Airflow 3.x: dataclass import publik adalah `airflow.sdk` (bukan
|
||||
`airflow.decorators`/`airflow.models`), dan `schedule_interval` sudah dihapus
|
||||
-> gunakan `schedule=None` untuk DAG yang hanya dipicu lewat REST.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import pendulum
|
||||
from airflow.sdk import dag, get_current_context, task
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Kunci conf yang WAJIB dikirim sitomat saat trigger (lihat F16 `flattenTriggerConf`).
|
||||
REQUIRED_CONF_KEYS = (
|
||||
"correlationId",
|
||||
"dataProductBuildId",
|
||||
"dataProductId",
|
||||
"useCaseId",
|
||||
)
|
||||
|
||||
|
||||
@dag(
|
||||
dag_id="sitomat_gold_build", # samakan dengan env `GOLD_DAG_ID` di sitomat
|
||||
description="sitomat F16 DAG-3 — connectivity smoke (tidak membangun gold)",
|
||||
schedule=None, # Airflow 3: hanya dipicu lewat REST (POST /api/v2/.../dagRuns)
|
||||
start_date=pendulum.datetime(2026, 1, 1, tz="UTC"),
|
||||
catchup=False,
|
||||
tags=["sitomat", "f16", "connectivity"],
|
||||
)
|
||||
def sitomat_gold_build():
|
||||
@task
|
||||
def parse_conf() -> dict:
|
||||
"""Baca `dag_run.conf` dari sitomat dan validasi kunci wajib."""
|
||||
ctx = get_current_context()
|
||||
conf = dict((ctx["dag_run"].conf or {})) if ctx else {}
|
||||
|
||||
missing = [key for key in REQUIRED_CONF_KEYS if not conf.get(key)]
|
||||
if missing:
|
||||
# Gagal jujur (BUKAN sukses palsu): conf tidak lengkap.
|
||||
raise ValueError(f"conf sitomat tidak lengkap, hilang: {missing}")
|
||||
|
||||
silver_refs = [ref for ref in str(conf.get("silverRefs", "")).split(",") if ref]
|
||||
log.info(
|
||||
"sitomat conf OK: correlationId=%s build=%s product=%s useCase=%s "
|
||||
"sources=%s silverRefs=%d",
|
||||
conf.get("correlationId"),
|
||||
conf.get("dataProductBuildId"),
|
||||
conf.get("dataProductId"),
|
||||
conf.get("useCaseId"),
|
||||
conf.get("sourceCount"),
|
||||
len(silver_refs),
|
||||
)
|
||||
if not silver_refs:
|
||||
log.warning(
|
||||
"silverRefs kosong — Tahap B (build nyata) belum bisa membaca sumber."
|
||||
)
|
||||
return {
|
||||
"correlationId": conf.get("correlationId"),
|
||||
"dataProductBuildId": conf.get("dataProductBuildId"),
|
||||
"silverRefs": silver_refs,
|
||||
}
|
||||
|
||||
@task
|
||||
def connectivity_report(payload: dict) -> None:
|
||||
"""TAHAP A: sengaja hanya melaporkan. TIDAK menulis gold.
|
||||
|
||||
Tahap B (ganti task ini):
|
||||
1. baca `payload["silverRefs"]` via Trino (koneksi `trino_default`),
|
||||
2. rakit & tulis tabel gold (Iceberg/Nessie),
|
||||
3. POST ke sitomat:
|
||||
{SITOMAT_BASE_URL}/data-products/builds/{dataProductBuildId}/gold
|
||||
header: `X-API-Key: <kunci scope pipeline>`
|
||||
body: {"goldRef": ..., "checksum": ..., "rowCount": ...}
|
||||
Simpan URL & kunci sebagai Airflow Variable/Connection (JANGAN hardcode).
|
||||
"""
|
||||
log.info(
|
||||
"sitomat DAG-3 connectivity OK untuk build=%s (Tahap A: tanpa gold)",
|
||||
payload.get("dataProductBuildId"),
|
||||
)
|
||||
|
||||
connectivity_report(parse_conf())
|
||||
|
||||
|
||||
sitomat_gold_build()
|
||||
@@ -0,0 +1,344 @@
|
||||
"""
|
||||
DAG-3 PRODUKSI — bangun GOLD untuk sebuah Data Product, lalu lapor ke sitomat.
|
||||
|
||||
Fitur F16/F15 (ADR-0004..0008). sitomat TIDAK memindahkan byte: DAG ini yang
|
||||
membangun artefak di data lake (Iceberg lewat Trino) dan memanggil callback.
|
||||
|
||||
Kontrak
|
||||
-------
|
||||
Trigger conf (dikirim F15 `createGoldBuildTriggerPort`; SEMUA nilai STRING):
|
||||
correlationId, dataProductBuildId, dataProductId, useCaseId,
|
||||
datasetVersionIds (dipisah koma), silverRefs (dipisah koma; posisinya
|
||||
sejajar dengan datasetVersionIds, slot kosong = sumber tanpa ref),
|
||||
sourceCount
|
||||
|
||||
Callback WAJIB (tanpa ini build sitomat tetap 'building'):
|
||||
POST {SITOMAT_BASE_URL}/data-products/builds/{dataProductBuildId}/gold
|
||||
Header: X-API-Key: <kunci F09-A scope `pipeline`>
|
||||
Body: {"goldRef": "...", "checksum": "sha256:...", "rowCount": N}
|
||||
|
||||
Konfigurasi — Airflow Variable dulu, lalu env, lalu default:
|
||||
sitomat_base_url SITOMAT_BASE_URL http://172.27.228.103:13000
|
||||
sitomat_api_key SITOMAT_API_KEY (WAJIB) kunci scope `pipeline`
|
||||
trino_url TRINO_URL https://sicabe-trino:8443
|
||||
trino_user TRINO_USER svc_seed
|
||||
trino_password TRINO_PASSWORD (WAJIB)
|
||||
gold_target GOLD_TARGET iceberg.gold.dp_<dataProductId>
|
||||
gold_sql GOLD_SQL opsional; bila kosong dirakit dari silverRefs
|
||||
gold_tls_verify GOLD_TLS_VERIFY false (sertifikat internal self-signed)
|
||||
checksum_max_rows CHECKSUM_MAX_ROWS 100000 (0 = tanpa batas)
|
||||
|
||||
**Presedensi:** `silverRefs` dari conf adalah jalur NYATA dan menang; `gold_sql`
|
||||
= **fallback** untuk dataset yang belum menyimpan `artifact_ref`.
|
||||
|
||||
**Kegagalan DILAPORKAN, bukan menggantung:** build NYATA yang sumbernya belum
|
||||
punya `artifact_ref` (atau tanpa sumber sama sekali) **GAGAL dengan sengaja** —
|
||||
DAG menolak membangun produk parsial dan task `report_failure` memanggil
|
||||
`POST …/fail`, sehingga build/produk/use case menjadi `failed`. Hanya run
|
||||
DIAGNOSTIK (`bun run check:airflow`) yang berhenti diam tanpa callback.
|
||||
|
||||
Cakupan OPA: stack ini memakai allowlist per-tabel (`default allow := false`).
|
||||
Menulis ke tabel gold BARU mensyaratkan policy OPA memberi grup `data-engineer`
|
||||
akses pada schema gold/silver (lihat README). Bila belum, set `GOLD_TARGET` ke
|
||||
tabel yang sudah diizinkan (mis. `iceberg.gold.orders_gold`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
import requests
|
||||
import urllib3
|
||||
from airflow.sdk import dag, get_current_context, task
|
||||
|
||||
# Sertifikat internal self-signed: jangan banjiri log dengan peringatan.
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
DEFAULT_BASE_URL = "http://172.27.228.103:13000"
|
||||
DEFAULT_TRINO_URL = "https://sicabe-trino:8443"
|
||||
DEFAULT_TRINO_USER = "svc_seed"
|
||||
DEFAULT_CHECKSUM_MAX_ROWS = 100000
|
||||
|
||||
# Nama identifier SQL yang aman; apa pun di luar ini DITOLAK (anti-injeksi).
|
||||
IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
|
||||
|
||||
# ── Pengaturan (Airflow Variable → env → default) ────────────────────────────
|
||||
|
||||
|
||||
def _variable_get(key: str) -> str | None:
|
||||
"""Baca Airflow Variable `key`; `None` bila tidak ada / tidak tersedia."""
|
||||
Variable = None
|
||||
try:
|
||||
from airflow.sdk import Variable # Airflow 3 SDK
|
||||
except Exception: # pragma: no cover - tergantung versi Airflow
|
||||
try:
|
||||
from airflow.models import Variable # fallback klasik
|
||||
except Exception:
|
||||
return None
|
||||
try:
|
||||
return Variable.get(key)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def setting(name: str, default: str | None = None) -> str | None:
|
||||
"""Airflow Variable `name` (huruf kecil) → env `NAME` (huruf besar) → default."""
|
||||
value = os.environ.get(name.upper())
|
||||
if value:
|
||||
return value
|
||||
variable = _variable_get(name.lower())
|
||||
return variable if variable else default
|
||||
|
||||
|
||||
# ── Util murni ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def qualified(identifier: str) -> str:
|
||||
"""Validasi `tabel` / `schema.tabel` / `catalog.schema.tabel`. Raise bila bukan."""
|
||||
raw = identifier.strip().strip('"').strip("`")
|
||||
parts = [part.strip().strip('"').strip("`") for part in raw.split(".")]
|
||||
if not 1 <= len(parts) <= 3 or not all(IDENTIFIER.match(part) for part in parts):
|
||||
raise ValueError(f"nama tabel tidak valid: {identifier!r}")
|
||||
return ".".join(parts)
|
||||
|
||||
|
||||
def safe_suffix(value: str) -> str:
|
||||
"""Jadikan nilai bebas menjadi sufiks identifier yang aman."""
|
||||
cleaned = re.sub(r"[^A-Za-z0-9_]", "_", value or "x")
|
||||
if not cleaned or cleaned[0].isdigit():
|
||||
cleaned = f"t_{cleaned}"
|
||||
return cleaned
|
||||
|
||||
|
||||
def _as_int(value) -> int:
|
||||
"""Integer non-negatif dari conf (semua nilai conf adalah STRING)."""
|
||||
try:
|
||||
return max(0, int(str(value).strip()))
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def target_and_sql(conf: dict, sources: list[str]) -> tuple[str, str]:
|
||||
"""(target_fqn, sql_ctas) dari conf + pengaturan.
|
||||
|
||||
`GOLD_SQL` (Variable `gold_sql`) adalah escape hatch yang disengaja: dipakai
|
||||
bila `silverRefs` kosong (mis. versi silver lama belum menyimpan `artifact_ref`)
|
||||
ATAU untuk mengganti transform. Tanpa keduanya, tidak ada yang bisa dibangun.
|
||||
"""
|
||||
configured = setting("gold_target")
|
||||
target = qualified(
|
||||
configured
|
||||
or f"iceberg.gold.dp_{safe_suffix(str(conf.get('dataProductId', 'x')))}"
|
||||
)
|
||||
explicit = setting("gold_sql")
|
||||
# PRESEDENSI: sumber dari conf (`silverRefs`) adalah jalur NYATA dan menang;
|
||||
# `gold_sql` hanya FALLBACK operator untuk dataset yang belum punya
|
||||
# `artifact_ref`. (Kebalikannya membuat override global menelan lineage.)
|
||||
if sources:
|
||||
select = " UNION ALL ".join(f"SELECT * FROM {source}" for source in sources)
|
||||
elif explicit:
|
||||
select = explicit.strip().rstrip(";")
|
||||
else:
|
||||
raise ValueError("tidak ada silverRefs dan tidak ada gold_sql — tidak ada yang dibangun")
|
||||
return target, f"CREATE OR REPLACE TABLE {target} AS {select}"
|
||||
|
||||
|
||||
# ── Klien Trino REST minimal (/v1/statement + paging nextUri) ─────────────────
|
||||
|
||||
|
||||
class Trino:
|
||||
"""Klien Trino HTTPS berbatas; tidak ada dependensi di luar `requests`."""
|
||||
|
||||
def __init__(self, url: str, user: str, password: str | None, verify: bool) -> None:
|
||||
self.url = url.rstrip("/")
|
||||
self.session = requests.Session()
|
||||
self.session.verify = verify
|
||||
self.session.auth = (user, password)
|
||||
self.session.headers.update({"X-Trino-User": user, "Content-Type": "text/plain"})
|
||||
|
||||
def pages(self, sql: str):
|
||||
"""Iterasi (columns, rows) per halaman; raise bila Trino melaporkan error."""
|
||||
response = self.session.post(f"{self.url}/v1/statement", data=sql, timeout=300)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
for _ in range(10_000): # pengaman halaman, bukan batas logis
|
||||
error = payload.get("error")
|
||||
if error:
|
||||
raise RuntimeError(f"trino error: {error.get('message', error)}")
|
||||
columns = [col.get("name") for col in payload.get("columns", [])]
|
||||
yield columns, payload.get("data", [])
|
||||
next_uri = payload.get("nextUri")
|
||||
if not next_uri:
|
||||
return
|
||||
payload = self.session.get(next_uri, timeout=300).json()
|
||||
raise RuntimeError("trino: terlalu banyak halaman tanpa akhir")
|
||||
|
||||
def execute(self, sql: str) -> None:
|
||||
"""Jalankan statement tanpa peduli baris (mis. CTAS)."""
|
||||
for _ in self.pages(sql):
|
||||
pass
|
||||
|
||||
def scalar(self, sql: str):
|
||||
"""Nilai pertama baris pertama."""
|
||||
for _columns, rows in self.pages(sql):
|
||||
if rows:
|
||||
return rows[0][0]
|
||||
return None
|
||||
|
||||
def checksum(self, table: str, max_rows: int) -> str:
|
||||
"""sha256 deterministik atas header + baris (JSON ringkas), opsional dibatasi."""
|
||||
digest = hashlib.sha256()
|
||||
header_written = False
|
||||
seen = 0
|
||||
for columns, rows in self.pages(f"SELECT * FROM {table}"):
|
||||
if not header_written and columns:
|
||||
digest.update((",".join(columns) + "\n").encode("utf-8"))
|
||||
header_written = True
|
||||
for row in rows:
|
||||
if max_rows and seen >= max_rows:
|
||||
return f"sha256:{digest.hexdigest()} (bounded:{max_rows})"
|
||||
digest.update(
|
||||
(json.dumps(row, separators=(",", ":"), default=str) + "\n").encode("utf-8")
|
||||
)
|
||||
seen += 1
|
||||
return f"sha256:{digest.hexdigest()}"
|
||||
|
||||
|
||||
def trino_client() -> Trino:
|
||||
password = setting("trino_password")
|
||||
if not password:
|
||||
raise RuntimeError("trino_password belum diatur (Airflow Variable atau TRINO_PASSWORD)")
|
||||
return Trino(
|
||||
setting("trino_url", DEFAULT_TRINO_URL) or DEFAULT_TRINO_URL,
|
||||
setting("trino_user", DEFAULT_TRINO_USER) or DEFAULT_TRINO_USER,
|
||||
password,
|
||||
str(setting("gold_tls_verify", "false")).lower() in {"1", "true", "yes"},
|
||||
)
|
||||
|
||||
|
||||
# ── DAG ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dag(
|
||||
dag_id="sitomat_gold_build_prod",
|
||||
schedule=None,
|
||||
catchup=False,
|
||||
tags=["sitomat", "f16", "gold"],
|
||||
doc_md=__doc__,
|
||||
)
|
||||
def sitomat_gold_build_prod():
|
||||
@task
|
||||
def build_gold() -> dict:
|
||||
"""CTAS gold dari silverRefs; kembalikan identitas artefak."""
|
||||
conf = get_current_context()["dag_run"].conf or {}
|
||||
build_id = str(conf.get("dataProductBuildId", "")).strip()
|
||||
# Run DIAGNOSTIK (`bun run check:airflow`) tidak punya build nyata.
|
||||
diagnostic = not build_id or build_id == "airflow-check"
|
||||
|
||||
source_count = _as_int(conf.get("sourceCount"))
|
||||
sources = [
|
||||
qualified(raw)
|
||||
for raw in str(conf.get("silverRefs") or "").split(",")
|
||||
if raw.strip()
|
||||
]
|
||||
missing = max(0, source_count - len(sources))
|
||||
explicit = setting("gold_sql")
|
||||
|
||||
# Run diagnostik tanpa sumber: tidak ada yang dibangun, tidak ada callback.
|
||||
if diagnostic and not sources and not explicit:
|
||||
print("[sitomat] run diagnostik tanpa sumber — tidak membangun, tidak callback")
|
||||
return {"built": False}
|
||||
|
||||
# PRODUK PARSIAL DITOLAK. Sumber yang belum punya `artifact_ref` TIDAK
|
||||
# boleh diam-diam dihilangkan (itu menghasilkan gold yang salah). Untuk
|
||||
# build NYATA kita GAGAL di sini, sehingga task `report_failure`
|
||||
# melaporkannya ke sitomat — bukan menggantung `building` selamanya.
|
||||
if missing > 0 and not explicit:
|
||||
raise ValueError(
|
||||
f"build {build_id}: {missing} dari {source_count} sumber tidak punya "
|
||||
"artifact_ref — menolak membangun produk parsial"
|
||||
)
|
||||
|
||||
target, sql = target_and_sql(conf, sources)
|
||||
client = trino_client()
|
||||
print(f"[sitomat] membangun {target} dari {len(sources)} sumber")
|
||||
client.execute(sql)
|
||||
|
||||
row_count = int(client.scalar(f"SELECT count(*) FROM {target}") or 0)
|
||||
checksum = client.checksum(target, int(setting("checksum_max_rows", str(DEFAULT_CHECKSUM_MAX_ROWS))))
|
||||
print(f"[sitomat] selesai: {target} rows={row_count} checksum={checksum}")
|
||||
return {
|
||||
"built": True,
|
||||
"goldRef": target,
|
||||
"rowCount": row_count,
|
||||
"checksum": checksum,
|
||||
"buildId": str(conf.get("dataProductBuildId", "")),
|
||||
}
|
||||
|
||||
@task
|
||||
def register_gold(build: dict) -> None:
|
||||
"""Laporkan artefak ke sitomat (callback `register_gold`)."""
|
||||
if not build.get("built"):
|
||||
return
|
||||
base = (setting("sitomat_base_url", DEFAULT_BASE_URL) or DEFAULT_BASE_URL).rstrip("/")
|
||||
api_key = setting("sitomat_api_key")
|
||||
if not api_key:
|
||||
raise RuntimeError("sitomat_api_key belum diatur (Airflow Variable atau SITOMAT_API_KEY)")
|
||||
url = f"{base}/data-products/builds/{build['buildId']}/gold"
|
||||
response = requests.post(
|
||||
url,
|
||||
json={
|
||||
"goldRef": build["goldRef"],
|
||||
"checksum": build["checksum"],
|
||||
"rowCount": build["rowCount"],
|
||||
},
|
||||
headers={"X-API-Key": api_key},
|
||||
timeout=60,
|
||||
)
|
||||
if response.status_code >= 300:
|
||||
raise RuntimeError(
|
||||
f"callback sitomat gagal: HTTP {response.status_code} {response.text[:300]}"
|
||||
)
|
||||
print(f"[sitomat] callback OK ({response.status_code}): {response.text[:200]}")
|
||||
|
||||
@task(trigger_rule="one_failed")
|
||||
def report_failure() -> None:
|
||||
"""Laporkan kegagalan ke sitomat agar build TIDAK menggantung `building`.
|
||||
|
||||
Best-effort: kegagalan pelaporan tidak boleh menutupi error asli DAG.
|
||||
"""
|
||||
context = get_current_context()
|
||||
conf = context["dag_run"].conf or {}
|
||||
build_id = str(conf.get("dataProductBuildId", "")).strip()
|
||||
# Run diagnostik (`bun run check:airflow`) tidak punya build nyata.
|
||||
if not build_id or build_id == "airflow-check":
|
||||
print("[sitomat] tidak ada build nyata untuk dilaporkan (mode konektivitas)")
|
||||
return
|
||||
base = (setting("sitomat_base_url", DEFAULT_BASE_URL) or DEFAULT_BASE_URL).rstrip("/")
|
||||
api_key = setting("sitomat_api_key")
|
||||
if not api_key:
|
||||
print("[sitomat] sitomat_api_key tidak diatur — kegagalan TIDAK dilaporkan")
|
||||
return
|
||||
cause = f"DAG-3 sitomat_gold_build_prod failed (run {context['dag_run'].run_id})"
|
||||
try:
|
||||
response = requests.post(
|
||||
f"{base}/data-products/builds/{build_id}/fail",
|
||||
json={"cause": cause},
|
||||
headers={"X-API-Key": api_key},
|
||||
timeout=60,
|
||||
)
|
||||
print(f"[sitomat] failure callback -> HTTP {response.status_code}")
|
||||
except Exception as error: # noqa: BLE001 - never mask the original failure
|
||||
print(f"[sitomat] failure callback GAGAL (diabaikan): {error}")
|
||||
|
||||
built = build_gold()
|
||||
registered = register_gold(built)
|
||||
# `one_failed` + kedua upstream: lapor bila SALAH SATU task gagal.
|
||||
[built, registered] >> report_failure()
|
||||
|
||||
|
||||
sitomat_gold_build_prod()
|
||||
@@ -0,0 +1,213 @@
|
||||
"""
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Contoh DAG Pipeline End-to-End
|
||||
#
|
||||
# Referensi arsitektur:
|
||||
# bab 5.4 (Ingestion → Processing: Pipeline Integration)
|
||||
# bab 8.1 (Skenario End-to-End: Batch Harian)
|
||||
# bab 8.2 (Skenario End-to-End: CDC Real-Time)
|
||||
# =============================================================================
|
||||
|
||||
Alur:
|
||||
1. [Ingestion] Sling tarik data dari PostgreSQL → Bronze (Iceberg via Nessie)
|
||||
2. [Processing] dbt via Trino: Bronze → Silver → Gold
|
||||
3. [Governance] Great Expectations validasi output pipeline DI ICEBERG
|
||||
|
||||
Prerequisites (dibuat oleh bootstrap.sh):
|
||||
- Airflow connection: minio_s3 (object storage)
|
||||
- Sling env: ~/.sling/env.yaml (koneksi POSTGRES_DB + ICEBERG)
|
||||
- dbt project: /opt/airflow/dbt/sicabe/ (bind-mount dari config/airflow/dbt) dengan model bronze_to_silver, silver_to_gold, incremental
|
||||
- Tabel sumber: public.orders di database sicabe_data (PostgreSQL)
|
||||
- Kredensial Trino service account: env DBT_TRINO_USER / DBT_TRINO_PASSWORD
|
||||
|
||||
Catatan penting (jangan diubah tanpa alasan):
|
||||
- Task dbt memakai `env=DBT_TASK_ENV, append_env=True`. Tanpa `append_env=True`,
|
||||
`env=` MENGGANTI seluruh environment sehingga PATH & HOME hilang dan `dbt`
|
||||
(terpasang di ~/.local/bin) tidak ditemukan: "dbt: command not found".
|
||||
- Validator Great Expectations membaca tabel Iceberg LEWAT TRINO (HTTPS 8443,
|
||||
service account), bukan tabel sumber di PostgreSQL — supaya hasil "LULUS"
|
||||
benar-benar membuktikan output pipeline (bronze/gold), bukan data mentah.
|
||||
GE tidak punya datasource Trino → dipakai Pandas datasource dari hasil query.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
import os
|
||||
|
||||
from airflow import DAG
|
||||
from airflow.providers.standard.operators.bash import BashOperator
|
||||
from airflow.providers.standard.operators.python import PythonOperator
|
||||
|
||||
|
||||
def _run_ge_to_om(table):
|
||||
"""Jalankan validasi Great Expectations DAN kirim hasilnya ke OpenMetadata.
|
||||
|
||||
Runner adalah SATU sumber logika validasi (dipakai juga di luar DAG) dan
|
||||
memakai action resmi OpenMetadata untuk GE 1.x, sehingga hasilnya muncul
|
||||
sebagai Data Quality Test Suite/Test Case di OpenMetadata — TERMASUK ketika
|
||||
GAGAL. Sebelumnya hasil GE hanya berhenti di log Airflow.
|
||||
|
||||
Fail-fast tetap dipertahankan: exit code non-nol → task FAILED.
|
||||
"""
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
script = os.getenv(
|
||||
"GE_OM_RUNNER", "/opt/airflow/great_expectations/validate_to_openmetadata.py"
|
||||
)
|
||||
proc = subprocess.run(
|
||||
[sys.executable, script, "--table", table], capture_output=True, text=True
|
||||
)
|
||||
for stream in (proc.stdout, proc.stderr):
|
||||
if stream:
|
||||
print(stream[-2000:])
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"Data quality check GAGAL untuk {table} (hasil tetap dikirim ke OpenMetadata)"
|
||||
)
|
||||
|
||||
|
||||
def validate_gold_data():
|
||||
"""Validasi GOLD (output akhir pipeline) — `gold.orders_gold` → hasil ke OM."""
|
||||
_run_ge_to_om("gold.orders_gold")
|
||||
|
||||
|
||||
def validate_bronze_data():
|
||||
"""Validasi BRONZE (hasil ingestion Sling) — `bronze.orders` → hasil ke OM."""
|
||||
_run_ge_to_om("bronze.orders")
|
||||
|
||||
|
||||
# Env untuk task dbt: autentikasi Trino via SERVICE ACCOUNT (HTTP Basic) langsung
|
||||
# ke HTTPS 8443 (lihat config/airflow/dbt/profiles.yml). Password dari env
|
||||
# DBT_TRINO_PASSWORD (di-set docker-compose untuk service Airflow).
|
||||
DBT_TASK_ENV = {
|
||||
"DBT_TRINO_USER": os.getenv("DBT_TRINO_USER", "svc_dbt"),
|
||||
"DBT_TRINO_PASSWORD": os.getenv("DBT_TRINO_PASSWORD", ""),
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Default Arguments
|
||||
# =============================================================================
|
||||
default_args = {
|
||||
"owner": "engineer",
|
||||
"depends_on_past": False,
|
||||
"email_on_failure": True,
|
||||
"email_on_retry": False,
|
||||
"retries": 2,
|
||||
"retry_delay": timedelta(minutes=5),
|
||||
"execution_timeout": timedelta(hours=2),
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# DAG — Batch Harian (bab 8.1)
|
||||
# =============================================================================
|
||||
BATCH_DAG_ID = "sicabe_batch_daily"
|
||||
BATCH_SCHEDULE = "0 2 * * *" # Setiap hari jam 02:00
|
||||
|
||||
with DAG(
|
||||
dag_id=BATCH_DAG_ID,
|
||||
default_args=default_args,
|
||||
description="SiCABE: Batch harian — Ingestion → Processing → Governance",
|
||||
schedule=BATCH_SCHEDULE,
|
||||
start_date=datetime(2026, 1, 1),
|
||||
catchup=False,
|
||||
tags=["sicabe", "batch", "bronze", "silver", "gold"],
|
||||
) as dag:
|
||||
|
||||
# ─── Step 1: Ingestion — Tarik data ke Bronze (Iceberg) ──────────────
|
||||
ingest_to_bronze = BashOperator(
|
||||
task_id="ingest_to_bronze",
|
||||
bash_command=(
|
||||
'sling run --src-conn POSTGRES_DB '
|
||||
'--src-stream public.orders '
|
||||
'--tgt-conn ICEBERG '
|
||||
'--tgt-object bronze.orders '
|
||||
'--mode full-refresh '
|
||||
'--stdout'
|
||||
),
|
||||
)
|
||||
|
||||
# ─── Step 2: Processing — Transform Bronze → Silver ──────────────────
|
||||
bronze_to_silver = BashOperator(
|
||||
task_id="bronze_to_silver",
|
||||
bash_command=(
|
||||
'cd /opt/airflow/dbt/sicabe && '
|
||||
'export DBT_PROFILES_DIR=/opt/airflow/dbt && '
|
||||
'dbt run --select bronze_to_silver --profile trino --target prod'
|
||||
),
|
||||
env=DBT_TASK_ENV,
|
||||
append_env=True,
|
||||
)
|
||||
|
||||
# ─── Step 3: Processing — Transform Silver → Gold ────────────────────
|
||||
silver_to_gold = BashOperator(
|
||||
task_id="silver_to_gold",
|
||||
bash_command=(
|
||||
'cd /opt/airflow/dbt/sicabe && '
|
||||
'export DBT_PROFILES_DIR=/opt/airflow/dbt && '
|
||||
'dbt run --select silver_to_gold --profile trino --target prod'
|
||||
),
|
||||
env=DBT_TASK_ENV,
|
||||
append_env=True,
|
||||
)
|
||||
|
||||
# ─── Step 4: Governance — Validasi GOLD di Iceberg ───────────────────
|
||||
validate_gold = PythonOperator(
|
||||
task_id="validate_gold",
|
||||
python_callable=validate_gold_data,
|
||||
)
|
||||
|
||||
# ─── Dependency Chain ────────────────────────────────────────────────
|
||||
ingest_to_bronze >> bronze_to_silver >> silver_to_gold >> validate_gold
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# DAG — CDC Real-Time (bab 8.2)
|
||||
# =============================================================================
|
||||
CDC_DAG_ID = "sicabe_cdc_realtime"
|
||||
CDC_SCHEDULE = "*/15 * * * *" # Setiap 15 menit
|
||||
|
||||
with DAG(
|
||||
dag_id=CDC_DAG_ID,
|
||||
default_args=default_args,
|
||||
description="SiCABE: CDC real-time — Incremental ingestion + processing",
|
||||
schedule=CDC_SCHEDULE,
|
||||
start_date=datetime(2026, 1, 1),
|
||||
catchup=False,
|
||||
tags=["sicabe", "cdc", "incremental", "realtime"],
|
||||
) as dag:
|
||||
|
||||
# ─── Step 1: CDC Ingestion via Sling ─────────────────────────────────
|
||||
cdc_ingest = BashOperator(
|
||||
task_id="cdc_ingest",
|
||||
bash_command=(
|
||||
'sling run --src-conn POSTGRES_DB '
|
||||
'--src-stream public.orders '
|
||||
'--tgt-conn ICEBERG '
|
||||
'--tgt-object bronze.orders '
|
||||
'--mode incremental '
|
||||
'--primary-key id '
|
||||
'--update-key updated_at '
|
||||
'--stdout'
|
||||
),
|
||||
)
|
||||
|
||||
# ─── Step 2: dbt incremental update ──────────────────────────────────
|
||||
dbt_incremental = BashOperator(
|
||||
task_id="dbt_incremental",
|
||||
bash_command=(
|
||||
'cd /opt/airflow/dbt/sicabe && '
|
||||
'export DBT_PROFILES_DIR=/opt/airflow/dbt && '
|
||||
'dbt run --select incremental --profile trino --target prod'
|
||||
),
|
||||
env=DBT_TASK_ENV,
|
||||
append_env=True,
|
||||
)
|
||||
|
||||
# ─── Step 3: Validasi BRONZE di Iceberg ──────────────────────────────
|
||||
cdc_validate = PythonOperator(
|
||||
task_id="cdc_validate",
|
||||
python_callable=validate_bronze_data,
|
||||
)
|
||||
|
||||
cdc_ingest >> dbt_incremental >> cdc_validate
|
||||
@@ -0,0 +1,58 @@
|
||||
# =============================================================================
|
||||
# SAMPLE — Lineage otomatis via operator SQL (tanpa inlets/outlets)
|
||||
# =============================================================================
|
||||
# Tujuan : membuktikan bahwa task yang memakai operator SQL memperoleh lineage
|
||||
# tabel SECARA OTOMATIS dari teks SQL, tanpa perlu menuliskan
|
||||
# inlets/outlets.
|
||||
#
|
||||
# Alur lineage yang diharapkan:
|
||||
# iceberg.bronze.orders ---> iceberg.silver.orders_clean
|
||||
#
|
||||
# Cara kerja:
|
||||
# Provider OpenLineage mem-parsing SQL task (lihat
|
||||
# airflow.providers.openlineage.utils.sql_hook_lineage) dan mengekstrak
|
||||
# tabel input/output, lalu mengirimnya ke OpenMetadata.
|
||||
#
|
||||
# Prasyarat di Airflow:
|
||||
# 1. Koneksi `trino_ol` (type trino) menunjuk ke Trino koordinator.
|
||||
# 2. Transport OpenLineage aktif + shim eventTime (lihat shim/ol_time_shim.py).
|
||||
# 3. AIRFLOW__OPENLINEAGE__EXECUTE_IN_THREAD=true (tanpa ini emisi HANG).
|
||||
#
|
||||
# Catatan: DAG ini tidak memakai inlets/outlets secara sengaja.
|
||||
# =============================================================================
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from airflow import DAG
|
||||
from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator
|
||||
|
||||
CONN_ID = "trino_ol"
|
||||
|
||||
CLEAN_SQL = """
|
||||
CREATE TABLE IF NOT EXISTS iceberg.silver.orders_clean AS
|
||||
SELECT
|
||||
id,
|
||||
order_date,
|
||||
komoditas,
|
||||
harga_per_kg,
|
||||
stok_kg,
|
||||
region
|
||||
FROM iceberg.bronze.orders
|
||||
WHERE id IS NOT NULL
|
||||
"""
|
||||
|
||||
with DAG(
|
||||
dag_id="sample_sql_lineage",
|
||||
description="SAMPLE: lineage otomatis via operator SQL (bronze.orders -> silver.orders_clean)",
|
||||
start_date=datetime(2026, 1, 1),
|
||||
schedule=None, # dijalankan manual
|
||||
catchup=False,
|
||||
tags=["sample", "lineage", "openlineage", "sql"],
|
||||
) as dag:
|
||||
|
||||
# Lineage bronze.orders -> silver.orders_clean diekstrak otomatis dari SQL ini.
|
||||
clean_orders = SQLExecuteQueryOperator(
|
||||
task_id="clean_orders",
|
||||
conn_id=CONN_ID,
|
||||
sql=CLEAN_SQL,
|
||||
)
|
||||
@@ -0,0 +1,9 @@
|
||||
"""
|
||||
This file has been generated from dag_runner.j2
|
||||
"""
|
||||
from airflow import DAG
|
||||
from openmetadata_managed_apis.workflows import workflow_factory
|
||||
|
||||
workflow = workflow_factory.WorkflowFactory.create("/opt/airflow/dag_generated_configs/sicabe_dq_gold_native.json")
|
||||
workflow.generate_dag(globals())
|
||||
dag = workflow.get_dag()
|
||||
@@ -0,0 +1,39 @@
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — dbt Profiles (Trino Connection)
|
||||
# =============================================================================
|
||||
# Service/pipeline memakai SERVICE ACCOUNT via HTTP Basic (dbt-trino method
|
||||
# 'ldap' = username/password Basic). Koneksi LANGSUNG ke Trino HTTPS
|
||||
# (self-signed) → cert: false (lewati verifikasi TLS) untuk dev.
|
||||
# Password dari env DBT_TRINO_PASSWORD.
|
||||
# =============================================================================
|
||||
|
||||
trino:
|
||||
target: prod
|
||||
outputs:
|
||||
prod:
|
||||
type: trino
|
||||
method: ldap
|
||||
user: "{{ env_var('DBT_TRINO_USER', 'svc_dbt') }}"
|
||||
password: "{{ env_var('DBT_TRINO_PASSWORD', '') }}"
|
||||
database: iceberg
|
||||
schema: bronze
|
||||
host: "{{ env_var('DBT_TRINO_HOST', 'trino-coordinator') }}"
|
||||
port: 8443
|
||||
threads: 4
|
||||
http_scheme: https
|
||||
cert: false
|
||||
suppress_cert_warning: true
|
||||
|
||||
dev:
|
||||
type: trino
|
||||
method: ldap
|
||||
user: "{{ env_var('DBT_TRINO_USER', 'svc_dbt') }}"
|
||||
password: "{{ env_var('DBT_TRINO_PASSWORD', '') }}"
|
||||
database: iceberg
|
||||
schema: bronze
|
||||
host: "{{ env_var('DBT_TRINO_HOST', 'localhost') }}"
|
||||
port: 12443
|
||||
threads: 2
|
||||
http_scheme: https
|
||||
cert: false
|
||||
suppress_cert_warning: true
|
||||
@@ -0,0 +1,56 @@
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — dbt Project Configuration
|
||||
# =============================================================================
|
||||
# Referensi arsitektur: bab 8.1 (Batch Harian), bab 8.2 (CDC Real-Time)
|
||||
# Pipeline: Bronze → Silver → Gold
|
||||
# =============================================================================
|
||||
|
||||
name: "sicabe"
|
||||
version: "1.0.0"
|
||||
config-version: 2
|
||||
require-dbt-version: [">=1.8.0", "<2.0.0"]
|
||||
|
||||
profile: "trino"
|
||||
|
||||
model-paths: ["models"]
|
||||
analysis-paths: ["analyses"]
|
||||
test-paths: ["tests"]
|
||||
seed-paths: ["seeds"]
|
||||
macro-paths: ["macros"]
|
||||
snapshot-paths: ["snapshots"]
|
||||
|
||||
target-path: "target"
|
||||
clean-targets:
|
||||
- "target"
|
||||
- "dbt_packages"
|
||||
|
||||
models:
|
||||
sicabe:
|
||||
bronze_to_silver:
|
||||
+materialized: table
|
||||
+schema: silver
|
||||
+database: iceberg
|
||||
silver_to_gold:
|
||||
+materialized: table
|
||||
+schema: gold
|
||||
+database: iceberg
|
||||
incremental:
|
||||
+materialized: incremental
|
||||
+schema: bronze
|
||||
+database: iceberg
|
||||
+unique_key: "id"
|
||||
+incremental_strategy: merge
|
||||
# ───────────────────────────────────────────────────────────────────────
|
||||
# WAJIB: relasi sementara harus berupa TABEL, bukan VIEW.
|
||||
#
|
||||
# Katalog Iceberg Trino di stack ini bertipe `nessie`, dan Trino TIDAK
|
||||
# mendukung view di sana. Tanpa setelan ini, setiap run incremental gagal:
|
||||
# TrinoUserError(NOT_SUPPORTED):
|
||||
# "createView is not supported for Iceberg Nessie catalogs"
|
||||
#
|
||||
# Sebabnya: untuk strategi `merge`/`append`, dbt-trino membuat relasi
|
||||
# sementara sebagai VIEW. `views_enabled: false` membuatnya memakai tabel.
|
||||
# Dibuktikan 2026-09-22: MERGE (1 rows) berhasil untuk baris baru maupun
|
||||
# baris yang diubah (upsert). Lihat reviews/sicabe-15-transform-service.md.
|
||||
# ───────────────────────────────────────────────────────────────────────
|
||||
+views_enabled: false
|
||||
@@ -0,0 +1,15 @@
|
||||
-- =============================================================================
|
||||
-- eBdesk SiCABE — dbt Macro: generate_schema_name
|
||||
-- =============================================================================
|
||||
-- Override default schema naming agar +schema menghasilkan nama persis
|
||||
-- (bukan {default}_{custom}). Contoh: +schema: gold → schema "gold"
|
||||
-- =============================================================================
|
||||
|
||||
{% macro generate_schema_name(custom_schema_name, node) -%}
|
||||
{%- set default_schema = target.schema -%}
|
||||
{%- if custom_schema_name is none -%}
|
||||
{{ default_schema }}
|
||||
{%- else -%}
|
||||
{{ custom_schema_name | trim }}
|
||||
{%- endif -%}
|
||||
{%- endmacro %}
|
||||
@@ -0,0 +1,36 @@
|
||||
-- =============================================================================
|
||||
-- eBdesk SiCABE — dbt Model: Bronze → Silver
|
||||
-- =============================================================================
|
||||
-- Transformasi data mentah (bronze.orders) menjadi data bersih (silver.orders):
|
||||
-- - Normalisasi tipe data
|
||||
-- - Filter data valid (harga > 0, stok >= 0)
|
||||
-- - Standarisasi format tanggal
|
||||
-- =============================================================================
|
||||
|
||||
WITH bronze_orders AS (
|
||||
SELECT
|
||||
id,
|
||||
CAST(order_date AS DATE) AS order_date,
|
||||
komoditas,
|
||||
CAST(harga_per_kg AS DOUBLE) AS harga_per_kg,
|
||||
CAST(stok_kg AS BIGINT) AS stok_kg,
|
||||
region,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM {{ source('iceberg', 'orders') }}
|
||||
)
|
||||
|
||||
SELECT
|
||||
id,
|
||||
order_date,
|
||||
komoditas,
|
||||
harga_per_kg,
|
||||
stok_kg,
|
||||
region,
|
||||
-- Nilai total per transaksi
|
||||
ROUND(harga_per_kg * stok_kg, 2) AS total_value,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM bronze_orders
|
||||
WHERE harga_per_kg > 0
|
||||
AND stok_kg >= 0
|
||||
@@ -0,0 +1,29 @@
|
||||
-- =============================================================================
|
||||
-- eBdesk SiCABE — dbt Model: Incremental (CDC)
|
||||
-- =============================================================================
|
||||
-- Incremental update untuk data bronze.orders:
|
||||
-- - Hanya memproses baris baru/berubah (berdasarkan updated_at)
|
||||
-- - Materialized: incremental dengan unique_key = id
|
||||
-- =============================================================================
|
||||
|
||||
{{ config(
|
||||
materialized='incremental',
|
||||
unique_key='id',
|
||||
incremental_strategy='merge'
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
id,
|
||||
CAST(order_date AS DATE) AS order_date,
|
||||
komoditas,
|
||||
CAST(harga_per_kg AS DOUBLE) AS harga_per_kg,
|
||||
CAST(stok_kg AS BIGINT) AS stok_kg,
|
||||
region,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM {{ source('iceberg', 'orders') }}
|
||||
|
||||
{% if is_incremental() %}
|
||||
-- Hanya ambil data yang baru/berubah sejak run terakhir
|
||||
WHERE updated_at > (SELECT MAX(updated_at) FROM {{ this }})
|
||||
{% endif %}
|
||||
@@ -0,0 +1,35 @@
|
||||
-- =============================================================================
|
||||
-- eBdesk SiCABE — dbt Model: Silver → Gold
|
||||
-- =============================================================================
|
||||
-- Agregasi data bersih (silver.orders) menjadi data analitik (gold.orders):
|
||||
-- - Rata-rata harga per komoditas per hari
|
||||
-- - Total stok per komoditas
|
||||
-- - Ranking komoditas berdasarkan nilai transaksi
|
||||
-- =============================================================================
|
||||
|
||||
WITH silver_orders AS (
|
||||
SELECT
|
||||
id,
|
||||
order_date,
|
||||
komoditas,
|
||||
harga_per_kg,
|
||||
stok_kg,
|
||||
region,
|
||||
total_value
|
||||
FROM {{ ref('orders_silver') }}
|
||||
)
|
||||
|
||||
SELECT
|
||||
order_date,
|
||||
komoditas,
|
||||
region,
|
||||
COUNT(*) AS jumlah_transaksi,
|
||||
ROUND(AVG(harga_per_kg), 2) AS avg_harga_per_kg,
|
||||
ROUND(MIN(harga_per_kg), 2) AS min_harga_per_kg,
|
||||
ROUND(MAX(harga_per_kg), 2) AS max_harga_per_kg,
|
||||
SUM(stok_kg) AS total_stok_kg,
|
||||
ROUND(SUM(total_value), 2) AS total_nilai,
|
||||
-- Harga rata-rata tertimbang (weighted average)
|
||||
ROUND(SUM(harga_per_kg * stok_kg) / NULLIF(SUM(stok_kg), 0), 2) AS weighted_avg_harga
|
||||
FROM silver_orders
|
||||
GROUP BY order_date, komoditas, region
|
||||
@@ -0,0 +1,32 @@
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — dbt Sources
|
||||
# =============================================================================
|
||||
# Mendefinisikan sumber data di Iceberg (via Trino catalog)
|
||||
# =============================================================================
|
||||
|
||||
version: 2
|
||||
|
||||
sources:
|
||||
- name: iceberg
|
||||
database: iceberg
|
||||
schema: bronze
|
||||
tables:
|
||||
- name: orders
|
||||
description: "Data mentah orders dari PostgreSQL (via Sling)"
|
||||
columns:
|
||||
- name: id
|
||||
description: "Primary key"
|
||||
- name: order_date
|
||||
description: "Tanggal order"
|
||||
- name: komoditas
|
||||
description: "Nama komoditas"
|
||||
- name: harga_per_kg
|
||||
description: "Harga per kilogram"
|
||||
- name: stok_kg
|
||||
description: "Stok dalam kilogram"
|
||||
- name: region
|
||||
description: "Wilayah"
|
||||
- name: created_at
|
||||
description: "Waktu dibuat"
|
||||
- name: updated_at
|
||||
description: "Waktu diupdate"
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"name": "cdc_quality_checkpoint",
|
||||
"config_version": 4.0,
|
||||
"template_name": null,
|
||||
"run_name_template": "cdc-quality-%Y%m%d-%H%M%S",
|
||||
"expectation_suite_name": "bronze_orders_suite",
|
||||
"batch_request": {
|
||||
"datasource_name": "trino_iceberg",
|
||||
"data_connector_name": "default_inferred_data_connector_name",
|
||||
"data_asset_name": "bronze.orders",
|
||||
"batch_spec_passthrough": {}
|
||||
},
|
||||
"action_list": [
|
||||
{
|
||||
"name": "store_validation_result",
|
||||
"action": {
|
||||
"class_name": "StoreValidationResultAction"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "update_data_docs",
|
||||
"action": {
|
||||
"class_name": "UpdateDataDocsAction"
|
||||
}
|
||||
}
|
||||
],
|
||||
"evaluation_parameters": {},
|
||||
"runtime_configuration": {},
|
||||
"validations": [
|
||||
{
|
||||
"batch_request": {
|
||||
"datasource_name": "trino_iceberg",
|
||||
"data_connector_name": "default_inferred_data_connector_name",
|
||||
"data_asset_name": "bronze.orders"
|
||||
},
|
||||
"expectation_suite_name": "bronze_orders_suite",
|
||||
"action_list": []
|
||||
}
|
||||
],
|
||||
"profilers": [],
|
||||
"ge_cloud_id": null,
|
||||
"expectation_suite_ge_cloud_id": null
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"name": "gold_quality_checkpoint",
|
||||
"config_version": 4.0,
|
||||
"template_name": null,
|
||||
"run_name_template": "gold-quality-%Y%m%d-%H%M%S",
|
||||
"expectation_suite_name": "gold_orders_suite",
|
||||
"batch_request": {
|
||||
"datasource_name": "trino_iceberg",
|
||||
"data_connector_name": "default_inferred_data_connector_name",
|
||||
"data_asset_name": "gold.orders",
|
||||
"batch_spec_passthrough": {}
|
||||
},
|
||||
"action_list": [
|
||||
{
|
||||
"name": "store_validation_result",
|
||||
"action": {
|
||||
"class_name": "StoreValidationResultAction"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "update_data_docs",
|
||||
"action": {
|
||||
"class_name": "UpdateDataDocsAction"
|
||||
}
|
||||
}
|
||||
],
|
||||
"evaluation_parameters": {},
|
||||
"runtime_configuration": {},
|
||||
"validations": [
|
||||
{
|
||||
"batch_request": {
|
||||
"datasource_name": "trino_iceberg",
|
||||
"data_connector_name": "default_inferred_data_connector_name",
|
||||
"data_asset_name": "gold.orders"
|
||||
},
|
||||
"expectation_suite_name": "gold_orders_suite",
|
||||
"action_list": []
|
||||
}
|
||||
],
|
||||
"profilers": [],
|
||||
"ge_cloud_id": null,
|
||||
"expectation_suite_ge_cloud_id": null
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"expectation_suite_name": "bronze_orders_suite",
|
||||
"ge_cloud_id": null,
|
||||
"meta": {
|
||||
"created_by": "sicabe-bootstrap",
|
||||
"created_at": "2026-08-31"
|
||||
},
|
||||
"expectations": [
|
||||
{
|
||||
"expectation_type": "expect_table_row_count_to_be_between",
|
||||
"kwargs": {
|
||||
"min_value": 1,
|
||||
"max_value": 100000
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_not_be_null",
|
||||
"kwargs": {
|
||||
"column": "id"
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_not_be_null",
|
||||
"kwargs": {
|
||||
"column": "komoditas"
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_be_between",
|
||||
"kwargs": {
|
||||
"column": "harga_per_kg",
|
||||
"min_value": 0,
|
||||
"max_value": 1000000
|
||||
},
|
||||
"meta": {}
|
||||
}
|
||||
],
|
||||
"data_asset_type": null
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"expectation_suite_name": "gold_orders_suite",
|
||||
"ge_cloud_id": null,
|
||||
"meta": {
|
||||
"created_by": "sicabe-bootstrap",
|
||||
"created_at": "2026-08-31"
|
||||
},
|
||||
"expectations": [
|
||||
{
|
||||
"expectation_type": "expect_table_row_count_to_be_between",
|
||||
"kwargs": {
|
||||
"min_value": 1,
|
||||
"max_value": 100000
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_not_be_null",
|
||||
"kwargs": {
|
||||
"column": "komoditas"
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_be_between",
|
||||
"kwargs": {
|
||||
"column": "harga_per_kg",
|
||||
"min_value": 0,
|
||||
"max_value": 1000000
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_be_between",
|
||||
"kwargs": {
|
||||
"column": "stok_kg",
|
||||
"min_value": 0,
|
||||
"max_value": 100000000
|
||||
},
|
||||
"meta": {}
|
||||
}
|
||||
],
|
||||
"data_asset_type": null
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Great Expectations Configuration
|
||||
# =============================================================================
|
||||
# Referensi arsitektur: bab 8.1 (Batch Harian), bab 8.2 (CDC Real-Time)
|
||||
# Validasi kualitas data di layer Gold sebelum dikonsumsi aplikasi.
|
||||
# =============================================================================
|
||||
|
||||
config_version: 4.0
|
||||
|
||||
datasources:
|
||||
trino_iceberg:
|
||||
class_name: Datasource
|
||||
execution_engine:
|
||||
class_name: SqlAlchemyExecutionEngine
|
||||
connection_string: "trino://{{ env_var('GE_TRINO_USER', 'admin') }}:{{ env_var('GE_TRINO_PASSWORD', '') }}@{{ env_var('GE_TRINO_HOST', 'trino-coordinator') }}:8080/iceberg/gold"
|
||||
data_connectors:
|
||||
default_inferred_data_connector_name:
|
||||
class_name: InferredAssetSqlDataConnector
|
||||
include_schema_name: true
|
||||
introspection_directives:
|
||||
schema_name: gold
|
||||
|
||||
expectations_store:
|
||||
class_name: ExpectationsStore
|
||||
store_backend:
|
||||
class_name: TupleFilesystemStoreBackend
|
||||
base_directory: expectations/
|
||||
|
||||
validations_store:
|
||||
class_name: ValidationsStore
|
||||
store_backend:
|
||||
class_name: TupleFilesystemStoreBackend
|
||||
base_directory: validations/
|
||||
|
||||
data_docs_sites:
|
||||
local:
|
||||
class_name: SiteBuilder
|
||||
show_how_to_buttons: true
|
||||
store_backend:
|
||||
class_name: TupleFilesystemStoreBackend
|
||||
base_directory: uncommitted/data_docs/local_site/
|
||||
site_index_builder:
|
||||
class_name: DefaultSiteIndexBuilder
|
||||
@@ -0,0 +1,18 @@
|
||||
# =============================================================================
|
||||
# Konfigurasi koneksi OpenMetadata untuk GE Action (dibaca oleh ometa_config_handler)
|
||||
# =============================================================================
|
||||
# Direktori ini berisi "config.yml" karena action membacanya sebagai TEMPLATE Jinja:
|
||||
# create_jinja_environment(<dir>) + render_template(env) → <dir>/config.yml
|
||||
# `env('VAR')` disediakan oleh handler, sehingga token TIDAK ditulis di repo.
|
||||
#
|
||||
# Dipakai oleh: metadata/great_expectations/action.py (OpenMetadata >= 2.0)
|
||||
# OpenMetadataValidationAction(config_file_path="/opt/airflow/great_expectations/om", ...)
|
||||
# Di OpenMetadata 1.13.x modul action-nya berbeda (generasi 1xx) dan dihapus di
|
||||
# 2.0; rincian migrasi ada di deploy/output/merge-om20x-rollback.md.
|
||||
#
|
||||
# Jalankan dengan OM_JWT_TOKEN tersedia di environment container.
|
||||
# =============================================================================
|
||||
hostPort: "{{ env('OM_API_URL') }}"
|
||||
authProvider: openmetadata
|
||||
securityConfig:
|
||||
jwtToken: "{{ env('OM_JWT_TOKEN') }}"
|
||||
@@ -0,0 +1,300 @@
|
||||
#!/usr/bin/env python
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Jalankan Great Expectations DAN kirim hasilnya ke OpenMetadata
|
||||
# =============================================================================
|
||||
# Jalur A (GE -> OpenMetadata): hasil GE tidak berhenti di log Airflow.
|
||||
# Action resmi OpenMetadata (metadata.great_expectations.action) dipasang pada
|
||||
# checkpoint GE 1.x, sehingga setiap kali checkpoint dijalankan, hasilnya dikirim
|
||||
# ke OpenMetadata dan muncul sebagai Data Quality Test Suite/Test Case dengan
|
||||
# platform "GreatExpectations".
|
||||
#
|
||||
# Catatan versi: OM 1.13.x memakai modul action generasi lama (kelas berakhiran
|
||||
# "1xx"); di OM 2.0 modul itu dihapus dan kelasnya kini bernama
|
||||
# OpenMetadataValidationAction di metadata.great_expectations.action.
|
||||
# Rincian migrasinya: deploy/output/merge-om20x-rollback.md.
|
||||
#
|
||||
# ⚠️ Sejak OM 2.0 action MENOLAK batch yang bukan dari database relasional:
|
||||
# `_get_checkpoint_batch_spec()` melempar ValueError bila batch_spec tidak
|
||||
# memuat kunci SQL (table_name/schema_name/query/temp_table_schema_name).
|
||||
# Karena itu ekspektasi dijalankan di Trino lewat DATASOURCE SQL — bukan atas
|
||||
# pandas dataframe — sehingga batch_spec memuat schema_name + table_name dan
|
||||
# action dapat menemukan tabel OpenMetadata-nya.
|
||||
#
|
||||
# Dijalankan DI DALAM container Airflow (butuh great_expectations + metadata):
|
||||
# docker exec -e OM_JWT_TOKEN=<token> -e DBT_TRINO_PASSWORD=<pw> \
|
||||
# sicabe-airflow-webserver python /opt/airflow/great_expectations/validate_to_openmetadata.py
|
||||
#
|
||||
# Exit code: 0 = semua ekspektasi lulus; 1 = ADA yang gagal (task Airflow akan
|
||||
# FAILED) — hasilnya TETAP terkirim ke OpenMetadata sebelum exit.
|
||||
# =============================================================================
|
||||
import os
|
||||
import sys
|
||||
|
||||
import great_expectations as gx
|
||||
import trino
|
||||
from great_expectations.core.expectation_suite import ExpectationSuite
|
||||
|
||||
from metadata.great_expectations.action import OpenMetadataValidationAction
|
||||
OM_CONFIG_DIR = os.getenv("OM_CONFIG_DIR", "/opt/airflow/great_expectations/om")
|
||||
# OM_API_URL = alamat OM dari DALAM jaringan container (bukan OPENMETADATA_URL yang
|
||||
# dipakai dari host). Dipakai juga oleh template config action (om/config.yml).
|
||||
OM_API_URL = os.getenv("OM_API_URL", "http://openmetadata-server:8585/api")
|
||||
DATABASE_SERVICE = os.getenv("OM_DATABASE_SERVICE", "trino_sicabe")
|
||||
DATABASE_NAME = os.getenv("OM_DATABASE_NAME", "iceberg")
|
||||
TRINO_CERT = os.getenv("TRINO_CA_BUNDLE", "/etc/ssl/trino/trino.crt")
|
||||
|
||||
# (schema, tabel, daftar ekspektasi)
|
||||
CHECKS = [
|
||||
("bronze", "orders", [
|
||||
gx.expectations.ExpectTableRowCountToBeBetween(min_value=1, max_value=1000000),
|
||||
gx.expectations.ExpectColumnValuesToNotBeNull(column="id"),
|
||||
gx.expectations.ExpectColumnValuesToNotBeNull(column="komoditas"),
|
||||
gx.expectations.ExpectColumnValuesToBeBetween(
|
||||
column="harga_per_kg", min_value=0, max_value=1000000),
|
||||
]),
|
||||
("silver", "orders_silver", [
|
||||
gx.expectations.ExpectTableRowCountToBeBetween(min_value=1, max_value=1000000),
|
||||
gx.expectations.ExpectColumnValuesToNotBeNull(column="id"),
|
||||
gx.expectations.ExpectColumnValuesToBeBetween(
|
||||
column="total_value", min_value=0, max_value=1000000000000),
|
||||
]),
|
||||
("gold", "orders_gold", [
|
||||
gx.expectations.ExpectTableRowCountToBeBetween(min_value=1, max_value=1000000),
|
||||
gx.expectations.ExpectColumnValuesToNotBeNull(column="komoditas"),
|
||||
gx.expectations.ExpectColumnValuesToBeBetween(
|
||||
column="avg_harga_per_kg", min_value=0, max_value=1000000),
|
||||
gx.expectations.ExpectColumnValuesToBeBetween(
|
||||
column="total_nilai", min_value=0, max_value=1000000000000),
|
||||
]),
|
||||
]
|
||||
|
||||
|
||||
def _trino_connection():
|
||||
user = os.getenv("DBT_TRINO_USER", "svc_dbt")
|
||||
password = os.getenv("DBT_TRINO_PASSWORD")
|
||||
if not password:
|
||||
raise RuntimeError("DBT_TRINO_PASSWORD tidak diset")
|
||||
return trino.dbapi.connect(
|
||||
host=os.getenv("DBT_TRINO_HOST", "trino-coordinator"),
|
||||
port=int(os.getenv("DBT_TRINO_PORT", "8443")),
|
||||
http_scheme="https",
|
||||
user=user,
|
||||
catalog=DATABASE_NAME,
|
||||
auth=trino.auth.BasicAuthentication(user, password),
|
||||
verify=TRINO_CERT,
|
||||
)
|
||||
|
||||
|
||||
def _trino_sqlalchemy_url():
|
||||
"""URL SQLAlchemy untuk datasource SQL Great Expectations (tanpa password).
|
||||
|
||||
Password TIDAK ditulis di URL — dikirim lewat objek auth pada connect_args,
|
||||
sehingga tidak muncul di connection string, log, atau pesan error.
|
||||
"""
|
||||
user = os.getenv("DBT_TRINO_USER", "svc_dbt")
|
||||
host = os.getenv("DBT_TRINO_HOST", "trino-coordinator")
|
||||
port = os.getenv("DBT_TRINO_PORT", "8443")
|
||||
return f"trino://{user}@{host}:{port}/{DATABASE_NAME}"
|
||||
|
||||
|
||||
def _trino_connect_args():
|
||||
"""Argumen koneksi untuk dialect SQLAlchemy Trino.
|
||||
|
||||
Penting: dialect Trino TIDAK membaca `http_scheme`/`verify` dari query URL —
|
||||
keduanya harus diberikan sebagai connect_args. `verify` juga harus berupa
|
||||
path sertifikat (bukan JSON string seperti bila lewat query URL).
|
||||
"""
|
||||
user = os.getenv("DBT_TRINO_USER", "svc_dbt")
|
||||
password = os.getenv("DBT_TRINO_PASSWORD")
|
||||
if not password:
|
||||
raise RuntimeError("DBT_TRINO_PASSWORD tidak diset")
|
||||
return {
|
||||
"http_scheme": "https",
|
||||
"verify": TRINO_CERT,
|
||||
"auth": trino.auth.BasicAuthentication(user, password),
|
||||
}
|
||||
|
||||
|
||||
def _assert_table_not_empty(schema, table):
|
||||
"""Fail-fast bila tabel kosong: tidak ada yang bermakna untuk divalidasi."""
|
||||
row = None
|
||||
with _trino_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(f"SELECT count(*) FROM {schema}.{table}")
|
||||
row = cursor.fetchone()
|
||||
if not row or not row[0]:
|
||||
raise RuntimeError(f"Tabel {schema}.{table} kosong — tidak ada yang divalidasi")
|
||||
|
||||
|
||||
def _build_checkpoint(context, schema, table, expectations):
|
||||
"""Satu checkpoint per tabel: suite + batch definition SQL + action OM."""
|
||||
_assert_table_not_empty(schema, table)
|
||||
|
||||
suite_name = f"{schema}_{table}_suite"
|
||||
data_source = context.data_sources.add_or_update_sql(
|
||||
name=f"trino_{schema}_{table}",
|
||||
connection_string=_trino_sqlalchemy_url(),
|
||||
kwargs={"connect_args": _trino_connect_args()},
|
||||
)
|
||||
asset = data_source.add_table_asset(name=table, table_name=table, schema_name=schema)
|
||||
batch_definition = asset.add_batch_definition_whole_table(f"bd_{schema}_{table}")
|
||||
|
||||
suite = ExpectationSuite(name=suite_name)
|
||||
for expectation in expectations:
|
||||
suite.add_expectation(expectation)
|
||||
context.suites.add_or_update(suite)
|
||||
|
||||
# ValidationDefinition memerlukan BatchDefinition (bukan Batch) + suite.
|
||||
validation_definition = context.validation_definitions.add(
|
||||
gx.ValidationDefinition(name=f"vd_{schema}_{table}", data=batch_definition, suite=suite)
|
||||
)
|
||||
|
||||
action = OpenMetadataValidationAction(
|
||||
config_file_path=OM_CONFIG_DIR,
|
||||
database_service_name=DATABASE_SERVICE,
|
||||
database_name=DATABASE_NAME,
|
||||
schema_name=schema,
|
||||
table_name=table,
|
||||
)
|
||||
checkpoint = context.checkpoints.add(
|
||||
gx.Checkpoint(
|
||||
name=f"cp_{schema}_{table}",
|
||||
validation_definitions=[validation_definition],
|
||||
actions=[action],
|
||||
)
|
||||
)
|
||||
# Batch "whole table" dijalankan di Trino (tanpa batch_parameters).
|
||||
return checkpoint
|
||||
|
||||
|
||||
def _ensure_om_token():
|
||||
"""Token OpenMetadata untuk action: pakai OM_JWT_TOKEN, atau login sendiri.
|
||||
|
||||
Login sendiri membuat skrip aman dipakai pada jadwal (DAG) — tidak bergantung
|
||||
pada token sementara yang ditempel di environment.
|
||||
"""
|
||||
if os.getenv("OM_JWT_TOKEN"):
|
||||
return True
|
||||
# Template config action membaca OM_API_URL → pastikan selalu terisi.
|
||||
os.environ.setdefault("OM_API_URL", OM_API_URL)
|
||||
email = os.getenv("OPENMETADATA_ADMIN_EMAIL", "admin@open-metadata.org")
|
||||
password = os.getenv("OPENMETADATA_ADMIN_PASSWORD")
|
||||
import base64
|
||||
import json
|
||||
import urllib.request
|
||||
|
||||
# DUA kandidat, MAKSIMAL DUA percobaan (jangan diulang dalam loop — akun admin bisa
|
||||
# terkunci): password .env, lalu default 'admin' yang dipakai OM 2.0.x sampai
|
||||
# bootstrap step 7 menulis hash bcrypt. Sama seperti ingest-openmetadata.sh.
|
||||
candidates = [c for c in dict.fromkeys([password or "", "admin"]) if c]
|
||||
err_last = ""
|
||||
for pw in candidates:
|
||||
body = json.dumps({"email": email,
|
||||
"password": base64.b64encode(pw.encode()).decode()}).encode()
|
||||
req = urllib.request.Request(f"{OM_API_URL}/v1/users/login", data=body,
|
||||
headers={"Content-Type": "application/json"})
|
||||
try:
|
||||
token = json.load(urllib.request.urlopen(req, timeout=20)).get("accessToken", "")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
err_last = str(exc)
|
||||
token = ""
|
||||
if token:
|
||||
os.environ["OM_JWT_TOKEN"] = token
|
||||
return True
|
||||
print(f"ERROR: login ke OpenMetadata gagal: {err_last or 'tanpa token'}")
|
||||
print(" dicoba: password .env + 'admin' — pastikan: ./bootstrap.sh --step 7")
|
||||
return False
|
||||
|
||||
|
||||
def _suite_exists_in_om(schema: str, table: str):
|
||||
"""Apakah test suite untuk tabel ini ADA di OpenMetadata?
|
||||
|
||||
Mengembalikan nama suite bila ada, False bila tidak, None bila tidak dapat
|
||||
diverifikasi.
|
||||
|
||||
Dua hal penting yang membuat versi pertama fungsi ini SALAH:
|
||||
1. `checkpoint.run()` hanya melaporkan status CHECKPOINT (validasi expectation),
|
||||
bukan status pengiriman action — jadi tidak boleh disimpulkan dari situ.
|
||||
2. Action OpenMetadataValidationAction menamai test suite menurut FQN TABEL:
|
||||
<service>.<database>.<schema>.<table>.testSuite
|
||||
BUKAN menurut nama ExpectationSuite GE (`<schema>_<table>_suite`). Memeriksa
|
||||
nama GE-nya menghasilkan false negative: DQ sudah ada di OM tapi dilaporkan
|
||||
tidak ada.
|
||||
"""
|
||||
token = os.environ.get("OM_JWT_TOKEN", "")
|
||||
if not token:
|
||||
return None
|
||||
import json
|
||||
import urllib.request
|
||||
req = urllib.request.Request(
|
||||
f"{OM_API_URL}/v1/dataQuality/testSuites?limit=200",
|
||||
headers={"Authorization": f"Bearer {token}"})
|
||||
try:
|
||||
d = json.load(urllib.request.urlopen(req, timeout=20))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"[GE] verifikasi ke OM gagal: {exc}")
|
||||
return None
|
||||
names = {s.get("name") for s in (d.get("data") or [])}
|
||||
table_fqn = f"{DATABASE_SERVICE}.{DATABASE_NAME}.{schema}.{table}"
|
||||
for expected in (f"{table_fqn}.testSuite", table_fqn):
|
||||
if expected in names:
|
||||
return expected
|
||||
# toleran terhadap variasi penamaan: suite apa pun yang berawalan FQN tabel ini
|
||||
for n in names:
|
||||
if n and n.startswith(table_fqn):
|
||||
return n
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
only = None
|
||||
if len(sys.argv) > 2 and sys.argv[1] == "--table":
|
||||
only = sys.argv[2] # format: schema.table
|
||||
|
||||
if not _ensure_om_token():
|
||||
print("ERROR: token OpenMetadata tidak tersedia (set OM_JWT_TOKEN, atau "
|
||||
"OPENMETADATA_ADMIN_PASSWORD untuk login otomatis)")
|
||||
return 1
|
||||
|
||||
checks = [c for c in CHECKS if only is None or f"{c[0]}.{c[1]}" == only]
|
||||
if not checks:
|
||||
print(f"ERROR: tidak ada pemeriksaan untuk '{only}'")
|
||||
return 1
|
||||
|
||||
context = gx.get_context(mode="ephemeral")
|
||||
checkpoints = [_build_checkpoint(context, s, t, e) for s, t, e in checks]
|
||||
|
||||
failed = 0
|
||||
not_in_om = 0
|
||||
for (schema, table, _expectations), checkpoint in zip(checks, checkpoints):
|
||||
result = checkpoint.run()
|
||||
success = bool(getattr(result, "success", False))
|
||||
in_om = _suite_exists_in_om(schema, table)
|
||||
if in_om is None:
|
||||
print(f"[GE] {checkpoint.name}: checkpoint success={success}; "
|
||||
f"keberadaan di OpenMetadata TIDAK dapat diverifikasi")
|
||||
elif in_om:
|
||||
print(f"[GE] {checkpoint.name}: checkpoint success={success} "
|
||||
f"→ test suite '{in_om}' ADA di OpenMetadata")
|
||||
else:
|
||||
print(f"[GE] {checkpoint.name}: checkpoint success={success} "
|
||||
f"TAPI belum ada test suite untuk {DATABASE_SERVICE}.{DATABASE_NAME}.{schema}.{table}")
|
||||
print("[GE] penyebab tersering: entitas TABEL-nya belum ada di katalog OM.")
|
||||
print("[GE] urutan yang benar: ./scripts/ingest-openmetadata.sh --all DULU,")
|
||||
print("[GE] baru DAG -- karena action GE menempel ke entitas tabel di OM.")
|
||||
not_in_om += 1
|
||||
if not success:
|
||||
failed += 1
|
||||
|
||||
total = len(checkpoints)
|
||||
print(f"[GE] ringkasan: {total - failed}/{total} checkpoint lulus"
|
||||
+ (f"; {not_in_om} suite TIDAK ADA di OpenMetadata" if not_in_om else ""))
|
||||
# PENTING: `success` hanyalah status CHECKPOINT, bukan status pengiriman. Pesan lama
|
||||
# ("success=True → hasil dikirim ke OpenMetadata") menyesatkan persis karena itu.
|
||||
# Suite yang tidak ada di OM juga membuat task gagal, supaya penyebabnya terlihat
|
||||
# di sini alih-alih muncul sebagai kegagalan gate beberapa langkah kemudian.
|
||||
return 1 if (failed or not_in_om) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,83 @@
|
||||
# Dremio OSS — Konfigurasi `deploy/`
|
||||
|
||||
Dremio ditambahkan sebagai **query engine paralel** di atas catalog Nessie yang
|
||||
sama dengan Trino & StarRocks (membuktikan dua engine berbagi satu catalog tanpa
|
||||
korupsi metadata).
|
||||
|
||||
| Item | Nilai |
|
||||
|---|---|
|
||||
| Image | `dremio/dremio-oss:${DREMIO_VERSION:-25.2.0}` |
|
||||
| Web UI | `http://localhost:${DREMIO_UI_PORT:-12947}` |
|
||||
| JDBC | `${DREMIO_JDBC_PORT:-12010}` → container `31010` |
|
||||
| Arrow Flight | `${DREMIO_FLIGHT_PORT:-12011}` → container `32010` |
|
||||
| Auth | **Lokal** (bukan OIDC — Dremio OSS tidak menyediakan OIDC) |
|
||||
| Penyimpanan internal | Volume `dremio-data` (bukan PostgreSQL) |
|
||||
|
||||
## First-run & Source Nessie — otomatis via `bootstrap.sh` (Step 9)
|
||||
|
||||
`bootstrap.sh` **Step 9** menangani first-run admin + registrasi source Nessie
|
||||
secara idempoten (aman diulang):
|
||||
|
||||
1. Tunggu Dremio siap.
|
||||
2. Buat admin first-run bila belum ada — `PUT /apiv2/bootstrap/firstuser`
|
||||
(body flat, tanpa auth; `400` = sudah pernah dibuat → dilewati).
|
||||
3. Panggil `scripts/dremio/register-nessie-source.sh` untuk source Nessie.
|
||||
|
||||
Kredensial default (override lewat `.env`):
|
||||
|
||||
| Variabel | Default |
|
||||
|---|---|
|
||||
| `DREMIO_USER` | `admin` |
|
||||
| `DREMIO_PASSWORD` | `dremio_admin_DevOnly1` |
|
||||
| `DREMIO_ADMIN_EMAIL` | `admin@sicabe.local` |
|
||||
|
||||
Jalankan hanya step ini:
|
||||
|
||||
```bash
|
||||
cd deploy
|
||||
./bootstrap.sh --step 9
|
||||
```
|
||||
|
||||
### Manual (alternatif)
|
||||
|
||||
1. Buka `http://localhost:12947` → buat akun admin pertama (form first-run).
|
||||
2. Daftarkan source Nessie lewat script:
|
||||
|
||||
```bash
|
||||
cd deploy
|
||||
set -a; source config/environment.defaults.sh; source .env; set +a
|
||||
DREMIO_USER=<admin> DREMIO_PASSWORD='<password>' \
|
||||
./scripts/dremio/register-nessie-source.sh
|
||||
```
|
||||
|
||||
Script: login (`/apiv2/login`, header `Authorization: _dremio<token>`) → cek
|
||||
existing (`/api/v3/catalog/by-path/<name>`) → `POST /api/v3/catalog`
|
||||
(`type=NESSIE`). Idempotent (source sudah ada → sukses tanpa buat ulang).
|
||||
|
||||
> **Temuan (diuji 2026-09-10, Dremio OSS 25.2.0):** klaim docs bahwa Catalog API
|
||||
> "unable to retrieve or manage Nessie sources" (DX-64994) **tidak akurat** —
|
||||
> create **dan** GET by-path berhasil. Kunci config yang benar:
|
||||
> - **JANGAN** sertakan `storageType` (ditolak: `Invalid value found at: storageType`).
|
||||
> - `awsRootPath` format **`/bucket/path`** (TANPA prefix `s3://`), mis. `/iceberg-data/warehouse`.
|
||||
> - `credentialType=ACCESS_KEY`, `awsAccessKey`/`awsAccessSecret`, `secure=false`.
|
||||
> - `propertyList`: `fs.s3a.path.style.access=true`, `fs.s3a.endpoint=minio:9000`,
|
||||
> `dremio.s3.compat=true`.
|
||||
|
||||
Verifikasi query (data yang sama ditulis Trino):
|
||||
```sql
|
||||
SELECT * FROM nessie.demo.t ORDER BY id; -- 2 baris: alpha, beta
|
||||
```
|
||||
|
||||
### Alternatif: via UI
|
||||
|
||||
**Add Source → Nessie** dengan field: endpoint `http://nessie:19120/api/v2`,
|
||||
Authentication `None`, Storage AWS S3/MinIO (path-style, endpoint `minio:9000`),
|
||||
access key/secret MinIO.
|
||||
|
||||
## Catatan
|
||||
|
||||
- **Dua engine berbagi satu Nessie catalog**: tulis dari Dremio (CTAS/INSERT)
|
||||
langsung terlihat di Trino dan sebaliknya. Lakukan tulis secara bergantian
|
||||
(bukan simultan) untuk menghindari konflik concurrency Nessie.
|
||||
- Nessie **tidak mendukung Iceberg spec v3** → gunakan `format-version=2`.
|
||||
- Referensi: `experiments/nessie-dremio/README.md` (H2 terbukti).
|
||||
@@ -0,0 +1,388 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Environment Defaults
|
||||
# Sourced by install.sh and bootstrap.sh
|
||||
# Override any value by exporting the variable before running install.sh
|
||||
# or by creating deploy/.env with: export VAR=value
|
||||
# =============================================================================
|
||||
|
||||
# ─── Global ─────────────────────────────────────────────────────────────────
|
||||
export SICABE_ENV="${SICABE_ENV:-development}"
|
||||
export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-sicabe}"
|
||||
export SICABE_DOMAIN="${SICABE_DOMAIN:-localhost}"
|
||||
export TIMEZONE="${TIMEZONE:-Asia/Jakarta}"
|
||||
|
||||
# ─── Secrets (ALWAYS override in production!) ───────────────────────────────
|
||||
# Password statis agar konsisten saat docker-compose memproses env vars.
|
||||
# Di production, overriede semua nilai berikut di file .env dengan password kuat.
|
||||
# openssl rand -hex 16 > .env
|
||||
|
||||
# PostgreSQL: 1 instance shared untuk semua service (beda database)
|
||||
export POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-sicabe_pg_super_DevOnly}"
|
||||
export KEYCLOAK_DB_PASSWORD="${KEYCLOAK_DB_PASSWORD:-sicabe_kc_db_DevOnly}"
|
||||
export NESSIE_DB_PASSWORD="${NESSIE_DB_PASSWORD:-sicabe_nessie_db_DevOnly}"
|
||||
export AIRFLOW_DB_PASSWORD="${AIRFLOW_DB_PASSWORD:-sicabe_af_db_DevOnly}"
|
||||
export OPENMETADATA_DB_PASSWORD="${OPENMETADATA_DB_PASSWORD:-sicabe_om_db_DevOnly}"
|
||||
|
||||
export KEYCLOAK_ADMIN_PASSWORD="${KEYCLOAK_ADMIN_PASSWORD:-admin123}"
|
||||
export MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:-minio12345}"
|
||||
export AIRFLOW_ADMIN_PASSWORD="${AIRFLOW_ADMIN_PASSWORD:-airflow123}"
|
||||
export GRAFANA_ADMIN_PASSWORD="${GRAFANA_ADMIN_PASSWORD:-grafana123}"
|
||||
export OPENMETADATA_ADMIN_PASSWORD="${OPENMETADATA_ADMIN_PASSWORD:-openmetadata123}"
|
||||
# OpenLDAP (identity — sumber grup untuk Trino & StarRocks)
|
||||
export LDAP_ADMIN_PASSWORD="${LDAP_ADMIN_PASSWORD:-ldap_admin_DevOnly}"
|
||||
export LDAP_CONFIG_PASSWORD="${LDAP_CONFIG_PASSWORD:-ldap_config_DevOnly}"
|
||||
|
||||
# OIDC Client secrets untuk setiap komponen (wajib sinkron dengan konfigurasi
|
||||
# Trino, Grafana, dll. yang menggunakan nilai hardcoded di file .properties)
|
||||
export TRINO_CLIENT_SECRET="${TRINO_CLIENT_SECRET:-trino-secret}"
|
||||
export STARROCKS_CLIENT_SECRET="${STARROCKS_CLIENT_SECRET:-starrocks-secret}"
|
||||
export AIRFLOW_CLIENT_SECRET="${AIRFLOW_CLIENT_SECRET:-airflow-secret}"
|
||||
export OPENMETADATA_CLIENT_SECRET="${OPENMETADATA_CLIENT_SECRET:-om-secret}"
|
||||
export GRAFANA_CLIENT_SECRET="${GRAFANA_CLIENT_SECRET:-grafana-secret}"
|
||||
|
||||
|
||||
# ─── Network ────────────────────────────────────────────────────────────────
|
||||
export HTTP_PORT="${HTTP_PORT:-80}"
|
||||
export HTTPS_PORT="${HTTPS_PORT:-443}"
|
||||
# Catatan: PostgreSQL TIDAK diexpose ke host (internal network saja).
|
||||
# Untuk debugging dari host: docker exec -it sicabe-postgres psql -U sicabe_admin -d keycloak
|
||||
|
||||
# ─── Skema Port Host (prefix 12xxx) ─────────────────────────────────────────
|
||||
# Aturan: host port = "12" + 3 digit port kanonik. Port internal container tetap.
|
||||
# Memudahkan pelatihan: seluruh endpoint ada di blok 12000-12999.
|
||||
|
||||
# ─── Keycloak (Phase 1 — Governance: IAM) ──────────────────────────────────
|
||||
export KEYCLOAK_VERSION="${KEYCLOAK_VERSION:-26.7}"
|
||||
export KEYCLOAK_PORT="${KEYCLOAK_PORT:-12080}"
|
||||
export KEYCLOAK_DB="${KEYCLOAK_DB:-keycloak}"
|
||||
export KEYCLOAK_DB_USER="${KEYCLOAK_DB_USER:-keycloak}"
|
||||
export KEYCLOAK_DB_HOST="${KEYCLOAK_DB_HOST:-postgres}"
|
||||
# CATATAN: KC_HOSTNAME_URL sudah DIHAPUS di Keycloak 26.0.0 — jangan dipasang lagi
|
||||
# (dulu diekspor di sini, dan tidak berpengaruh apa pun). Yang menentukan issuer,
|
||||
# discovery, dan redirect sekarang: KC_HOSTNAME = URL publik lengkap, yang di compose
|
||||
# diisi dari KEYCLOAK_PUBLIC_URL (lihat docker-compose.yml service keycloak).
|
||||
export SICABE_REALM="${SICABE_REALM:-sicabe}"
|
||||
export KEYCLOAK_ADMIN="${KEYCLOAK_ADMIN:-admin}"
|
||||
|
||||
# ─── Identity — OpenLDAP (Phase 1b) ─────────────────────────────────────────
|
||||
# LDAP = sumber kebenaran GRUP (Trino & StarRocks group provider).
|
||||
# User tetap didefinisikan di realm Keycloak (autentikasi OIDC).
|
||||
export OPENLDAP_VERSION="${OPENLDAP_VERSION:-1.5.0}"
|
||||
export OPENLDAP_PORT="${OPENLDAP_PORT:-12389}"
|
||||
export LDAP_ORGANISATION="${LDAP_ORGANISATION:-SiCABE}"
|
||||
export LDAP_DOMAIN="${LDAP_DOMAIN:-sicabe.local}"
|
||||
export LDAP_ADMIN_DN="${LDAP_ADMIN_DN:-cn=admin,dc=sicabe,dc=local}"
|
||||
export LDAP_BASE_DN="${LDAP_BASE_DN:-dc=sicabe,dc=local}"
|
||||
export LDAP_USERS_DN="${LDAP_USERS_DN:-ou=users,dc=sicabe,dc=local}"
|
||||
export LDAP_GROUPS_DN="${LDAP_GROUPS_DN:-ou=groups,dc=sicabe,dc=local}"
|
||||
|
||||
# ─── MinIO (Phase 2 — Storage) ──────────────────────────────────────────────
|
||||
# Image MinIO HANYA tersedia di quay.io — repo Docker Hub `minio/minio` dan
|
||||
# `minio/mc` sudah tidak ada (pull gagal: "repository does not exist").
|
||||
export MINIO_VERSION="${MINIO_VERSION:-RELEASE.2025-09-07T16-13-09Z}"
|
||||
export MINIO_MC_VERSION="${MINIO_MC_VERSION:-RELEASE.2025-08-13T08-35-41Z}"
|
||||
export MINIO_PORT="${MINIO_PORT:-12900}"
|
||||
export MINIO_CONSOLE_PORT="${MINIO_CONSOLE_PORT:-12901}"
|
||||
export MINIO_ROOT_USER="${MINIO_ROOT_USER:-minioadmin}"
|
||||
export MINIO_BUCKET_ICEBERG_DATA="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}"
|
||||
|
||||
# Apache Nessie (Phase 2 — Storage: Metadata Catalog)
|
||||
export NESSIE_VERSION="${NESSIE_VERSION:-0.108.4-java}"
|
||||
export NESSIE_PORT="${NESSIE_PORT:-12120}"
|
||||
# Nessie persistence database (PostgreSQL) — mengikuti konvensi shared Postgres
|
||||
export NESSIE_DB="${NESSIE_DB:-nessie}"
|
||||
export NESSIE_DB_USER="${NESSIE_DB_USER:-nessie}"
|
||||
|
||||
# ─── Alamat antar-container (multi-host) ────────────────────────────────────
|
||||
# B1 (Review 10 §4.4): hostname yang dilihat CONTAINER saat menghubungi komponen
|
||||
# lain. Kontrak: DEFAULT = nama container di docker-compose.yml → SINGLE-HOST
|
||||
# TIDAK BERUBAH (dibuktikan diff `docker compose config` sha256 identik).
|
||||
# Untuk menyebar komponen ke beberapa host, override ke alamat host pemilik:
|
||||
# POSTGRES_HOST=10.20.0.12 MINIO_HOST=10.20.0.12 KEYCLOAK_HOST=10.20.0.10
|
||||
# OPENMETADATA_HOST=10.20.0.10 STARROCKS_FE_HOST=10.20.0.11
|
||||
# Dibedakan dari komponen EXISTING milik sistem lain (lihat bagian di bawah):
|
||||
# yang itu memakai *_URL / S3_ENDPOINT + EXTERNAL_SERVICES.
|
||||
export POSTGRES_HOST="${POSTGRES_HOST:-postgres}"
|
||||
export MINIO_HOST="${MINIO_HOST:-minio}"
|
||||
export KEYCLOAK_HOST="${KEYCLOAK_HOST:-keycloak}"
|
||||
export OPENMETADATA_HOST="${OPENMETADATA_HOST:-openmetadata-server}"
|
||||
export AIRFLOW_WEBSERVER_HOST="${AIRFLOW_WEBSERVER_HOST:-airflow-webserver}"
|
||||
export STARROCKS_FE_HOST="${STARROCKS_FE_HOST:-starrocks-fe}"
|
||||
export STARROCKS_CN_HOST="${STARROCKS_CN_HOST:-starrocks-cn}"
|
||||
export OPENSEARCH_HOST="${OPENSEARCH_HOST:-openmetadata-opensearch}"
|
||||
|
||||
# ─── URL BROWSER-FACING (tls.mode != none) ──────────────────────────────────
|
||||
# Dipakai HANYA untuk apa yang dilihat BROWSER (issuer OIDC, redirect, link UI) —
|
||||
# BUKAN untuk lalu lintas container (itu memakai `*_HOST`/alamat di atas).
|
||||
# Default = nilai sekarang, jadi tanpa perubahan apa pun perilakunya identik.
|
||||
# Saat `tls.mode` = edge/full, generator menimpanya menjadi https://<label>.<domain>.
|
||||
# Perhatian: `iss` token memuat SKEMA, jadi berpindah none -> edge mengubah
|
||||
# issuer dan semua user harus login ulang (lihat R10 §5.7).
|
||||
export KEYCLOAK_PUBLIC_URL="${KEYCLOAK_PUBLIC_URL:-http://localhost:${KEYCLOAK_PORT:-12080}}"
|
||||
|
||||
# Kebijakan SSL realm Keycloak. Default Keycloak `external` menuntut HTTPS untuk
|
||||
# permintaan dari alamat NON-lokal — dan alamat Tailscale 100.64.0.0/10 (CGNAT)
|
||||
# TERMASUK non-lokal, sehingga admin console via http://<tailscale>:12080 gagal
|
||||
# dengan "HTTPS required". Keycloak 26 tidak punya opsi server-level untuk ini
|
||||
# (tidak ada KC_HTTPS_REQUIRED), jadi harus per realm. Diisi `external` oleh
|
||||
# generator saat tls.mode != none (ada Traefik/TLS di depan).
|
||||
# Default `external` = aman: admin console via HTTP hanya diizinkan dari alamat lokal.
|
||||
# Generator menulis `none` HANYA bila tls.mode != none... (mode `none` pada multi-host,
|
||||
# yaitu HTTP polos ala dev) — di situ akses dari alamat Tailscale/CGNAT memang perlu
|
||||
# diizinkan. Single-host (tanpa cluster.yml) tidak melewati generator, jadi ia memakai
|
||||
# default ini dan tetap aman.
|
||||
export KEYCLOAK_SSL_REQUIRED="${KEYCLOAK_SSL_REQUIRED:-external}"
|
||||
|
||||
# Cache Keycloak: `ispn` (default, registry di DB) atau `local` untuk instalasi
|
||||
# SATU Keycloak — mematikan klaster + pemeriksaan cluster health.
|
||||
export KEYCLOAK_CACHE="${KEYCLOAK_CACHE:-ispn}"
|
||||
|
||||
# ── Ingress / TLS (langkah 2, T-20) ──────────────────────────────────────────
|
||||
# Traefik berjalan hanya pada profile `edge` (docker compose --profile edge).
|
||||
# Tiap host menerbitkan sertifikatnya SENDIRI: acme.json tidak boleh dibagi antar
|
||||
# host (Traefik tidak punya penyimpanan ACME terdistribusi), dan batas Let's
|
||||
# Encrypt 5 sertifikat identik/minggu sebaiknya dihindari dengan staging dulu.
|
||||
export CLUSTER_DOMAIN="${CLUSTER_DOMAIN:-sicabe.com}"
|
||||
export TRAEFIK_VERSION="${TRAEFIK_VERSION:-v3.7}"
|
||||
export TRAEFIK_HTTP_PORT="${TRAEFIK_HTTP_PORT:-80}"
|
||||
export TRAEFIK_HTTPS_PORT="${TRAEFIK_HTTPS_PORT:-443}"
|
||||
# Alamat bind port 80/443. Di multi-host diisi generator dengan alamat host,
|
||||
# supaya ingress tidak terbuka di semua antarmuka.
|
||||
export TRAEFIK_BIND_ADDR="${TRAEFIK_BIND_ADDR:-0.0.0.0}"
|
||||
export TRAEFIK_ACME_EMAIL="${TRAEFIK_ACME_EMAIL:-admin@${CLUSTER_DOMAIN:-sicabe.com}}"
|
||||
export TRAEFIK_DNS_PROVIDER="${TRAEFIK_DNS_PROVIDER:-cloudflare}"
|
||||
# STAGING dulu (batas longgar). Produksi:
|
||||
# https://acme-v02.api.letsencrypt.org/directory
|
||||
# Jeda sebelum LE memeriksa TXT. Apex + wildcard memakai nama TXT yang SAMA
|
||||
# (_acme-challenge.<domain>), jadi propagasi perlu waktu — default Traefik 0s.
|
||||
export TRAEFIK_ACME_PROPAGATION_DELAY="${TRAEFIK_ACME_PROPAGATION_DELAY:-30s}"
|
||||
# Catatan lego/Cloudflare (opsional, diset sebagai env container bila perlu):
|
||||
# CLOUDFLARE_PROPAGATION_TIMEOUT (default 120), CLOUDFLARE_POLLING_INTERVAL (2), CLOUDFLARE_TTL (120)
|
||||
export TRAEFIK_ACME_CASERVER="${TRAEFIK_ACME_CASERVER:-https://acme-staging-v02.api.letsencrypt.org/directory}"
|
||||
# Cloudflare DNS-01: token API dengan izin Zone:DNS:Edit. WAJIB diisi sebelum
|
||||
# sertifikat bisa diterbitkan; kosong = Traefik tetap start dan memakai
|
||||
# sertifikat self-signed bawaan (routing masih bisa diuji).
|
||||
export CF_DNS_API_TOKEN="${CF_DNS_API_TOKEN:-}"
|
||||
export GRAFANA_PUBLIC_URL="${GRAFANA_PUBLIC_URL:-http://localhost:${GRAFANA_PORT:-12300}}"
|
||||
# URL browser-facing: default localhost (nama internal airflow-webserver:8080 tidak
|
||||
# bisa diresolusi browser). Bundle multi-host mengisinya dengan alamat host pemilik.
|
||||
export AIRFLOW_PUBLIC_URL="${AIRFLOW_PUBLIC_URL:-http://localhost:${AIRFLOW_PORT:-12082}}"
|
||||
export OPENMETADATA_PUBLIC_URL="${OPENMETADATA_PUBLIC_URL:-http://localhost:${OPENMETADATA_PORT:-12085}}"
|
||||
export TRINO_PUBLIC_URL="${TRINO_PUBLIC_URL:-https://localhost:${TRINO_HTTPS_PORT:-12443}}"
|
||||
export MINIO_CONSOLE_PUBLIC_URL="${MINIO_CONSOLE_PUBLIC_URL:-http://localhost:${MINIO_CONSOLE_PORT:-12901}}"
|
||||
|
||||
# ─── Endpoint Object Storage & Nessie ───────────────────────────────────────
|
||||
# Default = container lokal hasil deploy ini (perilaku tidak berubah). Untuk
|
||||
# memakai penyimpanan objek (S3/MinIO) atau Nessie EXISTING, override di .env:
|
||||
# S3_ENDPOINT → dilihat CONTAINER (Trino, Nessie, StarRocks)
|
||||
# NESSIE_URL → dilihat CONTAINER (Trino, Sling, StarRocks)
|
||||
# NESSIE_HOST/PORT → dilihat HOST (skrip verifikasi)
|
||||
# Contoh:
|
||||
# S3_ENDPOINT=https://s3.corp.internal
|
||||
# S3_ACCESS_KEY=... S3_SECRET_KEY=... S3_REGION=ap-southeast-1
|
||||
# NESSIE_URL=https://nessie.corp.internal
|
||||
export S3_ENDPOINT="${S3_ENDPOINT:-http://minio:9000}"
|
||||
export S3_REGION="${S3_REGION:-us-east-1}"
|
||||
# CATATAN: S3_ACCESS_KEY / S3_SECRET_KEY sengaja TIDAK di-default di sini.
|
||||
# Bila di-default dari MINIO_ROOT_*, nilainya terkunci pada nilai default sebelum
|
||||
# .env di-source (mis. minio12345) sehingga berbeda dari kredensial sebenarnya.
|
||||
# Semua pemakaian memakai fallback saat-pakai: ${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-...}}.
|
||||
export NESSIE_URL="${NESSIE_URL:-http://nessie:19120}"
|
||||
export NESSIE_HOST="${NESSIE_HOST:-localhost}"
|
||||
export NESSIE_PORT="${NESSIE_PORT:-12120}"
|
||||
|
||||
# ─── Trino (Phase 3 — Processing) ───────────────────────────────────────────
|
||||
export TRINO_VERSION="${TRINO_VERSION:-483}"
|
||||
# TRINO_PORT = port HTTP polos historis. Trino TIDAK melayani HTTP polos
|
||||
# (allow-insecure-over-http=false) dan port ini tidak di-expose. Dipakai gate Fase 3
|
||||
# (verify-phase3-processing.sh) sebagai target uji-negatif "HTTP polos harus mati".
|
||||
export TRINO_PORT="${TRINO_PORT:-12081}"
|
||||
export TRINO_HTTPS_PORT="${TRINO_HTTPS_PORT:-12443}"
|
||||
# Endpoint dari sisi HOST (gate verifikasi + bootstrap step 4). Untuk Trino
|
||||
# EXISTING: TRINO_URL=https://trino.corp:8443 dan TRINO_TLS_VERIFY=1 (verifikasi TLS aktif;
|
||||
# default 0 = lewati verifikasi, hanya untuk sertifikat self-signed dev).
|
||||
export TRINO_URL="${TRINO_URL:-https://localhost:${TRINO_HTTPS_PORT:-12443}}"
|
||||
export TRINO_TLS_VERIFY="${TRINO_TLS_VERIFY:-0}"
|
||||
# Service account (HTTP Basic) untuk service/pipeline. Password di-hash (bcrypt)
|
||||
# ke config/trino/password.db oleh bootstrap.sh/install.sh — pra-syarat: htpasswd.
|
||||
export TRINO_SVC_SEED_PASSWORD="${TRINO_SVC_SEED_PASSWORD:-svc_seed_dev}"
|
||||
export TRINO_SVC_DBT_PASSWORD="${TRINO_SVC_DBT_PASSWORD:-svc_dbt_dev}"
|
||||
export TRINO_SVC_AIRFLOW_PASSWORD="${TRINO_SVC_AIRFLOW_PASSWORD:-svc_airflow_dev}"
|
||||
export TRINO_SVC_OM_PASSWORD="${TRINO_SVC_OM_PASSWORD:-svc_om_dev}"
|
||||
|
||||
# ─── StarRocks (Phase 4 — Serving) ──────────────────────────────────────────
|
||||
export STARROCKS_VERSION="${STARROCKS_VERSION:-4.0.13}"
|
||||
export STARROCKS_FE_PORT="${STARROCKS_FE_PORT:-12930}"
|
||||
export STARROCKS_HTTP_PORT="${STARROCKS_HTTP_PORT:-12030}"
|
||||
# Endpoint dari sisi HOST (gate verifikasi + bootstrap step 5). Untuk StarRocks
|
||||
# EXISTING: STARROCKS_HOST=starrocks.corp (dan STARROCKS_PORT=9030, port MySQL
|
||||
# StarRocks — bukan port publish compose 12930).
|
||||
export STARROCKS_HOST="${STARROCKS_HOST:-localhost}"
|
||||
export STARROCKS_PORT="${STARROCKS_PORT:-${STARROCKS_FE_PORT:-12930}}"
|
||||
|
||||
# ─── Airflow (Phase 5 — Ingestion) ──────────────────────────────────────────
|
||||
export AIRFLOW_VERSION="${AIRFLOW_VERSION:-3.3.2}"
|
||||
export AIRFLOW_PORT="${AIRFLOW_PORT:-12082}"
|
||||
# Endpoint dari sisi HOST (gate Fase 5, test-verification). Untuk Airflow
|
||||
# EXISTING: AIRFLOW_HOST=airflow.corp (+ AIRFLOW_PORT sesuai) → install.sh tidak
|
||||
# menyalakan service Airflow, dan uji container menjadi SKIP.
|
||||
export AIRFLOW_HOST="${AIRFLOW_HOST:-localhost}"
|
||||
# Secret key untuk JWT internal Airflow (task runner auth) — WAJIB sama di semua container
|
||||
export AIRFLOW_SECRET_KEY="${AIRFLOW_SECRET_KEY:-sicabe_airflow_jwt_DevOnly_0123456789abcdef}"
|
||||
export AIRFLOW_DB="${AIRFLOW_DB:-airflow}"
|
||||
export AIRFLOW_DB_USER="${AIRFLOW_DB_USER:-airflow}"
|
||||
export AIRFLOW_DB_HOST="${AIRFLOW_DB_HOST:-postgres}"
|
||||
export AIRFLOW_EXECUTOR="${AIRFLOW_EXECUTOR:-LocalExecutor}"
|
||||
# Image Airflow kustom. Sejak OM 2.0.2 image ini memuat extra
|
||||
# `openmetadata-ingestion[airflow]` + plugin `openmetadata-managed-apis` yang
|
||||
# disajikan di /pluginsv2/ — inilah yang menghidupkan tombol "Test Connection"
|
||||
# di UI OpenMetadata (§8.11). Tag lama `3.3.1-slim` (tanpa plugin) tetap ada
|
||||
# sebagai jalur rollback: set AIRFLOW_IMAGE=sicabe/airflow:3.3.1-slim.
|
||||
# BATCH B (review R18, 2026-10-01): naik ke 3.3.2 (OM 2.0.3 mem-pin Airflow 3.3.2).
|
||||
# Jalur rollback: sicabe/airflow:3.3.1-om202 (atau 3.3.1-om202-presqlalchemy).
|
||||
# ⚠️ Image 3.3.2 di Python 3.13 DITOLAK pada uji go/no-go B3 (dbt mati) — yang
|
||||
# dipakai adalah varian Python 3.11. Lihat deploy/output/b3-hasil-uji.md.
|
||||
# 2026-10-02: tag `-ol` = 3.3.2 + provider **OpenLineage** (permintaan pemilik
|
||||
# proyek). Provider itu INERT tanpa backend OpenLineage (belum ada di stack) —
|
||||
# jadi ini mengaktifkan kemampuan, belum menghasilkan lineage.
|
||||
# Rollback: sicabe/airflow:3.3.2-om203 (tanpa OpenLineage).
|
||||
export AIRFLOW_IMAGE="${AIRFLOW_IMAGE:-sicabe/airflow:3.3.2-om203-ol}"
|
||||
|
||||
# ─── Image refs yang bisa diarahkan ke MIRROR/registry internal ──────────────
|
||||
# Repositori asal bisa berubah kebijakan: mis. image community MinIO kini menuntut
|
||||
# autentikasi → `pull` anonim GAGAL ("unauthorized"). Arahkan ke registry/mirror
|
||||
# internal via var FULL-IMAGE di bawah, TANPA menyunting docker-compose.yml.
|
||||
# Bila var ini tidak diset, default dibangun dari *_VERSION di atas (perilaku lama).
|
||||
export MINIO_IMAGE="${MINIO_IMAGE:-quay.io/minio/minio:${MINIO_VERSION}}"
|
||||
export MINIO_MC_IMAGE="${MINIO_MC_IMAGE:-quay.io/minio/mc:${MINIO_MC_VERSION}}"
|
||||
export NESSIE_IMAGE="${NESSIE_IMAGE:-ghcr.io/projectnessie/nessie:${NESSIE_VERSION}}"
|
||||
|
||||
# ─── OpenLineage → OpenMetadata (eksperimen experiments/lineage-airflow-om) ──
|
||||
# Provider OpenLineage sudah terpasang di image di atas, tetapi INERT sampai
|
||||
# diarahkan ke OpenMetadata. Konfigurasi transport dibaca dari KONEKSI Airflow
|
||||
# `openlineage_config` (bukan dari env) supaya token lineage TIDAK perlu
|
||||
# ditulis di deploy/.env — lihat blok koneksi di bootstrap.sh.
|
||||
#
|
||||
# AKTIF sejak 2026-10-02 (adopsi opsi C — keputusan pemilik proyek).
|
||||
# ⚠️ PRASYARAT: koneksi `openlineage_config` + `openmetadata_lineage_api` harus
|
||||
# SUDAH ADA. Pemuatan konfigurasi provider TIDAK dibungkus try/except, jadi
|
||||
# bila CONFIG_CONN_ID diisi sedangkan koneksinya belum ada, task dapat GAGAL.
|
||||
# Koneksi dibuat oleh `./bootstrap.sh` (langkah "Setup connections").
|
||||
# Mematikan kembali : AIRFLOW_OPENLINEAGE_CONFIG_CONN_ID= (kosongkan).
|
||||
export AIRFLOW_OPENLINEAGE_CONFIG_CONN_ID="${AIRFLOW_OPENLINEAGE_CONFIG_CONN_ID:-openlineage_config}"
|
||||
# Namespace job OpenLineage — label asal job; tidak memengaruhi pemetaan tabel
|
||||
# (pemetaan tabel berjalan otomatis lewat nama dataset + service OM).
|
||||
export AIRFLOW_OPENLINEAGE_NAMESPACE="${AIRFLOW_OPENLINEAGE_NAMESPACE:-airflow_sicabe}"
|
||||
|
||||
export AIRFLOW__CORE__PARALLELISM="${AIRFLOW__CORE__PARALLELISM:-4}"
|
||||
# Catatan (R18/A2): AIRFLOW_CONN_MINIO_ACCESS_KEY & AIRFLOW_CONN_MINIO_SECRET_KEY
|
||||
# DIHAPUS dari berkas ini. Keduanya tidak pernah dikonsumsi skrip/compose mana pun
|
||||
# (diverifikasi dengan grep seluruh repo) dan nilainya TIDAK SAH untuk MinIO
|
||||
# produksi — pengujian langsung di experiments/airflow332 menghasilkan
|
||||
# SignatureDoesNotMatch (access key `minioadmin`, sedangkan secret-nya hanya
|
||||
# 10 karakter padahal password root 32 karakter).
|
||||
# Jalur resmi kredensial MinIO adalah S3_ACCESS_KEY / S3_SECRET_KEY dengan
|
||||
# fallback saat-pakai ke MINIO_ROOT_* (lihat blok "S3 / MinIO" di atas) — itulah
|
||||
# yang dipakai bootstrap.sh untuk membangun connection `minio_s3`, dan cara itu
|
||||
# sudah terverifikasi bekerja. Ini sekaligus menutup item R02-L4.
|
||||
# Catatan: AIRFLOW_CONN_TRINO_HOST/PORT DIHAPUS pada review Fase 5 — tidak pernah
|
||||
# dikonsumsi skrip/compose mana pun, dan connection 'trino_default' yang dulu
|
||||
# dibangun darinya sudah tidak disediakan (tidak dipakai DAG; versi lamanya
|
||||
# menunjuk http://trino:8080 tanpa kredensial → 403). Lihat §7.6.
|
||||
|
||||
# ─── OpenMetadata (Phase 6 — Governance: Metadata) ─────────────────────────
|
||||
# BATCH B (review R18): 2.0.2 -> 2.0.3 (wajib seversi dengan openmetadata-ingestion
|
||||
# di image Airflow). Migrasi OM bersifat SATU ARAH — rollback = pulihkan dump
|
||||
# (lihat deploy/output/merge-airflow332-rollback.md).
|
||||
export OPENMETADATA_VERSION="${OPENMETADATA_VERSION:-2.0.3}"
|
||||
export OPENMETADATA_PORT="${OPENMETADATA_PORT:-12085}"
|
||||
export OPENMETADATA_DB="${OPENMETADATA_DB:-openmetadata_db}"
|
||||
export OPENMETADATA_DB_USER="${OPENMETADATA_DB_USER:-openmetadata}"
|
||||
# Endpoint dari sisi HOST (gate Fase 6, bootstrap step 7, test-verification).
|
||||
# Untuk OpenMetadata EXISTING: OPENMETADATA_URL=http://om.corp:8585
|
||||
export OPENMETADATA_URL="${OPENMETADATA_URL:-http://localhost:${OPENMETADATA_PORT:-12085}}"
|
||||
|
||||
# ─── Grafana (Phase 6 — Governance: Observability) ─────────────────────────
|
||||
export GRAFANA_VERSION="${GRAFANA_VERSION:-13.1}"
|
||||
export GRAFANA_PORT="${GRAFANA_PORT:-12300}"
|
||||
# Endpoint dari sisi HOST. Untuk Grafana EXISTING: GRAFANA_URL=http://grafana.corp:3000
|
||||
export GRAFANA_URL="${GRAFANA_URL:-http://localhost:${GRAFANA_PORT:-12300}}"
|
||||
# Prometheus — sumber metrik untuk datasource Grafana (fase 6).
|
||||
export PROMETHEUS_VERSION="${PROMETHEUS_VERSION:-v3.6.0}"
|
||||
export PROMETHEUS_PORT="${PROMETHEUS_PORT:-12090}"
|
||||
# statsd-exporter — metrik Airflow (StatsD/UDP 9125) → Prometheus :9102 (M2 observabilitas).
|
||||
export STATSD_EXPORTER_VERSION="${STATSD_EXPORTER_VERSION:-v0.28.0}"
|
||||
export STATSD_EXPORTER_PORT="${STATSD_EXPORTER_PORT:-12091}"
|
||||
# jmx-exporter — saturasi Trino (JMX RMI) → Prometheus :9404 (M3 observabilitas).
|
||||
export JMX_EXPORTER_VERSION="${JMX_EXPORTER_VERSION:-latest}"
|
||||
|
||||
# ─── Port tambahan hasil merge eksperimen (Fase 3-6) ────────────────────────
|
||||
export OPENSEARCH_PORT="${OPENSEARCH_PORT:-12920}"
|
||||
export OPA_VERSION="${OPA_VERSION:-1.20.2}"
|
||||
export OPA_PORT="${OPA_PORT:-12181}"
|
||||
# Endpoint OPA dari sisi HOST (gate verifikasi + bootstrap step 4). Untuk OPA
|
||||
# EXISTING: OPA_URL=http://opa.corp:8181 (policy path tetap /v1/data/trino/allow).
|
||||
export OPA_URL="${OPA_URL:-http://localhost:${OPA_PORT:-12181}}"
|
||||
export OPENLDAP_PORT="${OPENLDAP_PORT:-12389}"
|
||||
export DREMIO_UI_PORT="${DREMIO_UI_PORT:-12947}"
|
||||
export DREMIO_JDBC_PORT="${DREMIO_JDBC_PORT:-12010}"
|
||||
export DREMIO_FLIGHT_PORT="${DREMIO_FLIGHT_PORT:-12011}"
|
||||
|
||||
# ─── Endpoint IAM (Keycloak & OpenLDAP) ─────────────────────────────────────
|
||||
# Default = container lokal hasil deploy ini. Untuk memakai IAM EXISTING,
|
||||
# override di .env (tiga variabel):
|
||||
#
|
||||
# KEYCLOAK_URL=http://kc.internal:8080 # dijangkau dari HOST (bootstrap/verify)
|
||||
# LDAP_URL=ldap://ldap.internal:389 # dijangkau dari CONTAINER (Trino & StarRocks)
|
||||
# LDAP_HOST=ldap.internal # dijangkau dari HOST (bootstrap/verify)
|
||||
# LDAP_PORT=389
|
||||
#
|
||||
# Dua alamat diperlukan karena pada mode lokal hostname-nya berbeda:
|
||||
# container melihat `openldap:389`, host melihat `localhost:12389`.
|
||||
export KEYCLOAK_URL="${KEYCLOAK_URL:-http://localhost:${KEYCLOAK_PORT:-12080}}"
|
||||
export LDAP_URL="${LDAP_URL:-ldap://openldap:389}"
|
||||
export LDAP_HOST="${LDAP_HOST:-localhost}"
|
||||
export LDAP_PORT="${LDAP_PORT:-${OPENLDAP_PORT:-12389}}"
|
||||
# Dremio admin dibuat otomatis oleh bootstrap.sh (Step 9) via REST API first-run,
|
||||
# lalu dipakai script scripts/dremio/register-nessie-source.sh untuk source Nessie.
|
||||
export DREMIO_URL="${DREMIO_URL:-http://localhost:12947}"
|
||||
|
||||
# ─── Port INTERNAL (container ↔ container) — bisa diparameterkan ─────────────
|
||||
# Port yang dipakai ANTAR-container untuk menjangkau layanan; default = port
|
||||
# container standar. Ubah HANYA bila layanan yang dipakai berjalan di port berbeda
|
||||
# (mis. Keycloak/Postgres EXISTING) — cukup di .env, TANPA menyunting docker-compose.yml.
|
||||
# Layanan milik cluster sendiri selalu memakai DEFAULT ini (hostname service + port container),
|
||||
# jadi mengubahnya hanya relevan untuk layanan eksternal berport non-standar.
|
||||
export POSTGRES_INTERNAL_PORT="${POSTGRES_INTERNAL_PORT:-5432}"
|
||||
export KEYCLOAK_INTERNAL_PORT="${KEYCLOAK_INTERNAL_PORT:-8080}"
|
||||
export MINIO_INTERNAL_PORT="${MINIO_INTERNAL_PORT:-9000}"
|
||||
export NESSIE_INTERNAL_PORT="${NESSIE_INTERNAL_PORT:-19120}"
|
||||
export OPENLDAP_INTERNAL_PORT="${OPENLDAP_INTERNAL_PORT:-389}"
|
||||
export OPA_INTERNAL_PORT="${OPA_INTERNAL_PORT:-8181}"
|
||||
|
||||
# ─── Komponen EXISTING (dikelola sistem lain) ────────────────────────────────
|
||||
# Bila sebuah komponen sudah dikelola sistem lain, container-nya TIDAK perlu
|
||||
# (dan tidak boleh) dinyalakan oleh deploy ini. install.sh menyimpulkan ini
|
||||
# secara OTOMATIS dari endpoint di atas: bila nilainya berbeda dari nilai lokal
|
||||
# docker-compose.yml, komponen itu dianggap eksternal.
|
||||
#
|
||||
# keycloak lokal http://localhost:12080 → jadikan eksternal: KEYCLOAK_URL
|
||||
# openldap lokal ldap://openldap:389 → LDAP_URL
|
||||
# minio lokal http://minio:9000 → S3_ENDPOINT (+ minio-init dilewati)
|
||||
# nessie lokal http://nessie:19120 → NESSIE_URL
|
||||
# trino lokal https://localhost:12443 → TRINO_URL
|
||||
# opa lokal http://localhost:12181 → OPA_URL
|
||||
#
|
||||
# Postgres tidak punya variabel endpoint → set eksplisit bila dikelola DBA.
|
||||
# EXTERNAL_SERVICES menang atas auto-deteksi; LOCAL_SERVICES memaksa kembali lokal.
|
||||
# Nama service mengikuti docker-compose.yml (mis. starrocks-fe, airflow-webserver).
|
||||
export EXTERNAL_SERVICES="${EXTERNAL_SERVICES:-}"
|
||||
export LOCAL_SERVICES="${LOCAL_SERVICES:-}"
|
||||
export DREMIO_USER="${DREMIO_USER:-admin}"
|
||||
export DREMIO_PASSWORD="${DREMIO_PASSWORD:-dremio_admin_DevOnly1}"
|
||||
export DREMIO_ADMIN_EMAIL="${DREMIO_ADMIN_EMAIL:-admin@sicabe.local}"
|
||||
|
||||
# ─── StarRocks credentials (root + aplikasi) ────────────────────────────────
|
||||
export STARROCKS_ROOT_PASSWORD="${STARROCKS_ROOT_PASSWORD:-starrocks_root_DevOnly}"
|
||||
export STARROCKS_APP_PASSWORD="${STARROCKS_APP_PASSWORD:-starrocks_app_DevOnly}"
|
||||
@@ -0,0 +1,16 @@
|
||||
# =============================================================================
|
||||
# Grafana — Provisioning Dashboards
|
||||
# Referensi arsitektur: bab 4.6.3.3 (Setup dashboard bawaan)
|
||||
# =============================================================================
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: "SiCABE Data Quality"
|
||||
orgId: 1
|
||||
folder: "SiCABE"
|
||||
type: file
|
||||
disableDeletion: false
|
||||
editable: true
|
||||
updateIntervalSeconds: 60
|
||||
options:
|
||||
path: /etc/grafana/provisioning/dashboards/sicabe
|
||||
@@ -0,0 +1,260 @@
|
||||
{
|
||||
"title": "SiCABE — Host Docker (node_exporter)",
|
||||
"uid": "sicabe-hosts",
|
||||
"version": 1,
|
||||
"timezone": "Asia/Jakarta",
|
||||
"schemaVersion": 36,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"tags": ["sicabe", "host", "node_exporter"],
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Host Up",
|
||||
"type": "stat",
|
||||
"datasource": "Prometheus",
|
||||
"gridPos": { "h": 8, "w": 4, "x": 0, "y": 0 },
|
||||
"description": "Status node_exporter per host (:9100, pra-syarat host di luar installer). 1 = up, 0 = down. Job: node.",
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"textMode": "value_and_name",
|
||||
"orientation": "vertical"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"mappings": [
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"0": { "text": "DOWN", "color": "red", "index": 1 },
|
||||
"1": { "text": "UP", "color": "green", "index": 0 }
|
||||
}
|
||||
}
|
||||
],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{ "refId": "A", "expr": "up{job=\"node\"}", "legendFormat": "{{instance}}" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "CPU Usage %",
|
||||
"type": "timeseries",
|
||||
"datasource": "Prometheus",
|
||||
"gridPos": { "h": 8, "w": 10, "x": 4, "y": 0 },
|
||||
"description": "Pemakaian CPU per host = 100 × (1 - idle), rata-rata jendela 5m.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "100 * (1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])))",
|
||||
"legendFormat": "{{instance}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Load Average (1/5/15m)",
|
||||
"type": "timeseries",
|
||||
"datasource": "Prometheus",
|
||||
"gridPos": { "h": 8, "w": 10, "x": 14, "y": 0 },
|
||||
"description": "Load average 1/5/15 menit per host. Bandingkan dengan jumlah CPU; load > jumlah CPU = jenuh.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"custom": { "drawStyle": "line", "fillOpacity": 0, "lineWidth": 1, "showPoints": "never" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{ "refId": "A", "expr": "node_load1", "legendFormat": "{{instance}} load1" },
|
||||
{ "refId": "B", "expr": "node_load5", "legendFormat": "{{instance}} load5" },
|
||||
{ "refId": "C", "expr": "node_load15", "legendFormat": "{{instance}} load15" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Memory Used %",
|
||||
"type": "timeseries",
|
||||
"datasource": "Prometheus",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 8 },
|
||||
"description": "Pemakaian memori per host = 100 × (1 - MemAvailable/MemTotal). Ambang: 80% warning, 90% critical.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "orange", "value": 80 },
|
||||
{ "color": "red", "value": 90 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)",
|
||||
"legendFormat": "{{instance}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Disk I/O",
|
||||
"type": "timeseries",
|
||||
"datasource": "Prometheus",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 8 },
|
||||
"description": "Laju baca/tulis disk per host (rata-rata 5m, semua device digabung).",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "Bps",
|
||||
"custom": { "drawStyle": "line", "fillOpacity": 5, "lineWidth": 1, "showPoints": "never" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (instance) (rate(node_disk_read_bytes_total[5m]))",
|
||||
"legendFormat": "{{instance}} read"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum by (instance) (rate(node_disk_written_bytes_total[5m]))",
|
||||
"legendFormat": "{{instance}} write"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Network Traffic",
|
||||
"type": "timeseries",
|
||||
"datasource": "Prometheus",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 8 },
|
||||
"description": "Laju RX/TX per host, antarmuka fisik saja (lo/veth/docker/bridge dikecualikan).",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "Bps",
|
||||
"custom": { "drawStyle": "line", "fillOpacity": 5, "lineWidth": 1, "showPoints": "never" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|br-.*|virbr.*\"}[5m]))",
|
||||
"legendFormat": "{{instance}} RX"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|br-.*|virbr.*\"}[5m]))",
|
||||
"legendFormat": "{{instance}} TX"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Disk Usage % (per host & mount)",
|
||||
"type": "table",
|
||||
"datasource": "Prometheus",
|
||||
"gridPos": { "h": 8, "w": 16, "x": 0, "y": 16 },
|
||||
"description": "Persentase pemakaian tiap mount. Filesystem pseudo (tmpfs/overlay/squashfs/ramfs) dikecualikan. Ambang: 80% warning, 90% critical.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"decimals": 1,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "orange", "value": 80 },
|
||||
{ "color": "red", "value": 90 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Value" },
|
||||
"properties": [
|
||||
{ "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "gradient" } }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": { "showHeader": true },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "100 * (1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|ramfs\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|ramfs\"})",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Uptime",
|
||||
"type": "stat",
|
||||
"datasource": "Prometheus",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 16 },
|
||||
"description": "Lama host hidup sejak boot = time() - node_boot_time_seconds.",
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"colorMode": "none",
|
||||
"graphMode": "none",
|
||||
"textMode": "value_and_name",
|
||||
"orientation": "vertical"
|
||||
},
|
||||
"fieldConfig": { "defaults": { "unit": "dtdurations" }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "refId": "A", "expr": "time() - node_boot_time_seconds", "legendFormat": "{{instance}}" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
{
|
||||
"title": "SiCABE — Overview Lapisan & Tata Kelola Data",
|
||||
"version": 6,
|
||||
"timezone": "Asia/Jakarta",
|
||||
"schemaVersion": 36,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "DAG Failure Rate (Ingestion Layer)",
|
||||
"type": "stat",
|
||||
"datasource": "Prometheus",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 0 },
|
||||
"description": "KPI #7 — rasio DAG run gagal (jendela 1 jam). Ambang: 5% warning, 15% critical. Sumber: airflow_dagrun_duration_{failed,success}_count (Airflow StatsD → statsd-exporter).",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "orange", "value": 5 },
|
||||
{ "color": "red", "value": 15 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{ "refId": "A", "expr": "100 * (sum(rate(airflow_dagrun_duration_failed_count[1h])) or vector(0)) / clamp_min((sum(rate(airflow_dagrun_duration_failed_count[1h])) or vector(0)) + (sum(rate(airflow_dagrun_duration_success_count[1h])) or vector(0)), 0.0001)" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Disk Usage (MinIO — Storage Layer)",
|
||||
"type": "gauge",
|
||||
"datasource": "MinIO Metrics",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 0 },
|
||||
"description": "KPI #1 — kapasitas terpakai (%) = 1 - usable_free/usable_total. Ambang: 80% warning, 90% critical.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "orange", "value": 80 },
|
||||
{ "color": "red", "value": 90 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{ "refId": "A", "expr": "(1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) * 100" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Serving Query Latency P99 (StarRocks)",
|
||||
"type": "gauge",
|
||||
"datasource": "Prometheus",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 0 },
|
||||
"description": "KPI #6 — latensi query P99 StarRocks FE. Ambang: 500 ms warning, 2000 ms critical.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ms",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "orange", "value": 500 },
|
||||
{ "color": "red", "value": 2000 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{ "refId": "A", "expr": "starrocks_fe_query_latency_ms{quantile=\"0.99\"}" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Trino — Saturasi Query (coordinator)",
|
||||
"type": "timeseries",
|
||||
"datasource": "Prometheus",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 8 },
|
||||
"description": "KPI #4 (SATURASI) — antrean & kegagalan query Trino coordinator. ⚠️ P99 LATENCY tidak di sini: tidak tersedia via JMX/StatsD (metrik turunan, di luar lingkup M1–M3). Sumber: JMX → trino-jmx-exporter.",
|
||||
"targets": [
|
||||
{ "refId": "A", "legendFormat": "running query", "expr": "trino_execution_querymanager_runningqueries" },
|
||||
{ "refId": "B", "legendFormat": "queued query", "expr": "trino_execution_querymanager_queuedqueries" },
|
||||
{ "refId": "C", "legendFormat": "failed /s", "expr": "sum(rate(trino_execution_querymanager_failedqueries_totalcount[5m]))" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "S3 API Latency (MinIO) — % panggilan lambat",
|
||||
"type": "timeseries",
|
||||
"datasource": "MinIO Metrics",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 8 },
|
||||
"description": "KPI #2 — rasio panggilan S3 yang lebih lambat dari ambang (jendela 5m). Warning 50 ms; critical ~250 ms (bucket terdekat — MinIO tidak menyediakan bucket 200 ms). Basis metrik minio_s3_requests_ttfb_seconds_distribution bertipe gauge (bukan histogram), sehingga dipakai rasio barbucket, bukan histogram_quantile.",
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "percent", "min": 0 },
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"legendFormat": "% > 50 ms (warning)",
|
||||
"expr": "100 * (1 - sum(rate(minio_s3_requests_ttfb_seconds_distribution{le=\"0.050\"}[5m])) / sum(rate(minio_s3_requests_ttfb_seconds_distribution{le=\"+Inf\"}[5m])))"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"legendFormat": "% > 250 ms (critical)",
|
||||
"expr": "100 * (1 - sum(rate(minio_s3_requests_ttfb_seconds_distribution{le=\"0.250\"}[5m])) / sum(rate(minio_s3_requests_ttfb_seconds_distribution{le=\"+Inf\"}[5m])))"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Airflow — Durasi DAG & Task",
|
||||
"type": "timeseries",
|
||||
"datasource": "Prometheus",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 },
|
||||
"description": "KPI #5 (perkiraan) — durasi pipeline & task. Airflow pada StatsD ini TIDAK memecah durasi per-task (hanya summary global `airflow_task_duration`), jadi durasi dbt (silver_to_gold/bronze_to_silver) belum bisa diisolasi. Perbaikan menyusul: statsd mapping config (lihat Review 19). DAG P90 = durasi `sicabe_batch_daily`.",
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"legendFormat": "rata-rata task (dtk)",
|
||||
"expr": "sum(rate(airflow_task_duration_sum[1h])) / clamp_min(sum(rate(airflow_task_duration_count[1h])), 0.0001)"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"legendFormat": "DAG sicabe_batch_daily P90 (dtk)",
|
||||
"expr": "airflow_dagrun_duration_success_sicabe_batch_daily{quantile=\"0.9\"}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Trino — Query Throughput (coordinator)",
|
||||
"type": "timeseries",
|
||||
"datasource": "Prometheus",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 },
|
||||
"description": "Laju siklus hidup query Trino coordinator (per detik, jendela 5m). Sumber: JMX → trino-jmx-exporter (domain trino.execution).",
|
||||
"targets": [
|
||||
{ "refId": "A", "legendFormat": "submitted /s", "expr": "sum(rate(trino_execution_querymanager_submittedqueries_totalcount[5m]))" },
|
||||
{ "refId": "B", "legendFormat": "completed /s", "expr": "sum(rate(trino_execution_querymanager_completedqueries_totalcount[5m]))" },
|
||||
{ "refId": "C", "legendFormat": "failed /s", "expr": "sum(rate(trino_execution_querymanager_failedqueries_totalcount[5m]))" },
|
||||
{ "refId": "D", "legendFormat": "canceled /s", "expr": "sum(rate(trino_execution_querymanager_canceledqueries_totalcount[5m]))" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Grafana datasource (hasil provisioning)
|
||||
# =============================================================================
|
||||
# Datasource bertipe `prometheus` WAJIB menunjuk server Prometheus
|
||||
# (`http://prometheus:9090`). Menunjuk `http://minio:9000` akan selalu gagal
|
||||
# ("error querying the Prometheus API") karena MinIO adalah API S3, bukan
|
||||
# server Prometheus — metrik MinIO diambil Prometheus lewat scrape
|
||||
# (lihat config/prometheus/prometheus.yml), bukan dibaca langsung oleh Grafana.
|
||||
#
|
||||
# Datasource OpenMetadata DIHAPUS pada review Fase 6: plugin
|
||||
# `grafana-opendatasource-http` TIDAK ADA di katalog Grafana (404) sehingga
|
||||
# selalu "Plugin not registered". Tidak ada plugin resmi Grafana → OpenMetadata;
|
||||
# arah yang didukung OpenMetadata adalah sebaliknya (OM meng-ingest Grafana).
|
||||
# Bila data OM ingin ditampilkan di Grafana, pakai datasource HTTP generik
|
||||
# (mis. Infinity) + token API OpenMetadata.
|
||||
# =============================================================================
|
||||
apiVersion: 1
|
||||
|
||||
# Grafana provisioning hanya MENAMBAH/MENGUBAH — datasource yang dihapus dari
|
||||
# berkas ini TIDAK ikut terhapus. Karena itu penghapusan ditulis eksplisit di
|
||||
# bawah (idempoten, aman dijalankan berulang).
|
||||
deleteDatasources:
|
||||
- name: OpenMetadata
|
||||
orgId: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
|
||||
# Metrik MinIO tetap memakai instance Prometheus yang sama (job_name: minio),
|
||||
# dinamai terpisah karena dipakai panel "Storage Usage" di dashboard.
|
||||
- name: MinIO Metrics
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
editable: false
|
||||
jsonData:
|
||||
tlsSkipVerify: true
|
||||
@@ -0,0 +1,160 @@
|
||||
{
|
||||
"//": "⚠️ File referensi — gunakan bootstrap.sh untuk deployment otomatis.",
|
||||
"//2": "JSON tidak mendukung shell variable expansion. Ubah nilai secara manual jika perlu import via UI.",
|
||||
"realm": "sicabe",
|
||||
"displayName": "eBdesk SiCABE",
|
||||
"enabled": true,
|
||||
"sslRequired": "none",
|
||||
"registrationAllowed": false,
|
||||
"loginWithEmailAllowed": true,
|
||||
"resetPasswordAllowed": true,
|
||||
"accessTokenLifespan": 3600,
|
||||
"ssoSessionMaxLifespan": 86400,
|
||||
"clients": [
|
||||
{
|
||||
"clientId": "trino",
|
||||
"name": "Trino — Query Engine",
|
||||
"description": "OIDC client untuk Trino (Processing Layer)",
|
||||
"enabled": true,
|
||||
"protocol": "openid-connect",
|
||||
"clientAuthenticatorType": "client-secret",
|
||||
"secret": "trino-secret",
|
||||
"standardFlowEnabled": true,
|
||||
"serviceAccountsEnabled": true,
|
||||
"publicClient": false,
|
||||
"redirectUris": ["http://localhost:*", "https://localhost:*"],
|
||||
"webOrigins": ["http://localhost:*", "https://localhost:*"]
|
||||
},
|
||||
{
|
||||
"clientId": "starrocks",
|
||||
"name": "StarRocks — OLAP Engine",
|
||||
"description": "OIDC client untuk StarRocks (Serving Layer)",
|
||||
"enabled": true,
|
||||
"protocol": "openid-connect",
|
||||
"clientAuthenticatorType": "client-secret",
|
||||
"secret": "starrocks-secret",
|
||||
"standardFlowEnabled": true,
|
||||
"serviceAccountsEnabled": true,
|
||||
"publicClient": false,
|
||||
"redirectUris": ["http://localhost:12030/*", "http://starrocks-fe:8030/api/oauth2"],
|
||||
"webOrigins": ["http://localhost:12030"]
|
||||
},
|
||||
{
|
||||
"clientId": "airflow",
|
||||
"name": "Apache Airflow — Orchestrator",
|
||||
"description": "OIDC client untuk Airflow (Ingestion Layer)",
|
||||
"enabled": true,
|
||||
"protocol": "openid-connect",
|
||||
"clientAuthenticatorType": "client-secret",
|
||||
"secret": "airflow-secret",
|
||||
"standardFlowEnabled": true,
|
||||
"serviceAccountsEnabled": true,
|
||||
"publicClient": false,
|
||||
"redirectUris": ["http://localhost:12082/*"],
|
||||
"webOrigins": ["http://localhost:12082"]
|
||||
},
|
||||
{
|
||||
"clientId": "openmetadata",
|
||||
"name": "OpenMetadata — Metadata Platform",
|
||||
"description": "OIDC client untuk OpenMetadata (Governance Layer)",
|
||||
"enabled": true,
|
||||
"protocol": "openid-connect",
|
||||
"clientAuthenticatorType": "client-secret",
|
||||
"secret": "om-secret",
|
||||
"standardFlowEnabled": true,
|
||||
"serviceAccountsEnabled": true,
|
||||
"publicClient": false,
|
||||
"redirectUris": ["http://localhost:12085/*"],
|
||||
"webOrigins": ["http://localhost:12085"]
|
||||
},
|
||||
{
|
||||
"clientId": "grafana",
|
||||
"name": "Grafana — Observability",
|
||||
"description": "OIDC client untuk Grafana (Governance Layer)",
|
||||
"enabled": true,
|
||||
"protocol": "openid-connect",
|
||||
"clientAuthenticatorType": "client-secret",
|
||||
"secret": "grafana-secret",
|
||||
"standardFlowEnabled": true,
|
||||
"serviceAccountsEnabled": true,
|
||||
"publicClient": false,
|
||||
"redirectUris": ["http://localhost:12300/*"],
|
||||
"webOrigins": ["http://localhost:12300"]
|
||||
}
|
||||
],
|
||||
"roles": {
|
||||
"realm": [
|
||||
{
|
||||
"name": "admin",
|
||||
"description": "Administrator — akses penuh ke seluruh komponen"
|
||||
},
|
||||
{
|
||||
"name": "engineer",
|
||||
"description": "Data Engineer — akses ke Ingestion, Processing, Storage"
|
||||
},
|
||||
{
|
||||
"name": "analyst",
|
||||
"description": "Data Analyst — akses baca ke Serving dan Governance"
|
||||
},
|
||||
{
|
||||
"name": "viewer",
|
||||
"description": "Viewer — akses baca terbatas ke dashboard dan metadata"
|
||||
}
|
||||
]
|
||||
},
|
||||
"users": [
|
||||
{
|
||||
"username": "admin",
|
||||
"enabled": true,
|
||||
"email": "admin@sicabe.local",
|
||||
"firstName": "Administrator",
|
||||
"lastName": "SiCABE",
|
||||
"credentials": [{ "type": "password", "value": "admin123", "temporary": false }],
|
||||
"realmRoles": ["admin"],
|
||||
"clientRoles": {
|
||||
"trino": ["admin"],
|
||||
"starrocks": ["admin"],
|
||||
"airflow": ["admin"],
|
||||
"openmetadata": ["admin"],
|
||||
"grafana": ["Admin"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"username": "engineer",
|
||||
"enabled": true,
|
||||
"email": "engineer@sicabe.local",
|
||||
"firstName": "Data",
|
||||
"lastName": "Engineer",
|
||||
"credentials": [{ "type": "password", "value": "engineer123", "temporary": false }],
|
||||
"realmRoles": ["engineer"]
|
||||
},
|
||||
{
|
||||
"username": "analyst",
|
||||
"enabled": true,
|
||||
"email": "analyst@sicabe.local",
|
||||
"firstName": "Data",
|
||||
"lastName": "Analyst",
|
||||
"credentials": [{ "type": "password", "value": "analyst123", "temporary": false }],
|
||||
"realmRoles": ["analyst"]
|
||||
},
|
||||
{
|
||||
"username": "viewer",
|
||||
"enabled": true,
|
||||
"email": "viewer@sicabe.local",
|
||||
"firstName": "Viewer",
|
||||
"lastName": "SiCABE",
|
||||
"credentials": [{ "type": "password", "value": "viewer123", "temporary": false }],
|
||||
"realmRoles": ["viewer"]
|
||||
}
|
||||
],
|
||||
"scopeMappings": [
|
||||
{
|
||||
"client": "trino",
|
||||
"roles": ["admin", "engineer"]
|
||||
},
|
||||
{
|
||||
"client": "airflow",
|
||||
"roles": ["admin", "engineer"]
|
||||
}
|
||||
]
|
||||
}
|
||||
Executable
+30
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# MinIO — Inisialisasi Bucket dan Policy
|
||||
# Referensi arsitektur: bab 4.2.1 (MinIO — Object Storage)
|
||||
# - 4.2.1.7 Buat bucket awal (iceberg-data)
|
||||
# - 4.2.1.8 Buat service account dan policy akses
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
MINIO_ALIAS="${MINIO_ALIAS:-sicabe}"
|
||||
MINIO_ENDPOINT="${MINIO_ENDPOINT:-http://minio:9000}"
|
||||
MINIO_ROOT_USER="${MINIO_ROOT_USER:-minioadmin}"
|
||||
MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:?err}"
|
||||
MINIO_BUCKET_ICEBERG_DATA="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}"
|
||||
|
||||
echo "[MinIO] Configuring alias..."
|
||||
mc alias set "${MINIO_ALIAS}" "${MINIO_ENDPOINT}" "${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}"
|
||||
|
||||
# ─── Buat bucket awal (4.2.1.7) ──────────────────────────────────────────────
|
||||
echo "[MinIO] Creating buckets..."
|
||||
for bucket in "${MINIO_BUCKET_ICEBERG_DATA}"; do
|
||||
if mc ls "${MINIO_ALIAS}/${bucket}" >/dev/null 2>&1; then
|
||||
echo "[MinIO] Bucket '${bucket}' already exists, skipping."
|
||||
else
|
||||
mc mb "${MINIO_ALIAS}/${bucket}"
|
||||
echo "[MinIO] Bucket '${bucket}' created."
|
||||
fi
|
||||
done
|
||||
|
||||
echo "[MinIO] Buckets initialized successfully."
|
||||
@@ -0,0 +1,82 @@
|
||||
# Nessie — Konfigurasi `deploy/`
|
||||
|
||||
Nessie menggantikan Apache Polaris sebagai **Iceberg REST catalog** pada
|
||||
`deploy/`. Konfigurasi diberikan sepenuhnya melalui **environment variable
|
||||
di `docker-compose.yml`** (bukan file `application.properties`), dengan alasan:
|
||||
|
||||
- Kredensial (password PostgreSQL, access key MinIO) harus berasal dari `.env`
|
||||
yang di-generate `install.sh` — mengikuti pola komponen lain di SiCABE.
|
||||
- Env var adalah bentuk yang didokumentasikan resmi untuk image Nessie dan tidak
|
||||
bergantung pada lokasi mount file config di dalam image.
|
||||
|
||||
## Versi
|
||||
|
||||
| Item | Nilai |
|
||||
|---|---|
|
||||
| Image | `ghcr.io/projectnessie/nessie:${NESSIE_VERSION:-0.108.4-java}` |
|
||||
| Port API/Iceberg REST | `19120` (host `${NESSIE_PORT:-12120}`) |
|
||||
| Port management (health/metrics) | `9000` (internal, tidak dipublikasikan) |
|
||||
| Path Iceberg REST | `/iceberg/` (default warehouse, branch `main`) |
|
||||
| Path Nessie REST v2 | `/api/v2` |
|
||||
|
||||
## Persistensi: JDBC2 di PostgreSQL
|
||||
|
||||
Mengikuti **konvensi shared PostgreSQL** SiCABE (lihat `config/postgres/init/01-create-dbs.sh`):
|
||||
|
||||
```yaml
|
||||
NESSIE_VERSION_STORE_TYPE: "JDBC2"
|
||||
NESSIE_VERSION_STORE_PERSIST_JDBC_DATASOURCE: "postgresql"
|
||||
QUARKUS_DATASOURCE_POSTGRESQL_DB_KIND: "postgresql"
|
||||
QUARKUS_DATASOURCE_POSTGRESQL_ACTIVE: "true"
|
||||
QUARKUS_DATASOURCE_POSTGRESQL_DEVSERVICES_ENABLED: "false"
|
||||
QUARKUS_DATASOURCE_POSTGRESQL_JDBC_URL: "jdbc:postgresql://postgres:5432/nessie?currentSchema=nessie"
|
||||
QUARKUS_DATASOURCE_POSTGRESQL_USERNAME: "nessie"
|
||||
QUARKUS_DATASOURCE_POSTGRESQL_PASSWORD: "${NESSIE_DB_PASSWORD}"
|
||||
```
|
||||
|
||||
> **Penting**: Nessie **tidak** memakai Flyway. Database `nessie` **dan** schema
|
||||
> `nessie` harus dibuat lebih dulu (dilakukan `01-create-dbs.sh`); tabel-tabel
|
||||
> Nessie di-auto-create saat startup. Driver PostgreSQL sudah ter-bundle di image.
|
||||
|
||||
> **Catatan versi**: `JDBC` (tanpa `2`) masih ada tetapi **deprecated** di 0.108.x —
|
||||
> gunakan `JDBC2`.
|
||||
|
||||
## Warehouse & S3/MinIO (server-side)
|
||||
|
||||
```yaml
|
||||
NESSIE_CATALOG_DEFAULT_WAREHOUSE: "warehouse"
|
||||
NESSIE_CATALOG_WAREHOUSES_WAREHOUSE_LOCATION: "s3://${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/warehouse"
|
||||
NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_ENDPOINT: "http://minio:9000"
|
||||
NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_REGION: "us-east-1"
|
||||
NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_AUTH_TYPE: "STATIC"
|
||||
NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_PATH_STYLE_ACCESS: "true"
|
||||
NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_ACCESS_KEY: "urn:nessie-secret:quarkus:minio.creds"
|
||||
MINIO_CREDS_NAME: "${MINIO_ROOT_USER}"
|
||||
MINIO_CREDS_SECRET: "${MINIO_ROOT_PASSWORD}"
|
||||
```
|
||||
|
||||
> **Penting**: nama secret harus memakai **titik** (`minio.creds`), bukan hyphen.
|
||||
> Quarkus env mapping tidak men-decode hyphen dengan benar (temuan eksperimen
|
||||
> nessie-dremio) → error `Missing access key and secret for STATIC authentication mode`.
|
||||
|
||||
- Warehouse memakai bucket `iceberg-data` (konvensi base) dengan prefix `warehouse/`.
|
||||
- `default-options.` **wajib** untuk Nessie ≥ 0.92.
|
||||
- Kredensial S3 server-side **tidak diekspos ke klien**; engine (Trino/StarRocks/Dremio)
|
||||
mengakses MinIO dengan kredensialnya sendiri.
|
||||
- `external-endpoint` sengaja tidak di-set karena semua klien berada di dalam
|
||||
network Docker yang sama (`minio:9000`).
|
||||
|
||||
## Healthcheck
|
||||
|
||||
Image Nessie tidak menyediakan `curl`/`wget`, sehingga healthcheck memakai
|
||||
`bash /dev/tcp` ke **management port 9000**:
|
||||
|
||||
```yaml
|
||||
test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/localhost/9000 && printf 'GET /q/health/ready HTTP/1.0\\r\\n\\r\\n' >&3 && cat <&3 | grep -q '200 OK'"]
|
||||
```
|
||||
|
||||
## Referensi
|
||||
|
||||
- https://projectnessie.org/nessie-0-108-4/configuration/
|
||||
- https://projectnessie.org/guides/iceberg-rest/
|
||||
- Cache riset: `.tmp/external-context/nessie/`
|
||||
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"roles": {
|
||||
"data-admin": {
|
||||
"superuser": true
|
||||
},
|
||||
"data-engineer": {
|
||||
"tables": {
|
||||
"bronze_orders": { "select": true, "write": true, "ddl": true, "columns": "*" },
|
||||
"bronze_orders_incremental": { "select": true, "write": true, "ddl": true, "columns": "*" },
|
||||
"silver_orders_silver": { "select": true, "write": true, "ddl": true, "columns": "*" },
|
||||
"gold_orders_gold": { "select": true, "write": true, "ddl": true, "columns": "*" }
|
||||
}
|
||||
},
|
||||
"data-catalog": {
|
||||
"tables": {
|
||||
"bronze_orders": { "select": true, "columns": "*" },
|
||||
"bronze_orders_incremental": { "select": true, "columns": "*" },
|
||||
"silver_orders_silver": { "select": true, "columns": "*" },
|
||||
"silver_orders_clean": { "select": true, "columns": "*" },
|
||||
"gold_orders_gold": { "select": true, "columns": "*" }
|
||||
}
|
||||
},
|
||||
"data-analyst": {
|
||||
"tables": {
|
||||
"gold_orders_gold": { "select": true, "columns": "*" },
|
||||
"silver_orders_silver": { "select": true, "columns": ["id", "total_value"] }
|
||||
}
|
||||
},
|
||||
"data-viewer": {}
|
||||
},
|
||||
"users": {
|
||||
"svc_dbt": "data-engineer",
|
||||
"svc_airflow": "data-engineer",
|
||||
"svc_seed": "data-engineer",
|
||||
"svc_om": "data-catalog"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
# =============================================================================
|
||||
# OPA — policy Trino access-control (package trino) — OPA-ONLY
|
||||
# -----------------------------------------------------------------------------
|
||||
# Trino (access-control.name=opa) mengirim satu request per operasi:
|
||||
# POST http://opa:8181/v1/data/trino/allow
|
||||
# input.context.identity = { "user": "...", "groups": [...] }
|
||||
# input.action.operation = "SELECT" | "INSERT" | ...
|
||||
# input.action.resource = { catalogName, schemaName, tableName, columns }
|
||||
# OPA mengembalikan { "result": { "allow": true|false } }.
|
||||
#
|
||||
# Grup user berasal dari LDAP group provider Trino (OpenLDAP, cn = data-*).
|
||||
# Izin didefinisikan sebagai data di data.json → data.roles.<grup>.tables.<schema_table>.
|
||||
# grant = { select: bool, write: bool, ddl: bool, columns: "*" | [..] }
|
||||
# - data-admin : superuser (semua operasi)
|
||||
# - select : SELECT/SHOW COLUMNS/DESCRIBE
|
||||
# - write : INSERT/UPDATE/DELETE/MERGE
|
||||
# - ddl : CREATE/DROP/ALTER/... TABLE
|
||||
# Kolom: bila columns != "*", SELECT hanya boleh menyebut kolom tsb.
|
||||
# deny-by-default.
|
||||
# =============================================================================
|
||||
package trino
|
||||
|
||||
import rego.v1
|
||||
|
||||
default allow := false
|
||||
|
||||
# Normalisasi operasi: uppercase, buang spasi & underscore.
|
||||
op := upper(replace(replace(trim_space(input.action.operation), " ", ""), "_", ""))
|
||||
|
||||
identity_groups := input.context.identity.groups
|
||||
identity_user := input.context.identity.user
|
||||
|
||||
# Role efektif = grup (dari LDAP) ∪ role user (data.users) — untuk service account
|
||||
# yang tidak terdaftar di LDAP (mis. svc_dbt, svc_om dari password file Basic).
|
||||
effective_roles contains r if {
|
||||
some g in identity_groups
|
||||
r := g
|
||||
}
|
||||
effective_roles contains r if {
|
||||
r := data.users[identity_user]
|
||||
}
|
||||
|
||||
# Grup 'data-admin' = superuser (bypass semua pembatasan data/sistem).
|
||||
is_super if {
|
||||
some r in effective_roles
|
||||
r == "data-admin"
|
||||
}
|
||||
|
||||
# ── CREATE/DROP SCHEMA (DDL level-schema) ───────────────────────────────────
|
||||
# Trino mengirim operasi ini tanpa resource tabel → tak bisa memakai grant
|
||||
# per-tabel di op_caps. Izinkan untuk peran engineering (data-admin/engineer).
|
||||
allow if {
|
||||
op in {"CREATESCHEMA", "DROPSCHEMA"}
|
||||
some r in effective_roles
|
||||
r in {"data-admin", "data-engineer"}
|
||||
}
|
||||
|
||||
# ── Pemetaan operasi -> kemampuan yang dibutuhkan ───────────────────────────
|
||||
# SELECTFROMCOLUMNS TIDAK ada di sini (ditangani khusus + cek kolom).
|
||||
op_caps := {
|
||||
"SHOWCOLUMNS": "select",
|
||||
"DESCRIBE": "select",
|
||||
"INSERTINTOTABLE": "write",
|
||||
"UPDATE": "write",
|
||||
"DELETE": "write",
|
||||
"MERGE": "write",
|
||||
"CREATETABLE": "ddl",
|
||||
"DROPTABLE": "ddl",
|
||||
"TRUNCATETABLE": "ddl",
|
||||
"ALTERTABLE": "ddl",
|
||||
"RENAMETABLE": "ddl",
|
||||
"ADDCOLUMN": "ddl",
|
||||
"DROPCOLUMN": "ddl",
|
||||
"RENAMECOLUMN": "ddl",
|
||||
"COMMENTTABLE": "ddl",
|
||||
}
|
||||
|
||||
# ── Operasi metadata yang diizinkan utk user terautentikasi ────────────────
|
||||
meta_op(opx) if startswith(opx, "SHOW")
|
||||
meta_op(opx) if opx in {"USE", "SETSESSION", "RESETSESSION", "SETROLE", "SHOWROLES", "SHOWCURRENTROLES"}
|
||||
|
||||
# ── Eksekusi query (level sistem) ───────────────────────────────────────────
|
||||
allow if op == "EXECUTEQUERY"
|
||||
|
||||
# ── Operasi metadata SHOW/USE/... utk user terautentikasi ──────────────────
|
||||
allow if meta_op(op)
|
||||
|
||||
# ── Superuser ───────────────────────────────────────────────────────────────
|
||||
allow if is_super
|
||||
|
||||
# ── Schema sistem/metadata (listing/autocomplete; isi data tetap terbatas) ─
|
||||
allow if {
|
||||
op == "SELECTFROMCOLUMNS"
|
||||
res := input.action.resource
|
||||
res.table.catalogName == "system"
|
||||
res.table.schemaName in {"jdbc", "metadata", "information_schema"}
|
||||
}
|
||||
allow if {
|
||||
op == "SELECTFROMCOLUMNS"
|
||||
res := input.action.resource
|
||||
res.table.schemaName == "information_schema"
|
||||
}
|
||||
|
||||
# ── Akses & listing catalog / schema ────────────────────────────────────────
|
||||
allow if {
|
||||
op in {"ACCESSCATALOG", "FILTERCATALOGS"}
|
||||
res := input.action.resource
|
||||
res.catalog.name in {"iceberg", "system"}
|
||||
}
|
||||
allow if {
|
||||
op == "FILTERSCHEMAS"
|
||||
res := input.action.resource
|
||||
res.schema.catalogName in {"iceberg", "system"}
|
||||
}
|
||||
allow if {
|
||||
op == "FILTERTABLES"
|
||||
res := input.action.resource
|
||||
res.table.catalogName == "system"
|
||||
res.table.schemaName in {"jdbc", "metadata", "information_schema"}
|
||||
}
|
||||
|
||||
# ── Operasi data/metadata per tabel (non-SELECT) ────────────────────────────
|
||||
# dbt membuat tabel sementara <name>__dbt_tmp (dan backup <name>__dbt_backup):
|
||||
# cocokkan ke grant tabel dasarnya.
|
||||
dbt_suffixes := {"__dbt_tmp", "__dbt_backup"}
|
||||
|
||||
has_dbt_suffix(name) if {
|
||||
some s in dbt_suffixes
|
||||
endswith(name, s)
|
||||
}
|
||||
|
||||
base_table_name(name) := n if {
|
||||
some s in dbt_suffixes
|
||||
endswith(name, s)
|
||||
n := substring(name, 0, count(name) - count(s))
|
||||
}
|
||||
|
||||
base_table_name(name) := name if {
|
||||
not has_dbt_suffix(name)
|
||||
}
|
||||
|
||||
table_id(schema, table) := sprintf("%s_%s", [schema, base_table_name(table)])
|
||||
|
||||
allow if {
|
||||
cap := op_caps[op]
|
||||
res := input.action.resource
|
||||
tid := table_id(res.table.schemaName, res.table.tableName)
|
||||
some r in effective_roles
|
||||
grant := data.roles[r].tables[tid]
|
||||
grant[cap] == true
|
||||
}
|
||||
|
||||
# ── SELECT ... (per-kolom) ──────────────────────────────────────────────────
|
||||
# Semua kolom yang diminta harus diizinkan oleh grant.columns.
|
||||
cols_all_ok(columns, cols) if columns == "*"
|
||||
cols_all_ok(columns, cols) if {
|
||||
columns != "*"
|
||||
every c in cols {
|
||||
c in columns
|
||||
}
|
||||
}
|
||||
|
||||
allow if {
|
||||
op == "SELECTFROMCOLUMNS"
|
||||
res := input.action.resource
|
||||
res.table.catalogName == "iceberg"
|
||||
tid := table_id(res.table.schemaName, res.table.tableName)
|
||||
cols := res.table.columns
|
||||
count(cols) > 0
|
||||
some r in effective_roles
|
||||
grant := data.roles[r].tables[tid]
|
||||
grant.select == true
|
||||
cols_all_ok(grant.columns, cols)
|
||||
}
|
||||
|
||||
# ── SELECT tanpa kolom eksplisit (mis. count(*)) ────────────────────────────
|
||||
# Trino mengirim columns: [] untuk count(*)/agregasi tanpa kolom. Izinkan bila
|
||||
# user punya 'select' pada tabel tsb (tidak membocorkan isi kolom). Tanpa rule
|
||||
# ini, count(*) selalu ditolak karena rule SELECT di atas butuh count(cols) > 0.
|
||||
allow if {
|
||||
op == "SELECTFROMCOLUMNS"
|
||||
res := input.action.resource
|
||||
res.table.catalogName == "iceberg"
|
||||
tid := table_id(res.table.schemaName, res.table.tableName)
|
||||
count(res.table.columns) == 0
|
||||
some r in effective_roles
|
||||
grant := data.roles[r].tables[tid]
|
||||
grant.select == true
|
||||
}
|
||||
|
||||
# ── FilterColumns — daftar kolom utk metadata (information_schema/SHOW COLUMNS)
|
||||
# Trino memakai operasi ini untuk memfilter kolom yang boleh DILIHAT. Tanpa rule
|
||||
# ini (deny-by-default) SEMUA kolom tersembunyi dari information_schema,
|
||||
# `SHOW COLUMNS`, dan `DESCRIBE` — padahal SELECT-nya sendiri diizinkan. Akibat
|
||||
# nyata: katalog metadata (mis. ingestion OpenMetadata) melihat tabel TANPA kolom.
|
||||
# Batas per-kolom tetap dihormati lewat cols_all_ok (least privilege).
|
||||
allow if {
|
||||
op == "FILTERCOLUMNS"
|
||||
res := input.action.resource
|
||||
res.table.schemaName == "information_schema"
|
||||
}
|
||||
allow if {
|
||||
op == "FILTERCOLUMNS"
|
||||
res := input.action.resource
|
||||
res.table.catalogName == "system"
|
||||
}
|
||||
allow if {
|
||||
op == "FILTERCOLUMNS"
|
||||
res := input.action.resource
|
||||
tid := table_id(res.table.schemaName, res.table.tableName)
|
||||
some r in effective_roles
|
||||
grant := data.roles[r].tables[tid]
|
||||
grant.select == true
|
||||
grant.columns == "*"
|
||||
}
|
||||
allow if {
|
||||
op == "FILTERCOLUMNS"
|
||||
res := input.action.resource
|
||||
tid := table_id(res.table.schemaName, res.table.tableName)
|
||||
cols := res.table.columns
|
||||
count(cols) > 0
|
||||
some r in effective_roles
|
||||
grant := data.roles[r].tables[tid]
|
||||
grant.select == true
|
||||
cols_all_ok(grant.columns, cols)
|
||||
}
|
||||
|
||||
# ── Listing tabel (FilterTables): tabel terlihat bila ada grant apapun ─────
|
||||
any_cap(grant) if {
|
||||
some cap in ["select", "write", "ddl"]
|
||||
grant[cap] == true
|
||||
}
|
||||
|
||||
allow if {
|
||||
op == "FILTERTABLES"
|
||||
res := input.action.resource
|
||||
res.table.catalogName == "iceberg"
|
||||
tid := table_id(res.table.schemaName, res.table.tableName)
|
||||
some r in effective_roles
|
||||
grant := data.roles[r].tables[tid]
|
||||
any_cap(grant)
|
||||
}
|
||||
|
||||
allow if {
|
||||
op == "SetSystemSessionProperty"
|
||||
}
|
||||
|
||||
# ── DEV/EVAL: data-engineer bebas DDL + tulis di schema bronze ───────────────
|
||||
# Diminta eksplisit (2026-09-22) untuk mempermudah ingestion (Sling) dan
|
||||
# transform (dbt) di layer bronze.
|
||||
#
|
||||
# ⚠️ Cakupan blok ini SENGAJA LEBIH LUAS dari least-privilege per-tabel:
|
||||
# 1. Berlaku untuk OPERASI APA PUN yang menargetkan tabel di schema bronze —
|
||||
# termasuk operasi level-kolom yang dikirim Trino untuk UPDATE/MERGE, yang
|
||||
# tidak punya entri di op_caps sehingga sebelumnya jatuh ke deny-by-default.
|
||||
# Tanpa ini, `MERGE` (upsert) selalu gagal: "Access Denied: Cannot update
|
||||
# columns [...]".
|
||||
# 2. Berlaku untuk tabel apa pun, termasuk tabel sementara `<nama>_tmp` milik
|
||||
# Sling. Tabel itu tidak akan cocok dengan grant biasa karena policy hanya
|
||||
# men-strip sufiks milik dbt (`__dbt_tmp`, `__dbt_backup`).
|
||||
#
|
||||
# Efek: tabel baru di bronze tidak lagi perlu baris grant di data.json.
|
||||
# Mempersempit kembali: hapus blok ini, atau tambahkan syarat daftar nama tabel
|
||||
# eksplisit. Layer lain (silver/gold) TIDAK terpengaruh — tetap least-privilege.
|
||||
allow if {
|
||||
res := input.action.resource
|
||||
res.table.schemaName in {"bronze", "silver", "gold"}
|
||||
some r in effective_roles
|
||||
r == "data-engineer"
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
# =============================================================================
|
||||
# Seed OpenLDAP (osixia) — deploy-merged
|
||||
# ou=users → admin, engineer, analyst, viewer
|
||||
# ou=groups → data-admin, data-engineer, data-analyst, data-viewer
|
||||
#
|
||||
# Sumber kebenaran GRUP untuk:
|
||||
# - Trino → LDAP group provider (Fase 3)
|
||||
# - StarRocks → LDAP group provider (Fase 4)
|
||||
#
|
||||
# Catatan: user juga terdaftar lokal di realm Keycloak (autentikasi OIDC).
|
||||
# LDAP menyediakan pemetaan user → grup. Password LDAP di sini adalah nilai
|
||||
# dev (selaras dengan realm-export.json) untuk kemungkinan bind per-user.
|
||||
# =============================================================================
|
||||
|
||||
dn: ou=users,dc=sicabe,dc=local
|
||||
objectClass: organizationalUnit
|
||||
ou: users
|
||||
|
||||
dn: ou=groups,dc=sicabe,dc=local
|
||||
objectClass: organizationalUnit
|
||||
ou: groups
|
||||
|
||||
# ─── Users ──────────────────────────────────────────────────────────────────
|
||||
|
||||
dn: uid=admin,ou=users,dc=sicabe,dc=local
|
||||
objectClass: inetOrgPerson
|
||||
objectClass: posixAccount
|
||||
uid: admin
|
||||
cn: Administrator
|
||||
sn: SiCABE
|
||||
mail: admin@sicabe.local
|
||||
userPassword: admin123
|
||||
uidNumber: 1000
|
||||
gidNumber: 1000
|
||||
homeDirectory: /home/admin
|
||||
|
||||
dn: uid=engineer,ou=users,dc=sicabe,dc=local
|
||||
objectClass: inetOrgPerson
|
||||
objectClass: posixAccount
|
||||
uid: engineer
|
||||
cn: Data Engineer
|
||||
sn: SiCABE
|
||||
mail: engineer@sicabe.local
|
||||
userPassword: engineer123
|
||||
uidNumber: 1001
|
||||
gidNumber: 1001
|
||||
homeDirectory: /home/engineer
|
||||
|
||||
dn: uid=analyst,ou=users,dc=sicabe,dc=local
|
||||
objectClass: inetOrgPerson
|
||||
objectClass: posixAccount
|
||||
uid: analyst
|
||||
cn: Data Analyst
|
||||
sn: SiCABE
|
||||
mail: analyst@sicabe.local
|
||||
userPassword: analyst123
|
||||
uidNumber: 1002
|
||||
gidNumber: 1002
|
||||
homeDirectory: /home/analyst
|
||||
|
||||
dn: uid=viewer,ou=users,dc=sicabe,dc=local
|
||||
objectClass: inetOrgPerson
|
||||
objectClass: posixAccount
|
||||
uid: viewer
|
||||
cn: Viewer
|
||||
sn: SiCABE
|
||||
mail: viewer@sicabe.local
|
||||
userPassword: viewer123
|
||||
uidNumber: 1003
|
||||
gidNumber: 1003
|
||||
homeDirectory: /home/viewer
|
||||
|
||||
# ─── Groups ─────────────────────────────────────────────────────────────────
|
||||
|
||||
dn: cn=data-admin,ou=groups,dc=sicabe,dc=local
|
||||
objectClass: groupOfNames
|
||||
cn: data-admin
|
||||
member: uid=admin,ou=users,dc=sicabe,dc=local
|
||||
|
||||
dn: cn=data-engineer,ou=groups,dc=sicabe,dc=local
|
||||
objectClass: groupOfNames
|
||||
cn: data-engineer
|
||||
member: uid=engineer,ou=users,dc=sicabe,dc=local
|
||||
|
||||
dn: cn=data-analyst,ou=groups,dc=sicabe,dc=local
|
||||
objectClass: groupOfNames
|
||||
cn: data-analyst
|
||||
member: uid=analyst,ou=users,dc=sicabe,dc=local
|
||||
|
||||
dn: cn=data-viewer,ou=groups,dc=sicabe,dc=local
|
||||
objectClass: groupOfNames
|
||||
cn: data-viewer
|
||||
member: uid=viewer,ou=users,dc=sicabe,dc=local
|
||||
@@ -0,0 +1,55 @@
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Ingestion agent OpenMetadata: dbt (LINEAGE bronze→silver→gold)
|
||||
# =============================================================================
|
||||
# Mengapa file ini ada:
|
||||
# Ingestion metadata (`ingest-trino.yaml`) hanya mengambil tabel & kolom — TIDAK
|
||||
# menghasilkan lineage, karena lineage berasal dari definisi transformasi, bukan
|
||||
# dari katalog. Transformasi bronze→silver→gold dijalankan oleh dbt, sehingga
|
||||
# lineage-nya diambil dari ARTEFAK dbt:
|
||||
# target/manifest.json (grafik model: ref()/source() → relasi antar tabel)
|
||||
# target/catalog.json (skema kolom per model → lineage antar KOLOM)
|
||||
# target/run_results.json(hasil eksekusi/test)
|
||||
#
|
||||
# Artefak dibuat di dalam container Airflow (project di /home/airflow/dbt/sicabe):
|
||||
# dbt docs generate --profile trino --target prod
|
||||
#
|
||||
# ⚠️ Catatan: `target/` berada di layer container (bukan bind-mount), jadi artefak
|
||||
# HILANG bila container Airflow di-recreate → jalankan `dbt docs generate` lagi
|
||||
# sebelum ingestion dbt.
|
||||
#
|
||||
# Jalankan (butuh OM_JWT_TOKEN):
|
||||
# ./scripts/ingest-openmetadata.sh --workflow config/openmetadata/ingest-dbt.yaml
|
||||
# =============================================================================
|
||||
source:
|
||||
type: dbt
|
||||
# Service database tempat model dbt ditautkan (harus sama dengan service Trino).
|
||||
serviceName: trino_sicabe
|
||||
sourceConfig:
|
||||
config:
|
||||
# Enum-nya huruf besar: DbtConfigType.DBT = 'DBT' (bukan 'dbt').
|
||||
type: DBT
|
||||
dbtConfigSource:
|
||||
dbtConfigType: local
|
||||
# Artefak dbt dibaca dari DBT_TARGET_PATH (lihat docker-compose.yml), BUKAN dari
|
||||
# <project>/target di dalam bind-mount repo — bind-mount itu tidak bisa ditulis
|
||||
# container (uid host vs uid 50000), sehingga artefaknya memang tidak pernah ada
|
||||
# di sana dan ingestion lineage selalu menghasilkan 0 relasi.
|
||||
dbtManifestFilePath: /tmp/dbt/target/manifest.json
|
||||
dbtCatalogFilePath: /tmp/dbt/target/catalog.json
|
||||
dbtRunResultsFilePath: /tmp/dbt/target/run_results.json
|
||||
dbtUpdateDescriptions: true
|
||||
dbtUpdateOwners: true
|
||||
includeTags: false
|
||||
searchAcrossDatabases: false
|
||||
|
||||
sink:
|
||||
type: metadata-rest
|
||||
config: {}
|
||||
|
||||
workflowConfig:
|
||||
loggerLevel: INFO
|
||||
openMetadataServerConfig:
|
||||
hostPort: http://openmetadata-server:8585/api
|
||||
authProvider: openmetadata
|
||||
securityConfig:
|
||||
jwtToken: ${OM_JWT_TOKEN}
|
||||
@@ -0,0 +1,71 @@
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Ingestion agent OpenMetadata: Trino (Iceberg via Nessie)
|
||||
# =============================================================================
|
||||
# Menjelaskan mengapa file ini ada:
|
||||
# Membuat SERVICE di OpenMetadata hanya menyimpan konfigurasi koneksi. Metadata
|
||||
# sesungguhnya (database → schema → tabel → kolom) baru diambil oleh AGENT
|
||||
# (ingestion pipeline). Halaman Explore membaca search index, dan index itu
|
||||
# hanya terisi setelah agent berjalan. Tanpa file ini, service `trino_sicabe`
|
||||
# ada tetapi Explore kosong.
|
||||
#
|
||||
# Catatan skema (penting, berbeda dari skema API):
|
||||
# - di API OM 1.13.1 field TLS 'verifySSL' DITOLAK; di Ingestion Framework
|
||||
# yang benar adalah `verify` (string), di sini "false" karena sertifikat Trino
|
||||
# self-signed.
|
||||
# - `username` wajib di level config; `authType` memakai BasicAuth (username+password).
|
||||
# - Identitas agent = `svc_om` dengan role `data-catalog`: baca-saja
|
||||
# (select + kolom "*" untuk SEMUA tabel) sehingga katalog Explore LENGKAP,
|
||||
# namun TANPA hak write/ddl. (Sebelumnya identitas ini memakai `data-analyst`
|
||||
# sehingga katalog hanya sebagian.)
|
||||
#
|
||||
# Jalankan (butuh TRINO_SVC_OM_PASSWORD & OM_JWT_TOKEN di environment):
|
||||
# metadata ingest -c ingest-trino.yaml
|
||||
# =============================================================================
|
||||
source:
|
||||
type: trino
|
||||
serviceName: trino_sicabe
|
||||
serviceConnection:
|
||||
config:
|
||||
type: Trino
|
||||
scheme: trino
|
||||
hostPort: trino-coordinator:8443
|
||||
username: svc_om
|
||||
# BasicAuth hanya berisi `password` (username ada di level config) —
|
||||
# sesuai contoh resmi 'metadata ingest' untuk Trino.
|
||||
authType:
|
||||
password: ${TRINO_SVC_OM_PASSWORD}
|
||||
catalog: iceberg
|
||||
# `verify` adalah path CA bundle (string) — OM hanya meneruskannya bila
|
||||
# truthy, dan klien Trino memperlakukannya sebagai path. Memakai sertifikat
|
||||
# Trino (diekspor scripts/generate-trino-keystore.sh, di-mount ke
|
||||
# /etc/ssl/trino) membuat TLS DIVERIFIKASI, bukan dimatikan.
|
||||
verify: /etc/ssl/trino/trino.crt
|
||||
supportsMetadataExtraction: true
|
||||
supportsProfiler: true
|
||||
supportsQueryComment: true
|
||||
# Batasi ke namespace yang dikelola (bronze/silver/gold). Karena ini
|
||||
# pattern di level KONEKSI, filter schema juga tersedia di sourceConfig.
|
||||
schemaFilterPattern:
|
||||
includes:
|
||||
- bronze
|
||||
- silver
|
||||
- gold
|
||||
sourceConfig:
|
||||
config:
|
||||
type: DatabaseMetadata
|
||||
markDeletedTables: false
|
||||
includeTables: true
|
||||
includeViews: true
|
||||
includeTags: false
|
||||
|
||||
sink:
|
||||
type: metadata-rest
|
||||
config: {}
|
||||
|
||||
workflowConfig:
|
||||
loggerLevel: INFO
|
||||
openMetadataServerConfig:
|
||||
hostPort: http://openmetadata-server:8585/api
|
||||
authProvider: openmetadata
|
||||
securityConfig:
|
||||
jwtToken: ${OM_JWT_TOKEN}
|
||||
Executable
+95
@@ -0,0 +1,95 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — PostgreSQL Multi-Database Init
|
||||
# =============================================================================
|
||||
# Script ini dijalankan OTOMATIS oleh PostgreSQL Docker entrypoint pada
|
||||
# FIRST STARTUP saja (saat volume data masih kosong).
|
||||
#
|
||||
# Membuat database dan user terpisah untuk setiap komponen SiCABE:
|
||||
# - keycloak → Governance (IAM)
|
||||
# - nessie → Storage (Metadata Catalog: Iceberg REST + version store)
|
||||
# - airflow → Ingestion (Orchestrator)
|
||||
# - openmetadata_db → Governance (Metadata Platform)
|
||||
#
|
||||
# Variable environment yang harus tersedia (di-pass ke container postgres):
|
||||
# POSTGRES_USER, POSTGRES_PASSWORD (superuser — sudah built-in)
|
||||
# KEYCLOAK_DB_PASSWORD, NESSIE_DB_PASSWORD
|
||||
# AIRFLOW_DB_PASSWORD, OPENMETADATA_DB_PASSWORD
|
||||
# =============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
# ─── Fungsi: Buat database, user, dan grant schema permissions ────────────
|
||||
create_service_db() {
|
||||
local db_name="$1"
|
||||
local db_user="$2"
|
||||
local db_pass="$3"
|
||||
|
||||
echo "[SiCABE] Creating database '${db_name}' with user '${db_user}'..."
|
||||
|
||||
# Create user (if not exists) — connect to POSTGRES_DB (keycloak)
|
||||
psql -v ON_ERROR_STOP=1 --username "${POSTGRES_USER}" --dbname "${POSTGRES_DB}" <<-EOSQL
|
||||
DO \$\$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${db_user}') THEN
|
||||
CREATE ROLE ${db_user} LOGIN PASSWORD '${db_pass}';
|
||||
END IF;
|
||||
END
|
||||
\$\$;
|
||||
EOSQL
|
||||
|
||||
# Create database (if not exists) with owner
|
||||
psql -v ON_ERROR_STOP=1 --username "${POSTGRES_USER}" --dbname "${POSTGRES_DB}" <<-EOSQL
|
||||
SELECT 'CREATE DATABASE ${db_name} OWNER ${db_user}'
|
||||
WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '${db_name}')\gexec
|
||||
EOSQL
|
||||
|
||||
# Grant all privileges on the database
|
||||
psql -v ON_ERROR_STOP=1 --username "${POSTGRES_USER}" --dbname "${db_name}" <<-EOSQL
|
||||
GRANT ALL ON SCHEMA public TO ${db_user};
|
||||
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO ${db_user};
|
||||
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO ${db_user};
|
||||
ALTER SCHEMA public OWNER TO ${db_user};
|
||||
EOSQL
|
||||
|
||||
echo "[SiCABE] ✓ Database '${db_name}' siap."
|
||||
}
|
||||
|
||||
# ─── Validasi env vars ─────────────────────────────────────────────────────
|
||||
if [[ -z "${POSTGRES_USER}" ]]; then
|
||||
echo "[SiCABE] ERROR: POSTGRES_USER tidak terdefinisi!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ─── Buat database & user untuk setiap service ─────────────────────────────
|
||||
# Urutan: sesuai fase deployment
|
||||
|
||||
# Phase 1 — Governance (IAM)
|
||||
create_service_db "keycloak" "keycloak" "${KEYCLOAK_DB_PASSWORD}"
|
||||
|
||||
# Phase 2 — Storage (Metadata Catalog)
|
||||
create_service_db "nessie" "nessie" "${NESSIE_DB_PASSWORD}"
|
||||
|
||||
# Nessie (JDBC2 version store) membutuhkan schema dibuat lebih dulu.
|
||||
# Tabel-tabel Nessie akan di-auto-create oleh server saat startup.
|
||||
psql -v ON_ERROR_STOP=1 --username "${POSTGRES_USER}" --dbname "nessie" <<-EOSQL
|
||||
CREATE SCHEMA IF NOT EXISTS nessie AUTHORIZATION nessie;
|
||||
EOSQL
|
||||
|
||||
# Phase 5 — Ingestion (Orchestrator)
|
||||
create_service_db "airflow" "airflow" "${AIRFLOW_DB_PASSWORD}"
|
||||
|
||||
# Phase 6 — Governance (Metadata Platform)
|
||||
create_service_db "openmetadata_db" "openmetadata" "${OPENMETADATA_DB_PASSWORD}"
|
||||
|
||||
# Pipeline — database SUMBER contoh untuk DAG `sicabe_batch_daily`
|
||||
# (task `ingest_to_bronze` membaca public.orders via Sling). Dimiliki superuser
|
||||
# (POSTGRES_USER) sehingga tidak perlu role tambahan. Tabel + 6 baris contoh
|
||||
# diisi oleh `scripts/seed-source-postgres.sh` (dipanggil bootstrap step 6).
|
||||
psql -v ON_ERROR_STOP=1 --username "${POSTGRES_USER}" --dbname "${POSTGRES_DB}" <<-EOSQL
|
||||
SELECT 'CREATE DATABASE sicabe_data OWNER ${POSTGRES_USER}'
|
||||
WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'sicabe_data')\gexec
|
||||
EOSQL
|
||||
echo "[SiCABE] ✓ Database 'sicabe_data' siap (sumber contoh pipeline)."
|
||||
|
||||
echo "[SiCABE] ✅ Semua database dan user berhasil dibuat."
|
||||
@@ -0,0 +1,91 @@
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Konfigurasi Prometheus (Fase 6: Observability)
|
||||
# =============================================================================
|
||||
# Mengapa Prometheus ini ada:
|
||||
# Dashboard Grafana punya panel bertipe metrik (mis. minio_bucket_usage_object_total).
|
||||
# Datasource Grafana bertipe `prometheus` HARUS menunjuk SERVER Prometheus — versi
|
||||
# sebelumnya menunjuk `http://minio:9000` (API S3 MinIO) sehingga selalu
|
||||
# "400 Bad Request: error querying the Prometheus API". Prometheus ini yang
|
||||
# meng-scrape metrik, lalu Grafana membacanya.
|
||||
#
|
||||
# Target yang tersedia di stack ini:
|
||||
# - prometheus : self-scrape
|
||||
# - minio : metrik publik (MINIO_PROMETHEUS_AUTH_TYPE=public di compose)
|
||||
# - starrocks-fe : FE mengekspos /metrics pada port HTTP 8030
|
||||
#
|
||||
# Sudah tersedia (M2/M3 observabilitas — lihat reviews/sicabe-19-observability-metrics-plan.md):
|
||||
# - airflow : Airflow 3 mengirim metrik lewat StatsD → statsd-exporter:9102 (job `airflow`)
|
||||
# - trino : JMX coordinator (RMI :9080) → sidecar trino-jmx-exporter:9404 (job `trino`)
|
||||
# - node : node_exporter di SETIAP host (:9100). node_exporter adalah PRA-SYARAT HOST
|
||||
# (di luar installer); daftar host dibaca dari `targets/node.yml` via file_sd
|
||||
# (digenerate dari cluster.yml bila ada — lihat lib/cluster/generate.py).
|
||||
# =============================================================================
|
||||
global:
|
||||
scrape_interval: 30s
|
||||
evaluation_interval: 30s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ["localhost:9090"]
|
||||
|
||||
# MinIO memecah metrik per endpoint — satu job per metrics_path.
|
||||
# /cluster : 197 metrik (server/cluster)
|
||||
# /bucket : 140 metrik — termasuk 'minio_bucket_usage_object_total' yang
|
||||
# dipakai panel "Storage Usage" di dashboard Grafana. Metrik itu
|
||||
# TIDAK ada di /cluster, sehingga tanpa job ini panel menampilkan
|
||||
# "No data" walaupun target-nya 'up'.
|
||||
- job_name: minio
|
||||
metrics_path: /minio/v2/metrics/cluster
|
||||
scheme: http
|
||||
static_configs:
|
||||
- targets: ["minio:9000"]
|
||||
|
||||
- job_name: minio-bucket
|
||||
metrics_path: /minio/v2/metrics/bucket
|
||||
scheme: http
|
||||
static_configs:
|
||||
- targets: ["minio:9000"]
|
||||
|
||||
- job_name: starrocks-fe
|
||||
metrics_path: /metrics
|
||||
scheme: http
|
||||
static_configs:
|
||||
- targets: ["starrocks-fe:8030"]
|
||||
|
||||
# Airflow → StatsD → statsd-exporter (M2 observabilitas).
|
||||
# Metrik Airflow (dagrun duration, task duration, scheduler heartbeat) muncul di
|
||||
# :9102 setelah AIRFLOW__METRICS__STATSD_ON diaktifkan pada service Airflow.
|
||||
- job_name: airflow
|
||||
metrics_path: /metrics
|
||||
scheme: http
|
||||
static_configs:
|
||||
- targets: ["statsd-exporter:9102"]
|
||||
|
||||
# Trino → JMX (RMI :9080) → sidecar trino-jmx-exporter:9404 (M3 observabilitas).
|
||||
# Metrik saturasi coordinator (running/queued/failed query) + JVM.
|
||||
- job_name: trino
|
||||
metrics_path: /metrics
|
||||
scheme: http
|
||||
static_configs:
|
||||
- targets: ["trino-jmx-exporter:9404"]
|
||||
|
||||
# node_exporter di setiap host (:9100) — PRA-SYARAT HOST (bukan service compose).
|
||||
# Daftar host dibaca dari file ini dan bisa diubah TANPA restart Prometheus
|
||||
# (file_sd memantau perubahan berkas; refresh_interval hanya cadangan).
|
||||
#
|
||||
# - single-host : targets/node.yml default = "host.docker.internal:9100"
|
||||
# - multi-host : digenerate dari cluster.yml oleh `./install.sh --generate`
|
||||
# (semua host, alamat:9100, label host=<nama>)
|
||||
#
|
||||
# Label `host` dari berkas target dipromosikan ke semua seri; relabel di bawah
|
||||
# menyalinnya ke `instance` supaya panel menampilkan NAMA host, bukan alamat.
|
||||
- job_name: node
|
||||
metrics_path: /metrics
|
||||
scheme: http
|
||||
file_sd_configs:
|
||||
- files: ["/etc/prometheus/targets/node.yml"]
|
||||
refresh_interval: 30s
|
||||
relabel_configs:
|
||||
- source_labels: [host]
|
||||
target_label: instance
|
||||
@@ -0,0 +1,36 @@
|
||||
# =============================================================================
|
||||
# node_exporter targets — job `node` (Prometheus file_sd)
|
||||
# =============================================================================
|
||||
# node_exporter adalah PRA-SYARAT HOST: dipasang di SETIAP host (port 9100),
|
||||
# DI LUAR installer SiCABE. Berkas ini HANYA memberi tahu Prometheus host mana
|
||||
# yang harus di-scrape.
|
||||
#
|
||||
# FORMAT (file_sd): daftar objek { targets: [...], labels: {...} }.
|
||||
# - `targets` : "<alamat-host>:9100"
|
||||
# - `labels.host` : NAMA host — disalin ke label `instance` oleh relabel_configs
|
||||
# di prometheus.yml, sehingga panel menampilkan nama (bukan IP).
|
||||
#
|
||||
# CARA PAKAI
|
||||
# - Single-host (default): biarkan apa adanya. `host.docker.internal` menunjuk
|
||||
# Docker host lewat `extra_hosts: host-gateway` (lihat docker-compose.yml).
|
||||
# - Multi-host: TIDAK perlu disunting tangan bila memakai cluster.yml —
|
||||
# `./install.sh --generate` menulis ulang berkas ini di bundle host Prometheus
|
||||
# dengan SELURUH host (alamat:9100, label host=<nama>).
|
||||
# - Tanpa cluster.yml tapi ada host tambahan: tambahkan satu entri per host
|
||||
# (contoh di bawah). Perubahan terbaca otomatis tanpa restart Prometheus.
|
||||
# =============================================================================
|
||||
|
||||
- targets: ["host.docker.internal:9100"]
|
||||
labels:
|
||||
host: local
|
||||
- targets: ["100.123.123.71:9100"]
|
||||
labels:
|
||||
host: darkworld
|
||||
|
||||
# Contoh penambahan host (hapus komentar):
|
||||
# - targets: ["10.20.0.11:9100"]
|
||||
# labels:
|
||||
# host: compute
|
||||
# - targets: ["10.20.0.12:9100"]
|
||||
# labels:
|
||||
# host: state
|
||||
@@ -0,0 +1,32 @@
|
||||
# =============================================================================
|
||||
# StarRocks — Compute Node (CN) Configuration — SHARED-DATA
|
||||
# =============================================================================
|
||||
# CN TIDAK memakai parameter run_mode (hanya FE). Data utama tersimpan di
|
||||
# object storage; CN hanya menyimpan cache lokal di storage_root_path.
|
||||
# Start via ./bin/start_cn.sh (bukan start_be.sh).
|
||||
#
|
||||
# Diregistrasi ke FE dengan: ALTER SYSTEM ADD COMPUTE NODE "starrocks-cn:9050";
|
||||
# Referensi: docs.starrocks.io/docs/deployment/deploy_shared_data_manually/
|
||||
# =============================================================================
|
||||
sys_log_level = INFO
|
||||
|
||||
# Thrift server port utk koneksi FE
|
||||
be_port = 9060
|
||||
|
||||
# HTTP server port
|
||||
be_http_port = 8040
|
||||
|
||||
# Heartbeat port utk koneksi FE (diregistrasi sbg <host>:9050)
|
||||
heartbeat_service_port = 9050
|
||||
|
||||
# Brpc port komunikasi internal
|
||||
brpc_port = 8060
|
||||
|
||||
# Starlet port — komunikasi CN ↔ object storage (shared-data)
|
||||
starlet_port = 9070
|
||||
|
||||
# Direktori data cache lokal (shared-data). Volume cn-cache dipasang di sini.
|
||||
storage_root_path = /opt/starrocks/cn/cache
|
||||
|
||||
# Cluster configuration (subnet network deploy-merged)
|
||||
priority_networks = 172.28.0.0/16
|
||||
@@ -0,0 +1,42 @@
|
||||
# =============================================================================
|
||||
# StarRocks — Frontend (FE) Configuration — SHARED-DATA
|
||||
# =============================================================================
|
||||
# Mode shared-data: data tabel internal disimpan di object storage S3/MinIO
|
||||
# (bukan di disk lokal). Dua baris di bawah WAJIB di-set sebelum first start
|
||||
# dan TIDAK bisa diubah via SQL.
|
||||
#
|
||||
# Penyimpanan diatur via storage volume SQL:
|
||||
# CREATE STORAGE VOLUME s3_volume TYPE=S3 ...; SET s3_volume AS DEFAULT STORAGE VOLUME;
|
||||
#
|
||||
# External catalog Iceberg (via Nessie) tetap dikonfigurasi via SQL.
|
||||
# Referensi: docs.starrocks.io/docs/deployment/deploy_shared_data_manually/
|
||||
# =============================================================================
|
||||
|
||||
# --- Mode shared-data (WAJIB, sebelum first start) ---
|
||||
run_mode = shared_data
|
||||
cloud_native_storage_type = S3
|
||||
|
||||
# Meta data directory
|
||||
meta_dir = /opt/starrocks/fe/data
|
||||
|
||||
# HTTP server port (redirect_url OIDC: http://<fe>:8030/api/oauth2)
|
||||
http_port = 8030
|
||||
|
||||
# Thrift server port for CN connections
|
||||
rpc_port = 9020
|
||||
|
||||
# MySQL server port for client connections
|
||||
query_port = 9030
|
||||
|
||||
# Edit log port for FE cluster communication
|
||||
edit_log_port = 9010
|
||||
|
||||
# Port internal FE untuk shared-data metadata (default 6090)
|
||||
cloud_native_meta_port = 6090
|
||||
|
||||
# JVM configuration
|
||||
JAVA_HOME = /usr/lib/jvm/java-17-openjdk-amd64
|
||||
JAVA_OPTS = -Xms512m -Xmx1024m -XX:+UseG1GC
|
||||
|
||||
# Cluster configuration (subnet network deploy-merged)
|
||||
priority_networks = 172.28.0.0/16
|
||||
@@ -0,0 +1,14 @@
|
||||
# Konfigurasi dinamis Traefik (provider file).
|
||||
#
|
||||
# serversTransports TIDAK dapat didefinisikan lewat label Docker — percobaan memakai
|
||||
# label menghasilkan error runtime:
|
||||
# "error building proxy for server URL https://...: servers transport not found ..."
|
||||
# Karena itu didefinisikan di berkas ini.
|
||||
#
|
||||
# trino-insecure: Trino internal memakai HTTPS (http-server.allow-insecure-over-http=false)
|
||||
# dengan sertifikat self-signed untuk nama container, jadi Traefik perlu melewati
|
||||
# verifikasi HANYA untuk backend Trino — bukan global.
|
||||
http:
|
||||
serversTransports:
|
||||
trino-insecure:
|
||||
insecureSkipVerify: true
|
||||
@@ -0,0 +1,17 @@
|
||||
# =============================================================================
|
||||
# Trino — Access Control via OPA (built-in, deny-by-default)
|
||||
# =============================================================================
|
||||
# access-control.name=opa (built-in Trino). Policy Rego di OPA
|
||||
# (config/opa/policies/trino.rego) menerima input dari Trino dan memutuskan
|
||||
# per-objek/per-kolom secara LOKAL (ACL di config/opa/data.json).
|
||||
#
|
||||
# Grup user berasal dari LDAP group provider (lihat group-provider.properties),
|
||||
# lalu dicocokkan dengan data.roles.<grup> di data.json.
|
||||
#
|
||||
# OPA: http://opa:8181/v1/data/trino/allow
|
||||
# =============================================================================
|
||||
|
||||
access-control.name=opa
|
||||
opa.policy.uri=http://opa:8181/v1/data/trino/allow
|
||||
# Selama debugging biarkan true; matikan saat sudah stabil (log request OPA).
|
||||
opa.log-requests=true
|
||||
@@ -0,0 +1,30 @@
|
||||
# =============================================================================
|
||||
# Trino Iceberg Catalog — Project Nessie (Nessie REST API v2)
|
||||
# =============================================================================
|
||||
# Catalog ini menghubungkan Trino ke tabel Iceberg yang dikelola Nessie
|
||||
# (versioning git-like). Data file + metadata disimpan di MinIO (S3-compatible).
|
||||
#
|
||||
# Catatan: Nessie belum mendukung Iceberg spec v3 → selalu format-version=2
|
||||
# (default Trino = 2, dibiarkan).
|
||||
# =============================================================================
|
||||
|
||||
# --- Connector ---
|
||||
connector.name=iceberg
|
||||
|
||||
# --- Catalog Type: Nessie ---
|
||||
iceberg.catalog.type=nessie
|
||||
iceberg.nessie-catalog.uri=${ENV:NESSIE_URL}/api/v2
|
||||
iceberg.nessie-catalog.ref=main
|
||||
iceberg.nessie-catalog.default-warehouse-dir=s3://${ENV:MINIO_BUCKET_ICEBERG_DATA}/warehouse
|
||||
|
||||
# --- S3 Native Filesystem (MinIO / S3 existing) ---
|
||||
fs.s3.enabled=true
|
||||
s3.endpoint=${ENV:S3_ENDPOINT}
|
||||
s3.path-style-access=true
|
||||
s3.aws-access-key=${ENV:S3_ACCESS_KEY}
|
||||
s3.aws-secret-key=${ENV:S3_SECRET_KEY}
|
||||
s3.region=${ENV:S3_REGION}
|
||||
|
||||
# --- Table Defaults ---
|
||||
iceberg.file-format=PARQUET
|
||||
iceberg.compression-codec=ZSTD
|
||||
@@ -0,0 +1,20 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDVTCCAj2gAwIBAgIUETRg8AYPjCnKkFChXiQiP62JG9YwDQYJKoZIhvcNAQEL
|
||||
BQAwEDEOMAwGA1UEAwwFdHJpbm8wHhcNMjYwOTI4MTIwOTMxWhcNMjgxMjMxMTIw
|
||||
OTMxWjAQMQ4wDAYDVQQDDAV0cmlubzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
|
||||
AQoCggEBAMyEX/NwRqEuT3SweBZeHcUxFKgeFEY89trx9D2nkuP7OMiHRFyeKfuL
|
||||
Kmyyyta9QQDU1EU1cHyhgHwhXLzFxJ6QnlLgSxOP/hTQlF6qL2dvMgKDzBmo51/I
|
||||
e6nwNYlYC9wzOPLNVdqZkqraSk2xmnnMB4fM9L+xisZJOQ7VpprzTNIXigus7q8M
|
||||
CsWPXu+dr9XJSKYeqOvV5jIQGhSE7yyb3zC2nHG/qQ3YiBZc7OMtkUti+UT2/oGK
|
||||
tErYxBj8uCtsqLGCFjpGoioOIFAUmqriBzSrSrOwzkkeF3mI5cok4P4mBmwQcRZk
|
||||
3TBrQW8HrCB8oVq3kNgsuM8dRJZ3NzsCAwEAAaOBpjCBozAdBgNVHQ4EFgQUvcvH
|
||||
CUrnQiy1yYFCsrrcNcYukmswHwYDVR0jBBgwFoAUvcvHCUrnQiy1yYFCsrrcNcYu
|
||||
kmswDwYDVR0TAQH/BAUwAwEB/zBQBgNVHREESTBHggV0cmlub4IJbG9jYWxob3N0
|
||||
ggxzaWNhYmUtdHJpbm+CEXRyaW5vLWNvb3JkaW5hdG9yggx0cmluby13b3JrZXKH
|
||||
BH8AAAEwDQYJKoZIhvcNAQELBQADggEBAMrDCPDqEN/d+LhQ3bNoiVBdjoLemML5
|
||||
G+Torrv7VIcD2VDdUbWQ7WfzXmXS0WTuzVbqD5I6Fus1NK2GFKB0q2kP9whOrmqu
|
||||
lE6Sv2b1g38H3PnjzWQbY34lNTRzJr/PmxvxgTs1GIWFu4fcd7GsNocF3ZciLf/5
|
||||
4Ky51U8JG3ukzPjkFSdAY03aZ2d2Q2VqZkmLJyHcjJDH4+u3zveEbUxSSTgM9Kzx
|
||||
NxaYiqXeUzd65Y+9ALsECuDMQGLkMl9acmgJIO+vH/3sfM9WiDksSN9WJhR3R2UX
|
||||
B5zp85Reh+ehobxQsbIZU+IXR6cEy8SfgQNzxlFVyOQ1/V4MgFckFNw=
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,67 @@
|
||||
# =============================================================================
|
||||
# Trino COORDINATOR — config.properties
|
||||
# -----------------------------------------------------------------------------
|
||||
# Dipisah dari worker sejak awal (tier R11: coord). Coordinator melayani KLIEN
|
||||
# (HTTPS 8443 + OAuth2/Basic) dan mengatur penjadwalan; worker yang mengeksekusi
|
||||
# task (node-scheduler.include-coordinator=false).
|
||||
#
|
||||
# Klien TIDAK melihat worker; worker mendaftar lewat discovery.uri di bawah.
|
||||
# Perubahan dari konfigurasi single-node sebelumnya:
|
||||
# node-scheduler.include-coordinator : true → false
|
||||
# discovery.uri : http://localhost:8080 → http://trino-coordinator:8080
|
||||
# =============================================================================
|
||||
coordinator=true
|
||||
node-scheduler.include-coordinator=false
|
||||
http-server.http.port=8080
|
||||
# HTTPS untuk Web UI OAuth2 — OAuth2 Web UI WAJIB secure request (TLS),
|
||||
# jika tidak Trino mengalihkan /ui/ ke /ui/static/disabled.html.
|
||||
# HTTP (8080) TIDAK lagi melayani tanpa kredensial (allow-insecure-over-http=false);
|
||||
# klien service/pipeline lewat nginx proxy → HTTPS.
|
||||
http-server.https.enabled=true
|
||||
http-server.https.port=8443
|
||||
http-server.https.keystore.path=/etc/trino/keystore.p12
|
||||
http-server.https.keystore.key=trino_ssl_dev
|
||||
# HTTP 8080 tetap dipakai untuk komunikasi INTERNAL coordinator↔worker
|
||||
# (worker mendaftar ke coordinator lewat endpoint ini).
|
||||
discovery.uri=http://trino-coordinator:8080
|
||||
catalog.management=${ENV:CATALOG_MANAGEMENT}
|
||||
|
||||
# Authentication: hybrid — OAuth2 (end user, Keycloak SSO) + password/Basic
|
||||
# (service account). Keduanya berlaku di HTTPS (8443). HTTP polos DIMATIKAN
|
||||
# (allow-insecure-over-http=false) → klien service lewat nginx proxy (HTTP→HTTPS).
|
||||
http-server.authentication.type=oauth2,password
|
||||
http-server.authentication.allow-insecure-over-http=false
|
||||
# Di belakang reverse proxy TLS (Traefik), Trino harus memercayai X-Forwarded-* supaya
|
||||
# skema publik (https) dan alamat klien terbaca benar — termasuk untuk redirect OAuth.
|
||||
# Nilainya dari env: diisi `true` oleh generator saat tls.mode != none, `false` bila tidak.
|
||||
# Trino GAGAL START bila variabel ini tidak ada, jadi compose selalu mendefinisikannya.
|
||||
http-server.process-forwarded=${ENV:TRINO_PROCESS_FORWARDED}
|
||||
# Issuer = URL PUBLIK Keycloak — wajib SAMA PERSIS dengan iss claim JWT.
|
||||
# Diambil dari KEYCLOAK_PUBLIC_URL (bukan ditulis tetap): nilainya berubah antara
|
||||
# single-host (localhost) dan multi-host tanpa TLS (http://<alamat-host-control>:12080).
|
||||
# Kalau di-hardcode, issuer di klaster multi-host tidak cocok dan token ditolak.
|
||||
http-server.authentication.oauth2.issuer=${ENV:KEYCLOAK_PUBLIC_URL}/realms/sicabe
|
||||
# Matikan OIDC discovery — Trino tidak bisa mengakses URL publik dari dalam container.
|
||||
# Endpoint di-set manual: browser pakai URL publik, server-ke-server pakai keycloak:8080.
|
||||
http-server.authentication.oauth2.oidc.discovery=false
|
||||
# Endpoint browser-facing (auth redirect) — harus URL publik juga
|
||||
http-server.authentication.oauth2.auth-url=${ENV:KEYCLOAK_PUBLIC_URL}/realms/sicabe/protocol/openid-connect/auth
|
||||
# Endpoint server-side (token exchange & JWKS) — reachable dari container
|
||||
http-server.authentication.oauth2.token-url=http://keycloak:${ENV:KEYCLOAK_INTERNAL_PORT}/realms/sicabe/protocol/openid-connect/token
|
||||
http-server.authentication.oauth2.jwks-url=http://keycloak:${ENV:KEYCLOAK_INTERNAL_PORT}/realms/sicabe/protocol/openid-connect/certs
|
||||
http-server.authentication.oauth2.client-id=trino
|
||||
http-server.authentication.oauth2.client-secret=${ENV:TRINO_CLIENT_SECRET}
|
||||
# Username Trino = preferred_username (bukan 'sub') agar cocok dengan uid LDAP
|
||||
# pada LDAP group provider.
|
||||
http-server.authentication.oauth2.principal-field=preferred_username
|
||||
# Token Keycloak client 'trino' membawa aud='account'; trust audience ini.
|
||||
http-server.authentication.oauth2.additional-audiences=account
|
||||
|
||||
# Web UI: aktifkan OAuth2 untuk UI. Web UI OAuth2 WAJIB HTTPS (lihat blok https
|
||||
# di atas) — bila request tidak secure, Trino mengalihkan /ui/ ke disabled.html.
|
||||
# allow-insecure-over-http hanya berlaku untuk REST API/JDBC, BUKAN Web UI.
|
||||
web-ui.authentication.type=oauth2
|
||||
|
||||
# Shared secret untuk internal communication (required saat auth enabled)
|
||||
# WAJIB identik dengan worker — tanpa ini worker ditolak coordinator.
|
||||
internal-communication.shared-secret=trino-internal-secret-dev-only
|
||||
@@ -0,0 +1,18 @@
|
||||
# =============================================================================
|
||||
# Trino WORKER — config.properties
|
||||
# -----------------------------------------------------------------------------
|
||||
# Dipisah dari coordinator sejak awal (tier R11: engine). Worker TIDAK melayani
|
||||
# klien: tidak ada HTTPS, tidak ada autentikasi OAuth2/Basic, tidak ada port yang
|
||||
# di-expose ke host. Worker hanya:
|
||||
# 1. mendaftar & menerima task dari coordinator lewat HTTP 8080,
|
||||
# 2. membaca katalog yang sama (etc/catalog/*.properties).
|
||||
#
|
||||
# `internal-communication.shared-secret` WAJIB identik dengan coordinator —
|
||||
# inilah yang mengautentikasi worker (bukan password/OAuth2 klien).
|
||||
# =============================================================================
|
||||
coordinator=false
|
||||
http-server.http.port=8080
|
||||
# Daftar ke coordinator (bukan localhost — worker bukan coordinator-nya sendiri).
|
||||
discovery.uri=http://trino-coordinator:8080
|
||||
|
||||
internal-communication.shared-secret=trino-internal-secret-dev-only
|
||||
@@ -0,0 +1,28 @@
|
||||
# =============================================================================
|
||||
# Trino — LDAP Group Provider
|
||||
# =============================================================================
|
||||
# Grup user di-resolve dari OpenLDAP (source of truth grup). User Trino
|
||||
# = preferred_username dari Keycloak (lihat config.properties principal-field).
|
||||
# Nama grup yang dikembalikan = cn (mis. data-admin, data-engineer, ...),
|
||||
# lalu dicocokkan dengan data.roles.<grup> di config/opa/data.json.
|
||||
# =============================================================================
|
||||
|
||||
group-provider.name=ldap
|
||||
|
||||
# Alamat LDAP dari environment (LDAP_URL) supaya bisa diarahkan ke LDAP EXISTING.
|
||||
# Default compose: ldap://openldap:389 (container lokal).
|
||||
ldap.url=${ENV:LDAP_URL}
|
||||
ldap.allow-insecure=true
|
||||
# DN akun BIND (bukan harus admin): boleh akun read-only dgn hak baca ke ou=users/ou=groups.
|
||||
# Diarahkan dari .env (LDAP_ADMIN_DN) supaya LDAP EXISTING dgn bind DN berbeda bisa dipakai.
|
||||
# Env-nya WAJIB diteruskan ke container (lihat docker-compose.yml trino-coordinator).
|
||||
ldap.admin-user=${ENV:LDAP_ADMIN_DN}
|
||||
ldap.admin-password=${ENV:LDAP_ADMIN_PASSWORD}
|
||||
ldap.user-base-dn=ou=users,dc=sicabe,dc=local
|
||||
ldap.user-search-filter=(uid={0})
|
||||
ldap.group-name-attribute=cn
|
||||
|
||||
# Filtering mode: cari group di ou=groups yg member-nya = DN user
|
||||
ldap.use-group-filter=true
|
||||
ldap.group-base-dn=ou=groups,dc=sicabe,dc=local
|
||||
ldap.group-search-member-attribute=member
|
||||
@@ -0,0 +1,31 @@
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — konfigurasi sidecar `trino-jmx-exporter` (M3 observabilitas)
|
||||
# =============================================================================
|
||||
# Dibaca `jmx_prometheus_standalone.jar` (image bitnami/jmx-exporter):
|
||||
# java -jar jmx_prometheus_standalone.jar 9404 /config/config.yml
|
||||
#
|
||||
# hostPort : target JMX coordinator via RMI — HARUS sama dengan port &
|
||||
# java.rmi.server.hostname di config/trino/jvm-coordinator.config.
|
||||
#
|
||||
# includeObjectNames : PEMBATAS CARDINALITY. Tanpa ini (`rules: ".*"` saja)
|
||||
# jmx-exporter memaparkan SEMUA MBean (~9.700 metrik) — termasuk
|
||||
# io.trino.filesystem.s3, io.airlift.*, org.eclipse.jetty.*, java.lang.*,
|
||||
# trino.server, trino.security, dst yang tidak dipakai.
|
||||
# Di sini dibatasi ke domain EKSEKUSI Trino yang DIPAKAI saja:
|
||||
# trino.execution:* → QueryManager, QueryExecution, TaskManager
|
||||
# trino.execution.executor.* → task executor (timesharing/dedicated)
|
||||
# trino.execution.resourcegroups:* → resource group
|
||||
# SENGAJA DIKECUALIKAN: trino.execution.scheduler[.faulttolerant]*
|
||||
# (≈1.614 metrik statistik penjadwalan FT — stack ini tidak memakai
|
||||
# fault-tolerant execution). Tambahkan kembali bila FT dipakai.
|
||||
# (Konsekuensi: metrik JVM `java_lang_memory_*` juga tidak ada.)
|
||||
---
|
||||
hostPort: trino-coordinator:9080
|
||||
lowercaseOutputName: true
|
||||
lowercaseOutputLabelNames: true
|
||||
includeObjectNames:
|
||||
- "trino.execution:*"
|
||||
- "trino.execution.executor.*:*"
|
||||
- "trino.execution.resourcegroups:*"
|
||||
rules:
|
||||
- pattern: ".*"
|
||||
@@ -0,0 +1,32 @@
|
||||
# =============================================================================
|
||||
# Trino — JVM Tuning
|
||||
# Referensi arsitektur: bab 4.3.1.7
|
||||
# =============================================================================
|
||||
-server
|
||||
# Heap COORDINATOR: ringan (penjadwalan + auth); task dieksekusi worker.
|
||||
-Xmx512m
|
||||
-XX:+UseG1GC
|
||||
-XX:G1HeapRegionSize=32M
|
||||
-XX:+UseGCOverheadLimit
|
||||
-XX:+ExplicitGCInvokesConcurrent
|
||||
-XX:+HeapDumpOnOutOfMemoryError
|
||||
-XX:+ExitOnOutOfMemoryError
|
||||
-XX:HeapDumpPath=/data/trino/heapdump
|
||||
# Diperlukan oleh Trino 450 untuk Java < 22.0.2 (JDK-8329528)
|
||||
-XX:+UnlockDiagnosticVMOptions
|
||||
-XX:G1NumCollectionsKeepPinned=10000000
|
||||
-Djdk.attach.allowAttachSelf=true
|
||||
-Djava.security.egd=file:/dev/urandom
|
||||
# ─── JMX remote (M3 observabilitas, Review 19) ───────────────────────────────
|
||||
# Dibaca sidecar `trino-jmx-exporter` via RMI. Port 9080 hanya dalam jaringan
|
||||
# docker `sicabe-network` (tidak ada `ports:` yang mengeksposnya ke host), karena
|
||||
# itu authenticate=false dapat diterima di sini.
|
||||
# ⚠️ `java.rmi.server.hostname` WAJIB = nama container (bukan default = ID container),
|
||||
# agar stub RMI bisa diresolusi sidecar di jaringan yang sama.
|
||||
-Dcom.sun.management.jmxremote
|
||||
-Dcom.sun.management.jmxremote.port=9080
|
||||
-Dcom.sun.management.jmxremote.rmi.port=9080
|
||||
-Dcom.sun.management.jmxremote.authenticate=false
|
||||
-Dcom.sun.management.jmxremote.ssl=false
|
||||
-Dcom.sun.management.jmxremote.local.only=false
|
||||
-Djava.rmi.server.hostname=trino-coordinator
|
||||
@@ -0,0 +1,19 @@
|
||||
# =============================================================================
|
||||
# Trino — JVM Tuning
|
||||
# Referensi arsitektur: bab 4.3.1.7
|
||||
# =============================================================================
|
||||
-server
|
||||
# Heap WORKER: menampung eksekusi task (scan, join, agregasi).
|
||||
-Xmx1G
|
||||
-XX:+UseG1GC
|
||||
-XX:G1HeapRegionSize=32M
|
||||
-XX:+UseGCOverheadLimit
|
||||
-XX:+ExplicitGCInvokesConcurrent
|
||||
-XX:+HeapDumpOnOutOfMemoryError
|
||||
-XX:+ExitOnOutOfMemoryError
|
||||
-XX:HeapDumpPath=/data/trino/heapdump
|
||||
# Diperlukan oleh Trino 450 untuk Java < 22.0.2 (JDK-8329528)
|
||||
-XX:+UnlockDiagnosticVMOptions
|
||||
-XX:G1NumCollectionsKeepPinned=10000000
|
||||
-Djdk.attach.allowAttachSelf=true
|
||||
-Djava.security.egd=file:/dev/urandom
|
||||
Binary file not shown.
@@ -0,0 +1,6 @@
|
||||
# =============================================================================
|
||||
# Trino — Logging Configuration
|
||||
# =============================================================================
|
||||
io.trino=INFO
|
||||
io.trino.server.security=DEBUG
|
||||
io.trino.plugin.iceberg=INFO
|
||||
@@ -0,0 +1,13 @@
|
||||
# =============================================================================
|
||||
# Trino — Password authenticator (HTTP Basic) untuk SERVICE ACCOUNT
|
||||
# -----------------------------------------------------------------------------
|
||||
# Dipakai bersama OAuth2 (lihat config.properties: type=oauth2,password):
|
||||
# - End user : OAuth2 (Keycloak SSO) → HTTPS Web UI/JDBC
|
||||
# - Service : HTTP Basic (file bcrypt) → lewat nginx proxy (HTTP→HTTPS)
|
||||
#
|
||||
# password.db di-generate oleh bootstrap.sh/install.sh dari .env.
|
||||
# Pra-syarat: `htpasswd` tersedia di host. JANGAN commit (lihat .gitignore).
|
||||
# =============================================================================
|
||||
password-authenticator.name=file
|
||||
file.password-file=/etc/trino/password.db
|
||||
file.refresh-period=5s
|
||||
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"rootGroups": [
|
||||
{
|
||||
"name": "global",
|
||||
"softMemoryLimit": "80%",
|
||||
"hardConcurrencyLimit": 100,
|
||||
"maxQueued": 1000,
|
||||
"schedulingPolicy": "weighted",
|
||||
"jmxExport": true,
|
||||
"subGroups": [
|
||||
{
|
||||
"name": "data_definition",
|
||||
"softMemoryLimit": "10%",
|
||||
"hardConcurrencyLimit": 5,
|
||||
"maxQueued": 100,
|
||||
"schedulingWeight": 1
|
||||
},
|
||||
{
|
||||
"name": "data_analyst",
|
||||
"softMemoryLimit": "30%",
|
||||
"hardConcurrencyLimit": 20,
|
||||
"maxQueued": 200,
|
||||
"schedulingWeight": 2
|
||||
},
|
||||
{
|
||||
"name": "data_engineer",
|
||||
"softMemoryLimit": "40%",
|
||||
"hardConcurrencyLimit": 30,
|
||||
"maxQueued": 300,
|
||||
"schedulingWeight": 3
|
||||
},
|
||||
{
|
||||
"name": "pipeline",
|
||||
"softMemoryLimit": "20%",
|
||||
"hardConcurrencyLimit": 10,
|
||||
"maxQueued": 100,
|
||||
"schedulingWeight": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"selectors": [
|
||||
{
|
||||
"userGroup": "admin",
|
||||
"group": "global.data_engineer"
|
||||
},
|
||||
{
|
||||
"source": ".*pipeline.*",
|
||||
"group": "global.pipeline"
|
||||
},
|
||||
{
|
||||
"queryType": "SELECT",
|
||||
"group": "global.data_analyst"
|
||||
},
|
||||
{
|
||||
"queryType": "DATA_DEFINITION",
|
||||
"group": "global.data_definition"
|
||||
},
|
||||
{
|
||||
"user": "admin",
|
||||
"group": "global.data_engineer"
|
||||
},
|
||||
{
|
||||
"source": ".*",
|
||||
"group": "global.data_analyst"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
# =============================================================================
|
||||
# Trino — Resource Groups & Query Queuing
|
||||
# Referensi arsitektur: bab 4.3.1.5
|
||||
# =============================================================================
|
||||
resource-groups.configuration-manager=file
|
||||
resource-groups.config-file=/etc/trino/resource-groups.json
|
||||
Reference in New Issue
Block a user