sync dari sicabe-doc/deploy @ bc1a6e6
This commit is contained in:
@@ -215,3 +215,6 @@ deploy/generated/
|
||||
|
||||
# Trino keystore dipersistenkan saat --generate (jangan di-commit)
|
||||
deploy/.trino-keystore/
|
||||
|
||||
# Artefak dev (jangan di-commit)
|
||||
deploy/output/
|
||||
|
||||
+27
-5
@@ -227,10 +227,11 @@ di sini komponen tetap milik klaster ini, hanya dipindah host. Override variabel
|
||||
> sehingga path filesystem seperti `/etc/trino/` dan baris komentar tidak ikut berubah.
|
||||
> Rencana lengkap: `reviews/sicabe-10-multi-host-installer-plan.md`.
|
||||
|
||||
> ⚠️ **Pilot multi-host di host yang SAMA tidak boleh dua stack berjalan bersamaan.**
|
||||
> Memisah network/port saja **tidak cukup** — RAM tidak akan cukup (host pilot 15 GB,
|
||||
> satu stack penuh ±10 GB). **Matikan stack lama lebih dulu** sebelum menyalakan hasil
|
||||
> `generated/host-<x>/`.
|
||||
> ⚠️ **Satu host = satu stack aktif.** Stack single-host dan multi-host **tidak untuk
|
||||
> berjalan bersamaan** di host yang sama — **matikan stack lama lebih dulu** sebelum
|
||||
> menyalakan hasil `generated/host-<x>/`. Berdampingan **tidak cukup** hanya dipisah
|
||||
> network/port: pada host kecil hal itu juga menabrak RAM (pilot 15 GB, satu stack penuh
|
||||
> ±10 GB).
|
||||
|
||||
**Trino dipisah coordinator/worker.** Sejak 2026-09-28 Trino bukan lagi satu service:
|
||||
`trino-coordinator` (port host `12443`, HTTPS + OAuth2/Basic, client-facing) dan
|
||||
@@ -343,15 +344,36 @@ Menyalin seluruh `deploy/` — **termasuk `.env` dan `generated/`** — ke tiap
|
||||
```bash
|
||||
./install.sh --sync # ke SEMUA host lain (host ini dideteksi dari alamat lokal)
|
||||
./install.sh --sync --host state # hanya satu host
|
||||
./install.sh --sync --user admin --dest '~/sicabe-doc/deploy' # SSH user & tujuan
|
||||
./install.sh --sync --user admin --dest '~/lain/deploy' # SSH user & tujuan (timpa default)
|
||||
./install.sh --sync --delete # hapus di tujuan berkas yang tak ada di sumber
|
||||
```
|
||||
|
||||
**Tujuan default = cermin lokasi repo ini.** Nilai `--dest` diturunkan dari letak `install.sh`
|
||||
di bawah `$HOME` (`sicabe_default_dest`), jadi **nama repo tidak dipatok**:
|
||||
|
||||
| Repo di host control | Tujuan `--sync` default di host lain |
|
||||
|---|---|
|
||||
| `~/sicabe-doc/deploy` | `~/sicabe-doc/deploy` |
|
||||
| `~/sicabe-core/deploy` | `~/sicabe-core/deploy` |
|
||||
|
||||
Ini membuat **repo distribusi bersih** (mis. `sicabe-core` — hanya folder `deploy/`, tanpa
|
||||
`docs/`, `reviews/`, `experiments/`) bekerja **tanpa `--dest`**. `deploy/` **self-contained**:
|
||||
semua script memakai `SCRIPT_DIR`, dan bind-mount compose relatif (`./config/…`) — jadi nama
|
||||
atau lokasi repo tidak berpengaruh. Timpa tujuan dengan `--dest '~/…/deploy'` bila perlu.
|
||||
|
||||
Host yang sedang dijalankan **dikecualikan otomatis** (kecocokan `hostname -I` dengan
|
||||
`address` di inventory); bila tak cocok, ia menyinkron ke semua host. `.env` **wajib ikut**
|
||||
(rahasia bersama — OIDC client secret harus identik lintas host). Setelah sync, di tiap host
|
||||
jalankan `./scripts/prefetch-images.sh --host <nama>` lalu `./install.sh --host <nama>`.
|
||||
|
||||
> **Model distribusi: control = sumber tunggal · host lain = PENERIMA (bukan working tree git).**
|
||||
> `--sync` menimpa berkas yang di-track git di `deploy/` host tujuan dengan versi control,
|
||||
> sehingga `git status` di sana **selalu kotor** dan `git pull` akan **ditolak** ("local changes
|
||||
> would be overwritten"). **Jangan `git pull` di host tujuan** — perbarui di **control** lalu
|
||||
> `--sync` saja (clone repo di host tujuan cukup **sekali**, untuk `prepare-host.sh`).
|
||||
> Gunakan **`--sync --delete`** agar berkas yang dihapus/di-rename di repo ikut terhapus di
|
||||
> tujuan (`.git` & `output` dikecualikan, jadi keduanya aman).
|
||||
|
||||
Tanpa `cluster.yml`, installer berperilaku **persis** seperti single-host (tidak ada
|
||||
yang di-generate). Hasil generate berisi per host: `docker-compose.yml` (hanya service
|
||||
milik host itu), `.env` (endpoint → alamat host), dan `config/` ter-render.
|
||||
|
||||
+118
-17
@@ -94,6 +94,16 @@ ldap_endpoint() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Alamat BROWSER-reachable komponen (tempat user memakai UI di browser): nilai <host_var>
|
||||
# bila berupa ALAMAT (mengandung titik); kalau tidak → SICABE_HOST_ADDRESS (multi-host);
|
||||
# kalau tak ada → localhost (single-host). Dipakai redirect_url OIDC StarRocks & origin
|
||||
# redirectUris Keycloak (JANGAN nama container/hostname internal — browser tak bisa resolve).
|
||||
bootstrap_browser_addr() { # <host_var>
|
||||
local v="${!1:-}"
|
||||
if [[ "${v}" == *.* && "${v}" != "127.0.0.1" ]]; then echo "${v}"; return; fi
|
||||
echo "${SICABE_HOST_ADDRESS:-localhost}"
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# STEP 1: Keycloak — Realm, Client, User, Role (bab 5.1)
|
||||
# =============================================================================
|
||||
@@ -218,6 +228,28 @@ bootstrap_keycloak() {
|
||||
["grafana"]="${GRAFANA_PUBLIC_URL:-}"
|
||||
)
|
||||
|
||||
# Origin HOST (skema://alamat:port) — tempat browser MEMBUKA komponen. WAJIB ada di
|
||||
# redirectUris, kalau tidak login via ALAMAT (mis. https://100.123.123.71:12443) ditolak
|
||||
# "Invalid parameter: redirect_uri" (redirectUris hanya localhost). Sinkron dgn
|
||||
# scripts/verify-keycloak.sh.
|
||||
# <client>="<scheme> <host_var> <port_var> <default_port>"
|
||||
declare -A CLIENT_ORIGIN=(
|
||||
["trino"]="https TRINO_HOST TRINO_HTTPS_PORT 12443"
|
||||
["grafana"]="http GRAFANA_HOST GRAFANA_PORT 12300"
|
||||
["openmetadata"]="http OPENMETADATA_HOST OPENMETADATA_PORT 12085"
|
||||
["airflow"]="http AIRFLOW_HOST AIRFLOW_PORT 12082"
|
||||
["starrocks"]="http STARROCKS_FE_HOST STARROCKS_HTTP_PORT 12030"
|
||||
)
|
||||
_comp_addr() { # <host_var> → alamat efektif (IP/alamat, atau host ini bila co-located)
|
||||
local v="${!1:-}"
|
||||
if [[ "${v}" == *.* && "${v}" != "127.0.0.1" ]]; then echo "${v}"; else echo "${SICABE_HOST_ADDRESS:-localhost}"; fi
|
||||
}
|
||||
_client_origin() { # <client> → skema://alamat:port (atau kosong)
|
||||
local spec="${CLIENT_ORIGIN[$1]:-}"; [[ -n "${spec}" ]] || { echo ""; return; }
|
||||
local s hv pv dp; read -r s hv pv dp <<< "${spec}"
|
||||
printf '%s://%s:%s' "${s}" "$(_comp_addr "${hv}")" "${!pv:-${dp}}"
|
||||
}
|
||||
|
||||
for client_id in "${!CLIENTS[@]}"; do
|
||||
local entry="${CLIENTS[$client_id]}"
|
||||
IFS="|" read -r desc secret <<< "${entry}"
|
||||
@@ -260,25 +292,38 @@ bootstrap_keycloak() {
|
||||
# yang membuat login browser lewat alamat publik ditolak:
|
||||
# Invalid parameter: redirect_uri
|
||||
if [[ -n "${client_uuid}" ]]; then
|
||||
local _payload
|
||||
local _payload _host_origin
|
||||
_host_origin="$(_client_origin "${client_id}")"
|
||||
_payload=$(curl -sf -H "Authorization: Bearer ${token}" \
|
||||
"${kc_base}/admin/realms/${realm}/clients/${client_uuid}" \
|
||||
| PUBLIC_ORIGIN="${CLIENT_PUBLIC_URL[$client_id]:-}" python3 -c '
|
||||
| PUBLIC_ORIGIN="${CLIENT_PUBLIC_URL[$client_id]:-}" HOST_ORIGIN="${_host_origin}" \
|
||||
EXTRA_ORIGINS="${SICABE_CLIENT_REDIRECT_ORIGINS:-}" python3 -c '
|
||||
import json, os, sys
|
||||
c = json.load(sys.stdin)
|
||||
pub = (os.environ.get("PUBLIC_ORIGIN") or "").rstrip("/")
|
||||
host = (os.environ.get("HOST_ORIGIN") or "").rstrip("/")
|
||||
redirects = set(c.get("redirectUris") or []) | {"http://localhost:*", "https://localhost:*"}
|
||||
origins = set(c.get("webOrigins") or []) | {"http://localhost:*", "https://localhost:*"}
|
||||
if host:
|
||||
redirects.add(host + "/*")
|
||||
origins.add(host)
|
||||
if pub:
|
||||
redirects.add(pub + "/*")
|
||||
origins.add(pub)
|
||||
# Origin client dari CLUSTER (host existing tak tahu alamat host cluster) — daftar
|
||||
# spasi-dipisah, mis. SICABE_CLIENT_REDIRECT_ORIGINS="https://100.123.123.71:12443 ...".
|
||||
for o in (os.environ.get("EXTRA_ORIGINS") or "").split():
|
||||
o = o.rstrip("/")
|
||||
if o:
|
||||
redirects.add(o + "/*")
|
||||
origins.add(o)
|
||||
print(json.dumps({"redirectUris": sorted(redirects), "webOrigins": sorted(origins)}))
|
||||
' 2>/dev/null || echo "")
|
||||
if [[ -n "${_payload}" && -n "${CLIENT_PUBLIC_URL[$client_id]:-}" ]]; then
|
||||
if [[ -n "${_payload}" ]]; then
|
||||
curl -sf -X PUT -H "Authorization: Bearer ${token}" \
|
||||
-H "Content-Type: application/json" -d "${_payload}" \
|
||||
"${kc_base}/admin/realms/${realm}/clients/${client_uuid}" >/dev/null 2>&1 || true
|
||||
echo -e "${OK} Client '${client_id}' redirectUris disinkronkan (${CLIENT_PUBLIC_URL[$client_id]})."
|
||||
echo -e "${OK} Client '${client_id}' redirectUris disinkronkan (origin host ${_host_origin:-—}; publik ${CLIENT_PUBLIC_URL[$client_id]:-—})."
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -312,7 +357,8 @@ print(json.dumps({"redirectUris": sorted(redirects), "webOrigins": sorted(origin
|
||||
# StarRocks FE mengirim redirect internal (http://starrocks-fe:8030/api/oauth2)
|
||||
# saat membangun URL OAuth2 → WAJIB terdaftar, jika tidak Keycloak menolak
|
||||
# dengan "Invalid parameter: redirect_uri". Pola host dipakai untuk callback browser.
|
||||
local sr_client_id sr_client_json
|
||||
local sr_client_id sr_client_json sr_fe_browser_addr
|
||||
sr_fe_browser_addr="$(bootstrap_browser_addr STARROCKS_FE_HOST)"
|
||||
sr_client_id=$(curl -sf -H "Authorization: Bearer ${token}" \
|
||||
"${kc_base}/admin/realms/${realm}/clients?clientId=starrocks" | \
|
||||
python3 -c "import sys,json; d=json.load(sys.stdin); print(d[0]['id'] if d else '')" 2>/dev/null || echo "")
|
||||
@@ -322,7 +368,7 @@ print(json.dumps({"redirectUris": sorted(redirects), "webOrigins": sorted(origin
|
||||
python3 -c "
|
||||
import sys,json
|
||||
c=json.load(sys.stdin)
|
||||
extra=['http://starrocks-fe:8030/api/oauth2','http://localhost:${STARROCKS_HTTP_PORT:-12030}/*']
|
||||
extra=['http://starrocks-fe:8030/api/oauth2','http://localhost:${STARROCKS_HTTP_PORT:-12030}/*','http://${sr_fe_browser_addr}:${STARROCKS_HTTP_PORT:-12030}/*']
|
||||
c['redirectUris']=sorted(set((c.get('redirectUris') or [])+extra))
|
||||
print(json.dumps(c))
|
||||
" 2>/dev/null || true)
|
||||
@@ -451,12 +497,20 @@ bootstrap_minio() {
|
||||
local mc_endpoint="${S3_ENDPOINT:-http://minio:9000}"
|
||||
local mc_user="${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}"
|
||||
local mc_pass="${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-}}"
|
||||
# Nama bucket = config (MINIO_BUCKET_ICEBERG_DATA), bukan hardcoded.
|
||||
local bucket="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}"
|
||||
local mc_image="quay.io/minio/mc:${MINIO_MC_VERSION:-RELEASE.2025-08-13T08-35-41Z}"
|
||||
local local_minio_ep="http://minio:9000"
|
||||
|
||||
# Penantian via host hanya relevan untuk MinIO lokal (S3 existing tidak di localhost).
|
||||
if [[ "${mc_endpoint}" == "${local_minio_ep}" ]]; then
|
||||
# Pemilik storage ⇔ container MinIO ada di bundle host INI (single-host → selalu
|
||||
# lokal, krena SICABE_HOST_NAME kosong). Bila storage dikelola SISTEM LAIN
|
||||
# (mis. sisi existing P1/P3), host ini HANYA MEMVERIFIKASI bucket — pembuatan
|
||||
# bucket adalah tugas sisi PEMILIK storage. Ini menghindari cluster "menciptakan"
|
||||
# bucket di storage pihak lain tanpa otoritas.
|
||||
local mc_mode="create"
|
||||
sicabe_host_expects_container sicabe-minio || mc_mode="verify"
|
||||
|
||||
# Penantian via host hanya relevan untuk MinIO lokal milik host ini.
|
||||
if [[ "${mc_mode}" == "create" ]]; then
|
||||
echo -e "${INFO} Menunggu MinIO siap..."
|
||||
for i in $(seq 1 15); do
|
||||
if curl -sf "http://localhost:${MINIO_PORT:-12900}/minio/health/live" >/dev/null 2>&1; then
|
||||
@@ -467,11 +521,11 @@ bootstrap_minio() {
|
||||
sleep 2
|
||||
done
|
||||
else
|
||||
echo -e "${INFO} Object storage eksternal (${mc_endpoint}) — lewati penantian host."
|
||||
echo -e "${INFO} Object storage dikelola sistem lain (${mc_endpoint}) — mode VERIFIKASI bucket."
|
||||
fi
|
||||
|
||||
if [[ -z "${mc_pass}" ]]; then
|
||||
echo -e "${WARN} Kredensial storage kosong (S3_SECRET_KEY / MINIO_ROOT_PASSWORD) — lewati pembuatan bucket."
|
||||
echo -e "${WARN} Kredensial storage kosong (S3_SECRET_KEY / MINIO_ROOT_PASSWORD) — lewati cek bucket."
|
||||
return 0
|
||||
fi
|
||||
|
||||
@@ -488,12 +542,13 @@ bootstrap_minio() {
|
||||
local out
|
||||
out=$(docker run --rm -i \
|
||||
--network "${SICABE_NETWORK:-sicabe-network}" \
|
||||
-e S3E="${mc_endpoint}" -e S3U="${mc_user}" -e S3P="${mc_pass}" -e BUCKET="${bucket}" \
|
||||
-e S3E="${mc_endpoint}" -e S3U="${mc_user}" -e S3P="${mc_pass}" \
|
||||
-e BUCKET="${bucket}" -e MODE="${mc_mode}" \
|
||||
--entrypoint sh "${mc_image}" -c '
|
||||
if ! mc alias set probe "$S3E" "$S3U" "$S3P" >/dev/null 2>&1; then echo ALIAS_FAIL; exit 0; fi
|
||||
if mc ls "probe/$BUCKET" >/dev/null 2>&1; then echo BUCKET_OK
|
||||
elif mc mb "probe/$BUCKET" >/dev/null 2>&1; then echo BUCKET_CREATED
|
||||
else echo BUCKET_FAIL; fi' 2>&1)
|
||||
elif [ "$MODE" = "create" ] && mc mb "probe/$BUCKET" >/dev/null 2>&1; then echo BUCKET_CREATED
|
||||
else echo BUCKET_MISSING; fi' 2>&1)
|
||||
|
||||
case "${out}" in
|
||||
*ALIAS_FAIL*)
|
||||
@@ -507,9 +562,14 @@ else echo BUCKET_FAIL; fi' 2>&1)
|
||||
*BUCKET_CREATED*)
|
||||
echo -e "${OK} Bucket '${bucket}' dibuat."
|
||||
;;
|
||||
*)
|
||||
*BUCKET_MISSING*)
|
||||
if [[ "${mc_mode}" == "verify" ]]; then
|
||||
echo -e "${WARN} Bucket '${bucket}' belum ada — storage dikelola sistem lain; buat di sisi PEMILIK storage."
|
||||
echo " Bucket = MINIO_BUCKET_ICEBERG_DATA (config); konfirmasi ke admin storage existing."
|
||||
else
|
||||
echo -e "${WARN} Bucket '${bucket}' belum bisa dipastikan — buat manual, lalu:"
|
||||
echo " ./scripts/verify-phase2-storage.sh"
|
||||
fi
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
@@ -719,6 +779,12 @@ bootstrap_starrocks() {
|
||||
|
||||
# ── 2. Security integration OIDC (Keycloak) ─────────────────────────────
|
||||
echo -e "${INFO} [2/6] Security integration OIDC..."
|
||||
# redirect_url = URL yang dibuka BROWSER (host FE + port HTTP publik) — bukan localhost/
|
||||
# nama container. Keycloak memakai nilai ini LITERAL sebagai redirect_uri.
|
||||
local sr_redirect_addr sr_redirect_url
|
||||
sr_redirect_addr="$(bootstrap_browser_addr STARROCKS_FE_HOST)"
|
||||
sr_redirect_url="http://${sr_redirect_addr}:${STARROCKS_HTTP_PORT:-12030}/api/oauth2"
|
||||
echo -e "${INFO} redirect_url OIDC StarRocks: ${sr_redirect_url}"
|
||||
if ! sr_mysql "SHOW SECURITY INTEGRATIONS;" | grep -qi "sr_oidc"; then
|
||||
# redirect_url dipakai LITERAL sebagai redirect_uri & harus reachable dari
|
||||
# browser (DBeaver membuka browser ke sini). JANGAN pakai hostname internal.
|
||||
@@ -728,7 +794,7 @@ bootstrap_starrocks() {
|
||||
'token_server_url' = '${kc_internal}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/token',
|
||||
'client_id' = 'starrocks',
|
||||
'client_secret' = '${STARROCKS_CLIENT_SECRET}',
|
||||
'redirect_url' = 'http://localhost:${STARROCKS_HTTP_PORT:-12030}/api/oauth2',
|
||||
'redirect_url' = '${sr_redirect_url}',
|
||||
'jwks_url' = '${kc_internal}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/certs',
|
||||
'principal_field' = 'preferred_username',
|
||||
'required_issuer' = '${kc_base}/realms/${SICABE_REALM:-sicabe}',
|
||||
@@ -756,7 +822,7 @@ bootstrap_starrocks() {
|
||||
'ldap_cache_refresh_interval' = '900'
|
||||
);" && echo -e "${OK} Group provider 'sr_ldap' dibuat/disinkronkan." || echo -e "${WARN} CREATE GROUP PROVIDER gagal."
|
||||
sr_mysql "ALTER SECURITY INTEGRATION sr_oidc SET (
|
||||
'redirect_url' = 'http://localhost:${STARROCKS_HTTP_PORT:-12030}/api/oauth2',
|
||||
'redirect_url' = '${sr_redirect_url}',
|
||||
'group_provider' = 'sr_ldap',
|
||||
'permitted_groups' = 'data-admin,data-engineer,data-analyst,data-viewer',
|
||||
'auth_server_url' = '${kc_base}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/auth',
|
||||
@@ -1504,6 +1570,27 @@ step_tiers() {
|
||||
esac
|
||||
}
|
||||
|
||||
# Container yang menandai sebuah step BERJALAN di host ini. Dipakai mode `--host`
|
||||
# untuk MELEWATI step yang komponennya dikelola SISTEM LAIN (tidak ada di bundle host
|
||||
# ini) — mis. keycloak/nessie existing pada skenario P1/P3. Tanpa ini, `--host control`
|
||||
# tetap mencoba bootstrap Keycloak meski Keycloak ada di host existing.
|
||||
# Step 2 (MinIO) sengaja DIPERKECUALIKAN oleh pemanggil: bila storage dikelola sistem
|
||||
# lain, step itu tetap jalan dalam MODE VERIFIKASI (bukan skip).
|
||||
step_container() {
|
||||
case "$1" in
|
||||
1) echo sicabe-keycloak ;;
|
||||
2) echo sicabe-minio ;;
|
||||
3) echo sicabe-openldap ;;
|
||||
4) echo sicabe-trino-coordinator ;;
|
||||
5) echo sicabe-starrocks-fe ;;
|
||||
6) echo sicabe-airflow-webserver ;;
|
||||
7) echo sicabe-openmetadata ;;
|
||||
8) echo sicabe-grafana ;;
|
||||
9) echo sicabe-dremio ;;
|
||||
*) echo "" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
main() {
|
||||
local START_STEP=1
|
||||
@@ -1562,6 +1649,9 @@ main() {
|
||||
# sicabe_load_env(): tanpa ini sub-script menimpa endpoint host dengan deploy/.env
|
||||
# (nilai single-host) → menyentuh endpoint yang salah di multi-host.
|
||||
export SICABE_ENV_FILE="${henv}"
|
||||
# Bundle host ini — dipakai sicabe_host_expects_container() agar tahu komponen mana
|
||||
# yang dijalankan host ini (untuk melewati step milik sistem lain, lihat loop step).
|
||||
export SICABE_BUNDLE_DIR="$(dirname "${henv}")"
|
||||
# Generator menulis daftar tier dengan pemisah KOMA (SICABE_HOST_TIERS=
|
||||
# identity,policy,pipeline,governance); pencocokan di bawah memakai spasi.
|
||||
HOST_TIERS="${SICABE_HOST_TIERS//,/ }"
|
||||
@@ -1624,6 +1714,17 @@ main() {
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
# Komponen dikelola SISTEM LAIN (tak ada di bundle host ini) → step tak berlaku.
|
||||
# Step 2 (MinIO) DIPERKECUALIKAN: tetap jalan untuk memverifikasi bucket existing.
|
||||
if [[ -n "${HOST_NAME}" && "${step}" != "2" ]]; then
|
||||
local _step_ctr
|
||||
_step_ctr="$(step_container "${step}")"
|
||||
if [[ -n "${_step_ctr}" ]] && ! sicabe_host_expects_container "${_step_ctr}"; then
|
||||
echo -e "${INFO} [skip] step ${step} — '${_step_ctr}' dikelola sistem lain di host ini"
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
case ${step} in
|
||||
1) bootstrap_keycloak ;;
|
||||
2) bootstrap_minio ;;
|
||||
|
||||
@@ -8,6 +8,8 @@
|
||||
# - Cluster (control/compute/state) menjalankan SISANYA dan memakai keycloak/openldap/
|
||||
# minio/nessie EXISTING lewat alamat DI ATAS + port yang dipublikasikan.
|
||||
#
|
||||
# Runbook lengkap (teruji): docs/sicabe-p1-existing-runbook.md
|
||||
#
|
||||
# CARA PAKAI
|
||||
# # di control (sumber .env cluster), TAMBAHKAN ke deploy/.env:
|
||||
# KEYCLOAK_INTERNAL_PORT=12080 # <-- port HOST keycloak existing (bukan 8080)
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
# =============================================================================
|
||||
# cluster.p3.yml — Skenario P3: komponen EXISTING milik PIHAK LAIN (host/IP sendiri)
|
||||
# + DUA OpenLDAP independen (existing utk keycloak, cluster utk trino/starrocks).
|
||||
# -----------------------------------------------------------------------------
|
||||
# Kondisi client (nyata):
|
||||
# - keycloak, minio, nessie : EXISTING, dikelola pihak lain — masing-masing bisa di
|
||||
# host/IP SENDIRI (di sini ditandai `components.<c>.address`).
|
||||
# - openldap : DUA instance independen (analog postgres):
|
||||
# · sisi existing : openldap LOKAL (untuk Keycloak) — DI LUAR
|
||||
# inventory ini (setup single-host existing; JANGAN set LDAP_URL).
|
||||
# · cluster : openldap di `control` (tier identity) →
|
||||
# group-provider Trino/StarRocks.
|
||||
# - postgres (utk keycloak/nessie): sisi existing, BUKAN milik cluster → tak di inventory.
|
||||
# - postgres cluster : milik sendiri di `state`.
|
||||
#
|
||||
# Perbedaan dari P1: di P1 identity (keycloak+openldap+nessie) ada di host existing; di P3
|
||||
# `keycloak`+`nessie` eksternal (per-host) sementara `openldap` ada di KEDUA sisi (independen).
|
||||
# Penanda eksternal kini `components.<c>.address` (tanpa entri host) → tak dijalankan cluster.
|
||||
#
|
||||
# SIMULASI: ketiga komponen existing boleh dijalankan di SATU host (docker compose +
|
||||
# network sendiri) → isi `address` sama (mis. 100.93.54.108). REAL: isi alamat berbeda.
|
||||
#
|
||||
# .env cluster — override `*_INTERNAL_PORT` HANYA untuk komponen EKSTERNAL:
|
||||
# KEYCLOAK_INTERNAL_PORT=12080 MINIO_INTERNAL_PORT=12900 NESSIE_INTERNAL_PORT=12120
|
||||
# JANGAN override OPENLDAP_INTERNAL_PORT — openldap cluster LOKAL (di `control`), port 389.
|
||||
# (Plus: samakan secret existing — 5× *_CLIENT_SECRET, MINIO_ROOT_*; LDAP_ADMIN_PASSWORD
|
||||
# per-sisi, TIDAK perlu disamakan.)
|
||||
#
|
||||
# Langkah lengkap simulasi: docs/sicabe-p3-simulation-runbook.md
|
||||
#
|
||||
# Verifikasi (tanpa akses admin): hanya memeriksa apakah keycloak/minio/nessie memenuhi
|
||||
# kebutuhan (issuer, TLS, S3, secret) — lihat docs/sicabe-p1-existing-runbook.md §7.
|
||||
# =============================================================================
|
||||
|
||||
version: 1
|
||||
cluster:
|
||||
name: sicabe-p3
|
||||
env: simulation
|
||||
|
||||
tls:
|
||||
mode: none
|
||||
|
||||
preset: a3
|
||||
|
||||
hosts:
|
||||
# control mengelola OPENLDAP cluster (tier identity) — keycloak/nessie di-override eksternal.
|
||||
control:
|
||||
address: 100.112.76.124
|
||||
tiers: [identity, policy, pipeline, governance]
|
||||
compute:
|
||||
address: 100.123.123.71
|
||||
tiers: [coord, engine]
|
||||
state:
|
||||
address: 100.73.229.107
|
||||
tiers: [state]
|
||||
|
||||
components:
|
||||
# EXISTING (pihak lain) — TANPA host; endpoint = alamat ini di semua host.
|
||||
keycloak: { address: 100.93.54.108 }
|
||||
minio: { address: 100.93.54.108 }
|
||||
nessie: { address: 100.93.54.108 }
|
||||
dremio: { enabled: false }
|
||||
@@ -93,8 +93,11 @@ default_args = {
|
||||
"depends_on_past": False,
|
||||
"email_on_failure": True,
|
||||
"email_on_retry": False,
|
||||
"retries": 2,
|
||||
"retry_delay": timedelta(minutes=5),
|
||||
# Retries default 2 (produksi). Dapat di-override via env, mis. `-e SICABE_DAG_RETRIES=0`
|
||||
# pada `airflow dags test` — penting agar uji/uji-E2E TIDAK menggantung menunggu retry
|
||||
# saat task GE gagal (mis. DAG #1 pada cold start: entitas gold belum ada di OM).
|
||||
"retries": int(os.getenv("SICABE_DAG_RETRIES", "2")),
|
||||
"retry_delay": timedelta(seconds=int(os.getenv("SICABE_DAG_RETRY_DELAY_SECONDS", "300"))),
|
||||
"execution_timeout": timedelta(hours=2),
|
||||
}
|
||||
|
||||
|
||||
@@ -210,6 +210,10 @@ export TRINO_SVC_SEED_PASSWORD="${TRINO_SVC_SEED_PASSWORD:-svc_seed_dev}"
|
||||
export TRINO_SVC_DBT_PASSWORD="${TRINO_SVC_DBT_PASSWORD:-svc_dbt_dev}"
|
||||
export TRINO_SVC_AIRFLOW_PASSWORD="${TRINO_SVC_AIRFLOW_PASSWORD:-svc_airflow_dev}"
|
||||
export TRINO_SVC_OM_PASSWORD="${TRINO_SVC_OM_PASSWORD:-svc_om_dev}"
|
||||
# Kunci HMAC tetap untuk `state` OAuth2 Trino (HMAC-SHA256). Default Trino = ACAK tiap start
|
||||
# → sesi login OAuth2 yg sedang berjalan batal bila coordinator restart. Set tetap (acak per
|
||||
# instalasi) agar stabil. Dipakai config-coordinator.properties lewat ${ENV:TRINO_OAUTH_STATE_KEY}.
|
||||
export TRINO_OAUTH_STATE_KEY="${TRINO_OAUTH_STATE_KEY:-trino-oauth-state-dev-only}"
|
||||
|
||||
# ─── StarRocks (Phase 4 — Serving) ──────────────────────────────────────────
|
||||
export STARROCKS_VERSION="${STARROCKS_VERSION:-4.0.13}"
|
||||
|
||||
@@ -23,9 +23,6 @@
|
||||
- targets: ["host.docker.internal:9100"]
|
||||
labels:
|
||||
host: local
|
||||
- targets: ["100.123.123.71:9100"]
|
||||
labels:
|
||||
host: darkworld
|
||||
|
||||
# Contoh penambahan host (hapus komentar):
|
||||
# - targets: ["10.20.0.11:9100"]
|
||||
|
||||
@@ -42,13 +42,19 @@ http-server.process-forwarded=${ENV:TRINO_PROCESS_FORWARDED}
|
||||
# Kalau di-hardcode, issuer di klaster multi-host tidak cocok dan token ditolak.
|
||||
http-server.authentication.oauth2.issuer=${ENV:KEYCLOAK_PUBLIC_URL}/realms/sicabe
|
||||
# Matikan OIDC discovery — Trino tidak bisa mengakses URL publik dari dalam container.
|
||||
# Endpoint di-set manual: browser pakai URL publik, server-ke-server pakai keycloak:8080.
|
||||
# Endpoint di-set manual: browser pakai URL publik (auth-url); server-ke-server (token/jwks)
|
||||
# pakai KEYCLOAK_HOST:INTERNAL_PORT — di multi-host dgn Keycloak EKSTERNAL ini = ALAMAT host
|
||||
# pemilik (mis. 100.93.54.108:12080). JANGAN literal `keycloak` (nama container, tak resolve
|
||||
# di host lain → penukaran token gagal → "Authentication response could not be verified").
|
||||
http-server.authentication.oauth2.oidc.discovery=false
|
||||
# Endpoint browser-facing (auth redirect) — harus URL publik juga
|
||||
http-server.authentication.oauth2.auth-url=${ENV:KEYCLOAK_PUBLIC_URL}/realms/sicabe/protocol/openid-connect/auth
|
||||
# Endpoint server-side (token exchange & JWKS) — reachable dari container
|
||||
http-server.authentication.oauth2.token-url=http://keycloak:${ENV:KEYCLOAK_INTERNAL_PORT}/realms/sicabe/protocol/openid-connect/token
|
||||
http-server.authentication.oauth2.jwks-url=http://keycloak:${ENV:KEYCLOAK_INTERNAL_PORT}/realms/sicabe/protocol/openid-connect/certs
|
||||
http-server.authentication.oauth2.token-url=http://${ENV:KEYCLOAK_HOST}:${ENV:KEYCLOAK_INTERNAL_PORT}/realms/sicabe/protocol/openid-connect/token
|
||||
http-server.authentication.oauth2.jwks-url=http://${ENV:KEYCLOAK_HOST}:${ENV:KEYCLOAK_INTERNAL_PORT}/realms/sicabe/protocol/openid-connect/certs
|
||||
# Kunci HMAC tetap untuk `state` OAuth2 (default Trino = acak tiap coordinator start →
|
||||
# sesi login batal saat restart). Nilai acak-per-instalasi dari .env.
|
||||
http-server.authentication.oauth2.state-key=${ENV:TRINO_OAUTH_STATE_KEY}
|
||||
http-server.authentication.oauth2.client-id=trino
|
||||
http-server.authentication.oauth2.client-secret=${ENV:TRINO_CLIENT_SECRET}
|
||||
# Username Trino = preferred_username (bukan 'sub') agar cocok dengan uid LDAP
|
||||
|
||||
@@ -483,6 +483,12 @@ services:
|
||||
# Port internal Keycloak untuk config Trino (config-coordinator.properties
|
||||
# memakai ${ENV:KEYCLOAK_INTERNAL_PORT}) — wajib ada di container.
|
||||
KEYCLOAK_INTERNAL_PORT: ${KEYCLOAK_INTERNAL_PORT:-8080}
|
||||
# Host Keycloak untuk token-url/jwks-url (server-ke-server). Multi-host dgn Keycloak
|
||||
# EKSTERNAL: ini = ALAMAT host pemilik (mis. 100.93.54.108) — bukan nama container
|
||||
# `keycloak` yang tak bisa di-resolve → penukaran token gagal ("could not be verified").
|
||||
KEYCLOAK_HOST: ${KEYCLOAK_HOST:-keycloak}
|
||||
# Kunci HMAC tetap untuk state OAuth2 Trino (default Trino acak tiap start).
|
||||
TRINO_OAUTH_STATE_KEY: ${TRINO_OAUTH_STATE_KEY:-trino-oauth-state-dev-only}
|
||||
healthcheck:
|
||||
# Trino internal port adalah 8080 (http-server.http.port)
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8080/v1/info"]
|
||||
|
||||
+10
-3
@@ -69,7 +69,10 @@ TRINO_SVC_SEED_PASSWORD=$(openssl rand -hex 8)
|
||||
TRINO_SVC_DBT_PASSWORD=$(openssl rand -hex 8)
|
||||
TRINO_SVC_AIRFLOW_PASSWORD=$(openssl rand -hex 8)
|
||||
TRINO_SVC_OM_PASSWORD=$(openssl rand -hex 8)
|
||||
TRINO_OAUTH_STATE_KEY=$(openssl rand -hex 32)
|
||||
TRINO_OAUTH_STATE_KEY=$(openssl rand -hex 32)
|
||||
GRAFANA_ADMIN_PASSWORD=$(openssl rand -hex 8)
|
||||
STARROCKS_ROOT_PASSWORD=$(openssl rand -hex 16)
|
||||
# OM 2.0.x menegakkan kebijakan password (min 8 karakter + huruf besar + huruf
|
||||
# kecil + digit + karakter spesial). `openssl rand -hex` hanya menghasilkan
|
||||
# 0-9a-f sehingga TIDAK memenuhi kebijakan itu — password harus dirakit agar
|
||||
@@ -859,8 +862,8 @@ run_host() {
|
||||
# hilang → mis. Airflow IsADirectoryError). Isi ditulis setelah .env host di-source.
|
||||
sicabe_ensure_file_mount "${dir}/config/trino/password.db" 644
|
||||
echo -e "${WARN} Jalankan ini HANYA di host '${name}'."
|
||||
echo -e "${WARN} Pilot berdampingan di host yang SAMA tidak cukup dipisah network/port:"
|
||||
echo -e "${WARN} RAM tidak cukup untuk dua stack — matikan stack lama lebih dulu (README)."
|
||||
echo -e "${WARN} Stack single-host & multi-host TIDAK untuk berjalan bersamaan di satu host:"
|
||||
echo -e "${WARN} matikan stack lama lebih dulu sebelum menyalakan yang baru."
|
||||
# ── PRECEDENCE (bug nyata yang dilaporkan dari server) ──────────────────────
|
||||
# Compose memberi prioritas pada variabel **SHELL** di atas `--env-file`. Skrip ini
|
||||
# sudah men-source config/environment.defaults.sh, yang mengekspor
|
||||
@@ -905,9 +908,13 @@ run_host() {
|
||||
# over SSH. Dijalankan di host CONTROL (master installer). Target = semua host di
|
||||
# cluster.yml KECUALI host ini (dideteksi dari kecocokan alamat lokal) — atau satu
|
||||
# host saja via --host. Aman diulang (rsync idempoten).
|
||||
# Tujuan default (--dest) = CERMIN lokasi repo ini di bawah $HOME (lihat
|
||||
# sicabe_default_dest) — jadi nama repo TIDAK dipatok: `sicabe-core` (repo
|
||||
# distribusi bersih) → `~/sicabe-core/deploy`. Timpa dengan `--dest`.
|
||||
sync_hosts() {
|
||||
local inv="${SCRIPT_DIR}/cluster.yml" dest="~/sicabe-doc/deploy" user="${USER:-root}" \
|
||||
local inv="${SCRIPT_DIR}/cluster.yml" dest user="${USER:-root}" \
|
||||
only_host="" do_delete=0 dry=0
|
||||
dest="$(sicabe_default_dest "${SCRIPT_DIR}")"
|
||||
shift
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
|
||||
@@ -456,6 +456,10 @@ def endpoint_lines(doc: dict, host: str, plan: dict) -> list[str]:
|
||||
placement = component_placements(doc)
|
||||
lines: list[str] = []
|
||||
for var, component in ENDPOINT_VARS.items():
|
||||
ext = inventory.external_address(doc, component)
|
||||
if ext:
|
||||
lines.append(f"{var}={ext}") # EKSTERNAL (pihak lain) → alamat tetap
|
||||
continue
|
||||
owners = placement.get(component)
|
||||
if not owners:
|
||||
continue
|
||||
@@ -492,6 +496,10 @@ def public_url_lines(doc: dict, plan: dict | None = None) -> list[str]:
|
||||
placement = component_placements(doc)
|
||||
resolved: dict[str, str] = {}
|
||||
for var, (component, tmpl) in PUBLIC_URL_ENDPOINTS.items():
|
||||
ext = inventory.external_address(doc, component)
|
||||
if ext:
|
||||
resolved[var] = tmpl.format(addr=ext) # eksternal → alamat tetap
|
||||
continue
|
||||
owners = placement.get(component)
|
||||
if not owners:
|
||||
continue
|
||||
@@ -655,6 +663,10 @@ def url_endpoint_lines(doc: dict, host: str, plan: dict, env_path=None) -> list[
|
||||
placement = component_placements(doc)
|
||||
lines: list[str] = []
|
||||
for var, (component, template) in URL_ENDPOINT_VARS.items():
|
||||
ext = inventory.external_address(doc, component)
|
||||
if ext:
|
||||
lines.append(f"{var}={template.format(addr=ext, port=_endpoint_port(var, env_path))}")
|
||||
continue
|
||||
owners = placement.get(component)
|
||||
if not owners:
|
||||
continue
|
||||
@@ -684,6 +696,30 @@ def render_config_map(doc: dict, host: str, plan: dict) -> list[str]:
|
||||
return args
|
||||
|
||||
|
||||
def _rewrite_prometheus_scrapes(host_dir, doc, env_path) -> None:
|
||||
"""Ganti target scrape MinIO di prometheus.yml bila MinIO EKSTERNAL.
|
||||
|
||||
Prometheus TIDAK mendukung substitusi env (beda dgn config Trino), dan
|
||||
render-config.sh hanya memetakan hostname komponen yang dimiliki HOST LAIN —
|
||||
komponen EKSTERNAL (`components.minio.address`, skenario P3) TAK dipetakan →
|
||||
target tetap `minio:9000` (nama container) → scrape GAGAL dari host prometheus.
|
||||
Di sini diganti `<alamat-eksternal>:<MINIO_INTERNAL_PORT>`.
|
||||
|
||||
MinIO se-host (single-host / P1) → tak diubah (`minio:9000`, satu jaringan Docker).
|
||||
"""
|
||||
yml = host_dir / "config" / "prometheus" / "prometheus.yml"
|
||||
if not yml.is_file():
|
||||
return
|
||||
ext = inventory.external_address(doc, "minio")
|
||||
if not ext:
|
||||
return
|
||||
port = _source_env_value(env_path, "MINIO_INTERNAL_PORT") or "9000"
|
||||
txt = yml.read_text(encoding="utf-8")
|
||||
new = txt.replace("minio:9000", f"{ext}:{port}")
|
||||
if new != txt:
|
||||
yml.write_text(new, encoding="utf-8")
|
||||
|
||||
|
||||
def cmd_render(argv: list[str]) -> int:
|
||||
if len(argv) < 7:
|
||||
sys.stderr.write(__doc__ or "")
|
||||
@@ -797,6 +833,9 @@ def cmd_render(argv: list[str]) -> int:
|
||||
fh.write("# DIBANGKITKAN OTOMATIS dari cluster.yml oleh install.sh --generate\n")
|
||||
fh.write("# — jangan disunting tangan. Target node_exporter (job `node`) semua host.\n")
|
||||
yaml.safe_dump(node_targets_doc(plan), fh, sort_keys=False, allow_unicode=True)
|
||||
# Prometheus tak mendukung substitusi env → target MinIO eksternal
|
||||
# (`minio:9000`) di-rewrite di sini ke alamat:port publik.
|
||||
_rewrite_prometheus_scrapes(host_dir, doc, env_path)
|
||||
if rc.returncode != 0:
|
||||
sys.stderr.write(f"[host-{host}] render-config GAGAL:\n{rc.stdout}{rc.stderr}")
|
||||
return 1
|
||||
@@ -872,13 +911,38 @@ def _write_bundle_runtime_files(host_dir, env_path):
|
||||
elif not pw_json.is_file():
|
||||
pw_json.write_text("{}\n", encoding="utf-8")
|
||||
pw_json.chmod(0o666)
|
||||
# 2) Trino password.db — byte-identik dari sumber bila ada; kalau tidak, FILE kosong
|
||||
# (yang penting BUKAN direktori; bootstrap mengisinya).
|
||||
src_db = Path(env_path).resolve().parent / "config" / "trino" / "password.db"
|
||||
# 2) Trino password.db — DIBANGKITKAN dari TRINO_SVC_*_PASSWORD di .env (service
|
||||
# account, HTTP Basic/bcrypt). Dulu hanya MENYALIN config/trino/password.db
|
||||
# sumber — di mode cluster sumber itu sering KOSONG → bundle kosong → Trino
|
||||
# menolak dgn 401 "Access Denied: Invalid credentials" (mis. dbt `svc_dbt`).
|
||||
# Kini bundle MANDIRI & konsisten setelah --sync. Fallback: salin sumber bila
|
||||
# htpasswd tak tersedia; kalau tidak ada, file kosong (yang penting BUKAN direktori).
|
||||
dst_db = cfg / "trino" / "password.db"
|
||||
dst_db.parent.mkdir(parents=True, exist_ok=True)
|
||||
if dst_db.is_dir():
|
||||
shutil.rmtree(dst_db)
|
||||
svc_accounts = (
|
||||
("svc_seed", "TRINO_SVC_SEED_PASSWORD"),
|
||||
("svc_dbt", "TRINO_SVC_DBT_PASSWORD"),
|
||||
("svc_airflow", "TRINO_SVC_AIRFLOW_PASSWORD"),
|
||||
("svc_om", "TRINO_SVC_OM_PASSWORD"),
|
||||
)
|
||||
pw_lines = []
|
||||
htpasswd = shutil.which("htpasswd")
|
||||
if htpasswd:
|
||||
for user, key in svc_accounts:
|
||||
secret = _source_env_value(env_path, key)
|
||||
if not secret:
|
||||
continue
|
||||
proc = subprocess.run([htpasswd, "-nbB", "-C", "10", user, secret],
|
||||
capture_output=True, text=True)
|
||||
if proc.returncode == 0 and proc.stdout.strip():
|
||||
pw_lines.append(proc.stdout.strip())
|
||||
if pw_lines:
|
||||
dst_db.write_text("\n".join(pw_lines) + "\n", encoding="utf-8")
|
||||
dst_db.chmod(0o644)
|
||||
else:
|
||||
src_db = Path(env_path).resolve().parent / "config" / "trino" / "password.db"
|
||||
if src_db.is_file():
|
||||
shutil.copy2(src_db, dst_db)
|
||||
elif not dst_db.is_file():
|
||||
|
||||
@@ -127,14 +127,36 @@ def apply_preset(doc: dict) -> dict:
|
||||
return doc
|
||||
|
||||
|
||||
def external_components(doc: dict) -> set[str]:
|
||||
"""Komponen yang dikelola SISTEM LAIN: ditandai `components.<c>.address`.
|
||||
|
||||
Komponen ini TIDAK dijalankan cluster (tak masuk `placement`/`plan`); endpoint-nya
|
||||
diarahkan ke alamat itu di SEMUA host (lihat generate.endpoint_lines). Berguna untuk
|
||||
IAM/objek-storage milik pihak lain, termasuk di host/IP yang berbeda satu per komponen.
|
||||
"""
|
||||
return {n for n, s in (doc.get("components") or {}).items()
|
||||
if (s.get("address") or "").strip()}
|
||||
|
||||
|
||||
def external_address(doc: dict, service: str) -> str | None:
|
||||
"""Alamat eksternal komponen (components.<svc>.address) bila ada, selain itu None."""
|
||||
spec = (doc.get("components") or {}).get(service) or {}
|
||||
a = (spec.get("address") or "").strip()
|
||||
return a or None
|
||||
|
||||
|
||||
def resolve(doc: dict) -> dict[str, list[str]]:
|
||||
"""Komponen → daftar host (tier `engine` bisa >1 host)."""
|
||||
"""Komponen → daftar host (tier `engine` bisa >1 host). Komponen EKSTERNAL
|
||||
(ber-`address`) TIDAK masuk hasil — ia bukan milik host mana pun."""
|
||||
hosts = doc.get("hosts") or {}
|
||||
tiers_by_service = {svc: tier for tier, svcs in TIER_SERVICES.items() for svc in svcs}
|
||||
explicit = {name: spec.get("host") for name, spec in (doc.get("components") or {}).items()}
|
||||
external = external_components(doc)
|
||||
|
||||
placement: dict[str, list[str]] = {}
|
||||
for service, tier in tiers_by_service.items():
|
||||
if service in external:
|
||||
continue # eksternal (pihak lain) → bukan milik host mana pun
|
||||
if service in explicit and explicit[service]:
|
||||
placement[service] = [explicit[service]]
|
||||
continue
|
||||
@@ -246,18 +268,23 @@ def validate(doc: dict) -> list[str]:
|
||||
components = doc.get("components") or {}
|
||||
for name, spec in components.items():
|
||||
host = spec.get("host")
|
||||
addr = (spec.get("address") or "").strip()
|
||||
if host and host not in hosts:
|
||||
errors.append(f"komponen '{name}': host '{host}' tidak ada di 'hosts'")
|
||||
if host and addr:
|
||||
errors.append(f"komponen '{name}': 'host' dan 'address' tak boleh bersamaan "
|
||||
f"(host = dijalankan cluster; address = dikelola sistem lain)")
|
||||
if tier_of(name) is None and name not in ("dremio",):
|
||||
errors.append(f"komponen '{name}' tidak ada di taksonomi tier (TIER_SERVICES)")
|
||||
|
||||
# (c) tiap komponen harus bisa ditempatkan
|
||||
placement = resolve(doc)
|
||||
for service in (svc for svcs in TIER_SERVICES.values() for svc in svcs):
|
||||
if service not in placement:
|
||||
if service not in placement and not external_address(doc, service):
|
||||
errors.append(
|
||||
f"komponen '{service}' tidak bisa ditempatkan — tidak ada host bertier "
|
||||
f"'{tier_of(service)}' dan tidak ada components.{service}.host"
|
||||
f"'{tier_of(service)}', tidak ada components.{service}.host, "
|
||||
f"dan bukan eksternal (components.{service}.address)"
|
||||
)
|
||||
|
||||
# stateful: cegah pindah host tanpa migrasi (peringatan, bukan error)
|
||||
@@ -331,6 +358,8 @@ def cmd_components(path: str) -> int:
|
||||
doc = apply_preset(load(path))
|
||||
for service, owners in sorted(resolve(doc).items()):
|
||||
print(f"{service:24s} -> {','.join(owners)}")
|
||||
for service in sorted(external_components(doc)):
|
||||
print(f"{service:24s} -> (eksternal {external_address(doc, service)})")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -145,6 +145,24 @@ sicabe_load_env() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# ─── Helper: tujuan rsync default = CERMIN lokasi repo ini di bawah $HOME ────
|
||||
# Dipakai `install.sh --sync` dan `scripts/check-trino-keystore.sh` supaya nama
|
||||
# repo TIDAK dipatok (`~/sicabe-doc/deploy`). Repo distribusi bersih (mis.
|
||||
# `sicabe-core` — isinya hanya folder `deploy/`) pun bekerja tanpa `--dest`:
|
||||
# /home/u/sicabe-doc/deploy → ~/sicabe-doc/deploy
|
||||
# /home/u/sicabe-core/deploy → ~/sicabe-core/deploy
|
||||
# Bila repo di luar $HOME → ~/<nama-folder-induk>/deploy (fallback).
|
||||
# Mengembalikan path ber-`~` (diekspansi shell REMOTE saat rsync/SSH).
|
||||
sicabe_default_dest() { # <dir-absolut-dari-script> (default: $SCRIPT_DIR)
|
||||
local _sdd_dir="${1:-${SCRIPT_DIR:-}}" _sdd_home="${HOME:-}"
|
||||
[[ -n "${_sdd_dir}" ]] || { echo ""; return 1; }
|
||||
if [[ -n "${_sdd_home}" && "${_sdd_dir}" == "${_sdd_home}/"* ]]; then
|
||||
echo "~/${_sdd_dir#"${_sdd_home}/"}"
|
||||
else
|
||||
echo "~/$(basename "$(dirname "${_sdd_dir}")")/deploy"
|
||||
fi
|
||||
}
|
||||
|
||||
# ─── Helper: URL endpoint efektif (sadar EXTERNAL & multi-host) ─────────────
|
||||
# Urutan: (1) <url_var> eksplisit NON-localhost (komponen EXTERNAL) → pakai apa adanya;
|
||||
# (2) multi-host (bundle memuat SICABE_HOST_NAME) → scheme://<host_var>:<port>;
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
# A1 (R18) — verifikasi pin sqlalchemy<2.1 tidak menggeser paket
|
||||
# tanggal : 2026-10-01 21:04:37
|
||||
# metode : uv pip compile requirements.in --constraint <lock lama>
|
||||
# base : apache/airflow:slim-3.3.1-python3.11
|
||||
# lock lama : 1188 baris · md5 5936cf19f36004fd
|
||||
# lock baru : 1650 baris
|
||||
# baris PAKET berbeda : 0
|
||||
# sqlalchemy : sqlalchemy==2.0.54 -> sqlalchemy==2.0.54
|
||||
#
|
||||
# KESIMPULAN
|
||||
# 0 paket berubah — pin ini PREVENTIF (mencegah regenerasi lock mendatang
|
||||
# menarik SQLAlchemy 2.1). Lock dibiarkan utuh karena isi paketnya identik;
|
||||
# perbedaannya hanya baris komentar anotasi constraint (-c /old.lock).
|
||||
# Perlindungan ada di requirements.in (sumber), bukan di lock.
|
||||
@@ -1,114 +0,0 @@
|
||||
# Hasil Uji B3 — Titik Go/No-Go BATCH B (review R18)
|
||||
|
||||
> **Tanggal**: 2026-10-01 · **Image diuji**: `sicabe/airflow:3.3.2-om203-py313` (ID `33990a6c2b88`)
|
||||
> **Metode**: stack eksperimen terisolasi (`airflow332-exp`, port 12184/12185) + config & DAG
|
||||
> produksi di-mount **read-only** + uji koneksi/OPA/dbt dari container sekali pakai.
|
||||
> ⚠️ Produksi **tidak tersentuh** sepanjang uji.
|
||||
|
||||
---
|
||||
|
||||
## 1. VERDICT: 🔴 **NO-GO untuk Python 3.13**
|
||||
|
||||
BATCH B **tidak boleh dilanjutkan** dengan base Python 3.13. Penyebab tunggal: **dbt tidak
|
||||
berfungsi** di Python 3.13 pada tumpukan dependensi kita — dan dbt adalah tulang punggung
|
||||
lapisan silver/gold.
|
||||
|
||||
**Yang tetap sahih**: seluruh hasil B3 lainnya, serta A1 (pin `sqlalchemy<2.1`), A2, dan B1.
|
||||
**Yang perlu diulang**: hanya **B2** (build image) dengan base Python **3.11**.
|
||||
|
||||
---
|
||||
|
||||
## 2. Yang LOLOS ✅
|
||||
|
||||
| Uji | Hasil |
|
||||
|---|---|
|
||||
| **DAG produksi (4 berkas, dari volume produksi)** | ✅ `dag_sitomat_gold_build.py`, `dag_sitomat_gold_build_prod.py` **impor bersih**. `sicabe_dq_gold_native.py` berhasil dimuat sampai tahap *generate workflow* (gagal hanya karena `openmetadata-server` tidak ter-resolve dari container uji saya — bukan masalah versi) |
|
||||
| **Plugin OpenMetadata** | ✅ `/pluginsv2/api/v2/openmetadata/health-auth` → **200**; `/pluginsv2/` → **200** |
|
||||
| **Trino** | ✅ `SELECT 1` → `[[1]]`; `SHOW TABLES FROM iceberg.bronze` → `orders`, `orders_incremental` |
|
||||
| **MinIO** | ✅ bucket `['iceberg-data']` via `S3Hook` |
|
||||
| **MySQL (klien eksternal)** | ✅ `('8.0.40', 'asdp')` via `MySQLdb`/`MySqlHook` |
|
||||
| **PostgreSQL** | ✅ Postgres 15.18 via `PostgresHook` |
|
||||
| **OPA** | ✅ aturan bronze utuh (`bronze_orders`: ddl/select/write) + keputusan INSERT bronze = **true** |
|
||||
| **Metadata DB / migrasi** | ✅ `airflow db migrate` bersih; OM 2.0.3 (API `{"version":"2.0.3"}`) |
|
||||
|
||||
---
|
||||
|
||||
## 3. Yang GAGAL 🔴 — dbt
|
||||
|
||||
### 3.1 Gejala
|
||||
|
||||
```
|
||||
$ dbt debug --profile trino --target prod
|
||||
AttributeError: __class_getitem__
|
||||
mashumaro/core/meta/helpers.py:624 → typing_extensions.Annotated.__class_getitem__
|
||||
/usr/python/lib/python3.13/typing.py:548
|
||||
```
|
||||
`dbt compile` gagal dengan galat yang sama.
|
||||
|
||||
⚠️ **`dbt --version` justru berhasil** — sehingga pemeriksaan dangkal (`import dbt`,
|
||||
`dbt --version`) **tidak menangkap** kerusakan ini.
|
||||
|
||||
### 3.2 Akar masalah & tiga versi yang diuji
|
||||
|
||||
| `mashumaro` | dbt (`debug`/`compile`) | `pip check` |
|
||||
|---|---|---|
|
||||
| **3.10** (maksimum yang diizinkan `collate-data-diff <3.11`) | ❌ **GAGAL** | ✅ **bersih** ← *rusak, tapi terlihat sehat* |
|
||||
| **3.17** | ✅ **debug OK ("All checks passed!") + compile OK** (SQL `orders_silver` ter-compile) | ⚠️ melanggar `dbt-core <3.15` **dan** `collate-data-diff <3.11` |
|
||||
| **3.23** | ✅ berhasil | ⚠️ melanggar keduanya |
|
||||
|
||||
### 3.3 Kesimpulan teknis
|
||||
|
||||
**Tidak ada versi `mashumaro` yang sekaligus (a) memenuhi batas dependensi kita dan
|
||||
(b) berfungsi di Python 3.13.** Yang mengikat ketat adalah `collate-data-diff` (dipakai jalur
|
||||
lineage dbt OpenMetadata) yang menuntut `<3.11.0`, sedangkan perbaikan kompatibilitas 3.13
|
||||
berada di versi ≥ ~3.11.
|
||||
|
||||
➡️ Jadi masalahnya bukan pilihan paket, melainkan **kombinasi Python 3.13 + tumpukan ini**.
|
||||
|
||||
### 3.4 Pelajaran metodologis (penting)
|
||||
|
||||
`pip check` **bersih** pada konfigurasi yang **tidak berfungsi**. Artinya:
|
||||
> Batas dependensi yang dideklarasikan ≠ kebenaran fungsional.
|
||||
|
||||
Uji kelayakan untuk tumpukan ini **wajib menjalankan perintah nyata** (mis. `dbt debug`,
|
||||
`dbt compile`), bukan hanya mengimpor modul atau membaca `--version`.
|
||||
|
||||
---
|
||||
|
||||
## 4. Koreksi atas rekomendasi K1
|
||||
|
||||
Rekomendasi awal saya (K1 = Python **3.13**) dinilai **salah** dan dicabut. Dasarnya waktu itu
|
||||
adalah "artefak eksperimen sudah tervalidasi end-to-end" — padahal validasi itu **tidak
|
||||
lengkap**: eksperimen memeriksa `import dbt` dan `dbt --version`, tetapi **tidak pernah
|
||||
menjalankan `dbt` sungguhan**. B3 — yang memang dirancang sebagai gerbang — yang menemukannya.
|
||||
|
||||
**K1 direvisi menjadi Python 3.11**, dengan alasan:
|
||||
- tujuan BATCH B adalah **Airflow 3.3.2 + OM 2.0.3 + provider MySQL**, bukan Python 3.13;
|
||||
- `openmetadata-ingestion[airflow]` 2.0.3.0 mem-pin `apache-airflow==3.3.2` **terlepas dari
|
||||
Python** → 3.3.2 di Python 3.11 tetap koheren;
|
||||
- Python 3.11 mempertahankan seluruh dependensi lain di set yang **sudah terbukti di produksi**
|
||||
→ risiko jauh lebih rendah.
|
||||
|
||||
---
|
||||
|
||||
## 5. Langkah berikutnya (B2 revisi)
|
||||
|
||||
| Sub | Isi |
|
||||
|---|---|
|
||||
| B2r.1 | `Dockerfile.332` → base `slim-3.3.2-python3.11` (stage builder `mysqlclient` disesuaikan dengan kondisi header Python 3.11 pada base itu) |
|
||||
| B2r.2 | Regenerasi lock 3.3.2/py3.11 **dengan lock 3.3.1 sebagai `--constraint`** (platform identik → diff minimal; teknik yang terbukti di A1) |
|
||||
| B2r.3 | Build ulang image (tag tetap `sicabe/airflow:3.3.2-om203-py313`, menimpa image yang no-go) |
|
||||
| B3r | **Ulangi B3 dengan `dbt debug` + `dbt compile` sebagai syarat WAJIB** (bukan opsional) |
|
||||
|
||||
---
|
||||
|
||||
## 6. Catatan kesalahan uji saya (agar tidak terulang)
|
||||
|
||||
| # | Kesalahan | Akibat |
|
||||
|---|---|---|
|
||||
| 1 | Memakai `airflow dags list` untuk menguji folder yang di-mount | `dags list` membaca **metadata DB**, bukan folder → uji pertama tidak sah (diperbaiki dengan impor langsung tiap berkas DAG) |
|
||||
| 2 | Menguji DAG dari container yang tidak berada di jaringan yang tepat | `openmetadata-server` tidak ter-resolve → kegagalan palsu |
|
||||
| 3 | Mount dbt **read-only** | dbt perlu menulis `logs/` → keluar `exit 2` **tanpa keluaran sama sekali** (menyesatkan) |
|
||||
| 4 | `pip install "mashumaro>=3.9,<3.18"` tanpa `--upgrade` | pip tidak menaikkan (3.10 sudah memenuhi rentang) → uji sempat tidak konklusif |
|
||||
| 5 | Salah mount point volume config OM | kegagalan palsu pada DAG DQ |
|
||||
| 6 | Pemeriksaan awal hanya `import dbt` / `dbt --version` | kerusakan dbt lolos sampai B3 |
|
||||
@@ -1,72 +0,0 @@
|
||||
# Hasil B3 ULANG — Go/No-Go BATCH B pada Image Python 3.11 (review R18)
|
||||
|
||||
> **Tanggal**: 2026-10-01 · **Image diuji**: `sicabe/airflow:3.3.2-om203` (ID `a02620e1d0a8`)
|
||||
> — hasil **B2 revisi** (base Python 3.11), menggantikan image py3.13 yang ditolak.
|
||||
> **Metode**: sama dengan B3 pertama — stack eksperimen terisolasi + config & DAG produksi
|
||||
> read-only + container sekali pakai untuk koneksi/dbt. ⚠️ Produksi tidak tersentuh.
|
||||
|
||||
---
|
||||
|
||||
## 1. VERDICT: 🟢 **GO**
|
||||
|
||||
BATCH B boleh dilanjutkan ke B4. Penyebab no-go sebelumnya (dbt mati di Python 3.13)
|
||||
**teratasi** dengan kembali ke Python 3.11.
|
||||
|
||||
---
|
||||
|
||||
## 2. Hasil lengkap
|
||||
|
||||
| # | Uji | Hasil |
|
||||
|---|---|---|
|
||||
| 1 | `airflow version` | ✅ **3.3.2** |
|
||||
| 2 | **`dbt debug`** | ✅ **"Connection test: OK connection ok" + "All checks passed!"** |
|
||||
| 3 | **`dbt compile --select orders_silver`** | ✅ compiles (SQL model terbentuk) |
|
||||
| 4 | DAG produksi `dag_sitomat_gold_build.py` | ✅ impor bersih |
|
||||
| 5 | DAG produksi `dag_sitomat_gold_build_prod.py` | ✅ impor bersih |
|
||||
| 6 | DAG `sicabe_dq_gold_native.py` | ✅ dimuat sampai tahap *generate workflow*; gagal hanya karena `openmetadata-server` tidak ter-resolve dari container uji (bukan masalah versi) |
|
||||
| 7 | `airflow dags list-import-errors` | ✅ kosong |
|
||||
| 8 | Plugin OM `/pluginsv2/api/v2/openmetadata/health-auth` | ✅ **HTTP 200** |
|
||||
| 9 | Plugin OM `/pluginsv2/` | ✅ **HTTP 200** |
|
||||
| 10 | Trino (`TrinoHook`) | ✅ `SELECT 1` → `[[1]]`; bronze: `orders`, `orders_incremental` |
|
||||
| 11 | MinIO (`S3Hook`) | ✅ bucket `['iceberg-data']`, `check_for_bucket` True |
|
||||
| 12 | MySQL klien eksternal (`MySqlHook`) | ✅ `('8.0.40', 'asdp')`; tabel `passenger_productions`, `trx_booking` |
|
||||
| 13 | OPA (aturan bronze `data-engineer`) | ✅ utuh (`bronze_orders`: ddl/select/write) |
|
||||
|
||||
Verifikasi dari **build** (step #18–#24): 5 klien DB, `mysqlclient` C termuat +
|
||||
`libmariadb.so.3`, provider mysql, `dbt + great_expectations`, `flask_appbuilder` +
|
||||
**plugin OM dapat dimuat** (`RestApiPlugin`), `pip check` bersih.
|
||||
|
||||
---
|
||||
|
||||
## 3. Ringkasan perjalanan B2 (tiga percobaan)
|
||||
|
||||
| Percobaan | Masalah | Penyelesaian |
|
||||
|---|---|---|
|
||||
| **1** | Build memakai Python 3.13 → **dbt mati** (mashumaro/Annotated) | Diputuskan pindah ke Python 3.11 (lihat `b3-hasil-uji.md`) |
|
||||
| **2** | Build py3.11: pip install 62 menit (jaringan PyPI ~60–80 kB/s) lalu gagal di **penjaga regresi yang saya tulis** | Penjaga dipanggil dengan 1 argumen → `TypeError: Annotated[...] should be used with at least two arguments`. Diperbaiki menjadi `te.Annotated[int, 'meta']`. ⚠️ Galat itu justru membuktikan jalur Annotated **ada** di py3.11 |
|
||||
| **3** | — | ✅ **BERHASIL** (langkah pip ter-cache → selesai cepat) |
|
||||
|
||||
Catatan operasional: build percobaan 1 sempat **dibatalkan** karena client `docker build`
|
||||
ikut mati saat shell pengawas dihentikan (`nohup` tidak cukup — perlu `setsid`).
|
||||
|
||||
---
|
||||
|
||||
## 4. Konsekuensi & langkah berikutnya
|
||||
|
||||
- **Image target BATCH B**: `sicabe/airflow:3.3.2-om203` (ID `a02620e1d0a8`, 3.2 GB).
|
||||
- Image py3.13 (`3.3.2-om203-py313`) **tidak dipakai** — disimpan sebagai bukti no-go.
|
||||
- `Dockerfile.332` + `requirements.332.in/.lock` sudah final untuk py3.11; lock hanya
|
||||
**44 baris** berbeda dari produksi.
|
||||
- Berikutnya: **B4** — naikkan `AIRFLOW_IMAGE` + `OPENMETADATA_VERSION` di
|
||||
`environment.defaults.sh` lalu `docker compose up -d`.
|
||||
- Bukti rollback: `deploy/output/merge-airflow332-rollback.md` (dump sudah diuji pulih).
|
||||
|
||||
---
|
||||
|
||||
## 5. Kesalahan uji saya pada B3r (agar tidak terulang)
|
||||
|
||||
| # | Kesalahan | Akibat |
|
||||
|---|---|---|
|
||||
| 1 | Penjaga regresi `Annotated.__class_getitem__((int,))` — argumen kurang satu | **Positif palsu**: build gagal padahal sehat |
|
||||
| 2 | Uji MinIO hanya meng-`source deploy/.env`, tanpa `environment.defaults.sh` | `MINIO_ROOT_USER` kosong → `NoCredentialsError` (bukan masalah image) |
|
||||
| 3 | `pgrep -f` mencocokkan perintah saya sendiri | PID yang dilaporkan tidak sahih saat memantau build |
|
||||
@@ -1,178 +0,0 @@
|
||||
# Rollback — Merge Eksperimen `airflow332` ke `deploy/` (BATCH B / review R18)
|
||||
|
||||
> **Dibuat**: 2026-10-01 · **Oleh**: langkah **B1** dari `reviews/sicabe-18-merge-airflow332-plan.md`
|
||||
> **Status**: 🟢 **Backup siap & TERBUKTI DAPAT DIPULIHKAN.** BATCH B **belum dijalankan**.
|
||||
> ⚠️ **Berkas ini adalah satu-satunya jalan pulang** untuk langkah B4 (migrasi OpenMetadata
|
||||
> bersifat **satu arah** — lihat `docs/sicabe-install-phases-detail.md` §8.11.2).
|
||||
|
||||
---
|
||||
|
||||
## 1. Keadaan yang dibekukan (baseline sebelum BATCH B)
|
||||
|
||||
| Item | Nilai |
|
||||
|---|---|
|
||||
| Tanggal baseline | 2026-10-01 |
|
||||
| Airflow | **3.3.1** (`sicabe/airflow:3.3.1-om202`) |
|
||||
| OpenMetadata | **2.0.2** (`openmetadata/server:2.0.2`) |
|
||||
| OpenSearch | `opensearchproject/opensearch:3.8.0` |
|
||||
| Gate `test-verification.sh` | **60 PASS / 0 FAIL / 2 SKIP** |
|
||||
| Container produksi | 20 (Airflow webserver/scheduler/dag-processor, OM, OpenSearch, dst) |
|
||||
|
||||
---
|
||||
|
||||
## 2. Berkas backup
|
||||
|
||||
Lokasi: `deploy/output/merge-airflow332-backup/` — **gitignored** (berisi data produksi).
|
||||
|
||||
| Berkas | Isi | Ukuran | MD5 |
|
||||
|---|---|---|---|
|
||||
| `openmetadata_db.dump` | dump `openmetadata_db` (format `-Fc`) | 1.9 MB | `a8495339165cfbc286ea7e90490c12bb` |
|
||||
| `airflow.dump` | dump `airflow` (format `-Fc`) | 248 KB | `4a834cc43307b296b58c250909749d2a` |
|
||||
| `environment.defaults.sh.before-B` | salinan `deploy/config/environment.defaults.sh` | 22 KB | `2d341e008af43a7b` |
|
||||
|
||||
Ukuran DB saat baseline: `openmetadata_db` = **36 MB**, `airflow` = **12 MB**.
|
||||
|
||||
---
|
||||
|
||||
## 3. Bukti backup DAPAT DIPULIHKAN (dua kali diuji)
|
||||
|
||||
| Uji | Hasil |
|
||||
|---|---|
|
||||
| Restore `openmetadata_db` → DB coretan `_restorecheck_om` | **204 tabel** asli = **204 tabel** hasil restore ✅ |
|
||||
| Restore `airflow` → DB coretan `_restorecheck_af` | **5 DAG** asli = **5 DAG** hasil restore ✅ |
|
||||
| DB coretan setelah uji | dihapus ✅ (produksi tidak tersentuh: 20 container, OM HTTP 200) |
|
||||
|
||||
Perintah yang dipakai (nilai persis):
|
||||
```bash
|
||||
# contoh untuk openmetadata_db
|
||||
docker exec -e PGPASSWORD=<dari deploy/.env> sicabe-postgres \
|
||||
psql -U sicabe_admin -d postgres -c "CREATE DATABASE _restorecheck_om OWNER openmetadata;"
|
||||
docker cp deploy/output/merge-airflow332-backup/openmetadata_db.dump sicabe-postgres:/tmp/check.dump
|
||||
docker exec -e PGPASSWORD=<…> sicabe-postgres \
|
||||
pg_restore -U sicabe_admin -d _restorecheck_om --no-owner --no-privileges /tmp/check.dump
|
||||
docker exec -e PGPASSWORD=<…> sicabe-postgres \
|
||||
psql -U sicabe_admin -d postgres -c "DROP DATABASE _restorecheck_om;"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Digest image (untuk kembali ke versi lama)
|
||||
|
||||
| Image | Image ID (sha256, dipotong) |
|
||||
|---|---|
|
||||
| `sicabe/airflow:3.3.1-om202` (produksi saat ini) | `afc0421a0490…` |
|
||||
| `sicabe/airflow:3.3.1-om202-presqlalchemy` (tag rollback A1) | `afc0421a0490…` (identik — A1 tidak rebuild) |
|
||||
| **`sicabe/airflow:3.3.2-om203`** (target BATCH B — Python 3.11, hasil B2 revisi) | `a02620e1d0a8…` (3.2 GB) |
|
||||
| ~~`sicabe/airflow:3.3.2-om203-py313`~~ — **DITOLAK** di B3 (dbt mati di Python 3.13) | `33990a6c2b88…` (tidak dipakai) |
|
||||
| `openmetadata/server:2.0.2` | `3b6f52aeb30f…` |
|
||||
| `openmetadata/server:2.0.3` (target BATCH B) | `5f4d52bca2d7…` |
|
||||
| `opensearchproject/opensearch:3.8.0` | `fafe3fc35870…` |
|
||||
|
||||
---
|
||||
|
||||
## 5. PROSEDUR ROLLBACK
|
||||
|
||||
### 5.1 Kasus paling umum — B4/B5 gagal, skema OM masih sehat
|
||||
|
||||
Cukup kembalikan versi, tanpa menyentuh DB:
|
||||
|
||||
```bash
|
||||
cd /home/broto/sicabe-doc/deploy
|
||||
|
||||
# (1) kembalikan versi ke baseline
|
||||
# AIRFLOW_IMAGE -> sicabe/airflow:3.3.1-om202
|
||||
# OPENMETADATA_VERSION -> 2.0.2
|
||||
# (atau pulihkan berkas utuh: cp output/merge-airflow332-backup/environment.defaults.sh.before-B \
|
||||
# config/environment.defaults.sh)
|
||||
|
||||
# (2) jalankan ulang
|
||||
docker compose up -d airflow-webserver airflow-scheduler airflow-dag-processor \
|
||||
openmetadata-server openmetadata-migrate
|
||||
|
||||
# (3) verifikasi
|
||||
./test-verification.sh # harapan: 60 PASS / 0 FAIL / 2 SKIP
|
||||
```
|
||||
|
||||
### 5.2 Kasus berat — migrasi OM 2.0.3 merusak skema
|
||||
|
||||
⚠️ **Urutannya penting**: OM produksi harus **berhenti** dulu sebelum restore, kalau tidak
|
||||
koneksi aktif akan mengganggu `DROP/CREATE DATABASE`.
|
||||
|
||||
```bash
|
||||
cd /home/broto/sicabe-doc/deploy
|
||||
|
||||
# (1) HENTIKAN OpenMetadata (wajib, sebelum menyentuh DB-nya)
|
||||
docker stop sicabe-openmetadata
|
||||
|
||||
# (2) pulihkan versi image di config (OPENMETADATA_VERSION=2.0.2)
|
||||
cp output/merge-airflow332-backup/environment.defaults.sh.before-B config/environment.defaults.sh
|
||||
|
||||
# (3) restore openmetadata_db dari dump
|
||||
docker exec -e PGPASSWORD=<dari deploy/.env> sicabe-postgres \
|
||||
psql -U sicabe_admin -d postgres -c "DROP DATABASE IF EXISTS openmetadata_db;"
|
||||
docker exec -e PGPASSWORD=<…> sicabe-postgres \
|
||||
psql -U sicabe_admin -d postgres -c "CREATE DATABASE openmetadata_db OWNER openmetadata;"
|
||||
docker cp output/merge-airflow332-backup/openmetadata_db.dump sicabe-postgres:/tmp/om.dump
|
||||
docker exec -e PGPASSWORD=<…> sicabe-postgres \
|
||||
pg_restore -U sicabe_admin -d openmetadata_db --no-owner --no-privileges /tmp/om.dump
|
||||
|
||||
# (4) mulai ulang dengan image LAMA
|
||||
docker compose up -d openmetadata-server
|
||||
|
||||
# (5) verifikasi
|
||||
curl -s http://localhost:12085/api/v1/system/version # harapan: {"version":"2.0.2",...}
|
||||
```
|
||||
|
||||
### 5.3 Bila metadata Airflow perlu dipulihkan (mis. migrasi DB Airflow bermasalah)
|
||||
|
||||
```bash
|
||||
docker stop sicabe-airflow-webserver sicabe-airflow-scheduler sicabe-airflow-dag-processor
|
||||
docker exec -e PGPASSWORD=<…> sicabe-postgres psql -U sicabe_admin -d postgres \
|
||||
-c "DROP DATABASE IF EXISTS airflow;" -c "CREATE DATABASE airflow OWNER airflow;"
|
||||
docker cp output/merge-airflow332-backup/airflow.dump sicabe-postgres:/tmp/af.dump
|
||||
docker exec -e PGPASSWORD=<…> sicabe-postgres \
|
||||
pg_restore -U sicabe_admin -d airflow --no-owner --no-privileges /tmp/af.dump
|
||||
docker compose up -d airflow-webserver airflow-scheduler airflow-dag-processor
|
||||
```
|
||||
|
||||
### 5.4 Yang TIDAK perlu di-rollback
|
||||
|
||||
- **OpenSearch** — indeksnya **derived**: OM membangun ulang isinya. Tidak ada dump khusus
|
||||
untuk itu, dan itu memang disengaja.
|
||||
- **`deploy-legacy/`** — di luar lingkup (K6).
|
||||
- **Image** — tidak dihapus; dua versi hidup berdampingan, jadi rollback hanya soal menunjuk
|
||||
tag lama.
|
||||
|
||||
---
|
||||
|
||||
## 6. Setelah rollback
|
||||
|
||||
1. Catat penyebab kegagalan di berkas ini (bagian bawah) atau di `reviews/sicabe-18…`.
|
||||
2. Jangan jalankan B4 lagi sebelum penyebabnya dipahami.
|
||||
3. Indeks/tabel OM hasil migrasi yang tertinggal **tidak** dibiarkan: pastikan restore
|
||||
benar-benar menggantikan `openmetadata_db` (bukan menumpuk).
|
||||
|
||||
---
|
||||
|
||||
## 7. Catatan keamanan
|
||||
|
||||
⚠️ Berkas dump berisi **data produksi** (termasuk baris `user_entity` — hash kredensial, dan
|
||||
mungkin data pribadi). Perlakuan:
|
||||
- **gitignored** (`deploy/output/merge-airflow332-backup/`) — jangan pernah di-commit;
|
||||
- hapus setelah BATCH B dinyatakan stabil (dan simpan hanya bila memang diperlukan sebagai
|
||||
arsip, di lokasi terbatas akses).
|
||||
|
||||
---
|
||||
|
||||
## Riwayat
|
||||
|
||||
| Tanggal | Kejadian |
|
||||
|---|---|
|
||||
| 2026-10-01 | B1 dijalankan: dump dibuat, **diuji pulih (204=204 tabel; 5=5 DAG)**, digest & config dicatat, dokumen ini dibuat. BATCH B belum dimulai. |
|
||||
| 2026-10-01 | **B2** dijalankan: image `sicabe/airflow:3.3.2-om203-py313` dibangun **dari `deploy/`** (berdampingan — `Dockerfile.332` + `requirements.332.{in,lock}`), ID `33990a6c2b88…`. Seluruh verifikasi fatal lolos: 5 klien DB, `mysqlclient` benar-benar termuat, `sqlalchemy 2.0.54` (pin A1), **plugin OM dapat dimuat**, `pip check` bersih. Produksi **tetap** `3.3.1-om202` — belum tersentuh. |
|
||||
| 2026-10-01 | **B3 (go/no-go) = NO-GO** untuk Python 3.13: `dbt debug`/`dbt compile` gagal (mashumaro → `Annotated.__class_getitem__`). Tidak ada versi `mashumaro` yang memenuhi batas dependensi kita **dan** berfungsi di 3.13. Bukti: `b3-hasil-uji.md` |
|
||||
| 2026-10-01 | **B2 revisi**: base dikembalikan ke **Python 3.11**; lock di-resolve ulang dengan lock 3.3.1 sebagai constraint **tersaring** (`^apache-airflow`, `^openmetadata-`, `^collate-` dibuang) → hanya **44 baris** berbeda dari produksi. Image baru: **`sicabe/airflow:3.3.2-om203`** ID `a02620e1d0a8…` (3.2 GB) |
|
||||
| 2026-10-01 | **B3 ULANG = GO** 🟢: `dbt debug` **"All checks passed"**, `dbt compile` OK, DAG produksi impor bersih, plugin `/pluginsv2/` **200**, Trino/MinIO/MySQL OK, OPA utuh. Bukti: `b3r-hasil-uji.md`. Produksi tetap `3.3.1-om202` — siap untuk **B4** |
|
||||
| 2026-10-02 | **B4 dijalankan (produksi naik)**: migrasi OM 2.0.3 **exit 0**; `airflow-init` (DB migrate 3.3.2) **exit 0**; OM server + 3 service Airflow di-recreate ke versi baru — semuanya **healthy**. Image: `sicabe/airflow:3.3.2-om203`, `openmetadata/server:2.0.3` |
|
||||
| 2026-10-02 | **B5 verifikasi = LOLOS** ✅: `airflow version` **3.3.2**; OM API **2.0.3**; plugin `/pluginsv2/…health-auth` **200**; **0 error impor** DAG; `MySQLdb` + provider mysql tersedia di produksi; **GATE 60 PASS / 0 FAIL / 2 SKIP** (identik baseline). **B6 (rollback) tidak diperlukan.** |
|
||||
| — | ⚠️ **Dump DISIMPAN** sampai dinyatakan stabil. Bila beberapa hari produksi berjalan normal: hapus `deploy/output/merge-airflow332-backup/` (§7) — atau arsipkan di lokasi terbatas akses |
|
||||
@@ -1,114 +0,0 @@
|
||||
# Runbook Rollback — Integrasi OpenMetadata 2.0.2 ke `deploy/`
|
||||
|
||||
**Dibuat**: 2026-09-16 · **Tugas**: `merge-om20x` (subtask 01)
|
||||
**Tag titik kembali**: `pre-merge-om20x` @ `5627353`
|
||||
|
||||
Runbook ini adalah dokumen yang ditulis tangan (bukan keluaran uji otomatis), sengaja
|
||||
diletakkan di `deploy/output/` mengikuti penugasan subtask 01. Isinya harus tetap
|
||||
akurat selama integrasi berlangsung.
|
||||
|
||||
---
|
||||
|
||||
## 1. Kapan runbook ini dipakai
|
||||
|
||||
Saat integrasi ke OpenMetadata 2.0.2 + OpenSearch 3.x + Airflow ber-plugin gagal dan
|
||||
kita memutuskan kembali ke kondisi 1.13.1 yang stabil (sebelumnya gate Fase 1-6 hijau,
|
||||
`test-verification.sh` = 60 PASS / 0 FAIL / 2 SKIP).
|
||||
|
||||
## 2. Kondisi baseline (terverifikasi sebelum perubahan apa pun)
|
||||
|
||||
| Aspek | Nilai |
|
||||
|---|---|
|
||||
| git HEAD | `5627353` (tag `pre-merge-om20x`) |
|
||||
| OpenMetadata server | **1.13.1** (`openmetadata/server:1.13.1`) |
|
||||
| OpenSearch | **2.19.6** (`opensearchproject/opensearch:2.19.6`) |
|
||||
| Airflow image | **`sicabe/airflow:3.3.1-slim`** (id `15ae04b69228`), Airflow 3.3.1 |
|
||||
| Plugin OpenMetadata di Airflow | **TIDAK ADA** — `/api/v1/openmetadata/health` → 404 |
|
||||
| Data OM | database `openmetadata_db` di volume `sicabe_postgres-data` |
|
||||
| Data OpenSearch | **tidak di volume** — ada di layer container (index derived) |
|
||||
|
||||
Catatan penting soal "plugin tidak ada": di Airflow deploy,
|
||||
`/pluginsv2/api/v2/openmetadata/health-auth` menjawab **200 tetapi `content-type: text/html`**
|
||||
(HTML aplikasi Airflow) — itu fallback UI, bukan respons plugin. Bukti plugin absen
|
||||
adalah `/api/v1/openmetadata/health` → **404**. Jangan tertipu angka 200 di path
|
||||
`/pluginsv2/`.
|
||||
|
||||
## 3. Cara memverifikasi kita benar-benar sudah kembali ke baseline
|
||||
|
||||
```bash
|
||||
cd /home/broto/sicabe-doc
|
||||
git rev-parse --short HEAD # harus 5627353 (atau turunannya yg setara)
|
||||
docker inspect sicabe-openmetadata --format '{{.Config.Image}}' # openmetadata/server:1.13.1
|
||||
docker inspect sicabe-opensearch --format '{{.Config.Image}}' # opensearchproject/opensearch:2.19.6
|
||||
docker inspect sicabe-airflow-webserver --format '{{.Config.Image}}' # sicabe/airflow:3.3.1-slim
|
||||
curl -s -o /dev/null -w '%{http_code}\n' -u "admin:$AIRFLOW_ADMIN_PASSWORD" \
|
||||
http://localhost:12082/api/v1/openmetadata/health # harus 404
|
||||
cd deploy && ./test-verification.sh # harus 60 PASS / 0 FAIL / 2 SKIP
|
||||
```
|
||||
|
||||
## 4. Langkah rollback
|
||||
|
||||
### 4.1 Kembalikan berkas konfigurasi
|
||||
|
||||
```bash
|
||||
cd /home/broto/sicabe-doc
|
||||
git checkout pre-merge-om20x -- deploy/ docs/
|
||||
git status -s # tinjau: hanya berkas deploy/docs yang kembali
|
||||
```
|
||||
|
||||
Jangan `git reset --hard` — commit hasil eksperimen (`f47945d`, `5627353`) dan
|
||||
`experiments/om20x/` tidak perlu ikut hilang.
|
||||
|
||||
### 4.2 Nyalakan ulang container lama
|
||||
|
||||
```bash
|
||||
cd deploy
|
||||
set -a && . ./.env && set +a
|
||||
docker compose --profile all up -d --force-recreate \
|
||||
openmetadata-opensearch openmetadata-migrate openmetadata-server \
|
||||
airflow-init airflow-webserver airflow-scheduler airflow-dag-processor
|
||||
```
|
||||
|
||||
Image lama **tidak boleh dihapus**: `openmetadata/server:1.13.1`,
|
||||
`opensearchproject/opensearch:2.19.6`, `sicabe/airflow:3.3.1-slim`.
|
||||
OpenSearch akan membuat index baru dari nol saat container dibuat ulang
|
||||
(tidak ada volume; index memang derived dan OM akan mem-populasinya kembali).
|
||||
|
||||
### 4.3 Bila database OM sudah telanjur dikosongkan (subtask 12)
|
||||
|
||||
Setelah `openmetadata_db` di-drop, **data katalog OM tidak bisa dipulihkan dari
|
||||
deploy** — tidak ada dump (keputusan sadar: jaring pengaman cukup tag git). Yang bisa
|
||||
dilakukan: membuat ulang database kosong lalu membiarkan OM 1.13.1 membangun skemanya,
|
||||
lalu mengisi ulang katalog dari sumber yang reproducible:
|
||||
|
||||
```bash
|
||||
# buat ulang database kosong (pola sama dgn deploy/config/postgres/init/01-create-dbs.sh)
|
||||
docker exec sicabe-postgres psql -U postgres -c "DROP DATABASE IF EXISTS openmetadata_db;"
|
||||
docker exec sicabe-postgres psql -U postgres -c "CREATE DATABASE openmetadata_db OWNER openmetadata;"
|
||||
|
||||
cd deploy
|
||||
docker compose --profile all up -d --force-recreate openmetadata-migrate # migrate schema 1.13.1
|
||||
docker compose --profile all up -d openmetadata-server
|
||||
./scripts/ingest-openmetadata.sh # katalog Trino + lineage dbt
|
||||
# lalu jalankan DAG utk mengirim ulang hasil Great Expectations
|
||||
```
|
||||
|
||||
## 5. Yang bisa dan tidak bisa dipulihkan (jujur)
|
||||
|
||||
| Bisa dipulihkan | Tidak bisa dipulihkan |
|
||||
|---|---|
|
||||
| Versi komponen (kembali 1.13.1 / 2.19.6 / image slim) | Isi katalog OM bila DB sudah di-drop (reproducible, bukan mustahil) |
|
||||
| Konfigurasi (`deploy/`, `docs/`) via tag git | Riwayat DQ (test case + hasil) — dibangun ulang oleh DAG |
|
||||
| Lineage dbt | Dashboard/artefak yang hanya ada di OM |
|
||||
|
||||
Alasan ini dapat diterima: seluruh isi katalog berasal dari
|
||||
`scripts/ingest-openmetadata.sh` + DAG `sicabe_batch_daily`, jadi bisa dibangun ulang.
|
||||
|
||||
## 6. Titik tidak-balik (penting)
|
||||
|
||||
| Setelah subtask | Kondisi |
|
||||
|---|---|
|
||||
| 05 (+07) | Airflow memakai image ber-plugin; rollback = kembalikan 4 baris `AIRFLOW_IMAGE` |
|
||||
| 10 | OpenSearch 3.x; index lama sudah hilang saat container dibuat ulang |
|
||||
| **12** | **`openmetadata_db` di-drop → satu-satunya titik yang kehilangan data** |
|
||||
| 13-15 | OM 2.0.2 jalan dgn password admin dari `.env` |
|
||||
@@ -1,106 +0,0 @@
|
||||
# Hasil R18-6 — Uji End-to-End: OM membangkitkan DAG ingestion → Airflow 3.3.2
|
||||
|
||||
> **Tanggal**: 2026-10-01/02 · **Lingkungan**: PRODUKSI (Airflow 3.3.2 + OM 2.0.3)
|
||||
> **Tujuan**: membuktikan jalur "OpenMetadata mengirim DAG ke Airflow" secara **utuh** —
|
||||
> bukan hanya tingkat impor (itu sudah diukur di H7) — dan sekaligus menutup pertanyaan H7
|
||||
> secara definitif.
|
||||
> ⚠️ Uji ini **membuat** pipeline sementara di OM produksi lalu **menghapusnya kembali**.
|
||||
|
||||
---
|
||||
|
||||
## 1. VERDICT: 🟢 **BERHASIL PENUH — H7 tertutup secara definitif**
|
||||
|
||||
| Tahap | Hasil |
|
||||
|---|---|
|
||||
| OM membuat ingestion pipeline (`metadata`) untuk `trino_sicabe` | ✅ dibuat |
|
||||
| OM men-deploy DAG ke Airflow (lewat plugin `/pluginsv2/`) | ✅ **HTTP 200** |
|
||||
| DAG muncul di Airflow 3.3.2 | ✅ `r18_e2e_ingestion.py` |
|
||||
| **Error impor** | ✅ **0** (`No data found`) |
|
||||
| Trigger dari OM menjalankan DAG | ✅ HTTP 200 (percobaan ke-2) |
|
||||
| **`dag_run`** | ✅ **`success`** (17 detik) |
|
||||
| **`task_instance` (`ingestion_task`)** | ✅ **`success`** |
|
||||
|
||||
**Ringkasan dari log task (bukti terkuat):**
|
||||
|
||||
```
|
||||
Processed records: 4 · Updated records: 0 · Errors: 0 · Success %: 100.0
|
||||
Processed records: 5 · Updated records: 0 · Errors: 0 · Success %: 100.0
|
||||
Workflow Success %: 100.0
|
||||
Workflow finished in time: 6s 668.015ms
|
||||
Done.
|
||||
```
|
||||
|
||||
➡️ **Kesimpulan**: paket yang hilang akibat melepas extra `[airflow]`
|
||||
(`tornado`, `elasticsearch`, `opensearchpy`, `providers-elasticsearch`,
|
||||
`providers-opensearch`) **tidak mengganggu jalur ingestion metadata**. H7 kini bukan lagi
|
||||
"terukur di tingkat impor", melainkan **terbukti sampai eksekusi**.
|
||||
|
||||
---
|
||||
|
||||
## 2. TEMUAN BARU — plugin memvalidasi LEBIH KETAT daripada server OM
|
||||
|
||||
Ini temuan integrasi yang belum tercatat sebelumnya, dan menjelaskan kegagalan deploy yang
|
||||
sempat terjadi tiga kali:
|
||||
|
||||
```
|
||||
POST /v1/services/ingestionPipelines/deploy/<id> → 400
|
||||
{"error":"Extra parameter 'serviceName'"}
|
||||
{"error":"Extra parameter 'markDeletedTablesFromFilterOnly'"}
|
||||
{"error":"Extra parameter 'serviceConnections'"}
|
||||
```
|
||||
|
||||
**Artinya**: pipeline yang **diterima** server OM saat dibuat (`POST /v1/services/ingestionPipelines`
|
||||
→ 201) bisa **ditolak saat deploy** oleh plugin di sisi Airflow. Validasi plugin lebih ketat
|
||||
daripada skema server.
|
||||
|
||||
**Yang bekerja** — `sourceConfig.config` **paling minimal**:
|
||||
|
||||
```json
|
||||
{ "airflowConfig": { "scheduleInterval": "0 0 * * *", "retries": 0, "startDate": <epoch-ms> },
|
||||
"name": "<nama>",
|
||||
"pipelineType": "metadata",
|
||||
"service": { "id": "<id service>", "type": "databaseService" },
|
||||
"sourceConfig": { "config": { "type": "DatabaseMetadata" } } }
|
||||
```
|
||||
|
||||
- **Jangan** menambahkan `serviceName`, `serviceConnections`, `includeTables`,
|
||||
`markDeletedTablesFromFilterOnly`, dsb. pada `DatabaseMetadata` — ditolak sebagai
|
||||
*extra parameter*.
|
||||
- Koneksi diambil dari **service record** yang sudah ada (dibuat `bootstrap.sh`) — dan
|
||||
**bekerja** ✅. (Catatan: ini berbeda dari jalur SDK yang punya masalah koneksi tersimpan
|
||||
kehilangan secret → `401`; pada jalur OM-orchestrated koneksi tersimpan justru dipakai
|
||||
dengan benar.)
|
||||
- Untuk pipeline **TestSuite**, `serviceConnections` justru **diterima** (seeder DQ
|
||||
memakainya) — jadi aturan "extra parameter" itu **per tipe pipeline**, bukan global.
|
||||
|
||||
---
|
||||
|
||||
## 3. Perilaku lain yang terkonfirmasi
|
||||
|
||||
| Perilaku | Bukti |
|
||||
|---|---|
|
||||
| **Trigger gagal di percobaan pertama**, berhasil di percobaan kedua | DAG baru dibuat → `dag-processor` belum selesai mem-parsing. Perlu jeda ±20 detik (seeder DQ sudah mengantisipasi ini) |
|
||||
| `airflow dags list` membaca **metadata DB**, bukan folder | Sempat menyesatkan saat uji folder mount |
|
||||
| CLI Airflow 3: `airflow dags list-runs` **bukan** bentuk yang valid | Status run paling andal dibaca dari tabel `dag_run`/`task_instance` di metadata DB |
|
||||
| Menghapus pipeline (`hardDelete`) menghapus **berkas DAG** dari volume | File `r18_e2e_ingestion.py` hilang; catatan DAG di DB menjadi *stale* pada siklus parse berikutnya |
|
||||
|
||||
---
|
||||
|
||||
## 4. Kebersihan setelah uji
|
||||
|
||||
| Item | Status |
|
||||
|---|---|
|
||||
| Pipeline `r18_e2e_ingestion` di OM | ✅ **dihapus** (HTTP 200, `hardDelete`) |
|
||||
| Berkas DAG di volume `airflow-dags` | ✅ **hilang** (kembali 4 berkas seperti semula) |
|
||||
| Catatan DAG di metadata DB Airflow | ⏳ menjadi *stale* otomatis pada siklus parse berikutnya (berkasnya sudah tidak ada) |
|
||||
| DAG produksi lain | ✅ tidak tersentuh |
|
||||
|
||||
---
|
||||
|
||||
## 5. Kesalahan saya selama R18-6 (agar tidak terulang)
|
||||
|
||||
| # | Kesalahan | Akibat |
|
||||
|---|---|---|
|
||||
| 1 | Menyusun `sourceConfig.config` dengan field lengkap ala UI | Deploy ditolak 3× (`Extra parameter …`) — seharusnya mulai dari yang **minimal** |
|
||||
| 2 | Menganggap semua tipe pipeline menerima field yang sama | `serviceConnections` diterima di TestSuite, **ditolak** di DatabaseMetadata |
|
||||
| 3 | Memakai `airflow dags list-runs` | Bentuk CLI berbeda di Airflow 3 → pakai tabel DB |
|
||||
Executable
+171
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Cek konsistensi keystore TLS Trino + password.db ANTAR-HOST
|
||||
#
|
||||
# Dua hal yang WAJIB benar agar klien (Great Expectations / OpenMetadata / dbt)
|
||||
# bisa bicara ke Trino:
|
||||
# 1) keystore TLS (generated/host-<nama>/config/trino/keystore.p12) IDENTIK di
|
||||
# semua host → hasil SATU `--generate` + `--sync`.
|
||||
# 2) password.db punya service account (svc_dbt) & cocok dgn TRINO_SVC_DBT_PASSWORD
|
||||
# di bundle .env (kalau tidak → 401 "Access Denied: Invalid credentials").
|
||||
# Catatan: bcrypt memakai salt acak → berkas password.db TIDAK dibandingkan
|
||||
# byte-per-byte antar-host, melainkan DIVERIFIKASI thd password yang diharapkan.
|
||||
#
|
||||
# Pemakaian:
|
||||
# ./scripts/check-trino-keystore.sh # host ini saja
|
||||
# ./scripts/check-trino-keystore.sh --inventory cluster.p3.yml
|
||||
# ./scripts/check-trino-keystore.sh --expect <sha256-keystore> # CI (exit 1 bila beda)
|
||||
#
|
||||
# Opsi: --host <nama> (paksa host ini) · --user <u> · --dest <dir-remote>
|
||||
# default: user $USER, dest = cermin lokasi repo ini di bawah $HOME
|
||||
# (mis. /home/u/sicabe-core/deploy → ~/sicabe-core/deploy)
|
||||
#
|
||||
# Keluar: 0 = semua OK · 1 = ada BEDA/MISSING · 2 = error (berkas/inventory tak ada)
|
||||
# =============================================================================
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "${SCRIPT_DIR}"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/lib/common.sh" 2>/dev/null || true
|
||||
source "${SCRIPT_DIR}/config/environment.defaults.sh" 2>/dev/null || true
|
||||
if [[ -f "${SCRIPT_DIR}/.env" ]]; then set -a; source "${SCRIPT_DIR}/.env"; set +a; fi
|
||||
|
||||
INV=""; SSH_USER="${USER:-root}"; EXPECT=""; FORCE_HOST=""
|
||||
# Tujuan remote default = cermin lokasi repo ini di bawah $HOME (bebas nama repo:
|
||||
# `sicabe-core` → ~/sicabe-core/deploy). Fallback bila lib/common.sh gagal dimuat.
|
||||
if declare -F sicabe_default_dest >/dev/null 2>&1; then
|
||||
DEST="$(sicabe_default_dest "${SCRIPT_DIR}")"
|
||||
else
|
||||
DEST="~/$(basename "$(dirname "${SCRIPT_DIR}")")/deploy"
|
||||
fi
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--inventory) INV="${2:-}"; shift 2 ;;
|
||||
--user) SSH_USER="${2:-}"; shift 2 ;;
|
||||
--dest) DEST="${2:-}"; shift 2 ;;
|
||||
--expect) EXPECT="${2:-}"; shift 2 ;;
|
||||
--host) FORCE_HOST="${2:-}"; shift 2 ;;
|
||||
-h|--help) sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
*) echo "Argumen tidak dikenal: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
C_RED=$'\033[31m'; C_GRN=$'\033[32m'; C_YEL=$'\033[33m'; C_BLD=$'\033[1m'; C_RST=$'\033[0m'
|
||||
|
||||
# ── keystore ─────────────────────────────────────────────────────────────────
|
||||
ks_path() { echo "${SCRIPT_DIR}/generated/host-$1/config/trino/keystore.p12"; }
|
||||
ks_sha() { sha256sum "$1" 2>/dev/null | cut -d' ' -f1; }
|
||||
ks_pw() { sed -n 's/^http-server.https.keystore.key=//p' \
|
||||
"${SCRIPT_DIR}/generated/host-$1/config/trino/config-coordinator.properties" 2>/dev/null | head -1; }
|
||||
ks_fp() { openssl pkcs12 -in "$(ks_path "$1")" -nokeys -passin "pass:$(ks_pw "$1")" 2>/dev/null \
|
||||
| openssl x509 -noout -fingerprint -sha256 2>/dev/null | sed 's/^sha256 Fingerprint=//'; }
|
||||
|
||||
# ── password.db (service account) ────────────────────────────────────────────
|
||||
pw_path() { echo "${SCRIPT_DIR}/generated/host-$1/config/trino/password.db"; }
|
||||
pw_expect() { sed -n 's/^TRINO_SVC_DBT_PASSWORD=//p' \
|
||||
"${SCRIPT_DIR}/generated/host-$1/.env" 2>/dev/null | head -1; }
|
||||
# Status lokal: NOFILE / MISSING / PWBAD / OK
|
||||
pw_verify() { # <host>
|
||||
local f exp; f="$(pw_path "$1")"
|
||||
[[ -f "${f}" ]] || { echo NOFILE; return; }
|
||||
grep -q '^svc_dbt:' "${f}" 2>/dev/null || { echo MISSING; return; }
|
||||
exp="$(pw_expect "$1")"
|
||||
if command -v htpasswd >/dev/null 2>&1 && [[ -n "${exp}" ]]; then
|
||||
htpasswd -vb "${f}" svc_dbt "${exp}" >/dev/null 2>&1 && echo OK || echo PWBAD
|
||||
else
|
||||
echo OK # ada, tapi tak bisa diverifikasi (htpasswd/pw tak ada)
|
||||
fi
|
||||
}
|
||||
|
||||
# ── host ini ─────────────────────────────────────────────────────────────────
|
||||
detect_self() {
|
||||
local lip d addr; lip="$(hostname -I 2>/dev/null || true)"
|
||||
for d in "${SCRIPT_DIR}"/generated/host-*; do
|
||||
[[ -f "${d}/.env" ]] || continue
|
||||
addr="$(sed -n 's/^SICABE_HOST_ADDRESS=//p' "${d}/.env" | head -1)"
|
||||
[[ -n "${addr}" && " ${lip} " == *" ${addr} "* ]] && { basename "${d}" | sed 's/^host-//'; return 0; }
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
SELF="${FORCE_HOST}"
|
||||
[[ -z "${SELF}" ]] && SELF="$(detect_self || true)"
|
||||
if [[ -z "${SELF}" ]]; then
|
||||
mapfile -t _dirs < <(ls -d "${SCRIPT_DIR}"/generated/host-* 2>/dev/null)
|
||||
[[ ${#_dirs[@]} -eq 1 ]] && SELF="$(basename "${_dirs[0]}" | sed 's/^host-//')"
|
||||
fi
|
||||
if [[ -z "${SELF}" || ! -f "$(ks_path "${SELF}")" ]]; then
|
||||
echo "${C_RED}Keystore host ini tak ditemukan.${C_RST} Jalankan ./install.sh --generate dulu, atau pakai --host <nama>." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
SELF_SHA="$(ks_sha "$(ks_path "${SELF}")")"
|
||||
EXPECT_PW="$(pw_expect "${SELF}")"
|
||||
echo "${C_BLD}Cek konsistensi keystore + password.db Trino${C_RST}"
|
||||
echo " host ini : ${SELF}"
|
||||
echo " keystore : sha256=${SELF_SHA}"
|
||||
echo " cert : $(ks_fp "${SELF}")"
|
||||
echo " pw svc_dbt: $(pw_verify "${SELF}") (OK/MISSING/PWBAD/NOFILE)"
|
||||
echo ""
|
||||
|
||||
if [[ -n "${EXPECT}" ]]; then
|
||||
if [[ "${SELF_SHA}" == "${EXPECT}" ]]; then
|
||||
echo "${C_GRN}✓ sha256 keystore cocok dengan --expect.${C_RST}"; exit 0
|
||||
fi
|
||||
echo "${C_RED}✗ sha256 keystore BERBEDA dari --expect (${EXPECT}).${C_RST}"; exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${INV}" ]]; then
|
||||
echo " (tanpa --inventory — bandingkan di tiap host, atau pakai --inventory)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ -f "${INV}" ]] || { echo "${C_RED}Inventory tak ada: ${INV}${C_RST}" >&2; exit 2; }
|
||||
PAIRS="$(python3 "${SCRIPT_DIR}/lib/cluster/inventory.py" hosts "${INV}" 2>/dev/null)" \
|
||||
|| { echo "${C_RED}Gagal membaca inventory: ${INV}${C_RST}" >&2; exit 2; }
|
||||
|
||||
# Verifikasi password.db di host LAIN via SSH (bcrypt salt acak → verifikasi, bukan hash).
|
||||
remote_pw_status() { # <name> <addr>
|
||||
ssh -o ConnectTimeout=8 -o BatchMode=yes "${SSH_USER}@${2}" \
|
||||
"f='${DEST}/generated/host-${1}/config/trino/password.db'
|
||||
[ -f \"\$f\" ] || { echo NOFILE; exit 0; }
|
||||
grep -q '^svc_dbt:' \"\$f\" || { echo MISSING; exit 0; }
|
||||
if command -v htpasswd >/dev/null 2>&1; then htpasswd -vb \"\$f\" svc_dbt '${EXPECT_PW}' >/dev/null 2>&1 && echo OK || echo PWBAD; else echo OK; fi" \
|
||||
2>/dev/null || echo SSHFAIL
|
||||
}
|
||||
pw_icon() { case "$1" in OK) printf '%s%s%s' "${C_GRN}" "$1" "${C_RST}" ;; *) printf '%s%s%s' "${C_RED}" "$1" "${C_RST}" ;; esac; }
|
||||
|
||||
mismatch=0; checked=0
|
||||
while read -r name addr; do
|
||||
[[ -z "${name}" ]] && continue
|
||||
if [[ "${name}" == "${SELF}" ]]; then
|
||||
_ksok=1; _pw="$(pw_verify "${SELF}")"
|
||||
else
|
||||
_rsha="$(ssh -o ConnectTimeout=8 -o BatchMode=yes "${SSH_USER}@${addr}" \
|
||||
"sha256sum ${DEST}/generated/host-${name}/config/trino/keystore.p12 2>/dev/null | cut -d' ' -f1" 2>/dev/null || true)"
|
||||
if [[ -z "${_rsha}" ]]; then _ksok=-1; elif [[ "${_rsha}" == "${SELF_SHA}" ]]; then _ksok=1; else _ksok=0; fi
|
||||
_pw="$(remote_pw_status "${name}" "${addr}")"
|
||||
fi
|
||||
_kslabel="(host ini)"; _kscol=""
|
||||
case "${_ksok}" in
|
||||
1) [[ "${name}" != "${SELF}" ]] && { _kslabel="keystore ✓"; _kscol="${C_GRN}"; } ;;
|
||||
0) _kslabel="keystore ✗ BEDA"; _kscol="${C_RED}"; mismatch=1 ;;
|
||||
-1) _kslabel="keystore ? ssh gagal"; _kscol="${C_YEL}"; mismatch=1 ;;
|
||||
esac
|
||||
[[ "${_pw}" == "OK" ]] || mismatch=1
|
||||
printf ' %-9s %-15s %s%-16s%s pw svc_dbt %s\n' \
|
||||
"${name}" "${addr}" "${_kscol}" "${_kslabel}" "${C_RST}" "$(pw_icon "${_pw}")"
|
||||
checked=$((checked + 1))
|
||||
done <<< "${PAIRS}"
|
||||
|
||||
echo ""
|
||||
if [[ ${mismatch} -eq 0 ]]; then
|
||||
echo "${C_GRN}✓ keystore identik + password.db OK di semua host (${checked} host).${C_RST}"
|
||||
exit 0
|
||||
fi
|
||||
echo "${C_RED}✗ Ada ketidakcocokan (keystore BEDA / password.db MISSING/PWBAD).${C_RST}"
|
||||
echo " Perbaiki (di SATU host — control):"
|
||||
echo " ./install.sh --generate --inventory ${INV} && ./install.sh --sync"
|
||||
echo " lalu di tiap host: ./install.sh --host <nama> (atau recreate Trino & Airflow)"
|
||||
exit 1
|
||||
@@ -285,7 +285,7 @@ precheck() {
|
||||
ok "semua port 12xxx yang dibutuhkan masih bebas"
|
||||
else
|
||||
warn "port sudah terpakai:${busy}"
|
||||
echo " Bila itu stack SiCABE yang lama: matikan dulu (RAM tidak cukup untuk dua stack)."
|
||||
echo " Bila itu stack SiCABE yang lama: matikan dulu — stack single-host & multi-host tidak untuk berjalan bersamaan di satu host."
|
||||
fi
|
||||
else
|
||||
inf "daftar port tidak terbaca dari environment.defaults.sh — lewati pemeriksaan port"
|
||||
|
||||
Executable
+84
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Orkestrasi alur data E2E (satu perintah) — jalankan DI CONTROL.
|
||||
#
|
||||
# Menghilangkan langkah manual berulang pada cold start. Dua "chicken-egg" yang
|
||||
# ditangani otomatis:
|
||||
# (a) aksi Great Expectations menempel ke ENTITAS tabel di OM, tetapi
|
||||
# `gold.orders_gold` DIBUAT oleh DAG → DAG #1 gagal di GE (wajar).
|
||||
# (b) lineage dbt dibaca dari ARTEFAK (`manifest.json`+`catalog.json`+`run_results.json`
|
||||
# di /tmp/dbt/target) yang DIBUAT oleh `dbt docs generate` — DAG hanya `dbt run`
|
||||
# (tanpa catalog.json), dan artefak HILANG bila container Airflow di-recreate.
|
||||
#
|
||||
# Urutan: ingest metadata → DAG #1 (data) → dbt docs generate → ingest --all →
|
||||
# DAG #2 (GE) → verify-phase6.
|
||||
#
|
||||
# PRASYARAT (sekali, DI HOST MASING-MASING — di luar skrip ini):
|
||||
# (state) ./scripts/seed-source-postgres.sh --env-file generated/host-state/.env
|
||||
# (compute) ./scripts/seed-starrocks.sh --env-file generated/host-compute/.env
|
||||
# DQ yang bekerja = Jalur A (Great Expectations pada task validate_gold/cdc_validate di DAG).
|
||||
# seed-native-dq.sh = Jalur B (DQ native OM) opt-in & TERBLOKIR bug upstream §8.10 —
|
||||
# JANGAN dipakai di alur ini (menempel ke entitas gold yang baru ada setelah DAG+ingest).
|
||||
#
|
||||
# Pemakaian: ./scripts/run-e2e.sh [--env-file <path>] [--dag <nama>]
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "${SCRIPT_DIR}"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/lib/common.sh" 2>/dev/null || true
|
||||
|
||||
ENV_FILE="${SCRIPT_DIR}/generated/host-control/.env"
|
||||
DAG="${SICABE_DAG:-sicabe_batch_daily}"
|
||||
WEB="${SICABE_AIRFLOW_WEB:-sicabe-airflow-webserver}"
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env-file) ENV_FILE="${2:-}"; shift 2 ;;
|
||||
--dag) DAG="${2:-}"; shift 2 ;;
|
||||
-h|--help) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
*) echo "Argumen tidak dikenal: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
[[ -f "${ENV_FILE}" ]] || ENV_FILE="${SCRIPT_DIR}/.env"
|
||||
[[ -f "${ENV_FILE}" ]] || { echo "env file tak ditemukan: ${ENV_FILE}" >&2; exit 2; }
|
||||
|
||||
# Password service account dbt (untuk `dbt docs generate` di dalam container).
|
||||
DBT_PW="$(sed -n 's/^TRINO_SVC_DBT_PASSWORD=//p' "${ENV_FILE}" | head -1)"
|
||||
DBT_USER="${DBT_TRINO_USER:-svc_dbt}"
|
||||
|
||||
step() { echo; echo -e "\033[1m▶ $*\033[0m"; }
|
||||
|
||||
echo -e "\033[1mAlur data E2E (SiCABE)\033[0m"
|
||||
echo " env-file : ${ENV_FILE}"
|
||||
echo " DAG : ${DAG}"
|
||||
echo " Prasyarat: seed sudah dijalankan di tiap host (state/compute)."
|
||||
|
||||
step "1/6 ingest metadata (service + entitas Trino; TANPA dbt — artefak belum ada pra-DAG)"
|
||||
./scripts/ingest-openmetadata.sh --env-file "${ENV_FILE}" \
|
||||
|| echo " PERHATIAN: ingest metadata gagal (mis. tabel belum ada pra-DAG) — lanjut; di-ingest ulang di step 4."
|
||||
|
||||
step "2/6 DAG #1 — buat data (bronze→silver→gold). GE boleh gagal: entitas gold belum di OM."
|
||||
docker exec -e SICABE_DAG_RETRIES=0 "${WEB}" airflow dags test "${DAG}" \
|
||||
|| echo " PERHATIAN: DAG #1 gagal (biasanya di GE karena gold belum di OM) — lanjut; wajar pada cold start."
|
||||
|
||||
step "3/6 artefak dbt (manifest+catalog+run_results) untuk lineage"
|
||||
echo " (wajib: DAG hanya 'dbt run' → catalog.json absen; artefak hilang bila container di-recreate)"
|
||||
docker exec \
|
||||
-e DBT_TRINO_USER="${DBT_USER}" -e DBT_TRINO_PASSWORD="${DBT_PW}" \
|
||||
-e DBT_PROFILES_DIR=/opt/airflow/dbt -e DBT_TARGET_PATH=/tmp/dbt/target \
|
||||
-e DBT_LOG_PATH=/tmp/dbt/logs -w /opt/airflow/dbt/sicabe \
|
||||
"${WEB}" bash -lc 'dbt docs generate --profile trino --target prod' \
|
||||
|| echo " PERHATIAN: dbt docs generate gagal — lineage dbt mungkin terlewat."
|
||||
|
||||
step "4/6 ingest ulang — entitas gold + lineage dbt (artefak sudah ada)"
|
||||
./scripts/ingest-openmetadata.sh --env-file "${ENV_FILE}" --all
|
||||
|
||||
step "5/6 DAG #2 — GE kini menemukan entitas gold di OM (target: hijau)"
|
||||
docker exec -e SICABE_DAG_RETRIES=0 "${WEB}" airflow dags test "${DAG}"
|
||||
|
||||
step "6/6 verifikasi governance (target FAIL=0)"
|
||||
./scripts/verify-phase6-governance.sh --env-file "${ENV_FILE}"
|
||||
|
||||
echo
|
||||
echo -e "\033[1;32m✓ Alur E2E selesai.\033[0m"
|
||||
@@ -16,6 +16,10 @@
|
||||
#
|
||||
# Idempoten: test case & pipeline yang sudah ada dilewati.
|
||||
# Dijalankan dari host; pekerjaannya dieksekusi DI DALAM container Airflow.
|
||||
#
|
||||
# PRASYARAT: entitas tabel `trino_sicabe.iceberg.gold.orders_gold` SUDAH ada di OM
|
||||
# (setelah DAG + `ingest-openmetadata.sh --all`). Bila belum → worker gagal
|
||||
# `HTTP 404: table instance ... not found`.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
@@ -66,4 +70,8 @@ done
|
||||
docker exec \
|
||||
-e OPENMETADATA_ADMIN_PASSWORD="${OPENMETADATA_ADMIN_PASSWORD}" \
|
||||
-e TRINO_SVC_OM_PASSWORD="${TRINO_SVC_OM_PASSWORD}" \
|
||||
"${AIRFLOW_CONTAINER}" python "${WORKER}" "${WORKER_ARGS[@]+"${WORKER_ARGS[@]}"}"
|
||||
"${AIRFLOW_CONTAINER}" python "${WORKER}" "${WORKER_ARGS[@]+"${WORKER_ARGS[@]}"}" || {
|
||||
echo " (catatan) Bila error 404 'table instance ... not found' → tabel gold belum ada di OM." >&2
|
||||
echo " Jalankan DAG + ingest-openmetadata dulu, atau pakai Jalur A (GE di DAG)." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
@@ -82,6 +82,54 @@ else
|
||||
bad "step_tiers() belum memetakan step:${missing}"
|
||||
fi
|
||||
|
||||
# step_container() juga harus memetakan step 1..9 (dipakai mode --host untuk melewati
|
||||
# step yang komponennya dikelola SISTEM LAIN).
|
||||
sc_patterns="$(awk '/^step_container\(\) \{/{f=1} f{print} f&&/^\}/{exit}' "${DEPLOY_DIR}/bootstrap.sh" \
|
||||
| grep -E '^[[:space:]]*[0-9]+\)' | sed 's/).*//' | tr -d ' ')"
|
||||
sc_missing=""
|
||||
for s in 1 2 3 4 5 6 7 8 9; do
|
||||
grep -qx "${s}" <<<"${sc_patterns}" || sc_missing+=" ${s}"
|
||||
done
|
||||
if [[ -z "${sc_missing}" ]]; then
|
||||
ok "step_container() memetakan step 1..9"
|
||||
else
|
||||
bad "step_container() belum memetakan step:${sc_missing}"
|
||||
fi
|
||||
|
||||
# Mode --host HARUS melewati step milik sistem lain (bukan mencoba bootstrap-nya),
|
||||
# kecuali step 2 (MinIO → mode verifikasi).
|
||||
if grep -q 'sicabe_host_expects_container "${_step_ctr}"' "${DEPLOY_DIR}/bootstrap.sh" \
|
||||
&& grep -q '"${step}" != "2"' "${DEPLOY_DIR}/bootstrap.sh"; then
|
||||
ok "bootstrap --host melewati step komponen eksternal (step 2 dikecualikan)"
|
||||
else
|
||||
bad "bootstrap --host tidak melewati step komponen eksternal"
|
||||
fi
|
||||
|
||||
# Bundle host harus di-set agar sicabe_host_expects_container membaca bundle yang benar.
|
||||
if grep -q 'export SICABE_BUNDLE_DIR=' "${DEPLOY_DIR}/bootstrap.sh"; then
|
||||
ok "SICABE_BUNDLE_DIR di-set pada mode --host"
|
||||
else
|
||||
bad "SICABE_BUNDLE_DIR tidak di-set (deteksi komponen per-host salah)"
|
||||
fi
|
||||
|
||||
# Step 2 (MinIO): buat bucket hanya bila MinIO milik host ini; storage sistem lain → verifikasi.
|
||||
if grep -q 'mc_mode="create"' "${DEPLOY_DIR}/bootstrap.sh" \
|
||||
&& grep -q 'mc_mode="verify"' "${DEPLOY_DIR}/bootstrap.sh"; then
|
||||
ok "bootstrap_minio: create (lokal) vs verify (sistem lain)"
|
||||
else
|
||||
bad "bootstrap_minio tidak membedakan create vs verify"
|
||||
fi
|
||||
|
||||
# Sinkron redirectUris Keycloak harus memuat ORIGIN HOST client (bukan hanya localhost),
|
||||
# kalau tidak login lewat alamat (mis. https://<ip>:12443) ditolak "invalid redirect_uri".
|
||||
if grep -q 'CLIENT_ORIGIN' "${DEPLOY_DIR}/bootstrap.sh" \
|
||||
&& grep -q '_client_origin' "${DEPLOY_DIR}/bootstrap.sh" \
|
||||
&& grep -q 'host + "/\*"' "${DEPLOY_DIR}/bootstrap.sh"; then
|
||||
ok "bootstrap: sinkron redirectUris memuat ORIGIN HOST client"
|
||||
else
|
||||
bad "bootstrap: sinkron redirectUris TIDAK memuat origin host (login via alamat ditolak)"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "── B. penjagaan dinamis (butuh stack SiCABE berjalan) ──"
|
||||
if ! docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^sicabe-'; then
|
||||
|
||||
@@ -109,6 +109,170 @@ for pair in "MINIO_IMAGE:quay.io/minio/minio" "MINIO_MC_IMAGE:quay.io/minio/mc"
|
||||
fi
|
||||
done
|
||||
|
||||
# ─── 0e. Komponen EKSTERNAL ber-`address` (skenario P3) ──────────────────────
|
||||
# `components.<c>.address` = komponen dikelola SISTEM LAIN → TIDAK dijalankan cluster
|
||||
# (tak muncul di services host mana pun) + endpoint-nya = alamat itu. Diuji via cluster.p3.yml.
|
||||
p3_problems="$(python3 - "${DEPLOY_DIR}" <<'PY'
|
||||
import sys
|
||||
sys.path.insert(0, sys.argv[1] + "/lib/cluster")
|
||||
import inventory
|
||||
d = inventory.apply_preset(inventory.load(sys.argv[1] + "/cluster.p3.yml"))
|
||||
pl = inventory.plan(d)
|
||||
svc = {s for h in pl.values() for s in h["services"]}
|
||||
p = []
|
||||
for c in ("keycloak", "minio", "nessie"):
|
||||
if c in svc:
|
||||
p.append(c + " masih dijalankan")
|
||||
if not inventory.external_address(d, c):
|
||||
p.append(c + ": address hilang")
|
||||
if "openldap" not in pl.get("control", {}).get("services", []):
|
||||
p.append("openldap tak di control")
|
||||
print("; ".join(p))
|
||||
PY
|
||||
)"
|
||||
if [[ -z "${p3_problems}" ]]; then
|
||||
ok "P3: components.<c>.address → eksternal tak dijalankan, openldap lokal di control"
|
||||
else
|
||||
bad "P3: perilaku components.<c>.address salah — ${p3_problems}"
|
||||
fi
|
||||
|
||||
# ─── 0f. Deteksi keystore Trino BEDA antar-host ──────────────────────────────
|
||||
# Keystore Trino harus identik di semua host (satu --generate + --sync). Skrip ini
|
||||
# mendeteksi divergensi (mis. --generate dijalankan di >1 host).
|
||||
cks="${DEPLOY_DIR}/scripts/check-trino-keystore.sh"
|
||||
if [[ -x "${cks}" ]] && bash -n "${cks}" 2>/dev/null \
|
||||
&& grep -q -- '--inventory' "${cks}" && grep -q -- '--expect' "${cks}" \
|
||||
&& grep -q 'keystore.p12' "${cks}" && grep -q 'password.db' "${cks}" \
|
||||
&& grep -q 'svc_dbt' "${cks}"; then
|
||||
ok "check-trino-keystore.sh: deteksi keystore + password.db antar-host (--inventory/--expect)"
|
||||
else
|
||||
bad "check-trino-keystore.sh hilang/tidak lengkap (divergensi keystore/password.db tak terdeteksi)"
|
||||
fi
|
||||
# Fungsional (bila bundle host sudah ada): cetak sha256 & hormati --expect.
|
||||
_first_bundle="$(ls -d "${DEPLOY_DIR}"/generated/host-* 2>/dev/null | head -1)"
|
||||
if [[ -n "${_first_bundle}" ]]; then
|
||||
_bh="$(basename "${_first_bundle}" | sed 's/^host-//')"
|
||||
_sha="$("${cks}" --host "${_bh}" 2>/dev/null | sed -n 's/^ keystore : sha256=//p')"
|
||||
if [[ "${_sha}" =~ ^[0-9a-f]{64}$ ]]; then
|
||||
ok "check-trino-keystore.sh: sha256 host '${_bh}' terbaca"
|
||||
"${cks}" --host "${_bh}" --expect "${_sha}" >/dev/null 2>&1 \
|
||||
&& ok "check-trino-keystore.sh: --expect cocok → exit 0" \
|
||||
|| bad "check-trino-keystore.sh: --expect cocok seharusnya exit 0"
|
||||
else
|
||||
bad "check-trino-keystore.sh: sha256 tidak terbaca untuk host '${_bh}'"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ─── 0g. .env auto-generate memuat SEMUA var WAJIB compose ───────────────────
|
||||
# Regresi nyata: STARROCKS_ROOT_PASSWORD (dipakai `${VAR:?}` di compose) TIDAK dibuat
|
||||
# install.sh → `docker compose --env-file .env` (tanpa shell env) gagal
|
||||
# "required variable ... is missing a value".
|
||||
_required="$(grep -oE '\$\{[A-Z_][A-Z0-9_]*:\?' "${DEPLOY_DIR}/docker-compose.yml" | sed 's/\${//;s/:?//' | sort -u)"
|
||||
_envvars="$(awk '/^[[:space:]]*cat > .*\/\.env" <<EOF/{f=1;next} f&&/^EOF$/{exit} f' "${DEPLOY_DIR}/install.sh" \
|
||||
| grep -oE '^[A-Z_][A-Z0-9_]*=' | tr -d '=' | sort -u)"
|
||||
_miss_env=""
|
||||
for _v in ${_required}; do grep -qx "${_v}" <<<"${_envvars}" || _miss_env+=" ${_v}"; done
|
||||
if [[ -z "${_miss_env}" ]]; then
|
||||
ok ".env auto-generate memuat semua var wajib compose (${_required//$'\n'/ })"
|
||||
else
|
||||
bad "var wajib compose TIDAK di-generate ke .env:${_miss_env}"
|
||||
fi
|
||||
|
||||
# ─── 0h. Orkestrasi alur data E2E (run-e2e.sh) ───────────────────────────────
|
||||
e2e="${DEPLOY_DIR}/scripts/run-e2e.sh"
|
||||
if [[ -x "${e2e}" ]] && bash -n "${e2e}" 2>/dev/null \
|
||||
&& grep -q 'ingest-openmetadata.sh' "${e2e}" \
|
||||
&& grep -q 'airflow dags test' "${e2e}" \
|
||||
&& grep -q 'verify-phase6-governance.sh' "${e2e}"; then
|
||||
ok "run-e2e.sh: orkestrasi alur data E2E (ingest→DAG→ingest→DAG→verify)"
|
||||
else
|
||||
bad "run-e2e.sh hilang/tidak lengkap"
|
||||
fi
|
||||
|
||||
# ─── 0i. Prometheus: target MinIO EKSTERNAL → alamat:port publik ─────────────
|
||||
# Prometheus tak mendukung substitusi env, dan render-config hanya memetakan komponen
|
||||
# milik host lain — komponen EKSTERNAL (components.minio.address) harus di-rewrite
|
||||
# generate.py, kalau tidak target tetap `minio:9000` → scrape gagal dari host prometheus.
|
||||
_tmp_prom="$(mktemp -d)"; _prom_ok=""
|
||||
if [[ -f "${DEPLOY_DIR}/.env" ]]; then
|
||||
cp "${DEPLOY_DIR}/.env" "${_tmp_prom}/e.env"
|
||||
printf '\nMINIO_INTERNAL_PORT=12900\n' >> "${_tmp_prom}/e.env"
|
||||
if python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \
|
||||
"${DEPLOY_DIR}/cluster.p3.yml" "${DEPLOY_DIR}/docker-compose.yml" \
|
||||
"${_tmp_prom}/e.env" "${_tmp_prom}/out" "${DEPLOY_DIR}/scripts/render-config.sh" \
|
||||
>/dev/null 2>&1; then
|
||||
_pyml="${_tmp_prom}/out/host-control/config/prometheus/prometheus.yml"
|
||||
if grep -q 'targets: \["100.93.54.108:12900"\]' "${_pyml}" 2>/dev/null \
|
||||
&& ! grep -q 'targets: \["minio:9000"\]' "${_pyml}" 2>/dev/null; then
|
||||
_prom_ok=1
|
||||
fi
|
||||
fi
|
||||
if [[ -n "${_prom_ok}" ]]; then
|
||||
ok "prometheus: target MinIO eksternal → 100.93.54.108:12900 (bukan minio:9000)"
|
||||
else
|
||||
bad "prometheus: target MinIO eksternal salah (masih minio:9000?)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ─── 0j. Verifikasi Keycloak lengkap (verify-keycloak.sh) ────────────────────
|
||||
vk="${DEPLOY_DIR}/scripts/verify-keycloak.sh"
|
||||
if [[ -x "${vk}" ]] && bash -n "${vk}" 2>/dev/null \
|
||||
&& grep -q 'client_credentials' "${vk}" && grep -q 'redirectUris' "${vk}" \
|
||||
&& grep -q 'publicClient' "${vk}" && grep -q 'origin host' "${vk}" \
|
||||
&& grep -q 'kc_auth_probe' "${vk}" && grep -q 'password masih DEFAULT' "${vk}"; then
|
||||
ok "verify-keycloak.sh: realm/roles/client(redirectUri origin-host,secret,publicClient)/user/password-default + fungsional"
|
||||
else
|
||||
bad "verify-keycloak.sh hilang/tidak lengkap"
|
||||
fi
|
||||
|
||||
# ─── 0k. Trino OAuth2: token/jwks pakai KEYCLOAK_HOST (bukan literal `keycloak`) ──
|
||||
# Kls bug sama dgn target MinIO Prometheus: Trino server-ke-server ke Keycloak. Bila
|
||||
# Keycloak EKSTERNAL (P3), literal `keycloak:` tak resolve → penukaran token gagal →
|
||||
# callback `/oauth2/callback` balas "Authentication response could not be verified".
|
||||
_tc="${DEPLOY_DIR}/config/trino/config-coordinator.properties"
|
||||
if grep -q 'token-url=http://${ENV:KEYCLOAK_HOST}:${ENV:KEYCLOAK_INTERNAL_PORT}' "${_tc}" 2>/dev/null \
|
||||
&& grep -q 'jwks-url=http://${ENV:KEYCLOAK_HOST}:' "${_tc}" 2>/dev/null \
|
||||
&& grep -q 'oauth2.state-key=${ENV:TRINO_OAUTH_STATE_KEY}' "${_tc}" 2>/dev/null \
|
||||
&& grep -q 'KEYCLOAK_HOST: ${KEYCLOAK_HOST' "${DEPLOY_DIR}/docker-compose.yml" 2>/dev/null \
|
||||
&& grep -q 'TRINO_OAUTH_STATE_KEY: ${TRINO_OAUTH_STATE_KEY' "${DEPLOY_DIR}/docker-compose.yml" 2>/dev/null \
|
||||
&& ! grep -qE 'token-url=http://keycloak:' "${_tc}" 2>/dev/null; then
|
||||
ok "trino oauth2: token-url/jwks-url pakai KEYCLOAK_HOST + state-key tetap"
|
||||
else
|
||||
bad "trino oauth2: token-url/jwks-url literal 'keycloak:' atau state-key belum diset"
|
||||
fi
|
||||
|
||||
# ─── 0l. StarRocks OIDC: redirect_url = alamat browser (bukan localhost) ──────
|
||||
if ! grep -qE "redirect_url' = 'http://localhost" "${DEPLOY_DIR}/bootstrap.sh" 2>/dev/null \
|
||||
&& grep -q 'sr_redirect_url=' "${DEPLOY_DIR}/bootstrap.sh" 2>/dev/null \
|
||||
&& grep -q 'bootstrap_browser_addr' "${DEPLOY_DIR}/bootstrap.sh" 2>/dev/null; then
|
||||
ok "StarRocks OIDC: redirect_url pakai alamat browser (bukan localhost literal)"
|
||||
else
|
||||
bad "StarRocks OIDC: redirect_url masih localhost literal"
|
||||
fi
|
||||
|
||||
# ─── 0m. Tujuan `--sync` BEBAS nama repo (cermin $HOME, bukan ~/sicabe-doc) ───
|
||||
# Regresi: dest default dipatok `~/sicabe-doc/deploy` → repo distribusi bersih
|
||||
# (`sicabe-core`, isinya hanya deploy/) salah tujuan & membuat salinan kedua
|
||||
# `~/sicabe-doc/deploy`. Nilai harus DITURUNKAN dari lokasi script.
|
||||
dest_probe="$(HOME=/home/u bash -c '
|
||||
# shellcheck disable=SC1091
|
||||
source "'"${DEPLOY_DIR}"'/lib/common.sh" >/dev/null 2>&1 || exit 9
|
||||
for d in /home/u/sicabe-core/deploy /home/u/sicabe-doc/deploy /opt/apps/sicabe-core/deploy; do
|
||||
printf "%s|" "$(sicabe_default_dest "$d")"
|
||||
done' 2>/dev/null)"
|
||||
if [[ "${dest_probe}" == "~/sicabe-core/deploy|~/sicabe-doc/deploy|~/sicabe-core/deploy|" ]]; then
|
||||
ok "--sync: tujuan default mencerminkan nama repo (sicabe-core → ~/sicabe-core/deploy)"
|
||||
else
|
||||
bad "--sync: sicabe_default_dest salah → '${dest_probe}'"
|
||||
fi
|
||||
if ! grep -q 'dest="~/sicabe-doc/deploy"' "${DEPLOY_DIR}/install.sh" \
|
||||
&& ! grep -q 'DEST="~/sicabe-doc/deploy"' "${DEPLOY_DIR}/scripts/check-trino-keystore.sh" \
|
||||
&& grep -q 'sicabe_default_dest' "${DEPLOY_DIR}/install.sh"; then
|
||||
ok "--sync: tidak ada lagi path repo ter-hardcode (~/sicabe-doc/deploy)"
|
||||
else
|
||||
bad "--sync: masih ada path repo ter-hardcode (~/sicabe-doc/deploy)"
|
||||
fi
|
||||
|
||||
# ─── 1. generate ────────────────────────────────────────────────────────────
|
||||
if ! python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \
|
||||
"${INVENTORY}" "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" \
|
||||
|
||||
Executable
+355
@@ -0,0 +1,355 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Verifikasi Keycloak (realm, client, user) — LEBIH LENGKAP dari
|
||||
# verify-phase1. Tujuan: menjamin konfigurasi Keycloak siap dipakai fase berikutnya
|
||||
# & alur uji E2E (OIDC Trino/StarRocks/Airflow/OM/Grafana; login user engineer/…).
|
||||
#
|
||||
# Yang diperiksa:
|
||||
# A. Realm & roles : discovery/issuer; role admin/engineer/analyst/viewer.
|
||||
# B. Clients (statis) : per client — ada, enabled, publicClient=false,
|
||||
# standardFlow, serviceAccounts, client-secret,
|
||||
# redirectUris/webOrigins (localhost + PUBLIC_URL), audience mapper.
|
||||
# C. Users (statis) : per user — ada, enabled, realm role benar, dan
|
||||
# apakah password masih DEFAULT (hardcoded: engineer123/…).
|
||||
# D. Fungsional : client_credentials tiap service client → token OK (uji SECRET).
|
||||
# E. Fungsional : login user (password grant) → preferred_username + roles.
|
||||
# F. Redirect URI : validasi NYATA via /auth — Keycloak MENERIMA redirect alamat
|
||||
# host (dan MENOLAK yang ngawur). Bukan sekadar banding daftar.
|
||||
#
|
||||
# Mode READ-ONLY (tanpa admin Keycloak): bagian statis (B/C) DILEWATI; D/E tetap jalan.
|
||||
# Exit: 0 = semua gate lolos (WARN boleh), 1 = ada FAIL.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/verify-keycloak.sh
|
||||
# ./scripts/verify-keycloak.sh --env-file generated/host-control/.env
|
||||
# KEYCLOAK_URL=http://kc.corp:8080 ./scripts/verify-keycloak.sh # IdP existing
|
||||
# =============================================================================
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
# shellcheck source=../lib/common.sh
|
||||
source "${SCRIPT_DIR}/lib/common.sh"
|
||||
# shellcheck source=../config/environment.defaults.sh
|
||||
source "${SCRIPT_DIR}/config/environment.defaults.sh"
|
||||
sicabe_load_env "${SCRIPT_DIR}" "$@" || exit 1
|
||||
|
||||
KC="$(sicabe_local_url http KEYCLOAK_HOST "${KEYCLOAK_PORT:-12080}" KEYCLOAK_URL)"
|
||||
REALM="${SICABE_REALM:-sicabe}"
|
||||
|
||||
REQUIRED_CLIENTS=("trino" "starrocks" "airflow" "openmetadata" "grafana")
|
||||
REQUIRED_ROLES=("admin" "engineer" "analyst" "viewer")
|
||||
REQUIRED_USERS=("admin" "engineer" "analyst" "viewer")
|
||||
AUDIENCE_MAPPER="audience-account"
|
||||
|
||||
# Peran yang Diharapkan per user.
|
||||
declare -A USER_ROLE=([admin]=admin [engineer]=engineer [analyst]=analyst [viewer]=viewer)
|
||||
# Password DEFAULT (hardcoded oleh bootstrap.sh) — untuk mendeteksi password lemah.
|
||||
declare -A USER_DEFAULT_PW=(
|
||||
[admin]="${KEYCLOAK_ADMIN_PASSWORD:-}"
|
||||
[engineer]="engineer123"
|
||||
[analyst]="analyst123"
|
||||
[viewer]="viewer123"
|
||||
)
|
||||
# Secret client diharapkan (harus sama dgn env deploy) + URL publik (redirect/webOrigins).
|
||||
declare -A CLIENT_SECRET=(
|
||||
[trino]="${TRINO_CLIENT_SECRET:-}" [starrocks]="${STARROCKS_CLIENT_SECRET:-}"
|
||||
[airflow]="${AIRFLOW_CLIENT_SECRET:-}" [openmetadata]="${OPENMETADATA_CLIENT_SECRET:-}"
|
||||
[grafana]="${GRAFANA_CLIENT_SECRET:-}"
|
||||
)
|
||||
declare -A CLIENT_PUBLIC=(
|
||||
[trino]="${TRINO_PUBLIC_URL:-}" [starrocks]="${STARROCKS_PUBLIC_URL:-}"
|
||||
[airflow]="${AIRFLOW_PUBLIC_URL:-}" [openmetadata]="${OPENMETADATA_PUBLIC_URL:-}"
|
||||
[grafana]="${GRAFANA_PUBLIC_URL:-}"
|
||||
)
|
||||
# Origin (skema://alamat:port) tempat BROWSER membuka komponen — WAJIB ada di
|
||||
# redirectUris. Deteksi kesalahan umum: redirectUris hanya `https://localhost:12443/*`
|
||||
# padahal user membuka `https://<ip-host>:12443` → Keycloak menolak "invalid redirect_uri".
|
||||
# <client> = "<scheme> <host_var> <port_var> <default_port>"
|
||||
declare -A CLIENT_ORIGIN=(
|
||||
[trino]="https TRINO_HOST TRINO_HTTPS_PORT 12443"
|
||||
[grafana]="http GRAFANA_HOST GRAFANA_PORT 12300"
|
||||
[openmetadata]="http OPENMETADATA_HOST OPENMETADATA_PORT 12085"
|
||||
[airflow]="http AIRFLOW_HOST AIRFLOW_PORT 12082"
|
||||
[starrocks]="http STARROCKS_FE_HOST STARROCKS_HTTP_PORT 12030"
|
||||
)
|
||||
# Alamat efektif: nilai var bila berupa ALAMAT (mengandung titik); kalau kosong/co-located
|
||||
# → alamat host ini (SICABE_HOST_ADDRESS, multi-host). Kalau KEDUANYA tak ada → KOSONG
|
||||
# (JANGAN fallback ke localhost: itu membuat cek origin terhadap localhost selalu PASS).
|
||||
comp_addr() {
|
||||
local v="${!1:-}"
|
||||
if [[ "${v}" == *.* && "${v}" != "127.0.0.1" ]]; then echo "${v}"; return; fi
|
||||
echo "${SICABE_HOST_ADDRESS:-}"
|
||||
}
|
||||
expected_origin() { # <client> → "skema://alamat:port" (atau kosong bila alamat tak diketahui)
|
||||
local spec="${CLIENT_ORIGIN[$1]:-}"; [[ -n "${spec}" ]] || { echo ""; return; }
|
||||
local scheme hv pv dport addr; read -r scheme hv pv dport <<< "${spec}"
|
||||
addr="$(comp_addr "${hv}")"; [[ -n "${addr}" ]] || { echo ""; return; }
|
||||
printf '%s://%s:%s' "${scheme}" "${addr}" "${!pv:-${dport}}"
|
||||
}
|
||||
|
||||
PASS=0; FAIL_N=0; WARN_N=0
|
||||
pass() { echo -e " ${PASS} $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo -e " ${FAIL} $1"; FAIL_N=$((FAIL_N + 1)); }
|
||||
warn() { echo -e " ${WARN} $1"; WARN_N=$((WARN_N + 1)); }
|
||||
info() { echo -e " ${INFO} $1"; }
|
||||
|
||||
# ── Keycloak helper ──────────────────────────────────────────────────────────
|
||||
kc_admin_token() {
|
||||
curl -sf -X POST "${KC}/realms/master/protocol/openid-connect/token" \
|
||||
-d client_id=admin-cli -d "username=${KEYCLOAK_ADMIN:-admin}" \
|
||||
-d "password=${KEYCLOAK_ADMIN_PASSWORD:-}" -d grant_type=password 2>/dev/null |
|
||||
python3 -c "import sys,json;print(json.load(sys.stdin).get('access_token',''))" 2>/dev/null || echo ""
|
||||
}
|
||||
kc_get() { curl -sf -H "Authorization: Bearer ${KC_TOKEN}" "${KC}/admin/realms/${REALM}$1" 2>/dev/null; }
|
||||
|
||||
# client_credentials → cetak "OK" atau "HTTP<code>" atau "ERR".
|
||||
cc_grant() { # <client_id> <secret> → "OK" / "HTTP<code>"
|
||||
local code
|
||||
code="$(curl -s -o /dev/null -w '%{http_code}' -X POST \
|
||||
"${KC}/realms/${REALM}/protocol/openid-connect/token" \
|
||||
-d "client_id=$1" -d "client_secret=$2" -d grant_type=client_credentials 2>/dev/null || true)"
|
||||
code="${code:-000}"
|
||||
[[ "${code}" == "200" ]] && echo OK || echo "HTTP${code}"
|
||||
}
|
||||
|
||||
# Uji NYATA redirect_uri via endpoint /auth (publik — tak perlu admin): apakah Keycloak
|
||||
# MENERIMA redirect (→ halaman/redirect login) atau MENOLAK ("Invalid parameter: redirect_uri").
|
||||
kc_auth_probe() { # <client_id> <redirect_uri> → OK | REJECTED | HTTP<code>
|
||||
local out code
|
||||
out="$(curl -s -w $'\n%{http_code}' -G "${KC}/realms/${REALM}/protocol/openid-connect/auth" \
|
||||
--data-urlencode "client_id=$1" --data-urlencode "redirect_uri=$2" \
|
||||
--data-urlencode "response_type=code" --data-urlencode "scope=openid" 2>/dev/null || true)"
|
||||
code="${out##*$'\n'}"
|
||||
if printf '%s' "${out}" | grep -qi 'Invalid parameter: redirect_uri' || [[ "${code}" == "400" ]]; then
|
||||
echo REJECTED; return
|
||||
fi
|
||||
case "${code}" in 200|302|303) echo OK ;; *) echo "HTTP${code:-000}" ;; esac
|
||||
}
|
||||
# password grant → cetak "preferred_username|roles" (dari id_token) atau "".
|
||||
user_login() { # <user> <pw>
|
||||
curl -s -X POST "${KC}/realms/${REALM}/protocol/openid-connect/token" \
|
||||
-d client_id=admin-cli -d "username=$1" -d "password=$2" \
|
||||
-d grant_type=password -d scope=openid 2>/dev/null |
|
||||
python3 -c "
|
||||
import sys,json,base64
|
||||
try: r=json.load(sys.stdin)
|
||||
except Exception: print(''); raise SystemExit
|
||||
t=r.get('id_token') or r.get('access_token') or ''
|
||||
if not t: print(''); raise SystemExit
|
||||
p=t.split('.')[1]; p+='='*(-len(p)%4)
|
||||
try: c=json.loads(base64.urlsafe_b64decode(p))
|
||||
except Exception: print(''); raise SystemExit
|
||||
print(c.get('preferred_username','') + '|' + ','.join(c.get('realm_access',{}).get('roles') or []))
|
||||
" 2>/dev/null || echo ""
|
||||
}
|
||||
|
||||
# ── A. Realm & roles ─────────────────────────────────────────────────────────
|
||||
check_realm_roles() {
|
||||
local disc issuer
|
||||
disc="$(curl -sf "${KC}/realms/${REALM}/.well-known/openid-configuration" 2>/dev/null || true)"
|
||||
if [[ -z "${disc}" ]]; then
|
||||
fail "OIDC discovery realm '${REALM}' tidak merespon (${KC})"; return
|
||||
fi
|
||||
issuer="$(echo "${disc}" | python3 -c "import sys,json;print(json.load(sys.stdin).get('issuer',''))" 2>/dev/null)"
|
||||
pass "OIDC discovery OK (issuer=${issuer})"
|
||||
if [[ -z "${KC_TOKEN}" ]]; then
|
||||
warn "Tanpa admin Keycloak — cek statis (roles/client/user) DILEWATI"; return
|
||||
fi
|
||||
local roles missing=""
|
||||
roles="$(kc_get "/roles" | python3 -c "import sys,json;print(' '.join(r['name'] for r in json.load(sys.stdin)))" 2>/dev/null || echo "")"
|
||||
for r in "${REQUIRED_ROLES[@]}"; do [[ " ${roles} " == *" ${r} "* ]] || missing+=" ${r}"; done
|
||||
[[ -z "${missing}" ]] && pass "Roles lengkap: ${REQUIRED_ROLES[*]}" || fail "Role belum ada:${missing}"
|
||||
}
|
||||
|
||||
# ── B. Clients (statis) ──────────────────────────────────────────────────────
|
||||
check_clients() {
|
||||
[[ -n "${KC_TOKEN}" ]] || return 0
|
||||
local cid js ok
|
||||
# Bila origin host TAK BISA dihitung (env tanpa alamat cluster) → beri tahu SEKALI;
|
||||
# cek origin per client di-skip (bukan PASS palsu). Sumber benar: --env-file bundle
|
||||
# atau SICABE_CLIENT_REDIRECT_ORIGINS.
|
||||
local _c _have=0
|
||||
[[ -n "${SICABE_CLIENT_REDIRECT_ORIGINS:-}" ]] && _have=1
|
||||
for _c in "${REQUIRED_CLIENTS[@]}"; do [[ -n "$(expected_origin "${_c}")" ]] && { _have=1; break; }; done
|
||||
[[ ${_have} -eq 0 ]] && warn "Origin host tak bisa dihitung (env tanpa alamat cluster) — jalankan dgn --env-file generated/host-<nama>/.env, atau set SICABE_CLIENT_REDIRECT_ORIGINS. Cek origin DILEWATI (bukan PASS)."
|
||||
for cid in "${REQUIRED_CLIENTS[@]}"; do
|
||||
js="$(kc_get "/clients?clientId=${cid}")"
|
||||
if [[ -z "${js}" || "${js}" == "{}" || "${js}" == "[]" ]]; then
|
||||
fail "Client '${cid}' TIDAK ADA"; continue
|
||||
fi
|
||||
# Ambil bidang penting + secret + redirect + mapper dalam satu pass.
|
||||
read -r ok pub std svc auth secret redirects mapper < <(echo "${js}" | python3 -c "
|
||||
import sys,json
|
||||
d=json.load(sys.stdin) or []
|
||||
if not d: print('0 - - - - - - -'); raise SystemExit
|
||||
c=d[0]
|
||||
print(
|
||||
'1' if c.get('enabled') else '0',
|
||||
c.get('publicClient'),
|
||||
'1' if c.get('standardFlowEnabled') else '0',
|
||||
'1' if c.get('serviceAccountsEnabled') else '0',
|
||||
c.get('clientAuthenticatorType',''),
|
||||
c.get('secret',''),
|
||||
','.join(c.get('redirectUris') or []),
|
||||
','.join(c.get('redirectUris') or []))
|
||||
" 2>/dev/null)
|
||||
[[ "${ok}" == "1" ]] && pass "Client '${cid}' enabled" || fail "Client '${cid}' DISABLED"
|
||||
[[ "${pub}" == "False" ]] && pass " ${cid}: publicClient=false (confidential)" \
|
||||
|| fail " ${cid}: publicClient=${pub} (harus False)"
|
||||
[[ "${std}" == "1" ]] && pass " ${cid}: standardFlowEnabled" || fail " ${cid}: standardFlow OFF"
|
||||
[[ "${svc}" == "1" ]] && pass " ${cid}: serviceAccountsEnabled" || warn " ${cid}: serviceAccounts OFF"
|
||||
[[ "${auth}" == "client-secret" ]] && pass " ${cid}: clientAuthenticatorType=client-secret" \
|
||||
|| fail " ${cid}: authenticator='${auth}' (harus client-secret)"
|
||||
# Secret sinkron dgn env.
|
||||
if [[ -n "${CLIENT_SECRET[$cid]}" ]]; then
|
||||
[[ "${secret}" == "${CLIENT_SECRET[$cid]}" ]] && pass " ${cid}: secret sinkron dgn env" \
|
||||
|| fail " ${cid}: secret BEDA dari env (komponen akan gagal login)"
|
||||
else
|
||||
warn " ${cid}: *_CLIENT_SECRET kosong di env — tak bisa membandingkan secret"
|
||||
fi
|
||||
# redirectUris: localhost + (bila ada) PUBLIC_URL.
|
||||
if [[ "${redirects}" == *"http://localhost:*"* || "${redirects}" == *"https://localhost:*"* ]]; then
|
||||
pass " ${cid}: redirectUris memuat localhost"
|
||||
else
|
||||
warn " ${cid}: redirectUris TANPA localhost (${redirects})"
|
||||
fi
|
||||
local puburl="${CLIENT_PUBLIC[$cid]}"
|
||||
if [[ -n "${puburl}" ]]; then
|
||||
if [[ "${redirects}" == *"${puburl}"* ]]; then
|
||||
pass " ${cid}: redirectUris memuat PUBLIC_URL (${puburl})"
|
||||
else
|
||||
fail " ${cid}: redirectUris TANPA PUBLIC_URL '${puburl}' (login browser via alamat publik akan ditolak)"
|
||||
fi
|
||||
fi
|
||||
# Origin host (tempat browser membuka komponen) — WAJIB ada, bukan hanya localhost.
|
||||
local exp; exp="$(expected_origin "${cid}")"
|
||||
if [[ -n "${exp}" ]]; then
|
||||
if [[ "${redirects}" == *"${exp}"* ]]; then
|
||||
pass " ${cid}: redirectUris memuat origin host (${exp})"
|
||||
else
|
||||
fail " ${cid}: redirectUris TANPA origin host '${exp}' — login browser via alamat itu DITOLAK (invalid redirect_uri)"
|
||||
fi
|
||||
fi
|
||||
# Origin client dari CLUSTER (SICABE_CLIENT_REDIRECT_ORIGINS) — host existing yang
|
||||
# menjalankan bootstrap tak tahu alamat host cluster, jadi daftar ini disuplai ke .env.
|
||||
local o
|
||||
for o in ${SICABE_CLIENT_REDIRECT_ORIGINS:-}; do
|
||||
[[ -n "${o}" ]] || continue
|
||||
if [[ "${redirects}" == *"${o}"* ]]; then
|
||||
pass " ${cid}: redirectUris memuat origin cluster (${o})"
|
||||
else
|
||||
fail " ${cid}: redirectUris TANPA origin cluster '${o}'"
|
||||
fi
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
# ── C. Users (statis) + password default ─────────────────────────────────────
|
||||
check_users() {
|
||||
[[ -n "${KC_TOKEN}" ]] || return 0
|
||||
local u js
|
||||
for u in "${REQUIRED_USERS[@]}"; do
|
||||
js="$(kc_get "/users?username=${u}&exact=true")"
|
||||
if [[ -z "${js}" || "${js}" == "[]" ]]; then fail "User '${u}' TIDAK ADA"; continue; fi
|
||||
local uid enabled roles
|
||||
uid="$(echo "${js}" | python3 -c "import sys,json;d=json.load(sys.stdin);print(d[0]['id'] if d else '')" 2>/dev/null)"
|
||||
enabled="$(echo "${js}" | python3 -c "import sys,json;d=json.load(sys.stdin);print('1' if d and d[0].get('enabled') else '0')" 2>/dev/null)"
|
||||
roles="$(kc_get "/users/${uid}/role-mappings/realm" | python3 -c "import sys,json;print(' '.join(r['name'] for r in json.load(sys.stdin)))" 2>/dev/null || echo "")"
|
||||
[[ "${enabled}" == "1" ]] && pass "User '${u}' ada + enabled" || fail "User '${u}' DISABLED"
|
||||
[[ " ${roles} " == *" ${USER_ROLE[$u]} "* ]] && pass " ${u}: realm role '${USER_ROLE[$u]}'" \
|
||||
|| fail " ${u}: role '${USER_ROLE[$u]}' TIDAK ada (peran: ${roles:-—})"
|
||||
# Password masih DEFAULT (hardcoded)?
|
||||
local dp="${USER_DEFAULT_PW[$u]}"
|
||||
if [[ -n "${dp}" ]] && [[ "$(user_login "${u}" "${dp}")" == "${u}|"* ]]; then
|
||||
warn " ${u}: password masih DEFAULT (hardcoded) — ganti untuk produksi"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# ── D. client_credentials ────────────────────────────────────────────────────
|
||||
check_cc() {
|
||||
local cid res
|
||||
for cid in "${REQUIRED_CLIENTS[@]}"; do
|
||||
[[ -n "${CLIENT_SECRET[$cid]}" ]] || { warn "cc ${cid}: secret env kosong"; continue; }
|
||||
res="$(cc_grant "${cid}" "${CLIENT_SECRET[$cid]}")"
|
||||
[[ "${res}" == "OK" ]] && pass "client_credentials '${cid}' → token OK" \
|
||||
|| fail "client_credentials '${cid}' GAGAL (${res})"
|
||||
done
|
||||
}
|
||||
|
||||
# ── E2. Redirect URI — validasi NYATA via /auth (bukan sekadar bandingkan daftar) ──
|
||||
check_redirect_validation() {
|
||||
local cid o exp rs any=0
|
||||
for cid in "${REQUIRED_CLIENTS[@]}"; do
|
||||
exp="$(expected_origin "${cid}")"
|
||||
if [[ -n "${exp}" ]]; then
|
||||
any=1
|
||||
rs="$(kc_auth_probe "${cid}" "${exp}/")"
|
||||
case "${rs}" in
|
||||
OK) pass "${cid}: Keycloak MENERIMA redirect '${exp}/*' (uji nyata via /auth)";;
|
||||
REJECTED) fail "${cid}: Keycloak MENOLAK redirect '${exp}/*' — invalid redirect_uri (belum terdaftar di client)";;
|
||||
*) warn "${cid}: uji redirect '${exp}/*' tak konklusif (${rs})";;
|
||||
esac
|
||||
fi
|
||||
done
|
||||
# Origin dari daftar cluster (SICABE_CLIENT_REDIRECT_ORIGINS) — uji terhadap tiap client.
|
||||
for o in ${SICABE_CLIENT_REDIRECT_ORIGINS:-}; do
|
||||
[[ -n "${o}" ]] || continue
|
||||
any=1
|
||||
for cid in "${REQUIRED_CLIENTS[@]}"; do
|
||||
rs="$(kc_auth_probe "${cid}" "${o}/")"
|
||||
[[ "${rs}" == "OK" ]] && pass "${cid}: Keycloak MENERIMA origin cluster '${o}/*'" \
|
||||
|| fail "${cid}: Keycloak MENOLAK origin cluster '${o}/*' (${rs})"
|
||||
done
|
||||
done
|
||||
if [[ ${any} -eq 0 ]]; then
|
||||
warn "redirect_uri tak bisa diuji (env tanpa alamat cluster / SICABE_CLIENT_REDIRECT_ORIGINS)"
|
||||
return
|
||||
fi
|
||||
# Kontrol: redirect ngawur HARUS ditolak → membuktikan Keycloak memang memvalidasi.
|
||||
rs="$(kc_auth_probe "trino" "https://sicabe-invalid.example/blocked")"
|
||||
[[ "${rs}" == "REJECTED" ]] \
|
||||
&& pass "kontrol: redirect ngawur DITOLAK Keycloak (validasi redirect aktif)" \
|
||||
|| warn "kontrol: redirect ngawur '${rs}' — Keycloak mungkin TIDAK memvalidasi redirect_uri"
|
||||
}
|
||||
|
||||
# ── E. Login user (password grant) ───────────────────────────────────────────
|
||||
check_user_login() {
|
||||
local u out pref
|
||||
for u in "${REQUIRED_USERS[@]}"; do
|
||||
local pw="${USER_DEFAULT_PW[$u]}"
|
||||
[[ -n "${pw}" ]] || { warn "login '${u}': password tak diketahui — lewati"; continue; }
|
||||
out="$(user_login "${u}" "${pw}")"
|
||||
pref="${out%%|*}"
|
||||
if [[ "${pref}" == "${u}" ]]; then
|
||||
pass "login '${u}' OK (preferred_username=${pref}; roles=${out#*|})"
|
||||
elif [[ -z "${out}" ]]; then
|
||||
fail "login '${u}' GAGAL (user/password salah, atau user belum ada)"
|
||||
else
|
||||
fail "login '${u}' OK tapi preferred_username='${pref}' (harus = uid LDAP '${u}')"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# ── main ─────────────────────────────────────────────────────────────────────
|
||||
echo -e "${BOLD}Verifikasi Keycloak — realm '${REALM}' @ ${KC}${NC}"
|
||||
echo ""
|
||||
KC_TOKEN="$(kc_admin_token)"
|
||||
if [[ -z "${KC_TOKEN}" ]]; then
|
||||
warn "Mode READ-ONLY (login admin gagal / tanpa admin) — cek statis dilewati"
|
||||
else
|
||||
info "Token admin diperoleh — cek statis dijalankan"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
echo -e "${BOLD}A. Realm & roles${NC}"; check_realm_roles
|
||||
echo -e "${BOLD}B. Clients (statis)${NC}"; check_clients
|
||||
echo -e "${BOLD}C. Users (statis)${NC}"; check_users
|
||||
echo -e "${BOLD}D. Fungsional — client_credentials (uji SECRET client, bukan redirect)${NC}"; check_cc
|
||||
echo -e "${BOLD}E. Fungsional — login user${NC}"; check_user_login
|
||||
echo -e "${BOLD}F. Redirect URI — validasi NYATA via /auth${NC}"; check_redirect_validation
|
||||
echo ""
|
||||
echo -e "${BOLD}Ringkasan: ${PASS} PASS · ${FAIL_N} FAIL · ${WARN_N} WARN${NC}"
|
||||
[[ ${FAIL_N} -eq 0 ]] && { echo -e "${GREEN}✓ Keycloak siap.${NC}"; exit 0; }
|
||||
echo -e "${RED}✗ Ada ${FAIL_N} kegagalan — perbaiki sebelum melanjutkan.${NC}"
|
||||
exit 1
|
||||
@@ -97,7 +97,7 @@ try:
|
||||
except Exception:
|
||||
print(''); raise SystemExit
|
||||
roles=c.get('realm_access',{}).get('roles') or []
|
||||
print(c.get('preferred_username',''), '|', ','.join(roles))
|
||||
print(c.get('preferred_username','') + '|' + ','.join(roles))
|
||||
" 2>/dev/null || echo ""
|
||||
}
|
||||
|
||||
|
||||
@@ -268,10 +268,14 @@ check_security() {
|
||||
fail " ${OIDC_NAME}: tidak memuat ${pat} (cek bootstrap.sh --step 5)"
|
||||
fi
|
||||
done
|
||||
if [[ "${oidc}" == *"redirect_url\" = \"http://localhost:${STARROCKS_HTTP_PORT:-12030}/api/oauth2\""* ]]; then
|
||||
pass " ${OIDC_NAME}: redirect_url → http://localhost:${STARROCKS_HTTP_PORT:-12030}/api/oauth2"
|
||||
# redirect_url harus = alamat BROWSER (host FE) — bukan localhost/nama container.
|
||||
local _sraddr="${STARROCKS_FE_HOST:-}"
|
||||
[[ "${_sraddr}" == *.* && "${_sraddr}" != "127.0.0.1" ]] || _sraddr="${SICABE_HOST_ADDRESS:-localhost}"
|
||||
local _srexp="http://${_sraddr}:${STARROCKS_HTTP_PORT:-12030}/api/oauth2"
|
||||
if [[ "${oidc}" == *"redirect_url\" = \"${_srexp}\""* ]]; then
|
||||
pass " ${OIDC_NAME}: redirect_url → ${_srexp}"
|
||||
else
|
||||
warn " ${OIDC_NAME}: redirect_url tidak sesuai STARROCKS_HTTP_PORT (${STARROCKS_HTTP_PORT:-12030})"
|
||||
warn " ${OIDC_NAME}: redirect_url bukan '${_srexp}' (cek bootstrap.sh --step 5)"
|
||||
fi
|
||||
local g miss="" groups
|
||||
IFS=',' read -ra groups <<< "${EXPECTED_GROUPS}"
|
||||
|
||||
Reference in New Issue
Block a user