diff --git a/.gitignore b/.gitignore index 3f23316..41b8b8f 100644 --- a/.gitignore +++ b/.gitignore @@ -215,3 +215,6 @@ deploy/generated/ # Trino keystore dipersistenkan saat --generate (jangan di-commit) deploy/.trino-keystore/ + +# Artefak dev (jangan di-commit) +deploy/output/ diff --git a/deploy/README.md b/deploy/README.md index 50cdb6f..2016340 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -227,10 +227,11 @@ di sini komponen tetap milik klaster ini, hanya dipindah host. Override variabel > sehingga path filesystem seperti `/etc/trino/` dan baris komentar tidak ikut berubah. > Rencana lengkap: `reviews/sicabe-10-multi-host-installer-plan.md`. -> ⚠️ **Pilot multi-host di host yang SAMA tidak boleh dua stack berjalan bersamaan.** -> Memisah network/port saja **tidak cukup** — RAM tidak akan cukup (host pilot 15 GB, -> satu stack penuh ±10 GB). **Matikan stack lama lebih dulu** sebelum menyalakan hasil -> `generated/host-/`. +> ⚠️ **Satu host = satu stack aktif.** Stack single-host dan multi-host **tidak untuk +> berjalan bersamaan** di host yang sama — **matikan stack lama lebih dulu** sebelum +> menyalakan hasil `generated/host-/`. Berdampingan **tidak cukup** hanya dipisah +> network/port: pada host kecil hal itu juga menabrak RAM (pilot 15 GB, satu stack penuh +> ±10 GB). **Trino dipisah coordinator/worker.** Sejak 2026-09-28 Trino bukan lagi satu service: `trino-coordinator` (port host `12443`, HTTPS + OAuth2/Basic, client-facing) dan @@ -343,15 +344,36 @@ Menyalin seluruh `deploy/` — **termasuk `.env` dan `generated/`** — ke tiap ```bash ./install.sh --sync # ke SEMUA host lain (host ini dideteksi dari alamat lokal) ./install.sh --sync --host state # hanya satu host -./install.sh --sync --user admin --dest '~/sicabe-doc/deploy' # SSH user & tujuan +./install.sh --sync --user admin --dest '~/lain/deploy' # SSH user & tujuan (timpa default) ./install.sh --sync --delete # hapus di tujuan berkas yang tak ada di sumber ``` +**Tujuan default = cermin lokasi repo ini.** Nilai `--dest` diturunkan dari letak `install.sh` +di bawah `$HOME` (`sicabe_default_dest`), jadi **nama repo tidak dipatok**: + +| Repo di host control | Tujuan `--sync` default di host lain | +|---|---| +| `~/sicabe-doc/deploy` | `~/sicabe-doc/deploy` | +| `~/sicabe-core/deploy` | `~/sicabe-core/deploy` | + +Ini membuat **repo distribusi bersih** (mis. `sicabe-core` — hanya folder `deploy/`, tanpa +`docs/`, `reviews/`, `experiments/`) bekerja **tanpa `--dest`**. `deploy/` **self-contained**: +semua script memakai `SCRIPT_DIR`, dan bind-mount compose relatif (`./config/…`) — jadi nama +atau lokasi repo tidak berpengaruh. Timpa tujuan dengan `--dest '~/…/deploy'` bila perlu. + Host yang sedang dijalankan **dikecualikan otomatis** (kecocokan `hostname -I` dengan `address` di inventory); bila tak cocok, ia menyinkron ke semua host. `.env` **wajib ikut** (rahasia bersama — OIDC client secret harus identik lintas host). Setelah sync, di tiap host jalankan `./scripts/prefetch-images.sh --host ` lalu `./install.sh --host `. +> **Model distribusi: control = sumber tunggal · host lain = PENERIMA (bukan working tree git).** +> `--sync` menimpa berkas yang di-track git di `deploy/` host tujuan dengan versi control, +> sehingga `git status` di sana **selalu kotor** dan `git pull` akan **ditolak** ("local changes +> would be overwritten"). **Jangan `git pull` di host tujuan** — perbarui di **control** lalu +> `--sync` saja (clone repo di host tujuan cukup **sekali**, untuk `prepare-host.sh`). +> Gunakan **`--sync --delete`** agar berkas yang dihapus/di-rename di repo ikut terhapus di +> tujuan (`.git` & `output` dikecualikan, jadi keduanya aman). + Tanpa `cluster.yml`, installer berperilaku **persis** seperti single-host (tidak ada yang di-generate). Hasil generate berisi per host: `docker-compose.yml` (hanya service milik host itu), `.env` (endpoint → alamat host), dan `config/` ter-render. diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index d6f1189..86bf21b 100755 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -94,6 +94,16 @@ ldap_endpoint() { fi } +# Alamat BROWSER-reachable komponen (tempat user memakai UI di browser): nilai +# bila berupa ALAMAT (mengandung titik); kalau tidak → SICABE_HOST_ADDRESS (multi-host); +# kalau tak ada → localhost (single-host). Dipakai redirect_url OIDC StarRocks & origin +# redirectUris Keycloak (JANGAN nama container/hostname internal — browser tak bisa resolve). +bootstrap_browser_addr() { # + local v="${!1:-}" + if [[ "${v}" == *.* && "${v}" != "127.0.0.1" ]]; then echo "${v}"; return; fi + echo "${SICABE_HOST_ADDRESS:-localhost}" +} + # ============================================================================= # STEP 1: Keycloak — Realm, Client, User, Role (bab 5.1) # ============================================================================= @@ -218,6 +228,28 @@ bootstrap_keycloak() { ["grafana"]="${GRAFANA_PUBLIC_URL:-}" ) + # Origin HOST (skema://alamat:port) — tempat browser MEMBUKA komponen. WAJIB ada di + # redirectUris, kalau tidak login via ALAMAT (mis. https://100.123.123.71:12443) ditolak + # "Invalid parameter: redirect_uri" (redirectUris hanya localhost). Sinkron dgn + # scripts/verify-keycloak.sh. + # =" " + declare -A CLIENT_ORIGIN=( + ["trino"]="https TRINO_HOST TRINO_HTTPS_PORT 12443" + ["grafana"]="http GRAFANA_HOST GRAFANA_PORT 12300" + ["openmetadata"]="http OPENMETADATA_HOST OPENMETADATA_PORT 12085" + ["airflow"]="http AIRFLOW_HOST AIRFLOW_PORT 12082" + ["starrocks"]="http STARROCKS_FE_HOST STARROCKS_HTTP_PORT 12030" + ) + _comp_addr() { # → alamat efektif (IP/alamat, atau host ini bila co-located) + local v="${!1:-}" + if [[ "${v}" == *.* && "${v}" != "127.0.0.1" ]]; then echo "${v}"; else echo "${SICABE_HOST_ADDRESS:-localhost}"; fi + } + _client_origin() { # → skema://alamat:port (atau kosong) + local spec="${CLIENT_ORIGIN[$1]:-}"; [[ -n "${spec}" ]] || { echo ""; return; } + local s hv pv dp; read -r s hv pv dp <<< "${spec}" + printf '%s://%s:%s' "${s}" "$(_comp_addr "${hv}")" "${!pv:-${dp}}" + } + for client_id in "${!CLIENTS[@]}"; do local entry="${CLIENTS[$client_id]}" IFS="|" read -r desc secret <<< "${entry}" @@ -260,25 +292,38 @@ bootstrap_keycloak() { # yang membuat login browser lewat alamat publik ditolak: # Invalid parameter: redirect_uri if [[ -n "${client_uuid}" ]]; then - local _payload + local _payload _host_origin + _host_origin="$(_client_origin "${client_id}")" _payload=$(curl -sf -H "Authorization: Bearer ${token}" \ "${kc_base}/admin/realms/${realm}/clients/${client_uuid}" \ - | PUBLIC_ORIGIN="${CLIENT_PUBLIC_URL[$client_id]:-}" python3 -c ' + | PUBLIC_ORIGIN="${CLIENT_PUBLIC_URL[$client_id]:-}" HOST_ORIGIN="${_host_origin}" \ + EXTRA_ORIGINS="${SICABE_CLIENT_REDIRECT_ORIGINS:-}" python3 -c ' import json, os, sys c = json.load(sys.stdin) pub = (os.environ.get("PUBLIC_ORIGIN") or "").rstrip("/") +host = (os.environ.get("HOST_ORIGIN") or "").rstrip("/") redirects = set(c.get("redirectUris") or []) | {"http://localhost:*", "https://localhost:*"} origins = set(c.get("webOrigins") or []) | {"http://localhost:*", "https://localhost:*"} +if host: + redirects.add(host + "/*") + origins.add(host) if pub: redirects.add(pub + "/*") origins.add(pub) +# Origin client dari CLUSTER (host existing tak tahu alamat host cluster) — daftar +# spasi-dipisah, mis. SICABE_CLIENT_REDIRECT_ORIGINS="https://100.123.123.71:12443 ...". +for o in (os.environ.get("EXTRA_ORIGINS") or "").split(): + o = o.rstrip("/") + if o: + redirects.add(o + "/*") + origins.add(o) print(json.dumps({"redirectUris": sorted(redirects), "webOrigins": sorted(origins)})) ' 2>/dev/null || echo "") - if [[ -n "${_payload}" && -n "${CLIENT_PUBLIC_URL[$client_id]:-}" ]]; then + if [[ -n "${_payload}" ]]; then curl -sf -X PUT -H "Authorization: Bearer ${token}" \ -H "Content-Type: application/json" -d "${_payload}" \ "${kc_base}/admin/realms/${realm}/clients/${client_uuid}" >/dev/null 2>&1 || true - echo -e "${OK} Client '${client_id}' redirectUris disinkronkan (${CLIENT_PUBLIC_URL[$client_id]})." + echo -e "${OK} Client '${client_id}' redirectUris disinkronkan (origin host ${_host_origin:-—}; publik ${CLIENT_PUBLIC_URL[$client_id]:-—})." fi fi @@ -312,7 +357,8 @@ print(json.dumps({"redirectUris": sorted(redirects), "webOrigins": sorted(origin # StarRocks FE mengirim redirect internal (http://starrocks-fe:8030/api/oauth2) # saat membangun URL OAuth2 → WAJIB terdaftar, jika tidak Keycloak menolak # dengan "Invalid parameter: redirect_uri". Pola host dipakai untuk callback browser. - local sr_client_id sr_client_json + local sr_client_id sr_client_json sr_fe_browser_addr + sr_fe_browser_addr="$(bootstrap_browser_addr STARROCKS_FE_HOST)" sr_client_id=$(curl -sf -H "Authorization: Bearer ${token}" \ "${kc_base}/admin/realms/${realm}/clients?clientId=starrocks" | \ python3 -c "import sys,json; d=json.load(sys.stdin); print(d[0]['id'] if d else '')" 2>/dev/null || echo "") @@ -322,7 +368,7 @@ print(json.dumps({"redirectUris": sorted(redirects), "webOrigins": sorted(origin python3 -c " import sys,json c=json.load(sys.stdin) -extra=['http://starrocks-fe:8030/api/oauth2','http://localhost:${STARROCKS_HTTP_PORT:-12030}/*'] +extra=['http://starrocks-fe:8030/api/oauth2','http://localhost:${STARROCKS_HTTP_PORT:-12030}/*','http://${sr_fe_browser_addr}:${STARROCKS_HTTP_PORT:-12030}/*'] c['redirectUris']=sorted(set((c.get('redirectUris') or [])+extra)) print(json.dumps(c)) " 2>/dev/null || true) @@ -451,12 +497,20 @@ bootstrap_minio() { local mc_endpoint="${S3_ENDPOINT:-http://minio:9000}" local mc_user="${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}" local mc_pass="${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-}}" + # Nama bucket = config (MINIO_BUCKET_ICEBERG_DATA), bukan hardcoded. local bucket="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}" local mc_image="quay.io/minio/mc:${MINIO_MC_VERSION:-RELEASE.2025-08-13T08-35-41Z}" - local local_minio_ep="http://minio:9000" - # Penantian via host hanya relevan untuk MinIO lokal (S3 existing tidak di localhost). - if [[ "${mc_endpoint}" == "${local_minio_ep}" ]]; then + # Pemilik storage ⇔ container MinIO ada di bundle host INI (single-host → selalu + # lokal, krena SICABE_HOST_NAME kosong). Bila storage dikelola SISTEM LAIN + # (mis. sisi existing P1/P3), host ini HANYA MEMVERIFIKASI bucket — pembuatan + # bucket adalah tugas sisi PEMILIK storage. Ini menghindari cluster "menciptakan" + # bucket di storage pihak lain tanpa otoritas. + local mc_mode="create" + sicabe_host_expects_container sicabe-minio || mc_mode="verify" + + # Penantian via host hanya relevan untuk MinIO lokal milik host ini. + if [[ "${mc_mode}" == "create" ]]; then echo -e "${INFO} Menunggu MinIO siap..." for i in $(seq 1 15); do if curl -sf "http://localhost:${MINIO_PORT:-12900}/minio/health/live" >/dev/null 2>&1; then @@ -467,11 +521,11 @@ bootstrap_minio() { sleep 2 done else - echo -e "${INFO} Object storage eksternal (${mc_endpoint}) — lewati penantian host." + echo -e "${INFO} Object storage dikelola sistem lain (${mc_endpoint}) — mode VERIFIKASI bucket." fi if [[ -z "${mc_pass}" ]]; then - echo -e "${WARN} Kredensial storage kosong (S3_SECRET_KEY / MINIO_ROOT_PASSWORD) — lewati pembuatan bucket." + echo -e "${WARN} Kredensial storage kosong (S3_SECRET_KEY / MINIO_ROOT_PASSWORD) — lewati cek bucket." return 0 fi @@ -488,12 +542,13 @@ bootstrap_minio() { local out out=$(docker run --rm -i \ --network "${SICABE_NETWORK:-sicabe-network}" \ - -e S3E="${mc_endpoint}" -e S3U="${mc_user}" -e S3P="${mc_pass}" -e BUCKET="${bucket}" \ + -e S3E="${mc_endpoint}" -e S3U="${mc_user}" -e S3P="${mc_pass}" \ + -e BUCKET="${bucket}" -e MODE="${mc_mode}" \ --entrypoint sh "${mc_image}" -c ' if ! mc alias set probe "$S3E" "$S3U" "$S3P" >/dev/null 2>&1; then echo ALIAS_FAIL; exit 0; fi if mc ls "probe/$BUCKET" >/dev/null 2>&1; then echo BUCKET_OK -elif mc mb "probe/$BUCKET" >/dev/null 2>&1; then echo BUCKET_CREATED -else echo BUCKET_FAIL; fi' 2>&1) +elif [ "$MODE" = "create" ] && mc mb "probe/$BUCKET" >/dev/null 2>&1; then echo BUCKET_CREATED +else echo BUCKET_MISSING; fi' 2>&1) case "${out}" in *ALIAS_FAIL*) @@ -507,9 +562,14 @@ else echo BUCKET_FAIL; fi' 2>&1) *BUCKET_CREATED*) echo -e "${OK} Bucket '${bucket}' dibuat." ;; - *) - echo -e "${WARN} Bucket '${bucket}' belum bisa dipastikan — buat manual, lalu:" - echo " ./scripts/verify-phase2-storage.sh" + *BUCKET_MISSING*) + if [[ "${mc_mode}" == "verify" ]]; then + echo -e "${WARN} Bucket '${bucket}' belum ada — storage dikelola sistem lain; buat di sisi PEMILIK storage." + echo " Bucket = MINIO_BUCKET_ICEBERG_DATA (config); konfirmasi ke admin storage existing." + else + echo -e "${WARN} Bucket '${bucket}' belum bisa dipastikan — buat manual, lalu:" + echo " ./scripts/verify-phase2-storage.sh" + fi return 0 ;; esac @@ -719,6 +779,12 @@ bootstrap_starrocks() { # ── 2. Security integration OIDC (Keycloak) ───────────────────────────── echo -e "${INFO} [2/6] Security integration OIDC..." + # redirect_url = URL yang dibuka BROWSER (host FE + port HTTP publik) — bukan localhost/ + # nama container. Keycloak memakai nilai ini LITERAL sebagai redirect_uri. + local sr_redirect_addr sr_redirect_url + sr_redirect_addr="$(bootstrap_browser_addr STARROCKS_FE_HOST)" + sr_redirect_url="http://${sr_redirect_addr}:${STARROCKS_HTTP_PORT:-12030}/api/oauth2" + echo -e "${INFO} redirect_url OIDC StarRocks: ${sr_redirect_url}" if ! sr_mysql "SHOW SECURITY INTEGRATIONS;" | grep -qi "sr_oidc"; then # redirect_url dipakai LITERAL sebagai redirect_uri & harus reachable dari # browser (DBeaver membuka browser ke sini). JANGAN pakai hostname internal. @@ -728,7 +794,7 @@ bootstrap_starrocks() { 'token_server_url' = '${kc_internal}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/token', 'client_id' = 'starrocks', 'client_secret' = '${STARROCKS_CLIENT_SECRET}', - 'redirect_url' = 'http://localhost:${STARROCKS_HTTP_PORT:-12030}/api/oauth2', + 'redirect_url' = '${sr_redirect_url}', 'jwks_url' = '${kc_internal}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/certs', 'principal_field' = 'preferred_username', 'required_issuer' = '${kc_base}/realms/${SICABE_REALM:-sicabe}', @@ -756,7 +822,7 @@ bootstrap_starrocks() { 'ldap_cache_refresh_interval' = '900' );" && echo -e "${OK} Group provider 'sr_ldap' dibuat/disinkronkan." || echo -e "${WARN} CREATE GROUP PROVIDER gagal." sr_mysql "ALTER SECURITY INTEGRATION sr_oidc SET ( - 'redirect_url' = 'http://localhost:${STARROCKS_HTTP_PORT:-12030}/api/oauth2', + 'redirect_url' = '${sr_redirect_url}', 'group_provider' = 'sr_ldap', 'permitted_groups' = 'data-admin,data-engineer,data-analyst,data-viewer', 'auth_server_url' = '${kc_base}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/auth', @@ -1504,6 +1570,27 @@ step_tiers() { esac } +# Container yang menandai sebuah step BERJALAN di host ini. Dipakai mode `--host` +# untuk MELEWATI step yang komponennya dikelola SISTEM LAIN (tidak ada di bundle host +# ini) — mis. keycloak/nessie existing pada skenario P1/P3. Tanpa ini, `--host control` +# tetap mencoba bootstrap Keycloak meski Keycloak ada di host existing. +# Step 2 (MinIO) sengaja DIPERKECUALIKAN oleh pemanggil: bila storage dikelola sistem +# lain, step itu tetap jalan dalam MODE VERIFIKASI (bukan skip). +step_container() { + case "$1" in + 1) echo sicabe-keycloak ;; + 2) echo sicabe-minio ;; + 3) echo sicabe-openldap ;; + 4) echo sicabe-trino-coordinator ;; + 5) echo sicabe-starrocks-fe ;; + 6) echo sicabe-airflow-webserver ;; + 7) echo sicabe-openmetadata ;; + 8) echo sicabe-grafana ;; + 9) echo sicabe-dremio ;; + *) echo "" ;; + esac +} + # ============================================================================= main() { local START_STEP=1 @@ -1562,6 +1649,9 @@ main() { # sicabe_load_env(): tanpa ini sub-script menimpa endpoint host dengan deploy/.env # (nilai single-host) → menyentuh endpoint yang salah di multi-host. export SICABE_ENV_FILE="${henv}" + # Bundle host ini — dipakai sicabe_host_expects_container() agar tahu komponen mana + # yang dijalankan host ini (untuk melewati step milik sistem lain, lihat loop step). + export SICABE_BUNDLE_DIR="$(dirname "${henv}")" # Generator menulis daftar tier dengan pemisah KOMA (SICABE_HOST_TIERS= # identity,policy,pipeline,governance); pencocokan di bawah memakai spasi. HOST_TIERS="${SICABE_HOST_TIERS//,/ }" @@ -1624,6 +1714,17 @@ main() { echo "" continue fi + # Komponen dikelola SISTEM LAIN (tak ada di bundle host ini) → step tak berlaku. + # Step 2 (MinIO) DIPERKECUALIKAN: tetap jalan untuk memverifikasi bucket existing. + if [[ -n "${HOST_NAME}" && "${step}" != "2" ]]; then + local _step_ctr + _step_ctr="$(step_container "${step}")" + if [[ -n "${_step_ctr}" ]] && ! sicabe_host_expects_container "${_step_ctr}"; then + echo -e "${INFO} [skip] step ${step} — '${_step_ctr}' dikelola sistem lain di host ini" + echo "" + continue + fi + fi case ${step} in 1) bootstrap_keycloak ;; 2) bootstrap_minio ;; diff --git a/deploy/cluster.p1.yml b/deploy/cluster.p1.yml index b83941d..03726e6 100644 --- a/deploy/cluster.p1.yml +++ b/deploy/cluster.p1.yml @@ -8,6 +8,8 @@ # - Cluster (control/compute/state) menjalankan SISANYA dan memakai keycloak/openldap/ # minio/nessie EXISTING lewat alamat DI ATAS + port yang dipublikasikan. # +# Runbook lengkap (teruji): docs/sicabe-p1-existing-runbook.md +# # CARA PAKAI # # di control (sumber .env cluster), TAMBAHKAN ke deploy/.env: # KEYCLOAK_INTERNAL_PORT=12080 # <-- port HOST keycloak existing (bukan 8080) diff --git a/deploy/cluster.p3.yml b/deploy/cluster.p3.yml new file mode 100644 index 0000000..74111d8 --- /dev/null +++ b/deploy/cluster.p3.yml @@ -0,0 +1,62 @@ +# ============================================================================= +# cluster.p3.yml — Skenario P3: komponen EXISTING milik PIHAK LAIN (host/IP sendiri) +# + DUA OpenLDAP independen (existing utk keycloak, cluster utk trino/starrocks). +# ----------------------------------------------------------------------------- +# Kondisi client (nyata): +# - keycloak, minio, nessie : EXISTING, dikelola pihak lain — masing-masing bisa di +# host/IP SENDIRI (di sini ditandai `components..address`). +# - openldap : DUA instance independen (analog postgres): +# · sisi existing : openldap LOKAL (untuk Keycloak) — DI LUAR +# inventory ini (setup single-host existing; JANGAN set LDAP_URL). +# · cluster : openldap di `control` (tier identity) → +# group-provider Trino/StarRocks. +# - postgres (utk keycloak/nessie): sisi existing, BUKAN milik cluster → tak di inventory. +# - postgres cluster : milik sendiri di `state`. +# +# Perbedaan dari P1: di P1 identity (keycloak+openldap+nessie) ada di host existing; di P3 +# `keycloak`+`nessie` eksternal (per-host) sementara `openldap` ada di KEDUA sisi (independen). +# Penanda eksternal kini `components..address` (tanpa entri host) → tak dijalankan cluster. +# +# SIMULASI: ketiga komponen existing boleh dijalankan di SATU host (docker compose + +# network sendiri) → isi `address` sama (mis. 100.93.54.108). REAL: isi alamat berbeda. +# +# .env cluster — override `*_INTERNAL_PORT` HANYA untuk komponen EKSTERNAL: +# KEYCLOAK_INTERNAL_PORT=12080 MINIO_INTERNAL_PORT=12900 NESSIE_INTERNAL_PORT=12120 +# JANGAN override OPENLDAP_INTERNAL_PORT — openldap cluster LOKAL (di `control`), port 389. +# (Plus: samakan secret existing — 5× *_CLIENT_SECRET, MINIO_ROOT_*; LDAP_ADMIN_PASSWORD +# per-sisi, TIDAK perlu disamakan.) +# +# Langkah lengkap simulasi: docs/sicabe-p3-simulation-runbook.md +# +# Verifikasi (tanpa akses admin): hanya memeriksa apakah keycloak/minio/nessie memenuhi +# kebutuhan (issuer, TLS, S3, secret) — lihat docs/sicabe-p1-existing-runbook.md §7. +# ============================================================================= + +version: 1 +cluster: + name: sicabe-p3 + env: simulation + +tls: + mode: none + +preset: a3 + +hosts: + # control mengelola OPENLDAP cluster (tier identity) — keycloak/nessie di-override eksternal. + control: + address: 100.112.76.124 + tiers: [identity, policy, pipeline, governance] + compute: + address: 100.123.123.71 + tiers: [coord, engine] + state: + address: 100.73.229.107 + tiers: [state] + +components: + # EXISTING (pihak lain) — TANPA host; endpoint = alamat ini di semua host. + keycloak: { address: 100.93.54.108 } + minio: { address: 100.93.54.108 } + nessie: { address: 100.93.54.108 } + dremio: { enabled: false } diff --git a/deploy/config/airflow/dags/example_sicabe_pipeline.py b/deploy/config/airflow/dags/example_sicabe_pipeline.py index 6777c19..c7ce234 100644 --- a/deploy/config/airflow/dags/example_sicabe_pipeline.py +++ b/deploy/config/airflow/dags/example_sicabe_pipeline.py @@ -93,8 +93,11 @@ default_args = { "depends_on_past": False, "email_on_failure": True, "email_on_retry": False, - "retries": 2, - "retry_delay": timedelta(minutes=5), + # Retries default 2 (produksi). Dapat di-override via env, mis. `-e SICABE_DAG_RETRIES=0` + # pada `airflow dags test` — penting agar uji/uji-E2E TIDAK menggantung menunggu retry + # saat task GE gagal (mis. DAG #1 pada cold start: entitas gold belum ada di OM). + "retries": int(os.getenv("SICABE_DAG_RETRIES", "2")), + "retry_delay": timedelta(seconds=int(os.getenv("SICABE_DAG_RETRY_DELAY_SECONDS", "300"))), "execution_timeout": timedelta(hours=2), } diff --git a/deploy/config/environment.defaults.sh b/deploy/config/environment.defaults.sh index a95e8f3..05c8a07 100644 --- a/deploy/config/environment.defaults.sh +++ b/deploy/config/environment.defaults.sh @@ -210,6 +210,10 @@ export TRINO_SVC_SEED_PASSWORD="${TRINO_SVC_SEED_PASSWORD:-svc_seed_dev}" export TRINO_SVC_DBT_PASSWORD="${TRINO_SVC_DBT_PASSWORD:-svc_dbt_dev}" export TRINO_SVC_AIRFLOW_PASSWORD="${TRINO_SVC_AIRFLOW_PASSWORD:-svc_airflow_dev}" export TRINO_SVC_OM_PASSWORD="${TRINO_SVC_OM_PASSWORD:-svc_om_dev}" +# Kunci HMAC tetap untuk `state` OAuth2 Trino (HMAC-SHA256). Default Trino = ACAK tiap start +# → sesi login OAuth2 yg sedang berjalan batal bila coordinator restart. Set tetap (acak per +# instalasi) agar stabil. Dipakai config-coordinator.properties lewat ${ENV:TRINO_OAUTH_STATE_KEY}. +export TRINO_OAUTH_STATE_KEY="${TRINO_OAUTH_STATE_KEY:-trino-oauth-state-dev-only}" # ─── StarRocks (Phase 4 — Serving) ────────────────────────────────────────── export STARROCKS_VERSION="${STARROCKS_VERSION:-4.0.13}" diff --git a/deploy/config/prometheus/targets/node.yml b/deploy/config/prometheus/targets/node.yml index d132df5..f65fe8a 100644 --- a/deploy/config/prometheus/targets/node.yml +++ b/deploy/config/prometheus/targets/node.yml @@ -23,9 +23,6 @@ - targets: ["host.docker.internal:9100"] labels: host: local -- targets: ["100.123.123.71:9100"] - labels: - host: darkworld # Contoh penambahan host (hapus komentar): # - targets: ["10.20.0.11:9100"] diff --git a/deploy/config/trino/config-coordinator.properties b/deploy/config/trino/config-coordinator.properties index 569faf2..33509bf 100644 --- a/deploy/config/trino/config-coordinator.properties +++ b/deploy/config/trino/config-coordinator.properties @@ -42,13 +42,19 @@ http-server.process-forwarded=${ENV:TRINO_PROCESS_FORWARDED} # Kalau di-hardcode, issuer di klaster multi-host tidak cocok dan token ditolak. http-server.authentication.oauth2.issuer=${ENV:KEYCLOAK_PUBLIC_URL}/realms/sicabe # Matikan OIDC discovery — Trino tidak bisa mengakses URL publik dari dalam container. -# Endpoint di-set manual: browser pakai URL publik, server-ke-server pakai keycloak:8080. +# Endpoint di-set manual: browser pakai URL publik (auth-url); server-ke-server (token/jwks) +# pakai KEYCLOAK_HOST:INTERNAL_PORT — di multi-host dgn Keycloak EKSTERNAL ini = ALAMAT host +# pemilik (mis. 100.93.54.108:12080). JANGAN literal `keycloak` (nama container, tak resolve +# di host lain → penukaran token gagal → "Authentication response could not be verified"). http-server.authentication.oauth2.oidc.discovery=false # Endpoint browser-facing (auth redirect) — harus URL publik juga http-server.authentication.oauth2.auth-url=${ENV:KEYCLOAK_PUBLIC_URL}/realms/sicabe/protocol/openid-connect/auth # Endpoint server-side (token exchange & JWKS) — reachable dari container -http-server.authentication.oauth2.token-url=http://keycloak:${ENV:KEYCLOAK_INTERNAL_PORT}/realms/sicabe/protocol/openid-connect/token -http-server.authentication.oauth2.jwks-url=http://keycloak:${ENV:KEYCLOAK_INTERNAL_PORT}/realms/sicabe/protocol/openid-connect/certs +http-server.authentication.oauth2.token-url=http://${ENV:KEYCLOAK_HOST}:${ENV:KEYCLOAK_INTERNAL_PORT}/realms/sicabe/protocol/openid-connect/token +http-server.authentication.oauth2.jwks-url=http://${ENV:KEYCLOAK_HOST}:${ENV:KEYCLOAK_INTERNAL_PORT}/realms/sicabe/protocol/openid-connect/certs +# Kunci HMAC tetap untuk `state` OAuth2 (default Trino = acak tiap coordinator start → +# sesi login batal saat restart). Nilai acak-per-instalasi dari .env. +http-server.authentication.oauth2.state-key=${ENV:TRINO_OAUTH_STATE_KEY} http-server.authentication.oauth2.client-id=trino http-server.authentication.oauth2.client-secret=${ENV:TRINO_CLIENT_SECRET} # Username Trino = preferred_username (bukan 'sub') agar cocok dengan uid LDAP diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 082fe09..5599559 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -483,6 +483,12 @@ services: # Port internal Keycloak untuk config Trino (config-coordinator.properties # memakai ${ENV:KEYCLOAK_INTERNAL_PORT}) — wajib ada di container. KEYCLOAK_INTERNAL_PORT: ${KEYCLOAK_INTERNAL_PORT:-8080} + # Host Keycloak untuk token-url/jwks-url (server-ke-server). Multi-host dgn Keycloak + # EKSTERNAL: ini = ALAMAT host pemilik (mis. 100.93.54.108) — bukan nama container + # `keycloak` yang tak bisa di-resolve → penukaran token gagal ("could not be verified"). + KEYCLOAK_HOST: ${KEYCLOAK_HOST:-keycloak} + # Kunci HMAC tetap untuk state OAuth2 Trino (default Trino acak tiap start). + TRINO_OAUTH_STATE_KEY: ${TRINO_OAUTH_STATE_KEY:-trino-oauth-state-dev-only} healthcheck: # Trino internal port adalah 8080 (http-server.http.port) test: ["CMD", "curl", "-f", "http://localhost:8080/v1/info"] diff --git a/deploy/install.sh b/deploy/install.sh index 6b41094..83d8ef3 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -69,7 +69,10 @@ TRINO_SVC_SEED_PASSWORD=$(openssl rand -hex 8) TRINO_SVC_DBT_PASSWORD=$(openssl rand -hex 8) TRINO_SVC_AIRFLOW_PASSWORD=$(openssl rand -hex 8) TRINO_SVC_OM_PASSWORD=$(openssl rand -hex 8) +TRINO_OAUTH_STATE_KEY=$(openssl rand -hex 32) +TRINO_OAUTH_STATE_KEY=$(openssl rand -hex 32) GRAFANA_ADMIN_PASSWORD=$(openssl rand -hex 8) +STARROCKS_ROOT_PASSWORD=$(openssl rand -hex 16) # OM 2.0.x menegakkan kebijakan password (min 8 karakter + huruf besar + huruf # kecil + digit + karakter spesial). `openssl rand -hex` hanya menghasilkan # 0-9a-f sehingga TIDAK memenuhi kebijakan itu — password harus dirakit agar @@ -859,8 +862,8 @@ run_host() { # hilang → mis. Airflow IsADirectoryError). Isi ditulis setelah .env host di-source. sicabe_ensure_file_mount "${dir}/config/trino/password.db" 644 echo -e "${WARN} Jalankan ini HANYA di host '${name}'." - echo -e "${WARN} Pilot berdampingan di host yang SAMA tidak cukup dipisah network/port:" - echo -e "${WARN} RAM tidak cukup untuk dua stack — matikan stack lama lebih dulu (README)." + echo -e "${WARN} Stack single-host & multi-host TIDAK untuk berjalan bersamaan di satu host:" + echo -e "${WARN} matikan stack lama lebih dulu sebelum menyalakan yang baru." # ── PRECEDENCE (bug nyata yang dilaporkan dari server) ────────────────────── # Compose memberi prioritas pada variabel **SHELL** di atas `--env-file`. Skrip ini # sudah men-source config/environment.defaults.sh, yang mengekspor @@ -905,9 +908,13 @@ run_host() { # over SSH. Dijalankan di host CONTROL (master installer). Target = semua host di # cluster.yml KECUALI host ini (dideteksi dari kecocokan alamat lokal) — atau satu # host saja via --host. Aman diulang (rsync idempoten). +# Tujuan default (--dest) = CERMIN lokasi repo ini di bawah $HOME (lihat +# sicabe_default_dest) — jadi nama repo TIDAK dipatok: `sicabe-core` (repo +# distribusi bersih) → `~/sicabe-core/deploy`. Timpa dengan `--dest`. sync_hosts() { - local inv="${SCRIPT_DIR}/cluster.yml" dest="~/sicabe-doc/deploy" user="${USER:-root}" \ + local inv="${SCRIPT_DIR}/cluster.yml" dest user="${USER:-root}" \ only_host="" do_delete=0 dry=0 + dest="$(sicabe_default_dest "${SCRIPT_DIR}")" shift while [[ $# -gt 0 ]]; do case "$1" in diff --git a/deploy/lib/cluster/generate.py b/deploy/lib/cluster/generate.py index 9e362b8..71053c4 100644 --- a/deploy/lib/cluster/generate.py +++ b/deploy/lib/cluster/generate.py @@ -456,6 +456,10 @@ def endpoint_lines(doc: dict, host: str, plan: dict) -> list[str]: placement = component_placements(doc) lines: list[str] = [] for var, component in ENDPOINT_VARS.items(): + ext = inventory.external_address(doc, component) + if ext: + lines.append(f"{var}={ext}") # EKSTERNAL (pihak lain) → alamat tetap + continue owners = placement.get(component) if not owners: continue @@ -492,6 +496,10 @@ def public_url_lines(doc: dict, plan: dict | None = None) -> list[str]: placement = component_placements(doc) resolved: dict[str, str] = {} for var, (component, tmpl) in PUBLIC_URL_ENDPOINTS.items(): + ext = inventory.external_address(doc, component) + if ext: + resolved[var] = tmpl.format(addr=ext) # eksternal → alamat tetap + continue owners = placement.get(component) if not owners: continue @@ -655,6 +663,10 @@ def url_endpoint_lines(doc: dict, host: str, plan: dict, env_path=None) -> list[ placement = component_placements(doc) lines: list[str] = [] for var, (component, template) in URL_ENDPOINT_VARS.items(): + ext = inventory.external_address(doc, component) + if ext: + lines.append(f"{var}={template.format(addr=ext, port=_endpoint_port(var, env_path))}") + continue owners = placement.get(component) if not owners: continue @@ -684,6 +696,30 @@ def render_config_map(doc: dict, host: str, plan: dict) -> list[str]: return args +def _rewrite_prometheus_scrapes(host_dir, doc, env_path) -> None: + """Ganti target scrape MinIO di prometheus.yml bila MinIO EKSTERNAL. + + Prometheus TIDAK mendukung substitusi env (beda dgn config Trino), dan + render-config.sh hanya memetakan hostname komponen yang dimiliki HOST LAIN — + komponen EKSTERNAL (`components.minio.address`, skenario P3) TAK dipetakan → + target tetap `minio:9000` (nama container) → scrape GAGAL dari host prometheus. + Di sini diganti `:`. + + MinIO se-host (single-host / P1) → tak diubah (`minio:9000`, satu jaringan Docker). + """ + yml = host_dir / "config" / "prometheus" / "prometheus.yml" + if not yml.is_file(): + return + ext = inventory.external_address(doc, "minio") + if not ext: + return + port = _source_env_value(env_path, "MINIO_INTERNAL_PORT") or "9000" + txt = yml.read_text(encoding="utf-8") + new = txt.replace("minio:9000", f"{ext}:{port}") + if new != txt: + yml.write_text(new, encoding="utf-8") + + def cmd_render(argv: list[str]) -> int: if len(argv) < 7: sys.stderr.write(__doc__ or "") @@ -797,6 +833,9 @@ def cmd_render(argv: list[str]) -> int: fh.write("# DIBANGKITKAN OTOMATIS dari cluster.yml oleh install.sh --generate\n") fh.write("# — jangan disunting tangan. Target node_exporter (job `node`) semua host.\n") yaml.safe_dump(node_targets_doc(plan), fh, sort_keys=False, allow_unicode=True) + # Prometheus tak mendukung substitusi env → target MinIO eksternal + # (`minio:9000`) di-rewrite di sini ke alamat:port publik. + _rewrite_prometheus_scrapes(host_dir, doc, env_path) if rc.returncode != 0: sys.stderr.write(f"[host-{host}] render-config GAGAL:\n{rc.stdout}{rc.stderr}") return 1 @@ -872,17 +911,42 @@ def _write_bundle_runtime_files(host_dir, env_path): elif not pw_json.is_file(): pw_json.write_text("{}\n", encoding="utf-8") pw_json.chmod(0o666) - # 2) Trino password.db — byte-identik dari sumber bila ada; kalau tidak, FILE kosong - # (yang penting BUKAN direktori; bootstrap mengisinya). - src_db = Path(env_path).resolve().parent / "config" / "trino" / "password.db" + # 2) Trino password.db — DIBANGKITKAN dari TRINO_SVC_*_PASSWORD di .env (service + # account, HTTP Basic/bcrypt). Dulu hanya MENYALIN config/trino/password.db + # sumber — di mode cluster sumber itu sering KOSONG → bundle kosong → Trino + # menolak dgn 401 "Access Denied: Invalid credentials" (mis. dbt `svc_dbt`). + # Kini bundle MANDIRI & konsisten setelah --sync. Fallback: salin sumber bila + # htpasswd tak tersedia; kalau tidak ada, file kosong (yang penting BUKAN direktori). dst_db = cfg / "trino" / "password.db" dst_db.parent.mkdir(parents=True, exist_ok=True) if dst_db.is_dir(): shutil.rmtree(dst_db) - if src_db.is_file(): - shutil.copy2(src_db, dst_db) - elif not dst_db.is_file(): - dst_db.write_text("", encoding="utf-8") + svc_accounts = ( + ("svc_seed", "TRINO_SVC_SEED_PASSWORD"), + ("svc_dbt", "TRINO_SVC_DBT_PASSWORD"), + ("svc_airflow", "TRINO_SVC_AIRFLOW_PASSWORD"), + ("svc_om", "TRINO_SVC_OM_PASSWORD"), + ) + pw_lines = [] + htpasswd = shutil.which("htpasswd") + if htpasswd: + for user, key in svc_accounts: + secret = _source_env_value(env_path, key) + if not secret: + continue + proc = subprocess.run([htpasswd, "-nbB", "-C", "10", user, secret], + capture_output=True, text=True) + if proc.returncode == 0 and proc.stdout.strip(): + pw_lines.append(proc.stdout.strip()) + if pw_lines: + dst_db.write_text("\n".join(pw_lines) + "\n", encoding="utf-8") + dst_db.chmod(0o644) + else: + src_db = Path(env_path).resolve().parent / "config" / "trino" / "password.db" + if src_db.is_file(): + shutil.copy2(src_db, dst_db) + elif not dst_db.is_file(): + dst_db.write_text("", encoding="utf-8") def main(argv: list[str]) -> int: diff --git a/deploy/lib/cluster/inventory.py b/deploy/lib/cluster/inventory.py index e1c14de..57db6b2 100644 --- a/deploy/lib/cluster/inventory.py +++ b/deploy/lib/cluster/inventory.py @@ -127,14 +127,36 @@ def apply_preset(doc: dict) -> dict: return doc +def external_components(doc: dict) -> set[str]: + """Komponen yang dikelola SISTEM LAIN: ditandai `components..address`. + + Komponen ini TIDAK dijalankan cluster (tak masuk `placement`/`plan`); endpoint-nya + diarahkan ke alamat itu di SEMUA host (lihat generate.endpoint_lines). Berguna untuk + IAM/objek-storage milik pihak lain, termasuk di host/IP yang berbeda satu per komponen. + """ + return {n for n, s in (doc.get("components") or {}).items() + if (s.get("address") or "").strip()} + + +def external_address(doc: dict, service: str) -> str | None: + """Alamat eksternal komponen (components..address) bila ada, selain itu None.""" + spec = (doc.get("components") or {}).get(service) or {} + a = (spec.get("address") or "").strip() + return a or None + + def resolve(doc: dict) -> dict[str, list[str]]: - """Komponen → daftar host (tier `engine` bisa >1 host).""" + """Komponen → daftar host (tier `engine` bisa >1 host). Komponen EKSTERNAL + (ber-`address`) TIDAK masuk hasil — ia bukan milik host mana pun.""" hosts = doc.get("hosts") or {} tiers_by_service = {svc: tier for tier, svcs in TIER_SERVICES.items() for svc in svcs} explicit = {name: spec.get("host") for name, spec in (doc.get("components") or {}).items()} + external = external_components(doc) placement: dict[str, list[str]] = {} for service, tier in tiers_by_service.items(): + if service in external: + continue # eksternal (pihak lain) → bukan milik host mana pun if service in explicit and explicit[service]: placement[service] = [explicit[service]] continue @@ -246,18 +268,23 @@ def validate(doc: dict) -> list[str]: components = doc.get("components") or {} for name, spec in components.items(): host = spec.get("host") + addr = (spec.get("address") or "").strip() if host and host not in hosts: errors.append(f"komponen '{name}': host '{host}' tidak ada di 'hosts'") + if host and addr: + errors.append(f"komponen '{name}': 'host' dan 'address' tak boleh bersamaan " + f"(host = dijalankan cluster; address = dikelola sistem lain)") if tier_of(name) is None and name not in ("dremio",): errors.append(f"komponen '{name}' tidak ada di taksonomi tier (TIER_SERVICES)") # (c) tiap komponen harus bisa ditempatkan placement = resolve(doc) for service in (svc for svcs in TIER_SERVICES.values() for svc in svcs): - if service not in placement: + if service not in placement and not external_address(doc, service): errors.append( f"komponen '{service}' tidak bisa ditempatkan — tidak ada host bertier " - f"'{tier_of(service)}' dan tidak ada components.{service}.host" + f"'{tier_of(service)}', tidak ada components.{service}.host, " + f"dan bukan eksternal (components.{service}.address)" ) # stateful: cegah pindah host tanpa migrasi (peringatan, bukan error) @@ -331,6 +358,8 @@ def cmd_components(path: str) -> int: doc = apply_preset(load(path)) for service, owners in sorted(resolve(doc).items()): print(f"{service:24s} -> {','.join(owners)}") + for service in sorted(external_components(doc)): + print(f"{service:24s} -> (eksternal {external_address(doc, service)})") return 0 diff --git a/deploy/lib/common.sh b/deploy/lib/common.sh index cd256be..53ab5b7 100644 --- a/deploy/lib/common.sh +++ b/deploy/lib/common.sh @@ -145,6 +145,24 @@ sicabe_load_env() { return 0 } +# ─── Helper: tujuan rsync default = CERMIN lokasi repo ini di bawah $HOME ──── +# Dipakai `install.sh --sync` dan `scripts/check-trino-keystore.sh` supaya nama +# repo TIDAK dipatok (`~/sicabe-doc/deploy`). Repo distribusi bersih (mis. +# `sicabe-core` — isinya hanya folder `deploy/`) pun bekerja tanpa `--dest`: +# /home/u/sicabe-doc/deploy → ~/sicabe-doc/deploy +# /home/u/sicabe-core/deploy → ~/sicabe-core/deploy +# Bila repo di luar $HOME → ~//deploy (fallback). +# Mengembalikan path ber-`~` (diekspansi shell REMOTE saat rsync/SSH). +sicabe_default_dest() { # (default: $SCRIPT_DIR) + local _sdd_dir="${1:-${SCRIPT_DIR:-}}" _sdd_home="${HOME:-}" + [[ -n "${_sdd_dir}" ]] || { echo ""; return 1; } + if [[ -n "${_sdd_home}" && "${_sdd_dir}" == "${_sdd_home}/"* ]]; then + echo "~/${_sdd_dir#"${_sdd_home}/"}" + else + echo "~/$(basename "$(dirname "${_sdd_dir}")")/deploy" + fi +} + # ─── Helper: URL endpoint efektif (sadar EXTERNAL & multi-host) ───────────── # Urutan: (1) eksplisit NON-localhost (komponen EXTERNAL) → pakai apa adanya; # (2) multi-host (bundle memuat SICABE_HOST_NAME) → scheme://:; diff --git a/deploy/output/a1-verifikasi-lock.txt b/deploy/output/a1-verifikasi-lock.txt deleted file mode 100644 index 224b30f..0000000 --- a/deploy/output/a1-verifikasi-lock.txt +++ /dev/null @@ -1,14 +0,0 @@ -# A1 (R18) — verifikasi pin sqlalchemy<2.1 tidak menggeser paket -# tanggal : 2026-10-01 21:04:37 -# metode : uv pip compile requirements.in --constraint -# base : apache/airflow:slim-3.3.1-python3.11 -# lock lama : 1188 baris · md5 5936cf19f36004fd -# lock baru : 1650 baris -# baris PAKET berbeda : 0 -# sqlalchemy : sqlalchemy==2.0.54 -> sqlalchemy==2.0.54 -# -# KESIMPULAN -# 0 paket berubah — pin ini PREVENTIF (mencegah regenerasi lock mendatang -# menarik SQLAlchemy 2.1). Lock dibiarkan utuh karena isi paketnya identik; -# perbedaannya hanya baris komentar anotasi constraint (-c /old.lock). -# Perlindungan ada di requirements.in (sumber), bukan di lock. diff --git a/deploy/output/b3-hasil-uji.md b/deploy/output/b3-hasil-uji.md deleted file mode 100644 index 52d2001..0000000 --- a/deploy/output/b3-hasil-uji.md +++ /dev/null @@ -1,114 +0,0 @@ -# Hasil Uji B3 — Titik Go/No-Go BATCH B (review R18) - -> **Tanggal**: 2026-10-01 · **Image diuji**: `sicabe/airflow:3.3.2-om203-py313` (ID `33990a6c2b88`) -> **Metode**: stack eksperimen terisolasi (`airflow332-exp`, port 12184/12185) + config & DAG -> produksi di-mount **read-only** + uji koneksi/OPA/dbt dari container sekali pakai. -> ⚠️ Produksi **tidak tersentuh** sepanjang uji. - ---- - -## 1. VERDICT: 🔴 **NO-GO untuk Python 3.13** - -BATCH B **tidak boleh dilanjutkan** dengan base Python 3.13. Penyebab tunggal: **dbt tidak -berfungsi** di Python 3.13 pada tumpukan dependensi kita — dan dbt adalah tulang punggung -lapisan silver/gold. - -**Yang tetap sahih**: seluruh hasil B3 lainnya, serta A1 (pin `sqlalchemy<2.1`), A2, dan B1. -**Yang perlu diulang**: hanya **B2** (build image) dengan base Python **3.11**. - ---- - -## 2. Yang LOLOS ✅ - -| Uji | Hasil | -|---|---| -| **DAG produksi (4 berkas, dari volume produksi)** | ✅ `dag_sitomat_gold_build.py`, `dag_sitomat_gold_build_prod.py` **impor bersih**. `sicabe_dq_gold_native.py` berhasil dimuat sampai tahap *generate workflow* (gagal hanya karena `openmetadata-server` tidak ter-resolve dari container uji saya — bukan masalah versi) | -| **Plugin OpenMetadata** | ✅ `/pluginsv2/api/v2/openmetadata/health-auth` → **200**; `/pluginsv2/` → **200** | -| **Trino** | ✅ `SELECT 1` → `[[1]]`; `SHOW TABLES FROM iceberg.bronze` → `orders`, `orders_incremental` | -| **MinIO** | ✅ bucket `['iceberg-data']` via `S3Hook` | -| **MySQL (klien eksternal)** | ✅ `('8.0.40', 'asdp')` via `MySQLdb`/`MySqlHook` | -| **PostgreSQL** | ✅ Postgres 15.18 via `PostgresHook` | -| **OPA** | ✅ aturan bronze utuh (`bronze_orders`: ddl/select/write) + keputusan INSERT bronze = **true** | -| **Metadata DB / migrasi** | ✅ `airflow db migrate` bersih; OM 2.0.3 (API `{"version":"2.0.3"}`) | - ---- - -## 3. Yang GAGAL 🔴 — dbt - -### 3.1 Gejala - -``` -$ dbt debug --profile trino --target prod -AttributeError: __class_getitem__ - mashumaro/core/meta/helpers.py:624 → typing_extensions.Annotated.__class_getitem__ - /usr/python/lib/python3.13/typing.py:548 -``` -`dbt compile` gagal dengan galat yang sama. - -⚠️ **`dbt --version` justru berhasil** — sehingga pemeriksaan dangkal (`import dbt`, -`dbt --version`) **tidak menangkap** kerusakan ini. - -### 3.2 Akar masalah & tiga versi yang diuji - -| `mashumaro` | dbt (`debug`/`compile`) | `pip check` | -|---|---|---| -| **3.10** (maksimum yang diizinkan `collate-data-diff <3.11`) | ❌ **GAGAL** | ✅ **bersih** ← *rusak, tapi terlihat sehat* | -| **3.17** | ✅ **debug OK ("All checks passed!") + compile OK** (SQL `orders_silver` ter-compile) | ⚠️ melanggar `dbt-core <3.15` **dan** `collate-data-diff <3.11` | -| **3.23** | ✅ berhasil | ⚠️ melanggar keduanya | - -### 3.3 Kesimpulan teknis - -**Tidak ada versi `mashumaro` yang sekaligus (a) memenuhi batas dependensi kita dan -(b) berfungsi di Python 3.13.** Yang mengikat ketat adalah `collate-data-diff` (dipakai jalur -lineage dbt OpenMetadata) yang menuntut `<3.11.0`, sedangkan perbaikan kompatibilitas 3.13 -berada di versi ≥ ~3.11. - -➡️ Jadi masalahnya bukan pilihan paket, melainkan **kombinasi Python 3.13 + tumpukan ini**. - -### 3.4 Pelajaran metodologis (penting) - -`pip check` **bersih** pada konfigurasi yang **tidak berfungsi**. Artinya: -> Batas dependensi yang dideklarasikan ≠ kebenaran fungsional. - -Uji kelayakan untuk tumpukan ini **wajib menjalankan perintah nyata** (mis. `dbt debug`, -`dbt compile`), bukan hanya mengimpor modul atau membaca `--version`. - ---- - -## 4. Koreksi atas rekomendasi K1 - -Rekomendasi awal saya (K1 = Python **3.13**) dinilai **salah** dan dicabut. Dasarnya waktu itu -adalah "artefak eksperimen sudah tervalidasi end-to-end" — padahal validasi itu **tidak -lengkap**: eksperimen memeriksa `import dbt` dan `dbt --version`, tetapi **tidak pernah -menjalankan `dbt` sungguhan**. B3 — yang memang dirancang sebagai gerbang — yang menemukannya. - -**K1 direvisi menjadi Python 3.11**, dengan alasan: -- tujuan BATCH B adalah **Airflow 3.3.2 + OM 2.0.3 + provider MySQL**, bukan Python 3.13; -- `openmetadata-ingestion[airflow]` 2.0.3.0 mem-pin `apache-airflow==3.3.2` **terlepas dari - Python** → 3.3.2 di Python 3.11 tetap koheren; -- Python 3.11 mempertahankan seluruh dependensi lain di set yang **sudah terbukti di produksi** - → risiko jauh lebih rendah. - ---- - -## 5. Langkah berikutnya (B2 revisi) - -| Sub | Isi | -|---|---| -| B2r.1 | `Dockerfile.332` → base `slim-3.3.2-python3.11` (stage builder `mysqlclient` disesuaikan dengan kondisi header Python 3.11 pada base itu) | -| B2r.2 | Regenerasi lock 3.3.2/py3.11 **dengan lock 3.3.1 sebagai `--constraint`** (platform identik → diff minimal; teknik yang terbukti di A1) | -| B2r.3 | Build ulang image (tag tetap `sicabe/airflow:3.3.2-om203-py313`, menimpa image yang no-go) | -| B3r | **Ulangi B3 dengan `dbt debug` + `dbt compile` sebagai syarat WAJIB** (bukan opsional) | - ---- - -## 6. Catatan kesalahan uji saya (agar tidak terulang) - -| # | Kesalahan | Akibat | -|---|---|---| -| 1 | Memakai `airflow dags list` untuk menguji folder yang di-mount | `dags list` membaca **metadata DB**, bukan folder → uji pertama tidak sah (diperbaiki dengan impor langsung tiap berkas DAG) | -| 2 | Menguji DAG dari container yang tidak berada di jaringan yang tepat | `openmetadata-server` tidak ter-resolve → kegagalan palsu | -| 3 | Mount dbt **read-only** | dbt perlu menulis `logs/` → keluar `exit 2` **tanpa keluaran sama sekali** (menyesatkan) | -| 4 | `pip install "mashumaro>=3.9,<3.18"` tanpa `--upgrade` | pip tidak menaikkan (3.10 sudah memenuhi rentang) → uji sempat tidak konklusif | -| 5 | Salah mount point volume config OM | kegagalan palsu pada DAG DQ | -| 6 | Pemeriksaan awal hanya `import dbt` / `dbt --version` | kerusakan dbt lolos sampai B3 | diff --git a/deploy/output/b3r-hasil-uji.md b/deploy/output/b3r-hasil-uji.md deleted file mode 100644 index 380a9ef..0000000 --- a/deploy/output/b3r-hasil-uji.md +++ /dev/null @@ -1,72 +0,0 @@ -# Hasil B3 ULANG — Go/No-Go BATCH B pada Image Python 3.11 (review R18) - -> **Tanggal**: 2026-10-01 · **Image diuji**: `sicabe/airflow:3.3.2-om203` (ID `a02620e1d0a8`) -> — hasil **B2 revisi** (base Python 3.11), menggantikan image py3.13 yang ditolak. -> **Metode**: sama dengan B3 pertama — stack eksperimen terisolasi + config & DAG produksi -> read-only + container sekali pakai untuk koneksi/dbt. ⚠️ Produksi tidak tersentuh. - ---- - -## 1. VERDICT: 🟢 **GO** - -BATCH B boleh dilanjutkan ke B4. Penyebab no-go sebelumnya (dbt mati di Python 3.13) -**teratasi** dengan kembali ke Python 3.11. - ---- - -## 2. Hasil lengkap - -| # | Uji | Hasil | -|---|---|---| -| 1 | `airflow version` | ✅ **3.3.2** | -| 2 | **`dbt debug`** | ✅ **"Connection test: OK connection ok" + "All checks passed!"** | -| 3 | **`dbt compile --select orders_silver`** | ✅ compiles (SQL model terbentuk) | -| 4 | DAG produksi `dag_sitomat_gold_build.py` | ✅ impor bersih | -| 5 | DAG produksi `dag_sitomat_gold_build_prod.py` | ✅ impor bersih | -| 6 | DAG `sicabe_dq_gold_native.py` | ✅ dimuat sampai tahap *generate workflow*; gagal hanya karena `openmetadata-server` tidak ter-resolve dari container uji (bukan masalah versi) | -| 7 | `airflow dags list-import-errors` | ✅ kosong | -| 8 | Plugin OM `/pluginsv2/api/v2/openmetadata/health-auth` | ✅ **HTTP 200** | -| 9 | Plugin OM `/pluginsv2/` | ✅ **HTTP 200** | -| 10 | Trino (`TrinoHook`) | ✅ `SELECT 1` → `[[1]]`; bronze: `orders`, `orders_incremental` | -| 11 | MinIO (`S3Hook`) | ✅ bucket `['iceberg-data']`, `check_for_bucket` True | -| 12 | MySQL klien eksternal (`MySqlHook`) | ✅ `('8.0.40', 'asdp')`; tabel `passenger_productions`, `trx_booking` | -| 13 | OPA (aturan bronze `data-engineer`) | ✅ utuh (`bronze_orders`: ddl/select/write) | - -Verifikasi dari **build** (step #18–#24): 5 klien DB, `mysqlclient` C termuat + -`libmariadb.so.3`, provider mysql, `dbt + great_expectations`, `flask_appbuilder` + -**plugin OM dapat dimuat** (`RestApiPlugin`), `pip check` bersih. - ---- - -## 3. Ringkasan perjalanan B2 (tiga percobaan) - -| Percobaan | Masalah | Penyelesaian | -|---|---|---| -| **1** | Build memakai Python 3.13 → **dbt mati** (mashumaro/Annotated) | Diputuskan pindah ke Python 3.11 (lihat `b3-hasil-uji.md`) | -| **2** | Build py3.11: pip install 62 menit (jaringan PyPI ~60–80 kB/s) lalu gagal di **penjaga regresi yang saya tulis** | Penjaga dipanggil dengan 1 argumen → `TypeError: Annotated[...] should be used with at least two arguments`. Diperbaiki menjadi `te.Annotated[int, 'meta']`. ⚠️ Galat itu justru membuktikan jalur Annotated **ada** di py3.11 | -| **3** | — | ✅ **BERHASIL** (langkah pip ter-cache → selesai cepat) | - -Catatan operasional: build percobaan 1 sempat **dibatalkan** karena client `docker build` -ikut mati saat shell pengawas dihentikan (`nohup` tidak cukup — perlu `setsid`). - ---- - -## 4. Konsekuensi & langkah berikutnya - -- **Image target BATCH B**: `sicabe/airflow:3.3.2-om203` (ID `a02620e1d0a8`, 3.2 GB). -- Image py3.13 (`3.3.2-om203-py313`) **tidak dipakai** — disimpan sebagai bukti no-go. -- `Dockerfile.332` + `requirements.332.in/.lock` sudah final untuk py3.11; lock hanya - **44 baris** berbeda dari produksi. -- Berikutnya: **B4** — naikkan `AIRFLOW_IMAGE` + `OPENMETADATA_VERSION` di - `environment.defaults.sh` lalu `docker compose up -d`. -- Bukti rollback: `deploy/output/merge-airflow332-rollback.md` (dump sudah diuji pulih). - ---- - -## 5. Kesalahan uji saya pada B3r (agar tidak terulang) - -| # | Kesalahan | Akibat | -|---|---|---| -| 1 | Penjaga regresi `Annotated.__class_getitem__((int,))` — argumen kurang satu | **Positif palsu**: build gagal padahal sehat | -| 2 | Uji MinIO hanya meng-`source deploy/.env`, tanpa `environment.defaults.sh` | `MINIO_ROOT_USER` kosong → `NoCredentialsError` (bukan masalah image) | -| 3 | `pgrep -f` mencocokkan perintah saya sendiri | PID yang dilaporkan tidak sahih saat memantau build | diff --git a/deploy/output/merge-airflow332-rollback.md b/deploy/output/merge-airflow332-rollback.md deleted file mode 100644 index 878b1be..0000000 --- a/deploy/output/merge-airflow332-rollback.md +++ /dev/null @@ -1,178 +0,0 @@ -# Rollback — Merge Eksperimen `airflow332` ke `deploy/` (BATCH B / review R18) - -> **Dibuat**: 2026-10-01 · **Oleh**: langkah **B1** dari `reviews/sicabe-18-merge-airflow332-plan.md` -> **Status**: 🟢 **Backup siap & TERBUKTI DAPAT DIPULIHKAN.** BATCH B **belum dijalankan**. -> ⚠️ **Berkas ini adalah satu-satunya jalan pulang** untuk langkah B4 (migrasi OpenMetadata -> bersifat **satu arah** — lihat `docs/sicabe-install-phases-detail.md` §8.11.2). - ---- - -## 1. Keadaan yang dibekukan (baseline sebelum BATCH B) - -| Item | Nilai | -|---|---| -| Tanggal baseline | 2026-10-01 | -| Airflow | **3.3.1** (`sicabe/airflow:3.3.1-om202`) | -| OpenMetadata | **2.0.2** (`openmetadata/server:2.0.2`) | -| OpenSearch | `opensearchproject/opensearch:3.8.0` | -| Gate `test-verification.sh` | **60 PASS / 0 FAIL / 2 SKIP** | -| Container produksi | 20 (Airflow webserver/scheduler/dag-processor, OM, OpenSearch, dst) | - ---- - -## 2. Berkas backup - -Lokasi: `deploy/output/merge-airflow332-backup/` — **gitignored** (berisi data produksi). - -| Berkas | Isi | Ukuran | MD5 | -|---|---|---|---| -| `openmetadata_db.dump` | dump `openmetadata_db` (format `-Fc`) | 1.9 MB | `a8495339165cfbc286ea7e90490c12bb` | -| `airflow.dump` | dump `airflow` (format `-Fc`) | 248 KB | `4a834cc43307b296b58c250909749d2a` | -| `environment.defaults.sh.before-B` | salinan `deploy/config/environment.defaults.sh` | 22 KB | `2d341e008af43a7b` | - -Ukuran DB saat baseline: `openmetadata_db` = **36 MB**, `airflow` = **12 MB**. - ---- - -## 3. Bukti backup DAPAT DIPULIHKAN (dua kali diuji) - -| Uji | Hasil | -|---|---| -| Restore `openmetadata_db` → DB coretan `_restorecheck_om` | **204 tabel** asli = **204 tabel** hasil restore ✅ | -| Restore `airflow` → DB coretan `_restorecheck_af` | **5 DAG** asli = **5 DAG** hasil restore ✅ | -| DB coretan setelah uji | dihapus ✅ (produksi tidak tersentuh: 20 container, OM HTTP 200) | - -Perintah yang dipakai (nilai persis): -```bash -# contoh untuk openmetadata_db -docker exec -e PGPASSWORD= sicabe-postgres \ - psql -U sicabe_admin -d postgres -c "CREATE DATABASE _restorecheck_om OWNER openmetadata;" -docker cp deploy/output/merge-airflow332-backup/openmetadata_db.dump sicabe-postgres:/tmp/check.dump -docker exec -e PGPASSWORD=<…> sicabe-postgres \ - pg_restore -U sicabe_admin -d _restorecheck_om --no-owner --no-privileges /tmp/check.dump -docker exec -e PGPASSWORD=<…> sicabe-postgres \ - psql -U sicabe_admin -d postgres -c "DROP DATABASE _restorecheck_om;" -``` - ---- - -## 4. Digest image (untuk kembali ke versi lama) - -| Image | Image ID (sha256, dipotong) | -|---|---| -| `sicabe/airflow:3.3.1-om202` (produksi saat ini) | `afc0421a0490…` | -| `sicabe/airflow:3.3.1-om202-presqlalchemy` (tag rollback A1) | `afc0421a0490…` (identik — A1 tidak rebuild) | -| **`sicabe/airflow:3.3.2-om203`** (target BATCH B — Python 3.11, hasil B2 revisi) | `a02620e1d0a8…` (3.2 GB) | -| ~~`sicabe/airflow:3.3.2-om203-py313`~~ — **DITOLAK** di B3 (dbt mati di Python 3.13) | `33990a6c2b88…` (tidak dipakai) | -| `openmetadata/server:2.0.2` | `3b6f52aeb30f…` | -| `openmetadata/server:2.0.3` (target BATCH B) | `5f4d52bca2d7…` | -| `opensearchproject/opensearch:3.8.0` | `fafe3fc35870…` | - ---- - -## 5. PROSEDUR ROLLBACK - -### 5.1 Kasus paling umum — B4/B5 gagal, skema OM masih sehat - -Cukup kembalikan versi, tanpa menyentuh DB: - -```bash -cd /home/broto/sicabe-doc/deploy - -# (1) kembalikan versi ke baseline -# AIRFLOW_IMAGE -> sicabe/airflow:3.3.1-om202 -# OPENMETADATA_VERSION -> 2.0.2 -# (atau pulihkan berkas utuh: cp output/merge-airflow332-backup/environment.defaults.sh.before-B \ -# config/environment.defaults.sh) - -# (2) jalankan ulang -docker compose up -d airflow-webserver airflow-scheduler airflow-dag-processor \ - openmetadata-server openmetadata-migrate - -# (3) verifikasi -./test-verification.sh # harapan: 60 PASS / 0 FAIL / 2 SKIP -``` - -### 5.2 Kasus berat — migrasi OM 2.0.3 merusak skema - -⚠️ **Urutannya penting**: OM produksi harus **berhenti** dulu sebelum restore, kalau tidak -koneksi aktif akan mengganggu `DROP/CREATE DATABASE`. - -```bash -cd /home/broto/sicabe-doc/deploy - -# (1) HENTIKAN OpenMetadata (wajib, sebelum menyentuh DB-nya) -docker stop sicabe-openmetadata - -# (2) pulihkan versi image di config (OPENMETADATA_VERSION=2.0.2) -cp output/merge-airflow332-backup/environment.defaults.sh.before-B config/environment.defaults.sh - -# (3) restore openmetadata_db dari dump -docker exec -e PGPASSWORD= sicabe-postgres \ - psql -U sicabe_admin -d postgres -c "DROP DATABASE IF EXISTS openmetadata_db;" -docker exec -e PGPASSWORD=<…> sicabe-postgres \ - psql -U sicabe_admin -d postgres -c "CREATE DATABASE openmetadata_db OWNER openmetadata;" -docker cp output/merge-airflow332-backup/openmetadata_db.dump sicabe-postgres:/tmp/om.dump -docker exec -e PGPASSWORD=<…> sicabe-postgres \ - pg_restore -U sicabe_admin -d openmetadata_db --no-owner --no-privileges /tmp/om.dump - -# (4) mulai ulang dengan image LAMA -docker compose up -d openmetadata-server - -# (5) verifikasi -curl -s http://localhost:12085/api/v1/system/version # harapan: {"version":"2.0.2",...} -``` - -### 5.3 Bila metadata Airflow perlu dipulihkan (mis. migrasi DB Airflow bermasalah) - -```bash -docker stop sicabe-airflow-webserver sicabe-airflow-scheduler sicabe-airflow-dag-processor -docker exec -e PGPASSWORD=<…> sicabe-postgres psql -U sicabe_admin -d postgres \ - -c "DROP DATABASE IF EXISTS airflow;" -c "CREATE DATABASE airflow OWNER airflow;" -docker cp output/merge-airflow332-backup/airflow.dump sicabe-postgres:/tmp/af.dump -docker exec -e PGPASSWORD=<…> sicabe-postgres \ - pg_restore -U sicabe_admin -d airflow --no-owner --no-privileges /tmp/af.dump -docker compose up -d airflow-webserver airflow-scheduler airflow-dag-processor -``` - -### 5.4 Yang TIDAK perlu di-rollback - -- **OpenSearch** — indeksnya **derived**: OM membangun ulang isinya. Tidak ada dump khusus - untuk itu, dan itu memang disengaja. -- **`deploy-legacy/`** — di luar lingkup (K6). -- **Image** — tidak dihapus; dua versi hidup berdampingan, jadi rollback hanya soal menunjuk - tag lama. - ---- - -## 6. Setelah rollback - -1. Catat penyebab kegagalan di berkas ini (bagian bawah) atau di `reviews/sicabe-18…`. -2. Jangan jalankan B4 lagi sebelum penyebabnya dipahami. -3. Indeks/tabel OM hasil migrasi yang tertinggal **tidak** dibiarkan: pastikan restore - benar-benar menggantikan `openmetadata_db` (bukan menumpuk). - ---- - -## 7. Catatan keamanan - -⚠️ Berkas dump berisi **data produksi** (termasuk baris `user_entity` — hash kredensial, dan -mungkin data pribadi). Perlakuan: -- **gitignored** (`deploy/output/merge-airflow332-backup/`) — jangan pernah di-commit; -- hapus setelah BATCH B dinyatakan stabil (dan simpan hanya bila memang diperlukan sebagai - arsip, di lokasi terbatas akses). - ---- - -## Riwayat - -| Tanggal | Kejadian | -|---|---| -| 2026-10-01 | B1 dijalankan: dump dibuat, **diuji pulih (204=204 tabel; 5=5 DAG)**, digest & config dicatat, dokumen ini dibuat. BATCH B belum dimulai. | -| 2026-10-01 | **B2** dijalankan: image `sicabe/airflow:3.3.2-om203-py313` dibangun **dari `deploy/`** (berdampingan — `Dockerfile.332` + `requirements.332.{in,lock}`), ID `33990a6c2b88…`. Seluruh verifikasi fatal lolos: 5 klien DB, `mysqlclient` benar-benar termuat, `sqlalchemy 2.0.54` (pin A1), **plugin OM dapat dimuat**, `pip check` bersih. Produksi **tetap** `3.3.1-om202` — belum tersentuh. | -| 2026-10-01 | **B3 (go/no-go) = NO-GO** untuk Python 3.13: `dbt debug`/`dbt compile` gagal (mashumaro → `Annotated.__class_getitem__`). Tidak ada versi `mashumaro` yang memenuhi batas dependensi kita **dan** berfungsi di 3.13. Bukti: `b3-hasil-uji.md` | -| 2026-10-01 | **B2 revisi**: base dikembalikan ke **Python 3.11**; lock di-resolve ulang dengan lock 3.3.1 sebagai constraint **tersaring** (`^apache-airflow`, `^openmetadata-`, `^collate-` dibuang) → hanya **44 baris** berbeda dari produksi. Image baru: **`sicabe/airflow:3.3.2-om203`** ID `a02620e1d0a8…` (3.2 GB) | -| 2026-10-01 | **B3 ULANG = GO** 🟢: `dbt debug` **"All checks passed"**, `dbt compile` OK, DAG produksi impor bersih, plugin `/pluginsv2/` **200**, Trino/MinIO/MySQL OK, OPA utuh. Bukti: `b3r-hasil-uji.md`. Produksi tetap `3.3.1-om202` — siap untuk **B4** | -| 2026-10-02 | **B4 dijalankan (produksi naik)**: migrasi OM 2.0.3 **exit 0**; `airflow-init` (DB migrate 3.3.2) **exit 0**; OM server + 3 service Airflow di-recreate ke versi baru — semuanya **healthy**. Image: `sicabe/airflow:3.3.2-om203`, `openmetadata/server:2.0.3` | -| 2026-10-02 | **B5 verifikasi = LOLOS** ✅: `airflow version` **3.3.2**; OM API **2.0.3**; plugin `/pluginsv2/…health-auth` **200**; **0 error impor** DAG; `MySQLdb` + provider mysql tersedia di produksi; **GATE 60 PASS / 0 FAIL / 2 SKIP** (identik baseline). **B6 (rollback) tidak diperlukan.** | -| — | ⚠️ **Dump DISIMPAN** sampai dinyatakan stabil. Bila beberapa hari produksi berjalan normal: hapus `deploy/output/merge-airflow332-backup/` (§7) — atau arsipkan di lokasi terbatas akses | diff --git a/deploy/output/merge-om20x-rollback.md b/deploy/output/merge-om20x-rollback.md deleted file mode 100644 index 8d49bea..0000000 --- a/deploy/output/merge-om20x-rollback.md +++ /dev/null @@ -1,114 +0,0 @@ -# Runbook Rollback — Integrasi OpenMetadata 2.0.2 ke `deploy/` - -**Dibuat**: 2026-09-16 · **Tugas**: `merge-om20x` (subtask 01) -**Tag titik kembali**: `pre-merge-om20x` @ `5627353` - -Runbook ini adalah dokumen yang ditulis tangan (bukan keluaran uji otomatis), sengaja -diletakkan di `deploy/output/` mengikuti penugasan subtask 01. Isinya harus tetap -akurat selama integrasi berlangsung. - ---- - -## 1. Kapan runbook ini dipakai - -Saat integrasi ke OpenMetadata 2.0.2 + OpenSearch 3.x + Airflow ber-plugin gagal dan -kita memutuskan kembali ke kondisi 1.13.1 yang stabil (sebelumnya gate Fase 1-6 hijau, -`test-verification.sh` = 60 PASS / 0 FAIL / 2 SKIP). - -## 2. Kondisi baseline (terverifikasi sebelum perubahan apa pun) - -| Aspek | Nilai | -|---|---| -| git HEAD | `5627353` (tag `pre-merge-om20x`) | -| OpenMetadata server | **1.13.1** (`openmetadata/server:1.13.1`) | -| OpenSearch | **2.19.6** (`opensearchproject/opensearch:2.19.6`) | -| Airflow image | **`sicabe/airflow:3.3.1-slim`** (id `15ae04b69228`), Airflow 3.3.1 | -| Plugin OpenMetadata di Airflow | **TIDAK ADA** — `/api/v1/openmetadata/health` → 404 | -| Data OM | database `openmetadata_db` di volume `sicabe_postgres-data` | -| Data OpenSearch | **tidak di volume** — ada di layer container (index derived) | - -Catatan penting soal "plugin tidak ada": di Airflow deploy, -`/pluginsv2/api/v2/openmetadata/health-auth` menjawab **200 tetapi `content-type: text/html`** -(HTML aplikasi Airflow) — itu fallback UI, bukan respons plugin. Bukti plugin absen -adalah `/api/v1/openmetadata/health` → **404**. Jangan tertipu angka 200 di path -`/pluginsv2/`. - -## 3. Cara memverifikasi kita benar-benar sudah kembali ke baseline - -```bash -cd /home/broto/sicabe-doc -git rev-parse --short HEAD # harus 5627353 (atau turunannya yg setara) -docker inspect sicabe-openmetadata --format '{{.Config.Image}}' # openmetadata/server:1.13.1 -docker inspect sicabe-opensearch --format '{{.Config.Image}}' # opensearchproject/opensearch:2.19.6 -docker inspect sicabe-airflow-webserver --format '{{.Config.Image}}' # sicabe/airflow:3.3.1-slim -curl -s -o /dev/null -w '%{http_code}\n' -u "admin:$AIRFLOW_ADMIN_PASSWORD" \ - http://localhost:12082/api/v1/openmetadata/health # harus 404 -cd deploy && ./test-verification.sh # harus 60 PASS / 0 FAIL / 2 SKIP -``` - -## 4. Langkah rollback - -### 4.1 Kembalikan berkas konfigurasi - -```bash -cd /home/broto/sicabe-doc -git checkout pre-merge-om20x -- deploy/ docs/ -git status -s # tinjau: hanya berkas deploy/docs yang kembali -``` - -Jangan `git reset --hard` — commit hasil eksperimen (`f47945d`, `5627353`) dan -`experiments/om20x/` tidak perlu ikut hilang. - -### 4.2 Nyalakan ulang container lama - -```bash -cd deploy -set -a && . ./.env && set +a -docker compose --profile all up -d --force-recreate \ - openmetadata-opensearch openmetadata-migrate openmetadata-server \ - airflow-init airflow-webserver airflow-scheduler airflow-dag-processor -``` - -Image lama **tidak boleh dihapus**: `openmetadata/server:1.13.1`, -`opensearchproject/opensearch:2.19.6`, `sicabe/airflow:3.3.1-slim`. -OpenSearch akan membuat index baru dari nol saat container dibuat ulang -(tidak ada volume; index memang derived dan OM akan mem-populasinya kembali). - -### 4.3 Bila database OM sudah telanjur dikosongkan (subtask 12) - -Setelah `openmetadata_db` di-drop, **data katalog OM tidak bisa dipulihkan dari -deploy** — tidak ada dump (keputusan sadar: jaring pengaman cukup tag git). Yang bisa -dilakukan: membuat ulang database kosong lalu membiarkan OM 1.13.1 membangun skemanya, -lalu mengisi ulang katalog dari sumber yang reproducible: - -```bash -# buat ulang database kosong (pola sama dgn deploy/config/postgres/init/01-create-dbs.sh) -docker exec sicabe-postgres psql -U postgres -c "DROP DATABASE IF EXISTS openmetadata_db;" -docker exec sicabe-postgres psql -U postgres -c "CREATE DATABASE openmetadata_db OWNER openmetadata;" - -cd deploy -docker compose --profile all up -d --force-recreate openmetadata-migrate # migrate schema 1.13.1 -docker compose --profile all up -d openmetadata-server -./scripts/ingest-openmetadata.sh # katalog Trino + lineage dbt -# lalu jalankan DAG utk mengirim ulang hasil Great Expectations -``` - -## 5. Yang bisa dan tidak bisa dipulihkan (jujur) - -| Bisa dipulihkan | Tidak bisa dipulihkan | -|---|---| -| Versi komponen (kembali 1.13.1 / 2.19.6 / image slim) | Isi katalog OM bila DB sudah di-drop (reproducible, bukan mustahil) | -| Konfigurasi (`deploy/`, `docs/`) via tag git | Riwayat DQ (test case + hasil) — dibangun ulang oleh DAG | -| Lineage dbt | Dashboard/artefak yang hanya ada di OM | - -Alasan ini dapat diterima: seluruh isi katalog berasal dari -`scripts/ingest-openmetadata.sh` + DAG `sicabe_batch_daily`, jadi bisa dibangun ulang. - -## 6. Titik tidak-balik (penting) - -| Setelah subtask | Kondisi | -|---|---| -| 05 (+07) | Airflow memakai image ber-plugin; rollback = kembalikan 4 baris `AIRFLOW_IMAGE` | -| 10 | OpenSearch 3.x; index lama sudah hilang saat container dibuat ulang | -| **12** | **`openmetadata_db` di-drop → satu-satunya titik yang kehilangan data** | -| 13-15 | OM 2.0.2 jalan dgn password admin dari `.env` | diff --git a/deploy/output/r18-6-e2e-hasil.md b/deploy/output/r18-6-e2e-hasil.md deleted file mode 100644 index 9adf994..0000000 --- a/deploy/output/r18-6-e2e-hasil.md +++ /dev/null @@ -1,106 +0,0 @@ -# Hasil R18-6 — Uji End-to-End: OM membangkitkan DAG ingestion → Airflow 3.3.2 - -> **Tanggal**: 2026-10-01/02 · **Lingkungan**: PRODUKSI (Airflow 3.3.2 + OM 2.0.3) -> **Tujuan**: membuktikan jalur "OpenMetadata mengirim DAG ke Airflow" secara **utuh** — -> bukan hanya tingkat impor (itu sudah diukur di H7) — dan sekaligus menutup pertanyaan H7 -> secara definitif. -> ⚠️ Uji ini **membuat** pipeline sementara di OM produksi lalu **menghapusnya kembali**. - ---- - -## 1. VERDICT: 🟢 **BERHASIL PENUH — H7 tertutup secara definitif** - -| Tahap | Hasil | -|---|---| -| OM membuat ingestion pipeline (`metadata`) untuk `trino_sicabe` | ✅ dibuat | -| OM men-deploy DAG ke Airflow (lewat plugin `/pluginsv2/`) | ✅ **HTTP 200** | -| DAG muncul di Airflow 3.3.2 | ✅ `r18_e2e_ingestion.py` | -| **Error impor** | ✅ **0** (`No data found`) | -| Trigger dari OM menjalankan DAG | ✅ HTTP 200 (percobaan ke-2) | -| **`dag_run`** | ✅ **`success`** (17 detik) | -| **`task_instance` (`ingestion_task`)** | ✅ **`success`** | - -**Ringkasan dari log task (bukti terkuat):** - -``` -Processed records: 4 · Updated records: 0 · Errors: 0 · Success %: 100.0 -Processed records: 5 · Updated records: 0 · Errors: 0 · Success %: 100.0 -Workflow Success %: 100.0 -Workflow finished in time: 6s 668.015ms -Done. -``` - -➡️ **Kesimpulan**: paket yang hilang akibat melepas extra `[airflow]` -(`tornado`, `elasticsearch`, `opensearchpy`, `providers-elasticsearch`, -`providers-opensearch`) **tidak mengganggu jalur ingestion metadata**. H7 kini bukan lagi -"terukur di tingkat impor", melainkan **terbukti sampai eksekusi**. - ---- - -## 2. TEMUAN BARU — plugin memvalidasi LEBIH KETAT daripada server OM - -Ini temuan integrasi yang belum tercatat sebelumnya, dan menjelaskan kegagalan deploy yang -sempat terjadi tiga kali: - -``` -POST /v1/services/ingestionPipelines/deploy/ → 400 -{"error":"Extra parameter 'serviceName'"} -{"error":"Extra parameter 'markDeletedTablesFromFilterOnly'"} -{"error":"Extra parameter 'serviceConnections'"} -``` - -**Artinya**: pipeline yang **diterima** server OM saat dibuat (`POST /v1/services/ingestionPipelines` -→ 201) bisa **ditolak saat deploy** oleh plugin di sisi Airflow. Validasi plugin lebih ketat -daripada skema server. - -**Yang bekerja** — `sourceConfig.config` **paling minimal**: - -```json -{ "airflowConfig": { "scheduleInterval": "0 0 * * *", "retries": 0, "startDate": }, - "name": "", - "pipelineType": "metadata", - "service": { "id": "", "type": "databaseService" }, - "sourceConfig": { "config": { "type": "DatabaseMetadata" } } } -``` - -- **Jangan** menambahkan `serviceName`, `serviceConnections`, `includeTables`, - `markDeletedTablesFromFilterOnly`, dsb. pada `DatabaseMetadata` — ditolak sebagai - *extra parameter*. -- Koneksi diambil dari **service record** yang sudah ada (dibuat `bootstrap.sh`) — dan - **bekerja** ✅. (Catatan: ini berbeda dari jalur SDK yang punya masalah koneksi tersimpan - kehilangan secret → `401`; pada jalur OM-orchestrated koneksi tersimpan justru dipakai - dengan benar.) -- Untuk pipeline **TestSuite**, `serviceConnections` justru **diterima** (seeder DQ - memakainya) — jadi aturan "extra parameter" itu **per tipe pipeline**, bukan global. - ---- - -## 3. Perilaku lain yang terkonfirmasi - -| Perilaku | Bukti | -|---|---| -| **Trigger gagal di percobaan pertama**, berhasil di percobaan kedua | DAG baru dibuat → `dag-processor` belum selesai mem-parsing. Perlu jeda ±20 detik (seeder DQ sudah mengantisipasi ini) | -| `airflow dags list` membaca **metadata DB**, bukan folder | Sempat menyesatkan saat uji folder mount | -| CLI Airflow 3: `airflow dags list-runs` **bukan** bentuk yang valid | Status run paling andal dibaca dari tabel `dag_run`/`task_instance` di metadata DB | -| Menghapus pipeline (`hardDelete`) menghapus **berkas DAG** dari volume | File `r18_e2e_ingestion.py` hilang; catatan DAG di DB menjadi *stale* pada siklus parse berikutnya | - ---- - -## 4. Kebersihan setelah uji - -| Item | Status | -|---|---| -| Pipeline `r18_e2e_ingestion` di OM | ✅ **dihapus** (HTTP 200, `hardDelete`) | -| Berkas DAG di volume `airflow-dags` | ✅ **hilang** (kembali 4 berkas seperti semula) | -| Catatan DAG di metadata DB Airflow | ⏳ menjadi *stale* otomatis pada siklus parse berikutnya (berkasnya sudah tidak ada) | -| DAG produksi lain | ✅ tidak tersentuh | - ---- - -## 5. Kesalahan saya selama R18-6 (agar tidak terulang) - -| # | Kesalahan | Akibat | -|---|---|---| -| 1 | Menyusun `sourceConfig.config` dengan field lengkap ala UI | Deploy ditolak 3× (`Extra parameter …`) — seharusnya mulai dari yang **minimal** | -| 2 | Menganggap semua tipe pipeline menerima field yang sama | `serviceConnections` diterima di TestSuite, **ditolak** di DatabaseMetadata | -| 3 | Memakai `airflow dags list-runs` | Bentuk CLI berbeda di Airflow 3 → pakai tabel DB | diff --git a/deploy/scripts/check-trino-keystore.sh b/deploy/scripts/check-trino-keystore.sh new file mode 100755 index 0000000..79bb10e --- /dev/null +++ b/deploy/scripts/check-trino-keystore.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# ============================================================================= +# eBdesk SiCABE — Cek konsistensi keystore TLS Trino + password.db ANTAR-HOST +# +# Dua hal yang WAJIB benar agar klien (Great Expectations / OpenMetadata / dbt) +# bisa bicara ke Trino: +# 1) keystore TLS (generated/host-/config/trino/keystore.p12) IDENTIK di +# semua host → hasil SATU `--generate` + `--sync`. +# 2) password.db punya service account (svc_dbt) & cocok dgn TRINO_SVC_DBT_PASSWORD +# di bundle .env (kalau tidak → 401 "Access Denied: Invalid credentials"). +# Catatan: bcrypt memakai salt acak → berkas password.db TIDAK dibandingkan +# byte-per-byte antar-host, melainkan DIVERIFIKASI thd password yang diharapkan. +# +# Pemakaian: +# ./scripts/check-trino-keystore.sh # host ini saja +# ./scripts/check-trino-keystore.sh --inventory cluster.p3.yml +# ./scripts/check-trino-keystore.sh --expect # CI (exit 1 bila beda) +# +# Opsi: --host (paksa host ini) · --user · --dest +# default: user $USER, dest = cermin lokasi repo ini di bawah $HOME +# (mis. /home/u/sicabe-core/deploy → ~/sicabe-core/deploy) +# +# Keluar: 0 = semua OK · 1 = ada BEDA/MISSING · 2 = error (berkas/inventory tak ada) +# ============================================================================= +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${SCRIPT_DIR}" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/common.sh" 2>/dev/null || true +source "${SCRIPT_DIR}/config/environment.defaults.sh" 2>/dev/null || true +if [[ -f "${SCRIPT_DIR}/.env" ]]; then set -a; source "${SCRIPT_DIR}/.env"; set +a; fi + +INV=""; SSH_USER="${USER:-root}"; EXPECT=""; FORCE_HOST="" +# Tujuan remote default = cermin lokasi repo ini di bawah $HOME (bebas nama repo: +# `sicabe-core` → ~/sicabe-core/deploy). Fallback bila lib/common.sh gagal dimuat. +if declare -F sicabe_default_dest >/dev/null 2>&1; then + DEST="$(sicabe_default_dest "${SCRIPT_DIR}")" +else + DEST="~/$(basename "$(dirname "${SCRIPT_DIR}")")/deploy" +fi +while [[ $# -gt 0 ]]; do + case "$1" in + --inventory) INV="${2:-}"; shift 2 ;; + --user) SSH_USER="${2:-}"; shift 2 ;; + --dest) DEST="${2:-}"; shift 2 ;; + --expect) EXPECT="${2:-}"; shift 2 ;; + --host) FORCE_HOST="${2:-}"; shift 2 ;; + -h|--help) sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "Argumen tidak dikenal: $1" >&2; exit 2 ;; + esac +done + +C_RED=$'\033[31m'; C_GRN=$'\033[32m'; C_YEL=$'\033[33m'; C_BLD=$'\033[1m'; C_RST=$'\033[0m' + +# ── keystore ───────────────────────────────────────────────────────────────── +ks_path() { echo "${SCRIPT_DIR}/generated/host-$1/config/trino/keystore.p12"; } +ks_sha() { sha256sum "$1" 2>/dev/null | cut -d' ' -f1; } +ks_pw() { sed -n 's/^http-server.https.keystore.key=//p' \ + "${SCRIPT_DIR}/generated/host-$1/config/trino/config-coordinator.properties" 2>/dev/null | head -1; } +ks_fp() { openssl pkcs12 -in "$(ks_path "$1")" -nokeys -passin "pass:$(ks_pw "$1")" 2>/dev/null \ + | openssl x509 -noout -fingerprint -sha256 2>/dev/null | sed 's/^sha256 Fingerprint=//'; } + +# ── password.db (service account) ──────────────────────────────────────────── +pw_path() { echo "${SCRIPT_DIR}/generated/host-$1/config/trino/password.db"; } +pw_expect() { sed -n 's/^TRINO_SVC_DBT_PASSWORD=//p' \ + "${SCRIPT_DIR}/generated/host-$1/.env" 2>/dev/null | head -1; } +# Status lokal: NOFILE / MISSING / PWBAD / OK +pw_verify() { # + local f exp; f="$(pw_path "$1")" + [[ -f "${f}" ]] || { echo NOFILE; return; } + grep -q '^svc_dbt:' "${f}" 2>/dev/null || { echo MISSING; return; } + exp="$(pw_expect "$1")" + if command -v htpasswd >/dev/null 2>&1 && [[ -n "${exp}" ]]; then + htpasswd -vb "${f}" svc_dbt "${exp}" >/dev/null 2>&1 && echo OK || echo PWBAD + else + echo OK # ada, tapi tak bisa diverifikasi (htpasswd/pw tak ada) + fi +} + +# ── host ini ───────────────────────────────────────────────────────────────── +detect_self() { + local lip d addr; lip="$(hostname -I 2>/dev/null || true)" + for d in "${SCRIPT_DIR}"/generated/host-*; do + [[ -f "${d}/.env" ]] || continue + addr="$(sed -n 's/^SICABE_HOST_ADDRESS=//p' "${d}/.env" | head -1)" + [[ -n "${addr}" && " ${lip} " == *" ${addr} "* ]] && { basename "${d}" | sed 's/^host-//'; return 0; } + done + return 1 +} + +SELF="${FORCE_HOST}" +[[ -z "${SELF}" ]] && SELF="$(detect_self || true)" +if [[ -z "${SELF}" ]]; then + mapfile -t _dirs < <(ls -d "${SCRIPT_DIR}"/generated/host-* 2>/dev/null) + [[ ${#_dirs[@]} -eq 1 ]] && SELF="$(basename "${_dirs[0]}" | sed 's/^host-//')" +fi +if [[ -z "${SELF}" || ! -f "$(ks_path "${SELF}")" ]]; then + echo "${C_RED}Keystore host ini tak ditemukan.${C_RST} Jalankan ./install.sh --generate dulu, atau pakai --host ." >&2 + exit 2 +fi + +SELF_SHA="$(ks_sha "$(ks_path "${SELF}")")" +EXPECT_PW="$(pw_expect "${SELF}")" +echo "${C_BLD}Cek konsistensi keystore + password.db Trino${C_RST}" +echo " host ini : ${SELF}" +echo " keystore : sha256=${SELF_SHA}" +echo " cert : $(ks_fp "${SELF}")" +echo " pw svc_dbt: $(pw_verify "${SELF}") (OK/MISSING/PWBAD/NOFILE)" +echo "" + +if [[ -n "${EXPECT}" ]]; then + if [[ "${SELF_SHA}" == "${EXPECT}" ]]; then + echo "${C_GRN}✓ sha256 keystore cocok dengan --expect.${C_RST}"; exit 0 + fi + echo "${C_RED}✗ sha256 keystore BERBEDA dari --expect (${EXPECT}).${C_RST}"; exit 1 +fi + +if [[ -z "${INV}" ]]; then + echo " (tanpa --inventory — bandingkan di tiap host, atau pakai --inventory)" + exit 0 +fi + +[[ -f "${INV}" ]] || { echo "${C_RED}Inventory tak ada: ${INV}${C_RST}" >&2; exit 2; } +PAIRS="$(python3 "${SCRIPT_DIR}/lib/cluster/inventory.py" hosts "${INV}" 2>/dev/null)" \ + || { echo "${C_RED}Gagal membaca inventory: ${INV}${C_RST}" >&2; exit 2; } + +# Verifikasi password.db di host LAIN via SSH (bcrypt salt acak → verifikasi, bukan hash). +remote_pw_status() { # + ssh -o ConnectTimeout=8 -o BatchMode=yes "${SSH_USER}@${2}" \ + "f='${DEST}/generated/host-${1}/config/trino/password.db' +[ -f \"\$f\" ] || { echo NOFILE; exit 0; } +grep -q '^svc_dbt:' \"\$f\" || { echo MISSING; exit 0; } +if command -v htpasswd >/dev/null 2>&1; then htpasswd -vb \"\$f\" svc_dbt '${EXPECT_PW}' >/dev/null 2>&1 && echo OK || echo PWBAD; else echo OK; fi" \ + 2>/dev/null || echo SSHFAIL +} +pw_icon() { case "$1" in OK) printf '%s%s%s' "${C_GRN}" "$1" "${C_RST}" ;; *) printf '%s%s%s' "${C_RED}" "$1" "${C_RST}" ;; esac; } + +mismatch=0; checked=0 +while read -r name addr; do + [[ -z "${name}" ]] && continue + if [[ "${name}" == "${SELF}" ]]; then + _ksok=1; _pw="$(pw_verify "${SELF}")" + else + _rsha="$(ssh -o ConnectTimeout=8 -o BatchMode=yes "${SSH_USER}@${addr}" \ + "sha256sum ${DEST}/generated/host-${name}/config/trino/keystore.p12 2>/dev/null | cut -d' ' -f1" 2>/dev/null || true)" + if [[ -z "${_rsha}" ]]; then _ksok=-1; elif [[ "${_rsha}" == "${SELF_SHA}" ]]; then _ksok=1; else _ksok=0; fi + _pw="$(remote_pw_status "${name}" "${addr}")" + fi + _kslabel="(host ini)"; _kscol="" + case "${_ksok}" in + 1) [[ "${name}" != "${SELF}" ]] && { _kslabel="keystore ✓"; _kscol="${C_GRN}"; } ;; + 0) _kslabel="keystore ✗ BEDA"; _kscol="${C_RED}"; mismatch=1 ;; + -1) _kslabel="keystore ? ssh gagal"; _kscol="${C_YEL}"; mismatch=1 ;; + esac + [[ "${_pw}" == "OK" ]] || mismatch=1 + printf ' %-9s %-15s %s%-16s%s pw svc_dbt %s\n' \ + "${name}" "${addr}" "${_kscol}" "${_kslabel}" "${C_RST}" "$(pw_icon "${_pw}")" + checked=$((checked + 1)) +done <<< "${PAIRS}" + +echo "" +if [[ ${mismatch} -eq 0 ]]; then + echo "${C_GRN}✓ keystore identik + password.db OK di semua host (${checked} host).${C_RST}" + exit 0 +fi +echo "${C_RED}✗ Ada ketidakcocokan (keystore BEDA / password.db MISSING/PWBAD).${C_RST}" +echo " Perbaiki (di SATU host — control):" +echo " ./install.sh --generate --inventory ${INV} && ./install.sh --sync" +echo " lalu di tiap host: ./install.sh --host (atau recreate Trino & Airflow)" +exit 1 diff --git a/deploy/scripts/prepare-host.sh b/deploy/scripts/prepare-host.sh index 1966751..b4cd2ec 100755 --- a/deploy/scripts/prepare-host.sh +++ b/deploy/scripts/prepare-host.sh @@ -285,7 +285,7 @@ precheck() { ok "semua port 12xxx yang dibutuhkan masih bebas" else warn "port sudah terpakai:${busy}" - echo " Bila itu stack SiCABE yang lama: matikan dulu (RAM tidak cukup untuk dua stack)." + echo " Bila itu stack SiCABE yang lama: matikan dulu — stack single-host & multi-host tidak untuk berjalan bersamaan di satu host." fi else inf "daftar port tidak terbaca dari environment.defaults.sh — lewati pemeriksaan port" diff --git a/deploy/scripts/run-e2e.sh b/deploy/scripts/run-e2e.sh new file mode 100755 index 0000000..4c48833 --- /dev/null +++ b/deploy/scripts/run-e2e.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# ============================================================================= +# eBdesk SiCABE — Orkestrasi alur data E2E (satu perintah) — jalankan DI CONTROL. +# +# Menghilangkan langkah manual berulang pada cold start. Dua "chicken-egg" yang +# ditangani otomatis: +# (a) aksi Great Expectations menempel ke ENTITAS tabel di OM, tetapi +# `gold.orders_gold` DIBUAT oleh DAG → DAG #1 gagal di GE (wajar). +# (b) lineage dbt dibaca dari ARTEFAK (`manifest.json`+`catalog.json`+`run_results.json` +# di /tmp/dbt/target) yang DIBUAT oleh `dbt docs generate` — DAG hanya `dbt run` +# (tanpa catalog.json), dan artefak HILANG bila container Airflow di-recreate. +# +# Urutan: ingest metadata → DAG #1 (data) → dbt docs generate → ingest --all → +# DAG #2 (GE) → verify-phase6. +# +# PRASYARAT (sekali, DI HOST MASING-MASING — di luar skrip ini): +# (state) ./scripts/seed-source-postgres.sh --env-file generated/host-state/.env +# (compute) ./scripts/seed-starrocks.sh --env-file generated/host-compute/.env +# DQ yang bekerja = Jalur A (Great Expectations pada task validate_gold/cdc_validate di DAG). +# seed-native-dq.sh = Jalur B (DQ native OM) opt-in & TERBLOKIR bug upstream §8.10 — +# JANGAN dipakai di alur ini (menempel ke entitas gold yang baru ada setelah DAG+ingest). +# +# Pemakaian: ./scripts/run-e2e.sh [--env-file ] [--dag ] +# ============================================================================= +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${SCRIPT_DIR}" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/common.sh" 2>/dev/null || true + +ENV_FILE="${SCRIPT_DIR}/generated/host-control/.env" +DAG="${SICABE_DAG:-sicabe_batch_daily}" +WEB="${SICABE_AIRFLOW_WEB:-sicabe-airflow-webserver}" +while [[ $# -gt 0 ]]; do + case "$1" in + --env-file) ENV_FILE="${2:-}"; shift 2 ;; + --dag) DAG="${2:-}"; shift 2 ;; + -h|--help) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "Argumen tidak dikenal: $1" >&2; exit 2 ;; + esac +done +[[ -f "${ENV_FILE}" ]] || ENV_FILE="${SCRIPT_DIR}/.env" +[[ -f "${ENV_FILE}" ]] || { echo "env file tak ditemukan: ${ENV_FILE}" >&2; exit 2; } + +# Password service account dbt (untuk `dbt docs generate` di dalam container). +DBT_PW="$(sed -n 's/^TRINO_SVC_DBT_PASSWORD=//p' "${ENV_FILE}" | head -1)" +DBT_USER="${DBT_TRINO_USER:-svc_dbt}" + +step() { echo; echo -e "\033[1m▶ $*\033[0m"; } + +echo -e "\033[1mAlur data E2E (SiCABE)\033[0m" +echo " env-file : ${ENV_FILE}" +echo " DAG : ${DAG}" +echo " Prasyarat: seed sudah dijalankan di tiap host (state/compute)." + +step "1/6 ingest metadata (service + entitas Trino; TANPA dbt — artefak belum ada pra-DAG)" +./scripts/ingest-openmetadata.sh --env-file "${ENV_FILE}" \ + || echo " PERHATIAN: ingest metadata gagal (mis. tabel belum ada pra-DAG) — lanjut; di-ingest ulang di step 4." + +step "2/6 DAG #1 — buat data (bronze→silver→gold). GE boleh gagal: entitas gold belum di OM." +docker exec -e SICABE_DAG_RETRIES=0 "${WEB}" airflow dags test "${DAG}" \ + || echo " PERHATIAN: DAG #1 gagal (biasanya di GE karena gold belum di OM) — lanjut; wajar pada cold start." + +step "3/6 artefak dbt (manifest+catalog+run_results) untuk lineage" +echo " (wajib: DAG hanya 'dbt run' → catalog.json absen; artefak hilang bila container di-recreate)" +docker exec \ + -e DBT_TRINO_USER="${DBT_USER}" -e DBT_TRINO_PASSWORD="${DBT_PW}" \ + -e DBT_PROFILES_DIR=/opt/airflow/dbt -e DBT_TARGET_PATH=/tmp/dbt/target \ + -e DBT_LOG_PATH=/tmp/dbt/logs -w /opt/airflow/dbt/sicabe \ + "${WEB}" bash -lc 'dbt docs generate --profile trino --target prod' \ + || echo " PERHATIAN: dbt docs generate gagal — lineage dbt mungkin terlewat." + +step "4/6 ingest ulang — entitas gold + lineage dbt (artefak sudah ada)" +./scripts/ingest-openmetadata.sh --env-file "${ENV_FILE}" --all + +step "5/6 DAG #2 — GE kini menemukan entitas gold di OM (target: hijau)" +docker exec -e SICABE_DAG_RETRIES=0 "${WEB}" airflow dags test "${DAG}" + +step "6/6 verifikasi governance (target FAIL=0)" +./scripts/verify-phase6-governance.sh --env-file "${ENV_FILE}" + +echo +echo -e "\033[1;32m✓ Alur E2E selesai.\033[0m" diff --git a/deploy/scripts/seed-native-dq.sh b/deploy/scripts/seed-native-dq.sh index 35f7cf6..adf72b6 100755 --- a/deploy/scripts/seed-native-dq.sh +++ b/deploy/scripts/seed-native-dq.sh @@ -16,6 +16,10 @@ # # Idempoten: test case & pipeline yang sudah ada dilewati. # Dijalankan dari host; pekerjaannya dieksekusi DI DALAM container Airflow. +# +# PRASYARAT: entitas tabel `trino_sicabe.iceberg.gold.orders_gold` SUDAH ada di OM +# (setelah DAG + `ingest-openmetadata.sh --all`). Bila belum → worker gagal +# `HTTP 404: table instance ... not found`. # ============================================================================= set -euo pipefail @@ -66,4 +70,8 @@ done docker exec \ -e OPENMETADATA_ADMIN_PASSWORD="${OPENMETADATA_ADMIN_PASSWORD}" \ -e TRINO_SVC_OM_PASSWORD="${TRINO_SVC_OM_PASSWORD}" \ - "${AIRFLOW_CONTAINER}" python "${WORKER}" "${WORKER_ARGS[@]+"${WORKER_ARGS[@]}"}" + "${AIRFLOW_CONTAINER}" python "${WORKER}" "${WORKER_ARGS[@]+"${WORKER_ARGS[@]}"}" || { + echo " (catatan) Bila error 404 'table instance ... not found' → tabel gold belum ada di OM." >&2 + echo " Jalankan DAG + ingest-openmetadata dulu, atau pakai Jalur A (GE di DAG)." >&2 + exit 1 +} diff --git a/deploy/scripts/test-bootstrap-host.sh b/deploy/scripts/test-bootstrap-host.sh index 6dcf9d2..8345dd6 100755 --- a/deploy/scripts/test-bootstrap-host.sh +++ b/deploy/scripts/test-bootstrap-host.sh @@ -82,6 +82,54 @@ else bad "step_tiers() belum memetakan step:${missing}" fi +# step_container() juga harus memetakan step 1..9 (dipakai mode --host untuk melewati +# step yang komponennya dikelola SISTEM LAIN). +sc_patterns="$(awk '/^step_container\(\) \{/{f=1} f{print} f&&/^\}/{exit}' "${DEPLOY_DIR}/bootstrap.sh" \ + | grep -E '^[[:space:]]*[0-9]+\)' | sed 's/).*//' | tr -d ' ')" +sc_missing="" +for s in 1 2 3 4 5 6 7 8 9; do + grep -qx "${s}" <<<"${sc_patterns}" || sc_missing+=" ${s}" +done +if [[ -z "${sc_missing}" ]]; then + ok "step_container() memetakan step 1..9" +else + bad "step_container() belum memetakan step:${sc_missing}" +fi + +# Mode --host HARUS melewati step milik sistem lain (bukan mencoba bootstrap-nya), +# kecuali step 2 (MinIO → mode verifikasi). +if grep -q 'sicabe_host_expects_container "${_step_ctr}"' "${DEPLOY_DIR}/bootstrap.sh" \ + && grep -q '"${step}" != "2"' "${DEPLOY_DIR}/bootstrap.sh"; then + ok "bootstrap --host melewati step komponen eksternal (step 2 dikecualikan)" +else + bad "bootstrap --host tidak melewati step komponen eksternal" +fi + +# Bundle host harus di-set agar sicabe_host_expects_container membaca bundle yang benar. +if grep -q 'export SICABE_BUNDLE_DIR=' "${DEPLOY_DIR}/bootstrap.sh"; then + ok "SICABE_BUNDLE_DIR di-set pada mode --host" +else + bad "SICABE_BUNDLE_DIR tidak di-set (deteksi komponen per-host salah)" +fi + +# Step 2 (MinIO): buat bucket hanya bila MinIO milik host ini; storage sistem lain → verifikasi. +if grep -q 'mc_mode="create"' "${DEPLOY_DIR}/bootstrap.sh" \ + && grep -q 'mc_mode="verify"' "${DEPLOY_DIR}/bootstrap.sh"; then + ok "bootstrap_minio: create (lokal) vs verify (sistem lain)" +else + bad "bootstrap_minio tidak membedakan create vs verify" +fi + +# Sinkron redirectUris Keycloak harus memuat ORIGIN HOST client (bukan hanya localhost), +# kalau tidak login lewat alamat (mis. https://:12443) ditolak "invalid redirect_uri". +if grep -q 'CLIENT_ORIGIN' "${DEPLOY_DIR}/bootstrap.sh" \ + && grep -q '_client_origin' "${DEPLOY_DIR}/bootstrap.sh" \ + && grep -q 'host + "/\*"' "${DEPLOY_DIR}/bootstrap.sh"; then + ok "bootstrap: sinkron redirectUris memuat ORIGIN HOST client" +else + bad "bootstrap: sinkron redirectUris TIDAK memuat origin host (login via alamat ditolak)" +fi + echo echo "── B. penjagaan dinamis (butuh stack SiCABE berjalan) ──" if ! docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^sicabe-'; then diff --git a/deploy/scripts/test-generate.sh b/deploy/scripts/test-generate.sh index bf50381..ee47c80 100755 --- a/deploy/scripts/test-generate.sh +++ b/deploy/scripts/test-generate.sh @@ -109,6 +109,170 @@ for pair in "MINIO_IMAGE:quay.io/minio/minio" "MINIO_MC_IMAGE:quay.io/minio/mc" fi done +# ─── 0e. Komponen EKSTERNAL ber-`address` (skenario P3) ────────────────────── +# `components..address` = komponen dikelola SISTEM LAIN → TIDAK dijalankan cluster +# (tak muncul di services host mana pun) + endpoint-nya = alamat itu. Diuji via cluster.p3.yml. +p3_problems="$(python3 - "${DEPLOY_DIR}" <<'PY' +import sys +sys.path.insert(0, sys.argv[1] + "/lib/cluster") +import inventory +d = inventory.apply_preset(inventory.load(sys.argv[1] + "/cluster.p3.yml")) +pl = inventory.plan(d) +svc = {s for h in pl.values() for s in h["services"]} +p = [] +for c in ("keycloak", "minio", "nessie"): + if c in svc: + p.append(c + " masih dijalankan") + if not inventory.external_address(d, c): + p.append(c + ": address hilang") +if "openldap" not in pl.get("control", {}).get("services", []): + p.append("openldap tak di control") +print("; ".join(p)) +PY +)" +if [[ -z "${p3_problems}" ]]; then + ok "P3: components..address → eksternal tak dijalankan, openldap lokal di control" +else + bad "P3: perilaku components..address salah — ${p3_problems}" +fi + +# ─── 0f. Deteksi keystore Trino BEDA antar-host ────────────────────────────── +# Keystore Trino harus identik di semua host (satu --generate + --sync). Skrip ini +# mendeteksi divergensi (mis. --generate dijalankan di >1 host). +cks="${DEPLOY_DIR}/scripts/check-trino-keystore.sh" +if [[ -x "${cks}" ]] && bash -n "${cks}" 2>/dev/null \ + && grep -q -- '--inventory' "${cks}" && grep -q -- '--expect' "${cks}" \ + && grep -q 'keystore.p12' "${cks}" && grep -q 'password.db' "${cks}" \ + && grep -q 'svc_dbt' "${cks}"; then + ok "check-trino-keystore.sh: deteksi keystore + password.db antar-host (--inventory/--expect)" +else + bad "check-trino-keystore.sh hilang/tidak lengkap (divergensi keystore/password.db tak terdeteksi)" +fi +# Fungsional (bila bundle host sudah ada): cetak sha256 & hormati --expect. +_first_bundle="$(ls -d "${DEPLOY_DIR}"/generated/host-* 2>/dev/null | head -1)" +if [[ -n "${_first_bundle}" ]]; then + _bh="$(basename "${_first_bundle}" | sed 's/^host-//')" + _sha="$("${cks}" --host "${_bh}" 2>/dev/null | sed -n 's/^ keystore : sha256=//p')" + if [[ "${_sha}" =~ ^[0-9a-f]{64}$ ]]; then + ok "check-trino-keystore.sh: sha256 host '${_bh}' terbaca" + "${cks}" --host "${_bh}" --expect "${_sha}" >/dev/null 2>&1 \ + && ok "check-trino-keystore.sh: --expect cocok → exit 0" \ + || bad "check-trino-keystore.sh: --expect cocok seharusnya exit 0" + else + bad "check-trino-keystore.sh: sha256 tidak terbaca untuk host '${_bh}'" + fi +fi + +# ─── 0g. .env auto-generate memuat SEMUA var WAJIB compose ─────────────────── +# Regresi nyata: STARROCKS_ROOT_PASSWORD (dipakai `${VAR:?}` di compose) TIDAK dibuat +# install.sh → `docker compose --env-file .env` (tanpa shell env) gagal +# "required variable ... is missing a value". +_required="$(grep -oE '\$\{[A-Z_][A-Z0-9_]*:\?' "${DEPLOY_DIR}/docker-compose.yml" | sed 's/\${//;s/:?//' | sort -u)" +_envvars="$(awk '/^[[:space:]]*cat > .*\/\.env" </dev/null \ + && grep -q 'ingest-openmetadata.sh' "${e2e}" \ + && grep -q 'airflow dags test' "${e2e}" \ + && grep -q 'verify-phase6-governance.sh' "${e2e}"; then + ok "run-e2e.sh: orkestrasi alur data E2E (ingest→DAG→ingest→DAG→verify)" +else + bad "run-e2e.sh hilang/tidak lengkap" +fi + +# ─── 0i. Prometheus: target MinIO EKSTERNAL → alamat:port publik ───────────── +# Prometheus tak mendukung substitusi env, dan render-config hanya memetakan komponen +# milik host lain — komponen EKSTERNAL (components.minio.address) harus di-rewrite +# generate.py, kalau tidak target tetap `minio:9000` → scrape gagal dari host prometheus. +_tmp_prom="$(mktemp -d)"; _prom_ok="" +if [[ -f "${DEPLOY_DIR}/.env" ]]; then + cp "${DEPLOY_DIR}/.env" "${_tmp_prom}/e.env" + printf '\nMINIO_INTERNAL_PORT=12900\n' >> "${_tmp_prom}/e.env" + if python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \ + "${DEPLOY_DIR}/cluster.p3.yml" "${DEPLOY_DIR}/docker-compose.yml" \ + "${_tmp_prom}/e.env" "${_tmp_prom}/out" "${DEPLOY_DIR}/scripts/render-config.sh" \ + >/dev/null 2>&1; then + _pyml="${_tmp_prom}/out/host-control/config/prometheus/prometheus.yml" + if grep -q 'targets: \["100.93.54.108:12900"\]' "${_pyml}" 2>/dev/null \ + && ! grep -q 'targets: \["minio:9000"\]' "${_pyml}" 2>/dev/null; then + _prom_ok=1 + fi + fi + if [[ -n "${_prom_ok}" ]]; then + ok "prometheus: target MinIO eksternal → 100.93.54.108:12900 (bukan minio:9000)" + else + bad "prometheus: target MinIO eksternal salah (masih minio:9000?)" + fi +fi + +# ─── 0j. Verifikasi Keycloak lengkap (verify-keycloak.sh) ──────────────────── +vk="${DEPLOY_DIR}/scripts/verify-keycloak.sh" +if [[ -x "${vk}" ]] && bash -n "${vk}" 2>/dev/null \ + && grep -q 'client_credentials' "${vk}" && grep -q 'redirectUris' "${vk}" \ + && grep -q 'publicClient' "${vk}" && grep -q 'origin host' "${vk}" \ + && grep -q 'kc_auth_probe' "${vk}" && grep -q 'password masih DEFAULT' "${vk}"; then + ok "verify-keycloak.sh: realm/roles/client(redirectUri origin-host,secret,publicClient)/user/password-default + fungsional" +else + bad "verify-keycloak.sh hilang/tidak lengkap" +fi + +# ─── 0k. Trino OAuth2: token/jwks pakai KEYCLOAK_HOST (bukan literal `keycloak`) ── +# Kls bug sama dgn target MinIO Prometheus: Trino server-ke-server ke Keycloak. Bila +# Keycloak EKSTERNAL (P3), literal `keycloak:` tak resolve → penukaran token gagal → +# callback `/oauth2/callback` balas "Authentication response could not be verified". +_tc="${DEPLOY_DIR}/config/trino/config-coordinator.properties" +if grep -q 'token-url=http://${ENV:KEYCLOAK_HOST}:${ENV:KEYCLOAK_INTERNAL_PORT}' "${_tc}" 2>/dev/null \ + && grep -q 'jwks-url=http://${ENV:KEYCLOAK_HOST}:' "${_tc}" 2>/dev/null \ + && grep -q 'oauth2.state-key=${ENV:TRINO_OAUTH_STATE_KEY}' "${_tc}" 2>/dev/null \ + && grep -q 'KEYCLOAK_HOST: ${KEYCLOAK_HOST' "${DEPLOY_DIR}/docker-compose.yml" 2>/dev/null \ + && grep -q 'TRINO_OAUTH_STATE_KEY: ${TRINO_OAUTH_STATE_KEY' "${DEPLOY_DIR}/docker-compose.yml" 2>/dev/null \ + && ! grep -qE 'token-url=http://keycloak:' "${_tc}" 2>/dev/null; then + ok "trino oauth2: token-url/jwks-url pakai KEYCLOAK_HOST + state-key tetap" +else + bad "trino oauth2: token-url/jwks-url literal 'keycloak:' atau state-key belum diset" +fi + +# ─── 0l. StarRocks OIDC: redirect_url = alamat browser (bukan localhost) ────── +if ! grep -qE "redirect_url' = 'http://localhost" "${DEPLOY_DIR}/bootstrap.sh" 2>/dev/null \ + && grep -q 'sr_redirect_url=' "${DEPLOY_DIR}/bootstrap.sh" 2>/dev/null \ + && grep -q 'bootstrap_browser_addr' "${DEPLOY_DIR}/bootstrap.sh" 2>/dev/null; then + ok "StarRocks OIDC: redirect_url pakai alamat browser (bukan localhost literal)" +else + bad "StarRocks OIDC: redirect_url masih localhost literal" +fi + +# ─── 0m. Tujuan `--sync` BEBAS nama repo (cermin $HOME, bukan ~/sicabe-doc) ─── +# Regresi: dest default dipatok `~/sicabe-doc/deploy` → repo distribusi bersih +# (`sicabe-core`, isinya hanya deploy/) salah tujuan & membuat salinan kedua +# `~/sicabe-doc/deploy`. Nilai harus DITURUNKAN dari lokasi script. +dest_probe="$(HOME=/home/u bash -c ' + # shellcheck disable=SC1091 + source "'"${DEPLOY_DIR}"'/lib/common.sh" >/dev/null 2>&1 || exit 9 + for d in /home/u/sicabe-core/deploy /home/u/sicabe-doc/deploy /opt/apps/sicabe-core/deploy; do + printf "%s|" "$(sicabe_default_dest "$d")" + done' 2>/dev/null)" +if [[ "${dest_probe}" == "~/sicabe-core/deploy|~/sicabe-doc/deploy|~/sicabe-core/deploy|" ]]; then + ok "--sync: tujuan default mencerminkan nama repo (sicabe-core → ~/sicabe-core/deploy)" +else + bad "--sync: sicabe_default_dest salah → '${dest_probe}'" +fi +if ! grep -q 'dest="~/sicabe-doc/deploy"' "${DEPLOY_DIR}/install.sh" \ + && ! grep -q 'DEST="~/sicabe-doc/deploy"' "${DEPLOY_DIR}/scripts/check-trino-keystore.sh" \ + && grep -q 'sicabe_default_dest' "${DEPLOY_DIR}/install.sh"; then + ok "--sync: tidak ada lagi path repo ter-hardcode (~/sicabe-doc/deploy)" +else + bad "--sync: masih ada path repo ter-hardcode (~/sicabe-doc/deploy)" +fi + # ─── 1. generate ──────────────────────────────────────────────────────────── if ! python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \ "${INVENTORY}" "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" \ diff --git a/deploy/scripts/verify-keycloak.sh b/deploy/scripts/verify-keycloak.sh new file mode 100755 index 0000000..bba6fef --- /dev/null +++ b/deploy/scripts/verify-keycloak.sh @@ -0,0 +1,355 @@ +#!/usr/bin/env bash +# ============================================================================= +# eBdesk SiCABE — Verifikasi Keycloak (realm, client, user) — LEBIH LENGKAP dari +# verify-phase1. Tujuan: menjamin konfigurasi Keycloak siap dipakai fase berikutnya +# & alur uji E2E (OIDC Trino/StarRocks/Airflow/OM/Grafana; login user engineer/…). +# +# Yang diperiksa: +# A. Realm & roles : discovery/issuer; role admin/engineer/analyst/viewer. +# B. Clients (statis) : per client — ada, enabled, publicClient=false, +# standardFlow, serviceAccounts, client-secret, +# redirectUris/webOrigins (localhost + PUBLIC_URL), audience mapper. +# C. Users (statis) : per user — ada, enabled, realm role benar, dan +# apakah password masih DEFAULT (hardcoded: engineer123/…). +# D. Fungsional : client_credentials tiap service client → token OK (uji SECRET). +# E. Fungsional : login user (password grant) → preferred_username + roles. +# F. Redirect URI : validasi NYATA via /auth — Keycloak MENERIMA redirect alamat +# host (dan MENOLAK yang ngawur). Bukan sekadar banding daftar. +# +# Mode READ-ONLY (tanpa admin Keycloak): bagian statis (B/C) DILEWATI; D/E tetap jalan. +# Exit: 0 = semua gate lolos (WARN boleh), 1 = ada FAIL. +# +# Usage: +# ./scripts/verify-keycloak.sh +# ./scripts/verify-keycloak.sh --env-file generated/host-control/.env +# KEYCLOAK_URL=http://kc.corp:8080 ./scripts/verify-keycloak.sh # IdP existing +# ============================================================================= +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=../lib/common.sh +source "${SCRIPT_DIR}/lib/common.sh" +# shellcheck source=../config/environment.defaults.sh +source "${SCRIPT_DIR}/config/environment.defaults.sh" +sicabe_load_env "${SCRIPT_DIR}" "$@" || exit 1 + +KC="$(sicabe_local_url http KEYCLOAK_HOST "${KEYCLOAK_PORT:-12080}" KEYCLOAK_URL)" +REALM="${SICABE_REALM:-sicabe}" + +REQUIRED_CLIENTS=("trino" "starrocks" "airflow" "openmetadata" "grafana") +REQUIRED_ROLES=("admin" "engineer" "analyst" "viewer") +REQUIRED_USERS=("admin" "engineer" "analyst" "viewer") +AUDIENCE_MAPPER="audience-account" + +# Peran yang Diharapkan per user. +declare -A USER_ROLE=([admin]=admin [engineer]=engineer [analyst]=analyst [viewer]=viewer) +# Password DEFAULT (hardcoded oleh bootstrap.sh) — untuk mendeteksi password lemah. +declare -A USER_DEFAULT_PW=( + [admin]="${KEYCLOAK_ADMIN_PASSWORD:-}" + [engineer]="engineer123" + [analyst]="analyst123" + [viewer]="viewer123" +) +# Secret client diharapkan (harus sama dgn env deploy) + URL publik (redirect/webOrigins). +declare -A CLIENT_SECRET=( + [trino]="${TRINO_CLIENT_SECRET:-}" [starrocks]="${STARROCKS_CLIENT_SECRET:-}" + [airflow]="${AIRFLOW_CLIENT_SECRET:-}" [openmetadata]="${OPENMETADATA_CLIENT_SECRET:-}" + [grafana]="${GRAFANA_CLIENT_SECRET:-}" +) +declare -A CLIENT_PUBLIC=( + [trino]="${TRINO_PUBLIC_URL:-}" [starrocks]="${STARROCKS_PUBLIC_URL:-}" + [airflow]="${AIRFLOW_PUBLIC_URL:-}" [openmetadata]="${OPENMETADATA_PUBLIC_URL:-}" + [grafana]="${GRAFANA_PUBLIC_URL:-}" +) +# Origin (skema://alamat:port) tempat BROWSER membuka komponen — WAJIB ada di +# redirectUris. Deteksi kesalahan umum: redirectUris hanya `https://localhost:12443/*` +# padahal user membuka `https://:12443` → Keycloak menolak "invalid redirect_uri". +# = " " +declare -A CLIENT_ORIGIN=( + [trino]="https TRINO_HOST TRINO_HTTPS_PORT 12443" + [grafana]="http GRAFANA_HOST GRAFANA_PORT 12300" + [openmetadata]="http OPENMETADATA_HOST OPENMETADATA_PORT 12085" + [airflow]="http AIRFLOW_HOST AIRFLOW_PORT 12082" + [starrocks]="http STARROCKS_FE_HOST STARROCKS_HTTP_PORT 12030" +) +# Alamat efektif: nilai var bila berupa ALAMAT (mengandung titik); kalau kosong/co-located +# → alamat host ini (SICABE_HOST_ADDRESS, multi-host). Kalau KEDUANYA tak ada → KOSONG +# (JANGAN fallback ke localhost: itu membuat cek origin terhadap localhost selalu PASS). +comp_addr() { + local v="${!1:-}" + if [[ "${v}" == *.* && "${v}" != "127.0.0.1" ]]; then echo "${v}"; return; fi + echo "${SICABE_HOST_ADDRESS:-}" +} +expected_origin() { # → "skema://alamat:port" (atau kosong bila alamat tak diketahui) + local spec="${CLIENT_ORIGIN[$1]:-}"; [[ -n "${spec}" ]] || { echo ""; return; } + local scheme hv pv dport addr; read -r scheme hv pv dport <<< "${spec}" + addr="$(comp_addr "${hv}")"; [[ -n "${addr}" ]] || { echo ""; return; } + printf '%s://%s:%s' "${scheme}" "${addr}" "${!pv:-${dport}}" +} + +PASS=0; FAIL_N=0; WARN_N=0 +pass() { echo -e " ${PASS} $1"; PASS=$((PASS + 1)); } +fail() { echo -e " ${FAIL} $1"; FAIL_N=$((FAIL_N + 1)); } +warn() { echo -e " ${WARN} $1"; WARN_N=$((WARN_N + 1)); } +info() { echo -e " ${INFO} $1"; } + +# ── Keycloak helper ────────────────────────────────────────────────────────── +kc_admin_token() { + curl -sf -X POST "${KC}/realms/master/protocol/openid-connect/token" \ + -d client_id=admin-cli -d "username=${KEYCLOAK_ADMIN:-admin}" \ + -d "password=${KEYCLOAK_ADMIN_PASSWORD:-}" -d grant_type=password 2>/dev/null | + python3 -c "import sys,json;print(json.load(sys.stdin).get('access_token',''))" 2>/dev/null || echo "" +} +kc_get() { curl -sf -H "Authorization: Bearer ${KC_TOKEN}" "${KC}/admin/realms/${REALM}$1" 2>/dev/null; } + +# client_credentials → cetak "OK" atau "HTTP" atau "ERR". +cc_grant() { # → "OK" / "HTTP" + local code + code="$(curl -s -o /dev/null -w '%{http_code}' -X POST \ + "${KC}/realms/${REALM}/protocol/openid-connect/token" \ + -d "client_id=$1" -d "client_secret=$2" -d grant_type=client_credentials 2>/dev/null || true)" + code="${code:-000}" + [[ "${code}" == "200" ]] && echo OK || echo "HTTP${code}" +} + +# Uji NYATA redirect_uri via endpoint /auth (publik — tak perlu admin): apakah Keycloak +# MENERIMA redirect (→ halaman/redirect login) atau MENOLAK ("Invalid parameter: redirect_uri"). +kc_auth_probe() { # → OK | REJECTED | HTTP + local out code + out="$(curl -s -w $'\n%{http_code}' -G "${KC}/realms/${REALM}/protocol/openid-connect/auth" \ + --data-urlencode "client_id=$1" --data-urlencode "redirect_uri=$2" \ + --data-urlencode "response_type=code" --data-urlencode "scope=openid" 2>/dev/null || true)" + code="${out##*$'\n'}" + if printf '%s' "${out}" | grep -qi 'Invalid parameter: redirect_uri' || [[ "${code}" == "400" ]]; then + echo REJECTED; return + fi + case "${code}" in 200|302|303) echo OK ;; *) echo "HTTP${code:-000}" ;; esac +} +# password grant → cetak "preferred_username|roles" (dari id_token) atau "". +user_login() { # + curl -s -X POST "${KC}/realms/${REALM}/protocol/openid-connect/token" \ + -d client_id=admin-cli -d "username=$1" -d "password=$2" \ + -d grant_type=password -d scope=openid 2>/dev/null | + python3 -c " +import sys,json,base64 +try: r=json.load(sys.stdin) +except Exception: print(''); raise SystemExit +t=r.get('id_token') or r.get('access_token') or '' +if not t: print(''); raise SystemExit +p=t.split('.')[1]; p+='='*(-len(p)%4) +try: c=json.loads(base64.urlsafe_b64decode(p)) +except Exception: print(''); raise SystemExit +print(c.get('preferred_username','') + '|' + ','.join(c.get('realm_access',{}).get('roles') or [])) +" 2>/dev/null || echo "" +} + +# ── A. Realm & roles ───────────────────────────────────────────────────────── +check_realm_roles() { + local disc issuer + disc="$(curl -sf "${KC}/realms/${REALM}/.well-known/openid-configuration" 2>/dev/null || true)" + if [[ -z "${disc}" ]]; then + fail "OIDC discovery realm '${REALM}' tidak merespon (${KC})"; return + fi + issuer="$(echo "${disc}" | python3 -c "import sys,json;print(json.load(sys.stdin).get('issuer',''))" 2>/dev/null)" + pass "OIDC discovery OK (issuer=${issuer})" + if [[ -z "${KC_TOKEN}" ]]; then + warn "Tanpa admin Keycloak — cek statis (roles/client/user) DILEWATI"; return + fi + local roles missing="" + roles="$(kc_get "/roles" | python3 -c "import sys,json;print(' '.join(r['name'] for r in json.load(sys.stdin)))" 2>/dev/null || echo "")" + for r in "${REQUIRED_ROLES[@]}"; do [[ " ${roles} " == *" ${r} "* ]] || missing+=" ${r}"; done + [[ -z "${missing}" ]] && pass "Roles lengkap: ${REQUIRED_ROLES[*]}" || fail "Role belum ada:${missing}" +} + +# ── B. Clients (statis) ────────────────────────────────────────────────────── +check_clients() { + [[ -n "${KC_TOKEN}" ]] || return 0 + local cid js ok + # Bila origin host TAK BISA dihitung (env tanpa alamat cluster) → beri tahu SEKALI; + # cek origin per client di-skip (bukan PASS palsu). Sumber benar: --env-file bundle + # atau SICABE_CLIENT_REDIRECT_ORIGINS. + local _c _have=0 + [[ -n "${SICABE_CLIENT_REDIRECT_ORIGINS:-}" ]] && _have=1 + for _c in "${REQUIRED_CLIENTS[@]}"; do [[ -n "$(expected_origin "${_c}")" ]] && { _have=1; break; }; done + [[ ${_have} -eq 0 ]] && warn "Origin host tak bisa dihitung (env tanpa alamat cluster) — jalankan dgn --env-file generated/host-/.env, atau set SICABE_CLIENT_REDIRECT_ORIGINS. Cek origin DILEWATI (bukan PASS)." + for cid in "${REQUIRED_CLIENTS[@]}"; do + js="$(kc_get "/clients?clientId=${cid}")" + if [[ -z "${js}" || "${js}" == "{}" || "${js}" == "[]" ]]; then + fail "Client '${cid}' TIDAK ADA"; continue + fi + # Ambil bidang penting + secret + redirect + mapper dalam satu pass. + read -r ok pub std svc auth secret redirects mapper < <(echo "${js}" | python3 -c " +import sys,json +d=json.load(sys.stdin) or [] +if not d: print('0 - - - - - - -'); raise SystemExit +c=d[0] +print( + '1' if c.get('enabled') else '0', + c.get('publicClient'), + '1' if c.get('standardFlowEnabled') else '0', + '1' if c.get('serviceAccountsEnabled') else '0', + c.get('clientAuthenticatorType',''), + c.get('secret',''), + ','.join(c.get('redirectUris') or []), + ','.join(c.get('redirectUris') or [])) +" 2>/dev/null) + [[ "${ok}" == "1" ]] && pass "Client '${cid}' enabled" || fail "Client '${cid}' DISABLED" + [[ "${pub}" == "False" ]] && pass " ${cid}: publicClient=false (confidential)" \ + || fail " ${cid}: publicClient=${pub} (harus False)" + [[ "${std}" == "1" ]] && pass " ${cid}: standardFlowEnabled" || fail " ${cid}: standardFlow OFF" + [[ "${svc}" == "1" ]] && pass " ${cid}: serviceAccountsEnabled" || warn " ${cid}: serviceAccounts OFF" + [[ "${auth}" == "client-secret" ]] && pass " ${cid}: clientAuthenticatorType=client-secret" \ + || fail " ${cid}: authenticator='${auth}' (harus client-secret)" + # Secret sinkron dgn env. + if [[ -n "${CLIENT_SECRET[$cid]}" ]]; then + [[ "${secret}" == "${CLIENT_SECRET[$cid]}" ]] && pass " ${cid}: secret sinkron dgn env" \ + || fail " ${cid}: secret BEDA dari env (komponen akan gagal login)" + else + warn " ${cid}: *_CLIENT_SECRET kosong di env — tak bisa membandingkan secret" + fi + # redirectUris: localhost + (bila ada) PUBLIC_URL. + if [[ "${redirects}" == *"http://localhost:*"* || "${redirects}" == *"https://localhost:*"* ]]; then + pass " ${cid}: redirectUris memuat localhost" + else + warn " ${cid}: redirectUris TANPA localhost (${redirects})" + fi + local puburl="${CLIENT_PUBLIC[$cid]}" + if [[ -n "${puburl}" ]]; then + if [[ "${redirects}" == *"${puburl}"* ]]; then + pass " ${cid}: redirectUris memuat PUBLIC_URL (${puburl})" + else + fail " ${cid}: redirectUris TANPA PUBLIC_URL '${puburl}' (login browser via alamat publik akan ditolak)" + fi + fi + # Origin host (tempat browser membuka komponen) — WAJIB ada, bukan hanya localhost. + local exp; exp="$(expected_origin "${cid}")" + if [[ -n "${exp}" ]]; then + if [[ "${redirects}" == *"${exp}"* ]]; then + pass " ${cid}: redirectUris memuat origin host (${exp})" + else + fail " ${cid}: redirectUris TANPA origin host '${exp}' — login browser via alamat itu DITOLAK (invalid redirect_uri)" + fi + fi + # Origin client dari CLUSTER (SICABE_CLIENT_REDIRECT_ORIGINS) — host existing yang + # menjalankan bootstrap tak tahu alamat host cluster, jadi daftar ini disuplai ke .env. + local o + for o in ${SICABE_CLIENT_REDIRECT_ORIGINS:-}; do + [[ -n "${o}" ]] || continue + if [[ "${redirects}" == *"${o}"* ]]; then + pass " ${cid}: redirectUris memuat origin cluster (${o})" + else + fail " ${cid}: redirectUris TANPA origin cluster '${o}'" + fi + done + done +} + +# ── C. Users (statis) + password default ───────────────────────────────────── +check_users() { + [[ -n "${KC_TOKEN}" ]] || return 0 + local u js + for u in "${REQUIRED_USERS[@]}"; do + js="$(kc_get "/users?username=${u}&exact=true")" + if [[ -z "${js}" || "${js}" == "[]" ]]; then fail "User '${u}' TIDAK ADA"; continue; fi + local uid enabled roles + uid="$(echo "${js}" | python3 -c "import sys,json;d=json.load(sys.stdin);print(d[0]['id'] if d else '')" 2>/dev/null)" + enabled="$(echo "${js}" | python3 -c "import sys,json;d=json.load(sys.stdin);print('1' if d and d[0].get('enabled') else '0')" 2>/dev/null)" + roles="$(kc_get "/users/${uid}/role-mappings/realm" | python3 -c "import sys,json;print(' '.join(r['name'] for r in json.load(sys.stdin)))" 2>/dev/null || echo "")" + [[ "${enabled}" == "1" ]] && pass "User '${u}' ada + enabled" || fail "User '${u}' DISABLED" + [[ " ${roles} " == *" ${USER_ROLE[$u]} "* ]] && pass " ${u}: realm role '${USER_ROLE[$u]}'" \ + || fail " ${u}: role '${USER_ROLE[$u]}' TIDAK ada (peran: ${roles:-—})" + # Password masih DEFAULT (hardcoded)? + local dp="${USER_DEFAULT_PW[$u]}" + if [[ -n "${dp}" ]] && [[ "$(user_login "${u}" "${dp}")" == "${u}|"* ]]; then + warn " ${u}: password masih DEFAULT (hardcoded) — ganti untuk produksi" + fi + done +} + +# ── D. client_credentials ──────────────────────────────────────────────────── +check_cc() { + local cid res + for cid in "${REQUIRED_CLIENTS[@]}"; do + [[ -n "${CLIENT_SECRET[$cid]}" ]] || { warn "cc ${cid}: secret env kosong"; continue; } + res="$(cc_grant "${cid}" "${CLIENT_SECRET[$cid]}")" + [[ "${res}" == "OK" ]] && pass "client_credentials '${cid}' → token OK" \ + || fail "client_credentials '${cid}' GAGAL (${res})" + done +} + +# ── E2. Redirect URI — validasi NYATA via /auth (bukan sekadar bandingkan daftar) ── +check_redirect_validation() { + local cid o exp rs any=0 + for cid in "${REQUIRED_CLIENTS[@]}"; do + exp="$(expected_origin "${cid}")" + if [[ -n "${exp}" ]]; then + any=1 + rs="$(kc_auth_probe "${cid}" "${exp}/")" + case "${rs}" in + OK) pass "${cid}: Keycloak MENERIMA redirect '${exp}/*' (uji nyata via /auth)";; + REJECTED) fail "${cid}: Keycloak MENOLAK redirect '${exp}/*' — invalid redirect_uri (belum terdaftar di client)";; + *) warn "${cid}: uji redirect '${exp}/*' tak konklusif (${rs})";; + esac + fi + done + # Origin dari daftar cluster (SICABE_CLIENT_REDIRECT_ORIGINS) — uji terhadap tiap client. + for o in ${SICABE_CLIENT_REDIRECT_ORIGINS:-}; do + [[ -n "${o}" ]] || continue + any=1 + for cid in "${REQUIRED_CLIENTS[@]}"; do + rs="$(kc_auth_probe "${cid}" "${o}/")" + [[ "${rs}" == "OK" ]] && pass "${cid}: Keycloak MENERIMA origin cluster '${o}/*'" \ + || fail "${cid}: Keycloak MENOLAK origin cluster '${o}/*' (${rs})" + done + done + if [[ ${any} -eq 0 ]]; then + warn "redirect_uri tak bisa diuji (env tanpa alamat cluster / SICABE_CLIENT_REDIRECT_ORIGINS)" + return + fi + # Kontrol: redirect ngawur HARUS ditolak → membuktikan Keycloak memang memvalidasi. + rs="$(kc_auth_probe "trino" "https://sicabe-invalid.example/blocked")" + [[ "${rs}" == "REJECTED" ]] \ + && pass "kontrol: redirect ngawur DITOLAK Keycloak (validasi redirect aktif)" \ + || warn "kontrol: redirect ngawur '${rs}' — Keycloak mungkin TIDAK memvalidasi redirect_uri" +} + +# ── E. Login user (password grant) ─────────────────────────────────────────── +check_user_login() { + local u out pref + for u in "${REQUIRED_USERS[@]}"; do + local pw="${USER_DEFAULT_PW[$u]}" + [[ -n "${pw}" ]] || { warn "login '${u}': password tak diketahui — lewati"; continue; } + out="$(user_login "${u}" "${pw}")" + pref="${out%%|*}" + if [[ "${pref}" == "${u}" ]]; then + pass "login '${u}' OK (preferred_username=${pref}; roles=${out#*|})" + elif [[ -z "${out}" ]]; then + fail "login '${u}' GAGAL (user/password salah, atau user belum ada)" + else + fail "login '${u}' OK tapi preferred_username='${pref}' (harus = uid LDAP '${u}')" + fi + done +} + +# ── main ───────────────────────────────────────────────────────────────────── +echo -e "${BOLD}Verifikasi Keycloak — realm '${REALM}' @ ${KC}${NC}" +echo "" +KC_TOKEN="$(kc_admin_token)" +if [[ -z "${KC_TOKEN}" ]]; then + warn "Mode READ-ONLY (login admin gagal / tanpa admin) — cek statis dilewati" +else + info "Token admin diperoleh — cek statis dijalankan" +fi +echo "" + +echo -e "${BOLD}A. Realm & roles${NC}"; check_realm_roles +echo -e "${BOLD}B. Clients (statis)${NC}"; check_clients +echo -e "${BOLD}C. Users (statis)${NC}"; check_users +echo -e "${BOLD}D. Fungsional — client_credentials (uji SECRET client, bukan redirect)${NC}"; check_cc +echo -e "${BOLD}E. Fungsional — login user${NC}"; check_user_login +echo -e "${BOLD}F. Redirect URI — validasi NYATA via /auth${NC}"; check_redirect_validation +echo "" +echo -e "${BOLD}Ringkasan: ${PASS} PASS · ${FAIL_N} FAIL · ${WARN_N} WARN${NC}" +[[ ${FAIL_N} -eq 0 ]] && { echo -e "${GREEN}✓ Keycloak siap.${NC}"; exit 0; } +echo -e "${RED}✗ Ada ${FAIL_N} kegagalan — perbaiki sebelum melanjutkan.${NC}" +exit 1 diff --git a/deploy/scripts/verify-phase1-iam.sh b/deploy/scripts/verify-phase1-iam.sh index 12ed044..d819547 100755 --- a/deploy/scripts/verify-phase1-iam.sh +++ b/deploy/scripts/verify-phase1-iam.sh @@ -97,7 +97,7 @@ try: except Exception: print(''); raise SystemExit roles=c.get('realm_access',{}).get('roles') or [] -print(c.get('preferred_username',''), '|', ','.join(roles)) +print(c.get('preferred_username','') + '|' + ','.join(roles)) " 2>/dev/null || echo "" } @@ -216,7 +216,7 @@ check_keycloak_users_login() { *) pw="${user}123" ;; esac claim=$(kc_user_login_probe "${user}" "${pw}") - preferred="${claim%% |*}" + preferred="${claim%%|*}" if [[ "${preferred}" == "${user}" ]]; then pass "Login '${user}' OK (preferred_username=${preferred})" elif [[ -n "${preferred}" ]]; then diff --git a/deploy/scripts/verify-phase4-serving.sh b/deploy/scripts/verify-phase4-serving.sh index f98479d..18fe58c 100755 --- a/deploy/scripts/verify-phase4-serving.sh +++ b/deploy/scripts/verify-phase4-serving.sh @@ -268,10 +268,14 @@ check_security() { fail " ${OIDC_NAME}: tidak memuat ${pat} (cek bootstrap.sh --step 5)" fi done - if [[ "${oidc}" == *"redirect_url\" = \"http://localhost:${STARROCKS_HTTP_PORT:-12030}/api/oauth2\""* ]]; then - pass " ${OIDC_NAME}: redirect_url → http://localhost:${STARROCKS_HTTP_PORT:-12030}/api/oauth2" + # redirect_url harus = alamat BROWSER (host FE) — bukan localhost/nama container. + local _sraddr="${STARROCKS_FE_HOST:-}" + [[ "${_sraddr}" == *.* && "${_sraddr}" != "127.0.0.1" ]] || _sraddr="${SICABE_HOST_ADDRESS:-localhost}" + local _srexp="http://${_sraddr}:${STARROCKS_HTTP_PORT:-12030}/api/oauth2" + if [[ "${oidc}" == *"redirect_url\" = \"${_srexp}\""* ]]; then + pass " ${OIDC_NAME}: redirect_url → ${_srexp}" else - warn " ${OIDC_NAME}: redirect_url tidak sesuai STARROCKS_HTTP_PORT (${STARROCKS_HTTP_PORT:-12030})" + warn " ${OIDC_NAME}: redirect_url bukan '${_srexp}' (cek bootstrap.sh --step 5)" fi local g miss="" groups IFS=',' read -ra groups <<< "${EXPECTED_GROUPS}"