sicabe-core/deploy/scripts/test-generate.sh

#!/usr/bin/env bash
# =============================================================================
# test-generate.sh — verifikasi generator multi-host (Fase 1 R10 §4.5)
# -----------------------------------------------------------------------------
# Yang dibuktikan (bukan sekadar "berkasnya ada"):
#   1. `docker compose config` VALID untuk setiap host hasil generate.
#   2. Service yang ikut **identik secara semantik** dengan single-host pada semua
#      field (image, command, ports, healthcheck, profiles, volumes, ...) — ini yang
#      menangkap kerusakan akibat round-trip YAML (mis. `"true"` jadi boolean).
#   3. Perbedaan `environment` HANYA pada variabel topologi yang memang diarahkan
#      generator (alamat host) — tidak ada kebocoran lain.
#   4. `depends_on` lintas host benar-benar HABIS; intra-host tetap.
#   5. Tidak ada variabel topologi yang DUPLIKAT di `.env` (dotenv: yang terakhir
#      menang → duplikat = nilai inventory bisa hilang diam-diam).
#
# Jalankan: ./scripts/test-generate.sh [cluster.yml]
# =============================================================================
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DEPLOY_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
INVENTORY="${1:-${DEPLOY_DIR}/cluster.example.yml}"
OUT="$(mktemp -d)"
trap 'rm -rf "${OUT}"' EXIT

GRN=$'\e[32m'; RED=$'\e[31m'; BLD=$'\e[1m'; NC=$'\e[0m'
PASS_N=0; FAIL_N=0
ok()   { printf "  ${GRN}✅${NC} %s\n" "$1"; PASS_N=$((PASS_N + 1)); }
bad()  { printf "  ${RED}❌${NC} %s\n" "$1"; FAIL_N=$((FAIL_N + 1)); }

echo -e "${BLD}Uji generator multi-host (lib/cluster/generate.py)${NC}"
echo "  inventory: ${INVENTORY}"
echo

# ─── 0. konsistensi RENDER_HOSTS ⊆ KNOWN_HOSTS ──────────────────────────────
# generate.py meng-`--map` SETIAP entri RENDER_HOSTS, sedangkan render-config.sh
# hanya menerima hostname di KNOWN_HOSTS. Bila berbeda, `install.sh --generate`
# gagal total (bug M2/M3: statsd-exporter/trino-jmx-exporter). Dijaga di sini agar
# tidak terulang diam-diam.
map_gap="$(python3 - "${DEPLOY_DIR}" <<'PY'
import re, sys
d = sys.argv[1]
gen = open(f"{d}/lib/cluster/generate.py", encoding="utf-8").read()
rc = open(f"{d}/scripts/render-config.sh", encoding="utf-8").read()
render_hosts = set(re.findall(r'"([^"]+)"', re.search(r"RENDER_HOSTS = \((.*?)\)", gen, re.S).group(1)))
# Blok KNOWN_HOSTS berakhir pada baris ")" tersendiri; komentar dibuang sebelum split
# (komentar boleh memuat tanda kurung, mis. "(M2/M3)", yang merusak regex sederhana).
block = re.search(r"KNOWN_HOSTS=\((.*?)\n\)", rc, re.S).group(1)
known = set()
for line in block.splitlines():
    known.update(line.split("#", 1)[0].split())
print(" ".join(sorted(render_hosts - known)))
PY
)"
if [[ -z "${map_gap}" ]]; then
  ok "RENDER_HOSTS ⊆ KNOWN_HOSTS (render-config menerima semua --map)"
else
  bad "RENDER_HOSTS memuat hostname yang TIDAK ada di KNOWN_HOSTS: ${map_gap}"
fi

# ─── 0b. Trino group-provider memakai LDAP_ADMIN_DN (bukan DN literal) ───────
# LDAP existing bisa punya bind DN ≠ cn=admin,dc=sicabe,dc=local. Bila
# group-provider.properties menulis DN literal, Trino mem-bind dengan DN SALAH →
# user terautentikasi tetapi tanpa grup. Pastikan ia memakai ${ENV:LDAP_ADMIN_DN}
# DAN compose meneruskan env itu ke trino-coordinator (Trino gagal start bila tak ada).
gp="${DEPLOY_DIR}/config/trino/group-provider.properties"
if grep -q 'ldap.admin-user=${ENV:LDAP_ADMIN_DN}' "${gp}" 2>/dev/null; then
  ok "group-provider.properties: ldap.admin-user dari \${ENV:LDAP_ADMIN_DN}"
else
  bad "group-provider.properties: ldap.admin-user BUKAN \${ENV:LDAP_ADMIN_DN} (bind DN existing tak akan jalan)"
fi
if awk '/^  trino-coordinator:/{f=1;next} f && /^  [a-zA-Z0-9_-]+:/{exit} f' \
     "${DEPLOY_DIR}/docker-compose.yml" | grep -q 'LDAP_ADMIN_DN:'; then
  ok "docker-compose: trino-coordinator menerima LDAP_ADMIN_DN"
else
  bad "docker-compose: trino-coordinator TIDAK menerima LDAP_ADMIN_DN (Trino gagal start)"
fi

# ─── 0c. Port internal terparameter (bukan literal) ──────────────────────────
# Port container harus bisa di-override via .env (mis. layanan existon berport
# non-standar) tanpa menyunting docker-compose.yml.
dc="${DEPLOY_DIR}/docker-compose.yml"
miss=""
grep -q '${POSTGRES_INTERNAL_PORT:-5432}' "${dc}" || miss="${miss} POSTGRES"
grep -q '${KEYCLOAK_INTERNAL_PORT:-8080}' "${dc}" || miss="${miss} KEYCLOAK"
grep -q '${MINIO_INTERNAL_PORT:-9000}'    "${dc}" || miss="${miss} MINIO"
[[ -z "${miss}" ]] && ok "docker-compose: port internal terparameter (POSTGRES/KEYCLOAK/MINIO)" \
                   || bad "docker-compose: port internal BUKAN parameter:${miss}"
hard="$(grep -nE '\$\{(POSTGRES_HOST|KEYCLOAK_HOST|MINIO_HOST)[^}]*\}:(5432|8080|9000)' "${dc}" || true)"
[[ -z "${hard}" ]] && ok "docker-compose: tak ada port internal literal di var *_HOST" \
                    || bad "docker-compose: masih ada port literal: ${hard}"
grep -q '${ENV:KEYCLOAK_INTERNAL_URL}' "${DEPLOY_DIR}/config/trino/config-coordinator.properties" \
  && ok "Trino config: token/jwks-url pakai \${ENV:KEYCLOAK_INTERNAL_URL} (skema+host+port)" \
  || bad "Trino config: URL Keycloak belum terparameter (token/jwks literal?)"
grep -q ':{port}' "${DEPLOY_DIR}/lib/cluster/generate.py" \
  && ok "generate.py: URL_ENDPOINT_VARS memakai {port} (dari .env)" \
  || bad "generate.py: URL_ENDPOINT_VARS masih port literal"

# ─── 0d. Image ref terparameter (bisa diarahkan ke mirror/registry internal) ─
# Repositori asal bisa mengubah kebijakan (mis. MinIO community butuh auth →
# pull anonim GAGAL). Var *_IMAGE (full ref) harus ada agar bisa di-mirror tanpa
# menyunting docker-compose.yml.
for pair in "MINIO_IMAGE:quay.io/minio/minio" "MINIO_MC_IMAGE:quay.io/minio/mc" "NESSIE_IMAGE:ghcr.io/projectnessie/nessie"; do
  var="${pair%%:*}"; repo="${pair#*:}"
  if grep -q "\${${var}:-${repo}" "${DEPLOY_DIR}/docker-compose.yml"; then
    ok "docker-compose: image ${repo} lewat \${${var}}"
  else
    bad "docker-compose: image ${repo} masih hardcoded (tidak bisa di-mirror)"
  fi
done

# ─── 0e. Komponen EKSTERNAL ber-`address` (skenario P3) ──────────────────────
# `components.<c>.address` = komponen dikelola SISTEM LAIN → TIDAK dijalankan cluster
# (tak muncul di services host mana pun) + endpoint-nya = alamat itu. Diuji via cluster.p3.yml.
p3_problems="$(python3 - "${DEPLOY_DIR}" <<'PY'
import sys
sys.path.insert(0, sys.argv[1] + "/lib/cluster")
import inventory
d = inventory.apply_preset(inventory.load(sys.argv[1] + "/cluster.p3.yml"))
pl = inventory.plan(d)
svc = {s for h in pl.values() for s in h["services"]}
p = []
for c in ("keycloak", "minio", "nessie"):
    if c in svc:
        p.append(c + " masih dijalankan")
    if not inventory.external_address(d, c):
        p.append(c + ": address hilang")
if "openldap" not in pl.get("control", {}).get("services", []):
    p.append("openldap tak di control")
print("; ".join(p))
PY
)"
if [[ -z "${p3_problems}" ]]; then
  ok "P3: components.<c>.address → eksternal tak dijalankan, openldap lokal di control"
else
  bad "P3: perilaku components.<c>.address salah — ${p3_problems}"
fi

# ─── 0f. Deteksi keystore Trino BEDA antar-host ──────────────────────────────
# Keystore Trino harus identik di semua host (satu --generate + --sync). Skrip ini
# mendeteksi divergensi (mis. --generate dijalankan di >1 host).
cks="${DEPLOY_DIR}/scripts/check-trino-keystore.sh"
if [[ -x "${cks}" ]] && bash -n "${cks}" 2>/dev/null \
   && grep -q -- '--inventory' "${cks}" && grep -q -- '--expect' "${cks}" \
   && grep -q 'keystore.p12' "${cks}" && grep -q 'password.db' "${cks}" \
   && grep -q 'svc_dbt' "${cks}"; then
  ok "check-trino-keystore.sh: deteksi keystore + password.db antar-host (--inventory/--expect)"
else
  bad "check-trino-keystore.sh hilang/tidak lengkap (divergensi keystore/password.db tak terdeteksi)"
fi
# Fungsional (bila bundle host sudah ada): cetak sha256 & hormati --expect.
_first_bundle="$(ls -d "${DEPLOY_DIR}"/generated/host-* 2>/dev/null | head -1)"
if [[ -n "${_first_bundle}" ]]; then
  _bh="$(basename "${_first_bundle}" | sed 's/^host-//')"
  _sha="$("${cks}" --host "${_bh}" 2>/dev/null | sed -n 's/^  keystore  : sha256=//p')"
  if [[ "${_sha}" =~ ^[0-9a-f]{64}$ ]]; then
    ok "check-trino-keystore.sh: sha256 host '${_bh}' terbaca"
    "${cks}" --host "${_bh}" --expect "${_sha}" >/dev/null 2>&1 \
      && ok "check-trino-keystore.sh: --expect cocok → exit 0" \
      || bad "check-trino-keystore.sh: --expect cocok seharusnya exit 0"
  else
    bad "check-trino-keystore.sh: sha256 tidak terbaca untuk host '${_bh}'"
  fi
fi

# ─── 0g. .env auto-generate memuat SEMUA var WAJIB compose ───────────────────
# Regresi nyata: STARROCKS_ROOT_PASSWORD (dipakai `${VAR:?}` di compose) TIDAK dibuat
# install.sh → `docker compose --env-file .env` (tanpa shell env) gagal
# "required variable ... is missing a value".
_required="$(grep -oE '\$\{[A-Z_][A-Z0-9_]*:\?' "${DEPLOY_DIR}/docker-compose.yml" | sed 's/\${//;s/:?//' | sort -u)"
_envvars="$(awk '/^[[:space:]]*cat > .*\/\.env" <<EOF/{f=1;next} f&&/^EOF$/{exit} f' "${DEPLOY_DIR}/install.sh" \
            | grep -oE '^[A-Z_][A-Z0-9_]*=' | tr -d '=' | sort -u)"
_miss_env=""
for _v in ${_required}; do grep -qx "${_v}" <<<"${_envvars}" || _miss_env+=" ${_v}"; done
if [[ -z "${_miss_env}" ]]; then
  ok ".env auto-generate memuat semua var wajib compose (${_required//$'\n'/ })"
else
  bad "var wajib compose TIDAK di-generate ke .env:${_miss_env}"
fi

# ─── 0h. Orkestrasi alur data E2E (run-e2e.sh) ───────────────────────────────
e2e="${DEPLOY_DIR}/scripts/run-e2e.sh"
if [[ -x "${e2e}" ]] && bash -n "${e2e}" 2>/dev/null \
   && grep -q 'ingest-openmetadata.sh' "${e2e}" \
   && grep -q 'airflow dags test' "${e2e}" \
   && grep -q 'verify-phase6-governance.sh' "${e2e}"; then
  ok "run-e2e.sh: orkestrasi alur data E2E (ingest→DAG→ingest→DAG→verify)"
else
  bad "run-e2e.sh hilang/tidak lengkap"
fi

# ─── 0i. Prometheus: target MinIO EKSTERNAL → alamat:port publik ─────────────
# Prometheus tak mendukung substitusi env, dan render-config hanya memetakan komponen
# milik host lain — komponen EKSTERNAL (components.minio.address) harus di-rewrite
# generate.py, kalau tidak target tetap `minio:9000` → scrape gagal dari host prometheus.
_tmp_prom="$(mktemp -d)"; _prom_ok=""
if [[ -f "${DEPLOY_DIR}/.env" ]]; then
  cp "${DEPLOY_DIR}/.env" "${_tmp_prom}/e.env"
  printf '\nMINIO_INTERNAL_PORT=12900\n' >> "${_tmp_prom}/e.env"
  if python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \
       "${DEPLOY_DIR}/cluster.p3.yml" "${DEPLOY_DIR}/docker-compose.yml" \
       "${_tmp_prom}/e.env" "${_tmp_prom}/out" "${DEPLOY_DIR}/scripts/render-config.sh" \
       >/dev/null 2>&1; then
    _pyml="${_tmp_prom}/out/host-control/config/prometheus/prometheus.yml"
    if grep -q 'targets: \["100.93.54.108:12900"\]' "${_pyml}" 2>/dev/null \
       && ! grep -q 'targets: \["minio:9000"\]' "${_pyml}" 2>/dev/null; then
      _prom_ok=1
    fi
  fi
  if [[ -n "${_prom_ok}" ]]; then
    ok "prometheus: target MinIO eksternal → 100.93.54.108:12900 (bukan minio:9000)"
  else
    bad "prometheus: target MinIO eksternal salah (masih minio:9000?)"
  fi
fi

# ─── 0j. Verifikasi Keycloak lengkap (verify-keycloak.sh) ────────────────────
vk="${DEPLOY_DIR}/scripts/verify-keycloak.sh"
  if [[ -x "${vk}" ]] && bash -n "${vk}" 2>/dev/null \
     && grep -q 'client_credentials' "${vk}" && grep -q 'redirectUris' "${vk}" \
     && grep -q 'publicClient' "${vk}" && grep -q 'origin host' "${vk}" \
     && grep -q 'kc_auth_probe' "${vk}" && grep -q 'password masih DEFAULT' "${vk}"; then
    ok "verify-keycloak.sh: realm/roles/client(redirectUri origin-host,secret,publicClient)/user/password-default + fungsional"
else
  bad "verify-keycloak.sh hilang/tidak lengkap"
fi

# ─── 0k. Trino OAuth2: token/jwks pakai KEYCLOAK_INTERNAL_URL (bukan literal) ──
# Kls bug sama dgn target MinIO Prometheus: Trino server-ke-server ke Keycloak. Bila
# Keycloak EKSTERNAL (P3), literal `keycloak:` tak resolve → penukaran token gagal →
# callback `/oauth2/callback` balas "Authentication response could not be verified".
_tc="${DEPLOY_DIR}/config/trino/config-coordinator.properties"
if grep -q 'token-url=${ENV:KEYCLOAK_INTERNAL_URL}' "${_tc}" 2>/dev/null \
   && grep -q 'jwks-url=${ENV:KEYCLOAK_INTERNAL_URL}' "${_tc}" 2>/dev/null \
   && grep -q 'oauth2.state-key=${ENV:TRINO_OAUTH_STATE_KEY}' "${_tc}" 2>/dev/null \
   && grep -q 'KEYCLOAK_INTERNAL_URL: ${KEYCLOAK_INTERNAL_URL' "${DEPLOY_DIR}/docker-compose.yml" 2>/dev/null \
   && grep -q 'TRINO_OAUTH_STATE_KEY: ${TRINO_OAUTH_STATE_KEY' "${DEPLOY_DIR}/docker-compose.yml" 2>/dev/null \
   && ! grep -qE 'token-url=http://keycloak:' "${_tc}" 2>/dev/null; then
  ok "trino oauth2: token/jwks pakai KEYCLOAK_INTERNAL_URL + state-key tetap"
else
  bad "trino oauth2: token/jwks belum pakai KEYCLOAK_INTERNAL_URL atau state-key belum diset"
fi

# ─── 0l. StarRocks OIDC: redirect_url = alamat browser (bukan localhost) ──────
if ! grep -qE "redirect_url' = 'http://localhost" "${DEPLOY_DIR}/bootstrap.sh" 2>/dev/null \
   && grep -q 'sr_redirect_url=' "${DEPLOY_DIR}/bootstrap.sh" 2>/dev/null \
   && grep -q 'bootstrap_browser_addr' "${DEPLOY_DIR}/bootstrap.sh" 2>/dev/null; then
  ok "StarRocks OIDC: redirect_url pakai alamat browser (bukan localhost literal)"
else
  bad "StarRocks OIDC: redirect_url masih localhost literal"
fi

# ─── 0m. Tujuan `--sync` BEBAS nama repo (cermin $HOME, bukan ~/sicabe-doc) ───
# Regresi: dest default dipatok `~/sicabe-doc/deploy` → repo distribusi bersih
# (`sicabe-core`, isinya hanya deploy/) salah tujuan & membuat salinan kedua
# `~/sicabe-doc/deploy`. Nilai harus DITURUNKAN dari lokasi script.
dest_probe="$(HOME=/home/u bash -c '
  # shellcheck disable=SC1091
  source "'"${DEPLOY_DIR}"'/lib/common.sh" >/dev/null 2>&1 || exit 9
  for d in /home/u/sicabe-core/deploy /home/u/sicabe-doc/deploy /opt/apps/sicabe-core/deploy; do
    printf "%s|" "$(sicabe_default_dest "$d")"
  done' 2>/dev/null)"
if [[ "${dest_probe}" == "~/sicabe-core/deploy|~/sicabe-doc/deploy|~/sicabe-core/deploy|" ]]; then
  ok "--sync: tujuan default mencerminkan nama repo (sicabe-core → ~/sicabe-core/deploy)"
else
  bad "--sync: sicabe_default_dest salah → '${dest_probe}'"
fi
if ! grep -q 'dest="~/sicabe-doc/deploy"' "${DEPLOY_DIR}/install.sh" \
   && ! grep -q 'DEST="~/sicabe-doc/deploy"' "${DEPLOY_DIR}/scripts/check-trino-keystore.sh" \
   && grep -q 'sicabe_default_dest' "${DEPLOY_DIR}/install.sh"; then
  ok "--sync: tidak ada lagi path repo ter-hardcode (~/sicabe-doc/deploy)"
else
  bad "--sync: masih ada path repo ter-hardcode (~/sicabe-doc/deploy)"
fi

# ─── 0n. Bucket & warehouse Iceberg terkonfigurasi (bukan literal) ───────────
# Bucket = MINIO_BUCKET_ICEBERG_DATA; warehouse = ICEBERG_WAREHOUSE (nama REST
# Nessie + prefix path S3, IDENTIK). Client boleh punya nama berbeda → tak boleh
# ada literal 'warehouse' / region Nessie hardcoded.
_wh_dc="${DEPLOY_DIR}/docker-compose.yml"; _wh_ed="${DEPLOY_DIR}/config/environment.defaults.sh"
_wh_ip="${DEPLOY_DIR}/config/trino/catalog/iceberg.properties"; _wh_bs="${DEPLOY_DIR}/bootstrap.sh"
_wh_se="${DEPLOY_DIR}/scripts/seed-starrocks.sh"; _wh_v2="${DEPLOY_DIR}/scripts/verify-phase2-storage.sh"
_wh_ins="${DEPLOY_DIR}/install.sh"
wh_bad=""
have() { grep -qF "$2" "$1" 2>/dev/null || wh_bad="${wh_bad} $(basename "$1"):$3"; }
lack() { grep -qF "$2" "$1" 2>/dev/null && wh_bad="${wh_bad} $(basename "$1"):$3"; }
have "${_wh_ed}" 'export ICEBERG_WAREHOUSE=' "env-default"
have "${_wh_dc}" 'NESSIE_CATALOG_DEFAULT_WAREHOUSE: "${ICEBERG_WAREHOUSE:-warehouse}"' "nessie-name"
have "${_wh_dc}" 's3://${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/${ICEBERG_WAREHOUSE:-warehouse}' "nessie-loc"
have "${_wh_dc}" 'DEFAULT_OPTIONS_REGION: "${S3_REGION:-us-east-1}"' "nessie-region"
have "${_wh_dc}" 'ICEBERG_WAREHOUSE: ${ICEBERG_WAREHOUSE:-warehouse}' "trino-env"
have "${_wh_ip}" '${ENV:ICEBERG_WAREHOUSE}' "trino-catalog"
have "${_wh_bs}" "'iceberg.catalog.warehouse' = '\${ICEBERG_WAREHOUSE:-warehouse}'" "sr-catalog"
have "${_wh_bs}" "'iceberg.catalog.warehouse'='\${ICEBERG_WAREHOUSE:-warehouse}'" "sr-catalog2"
have "${_wh_bs}" 'rest_warehouse: ${ICEBERG_WAREHOUSE:-warehouse}' "sling"
have "${_wh_v2}" 'WAREHOUSE="${ICEBERG_WAREHOUSE:-warehouse}"' "gate-def"
have "${_wh_v2}" 's3://${BUCKET}/${WAREHOUSE}' "gate-expect"
have "${_wh_ins}" 'MINIO_BUCKET_ICEBERG_DATA=iceberg-data' "env-bucket"
have "${_wh_ins}" 'ICEBERG_WAREHOUSE=warehouse' "env-warehouse"
lack "${_wh_bs}" "'iceberg.catalog.warehouse' = 'warehouse'" "sr-literal"
lack "${_wh_bs}" "'iceberg.catalog.warehouse'='warehouse'" "sr-literal2"
lack "${_wh_bs}" 'rest_warehouse: warehouse' "sling-literal"
lack "${_wh_se}" "'iceberg.catalog.warehouse' = 'warehouse'" "seed-literal"
if [[ -z "${wh_bad}" ]]; then
  ok "storage: bucket+warehouse terkonfigurasi (nama & prefix), region Nessie = S3_REGION"
else
  bad "storage: konfigurasi bucket/warehouse belum lengkap →${wh_bad}"
fi

# ─── 0o. Komponen EKSTERNAL ber-URL penuh (domain + skema) ────────────────────
# Client existing (mis. Keycloak) bisa sudah berdomain `https://sso.example.com`.
# Alamat itu WAJIB didukung: skema+port dihormati (tak ditempeli http://…:12xxx) dan
# server→server diarahkan lewat KEYCLOAK_INTERNAL_URL (token/jwks Trino & Grafana).
_uo=1
grep -q 'def parse_address' "${DEPLOY_DIR}/lib/cluster/inventory.py" || _uo=0
grep -q 'def _ext_url' "${DEPLOY_DIR}/lib/cluster/generate.py" || _uo=0
grep -q 'token-url=${ENV:KEYCLOAK_INTERNAL_URL}' "${DEPLOY_DIR}/config/trino/config-coordinator.properties" || _uo=0
grep -q 'KEYCLOAK_INTERNAL_URL' "${DEPLOY_DIR}/docker-compose.yml" || _uo=0
_uo_parse="$(python3 - "${DEPLOY_DIR}" <<'PY'
import importlib.util, sys
d = sys.argv[1]
s = importlib.util.spec_from_file_location("inv", f"{d}/lib/cluster/inventory.py")
inv = importlib.util.module_from_spec(s); s.loader.exec_module(inv)
cases = {"https://sso.example.com": ("https", "sso.example.com", 443),
         "http://minio.corp:9000": ("http", "minio.corp", 9000),
         "https://sso.example.com:8443": ("https", "sso.example.com", 8443),
         "sso.example.com": ("", "sso.example.com", None),
         "10.20.0.10": ("", "10.20.0.10", None)}
print("1" if all(inv.parse_address(k) == v for k, v in cases.items()) else "0")
PY
)"
cat > "${OUT}/inv-url.yml" <<'YML'
version: 1
tls: { mode: none }
hosts:
  control: { address: 10.20.0.10, tiers: [identity, policy, pipeline, governance] }
  compute: { address: 10.20.0.11, tiers: [coord, engine] }
  state:   { address: 10.20.0.12, tiers: [state] }
components:
  keycloak: { address: https://sso.example.com }
YML
_uo_gen=0
if python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \
      "${OUT}/inv-url.yml" "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" \
      "${OUT}/url" "${DEPLOY_DIR}/scripts/render-config.sh" > "${OUT}/uri.log" 2>&1; then
  _ue="${OUT}/url/host-control/.env"
  if grep -qx 'KEYCLOAK_HOST=sso.example.com' "${_ue}" 2>/dev/null \
     && grep -qx 'KEYCLOAK_PUBLIC_URL=https://sso.example.com' "${_ue}" 2>/dev/null \
     && grep -qx 'KEYCLOAK_INTERNAL_URL=https://sso.example.com' "${_ue}" 2>/dev/null \
     && grep -qx 'KEYCLOAK_SSL_REQUIRED=external' "${_ue}" 2>/dev/null; then
    _uo_gen=1
  fi
fi
if [[ "${_uo}" == "1" && "${_uo_parse}" == "1" && "${_uo_gen}" == "1" ]]; then
  ok "eksternal URL penuh: domain https dihormati + KEYCLOAK_INTERNAL_URL + sslRequired=external"
else
  bad "eksternal URL penuh belum didukung (wire=${_uo} parse=${_uo_parse} gen=${_uo_gen})"
fi

# ─── 0p. Origin client 'airflow' pakai AIRFLOW_WEBSERVER_HOST ─────────────────
# Regresi: memakai `AIRFLOW_HOST` (penanda komponen external, SENGAJA tak diisi
# generator) → _comp_addr/comp_addr jatuh ke SICABE_HOST_ADDRESS (alamat host LOKAL),
# sehingga di host non-pemilik (compute/state) origin jadi alamat host ITU, bukan
# alamat control pemilik airflow → verify-keycloak FAIL "belum terdaftar di client".
_ap=1
grep -q '\[airflow\]="http AIRFLOW_WEBSERVER_HOST' "${DEPLOY_DIR}/scripts/verify-keycloak.sh" || _ap=0
grep -q '\["airflow"\]="http AIRFLOW_WEBSERVER_HOST' "${DEPLOY_DIR}/bootstrap.sh" || _ap=0
grep -q '\[airflow\]="http AIRFLOW_HOST ' "${DEPLOY_DIR}/scripts/verify-keycloak.sh" && _ap=0
grep -q '\["airflow"\]="http AIRFLOW_HOST ' "${DEPLOY_DIR}/bootstrap.sh" && _ap=0
if [[ "${_ap}" == "1" ]]; then
  ok "client origin airflow: pakai AIRFLOW_WEBSERVER_HOST (verify-keycloak + bootstrap)"
else
  bad "client origin airflow masih pakai AIRFLOW_HOST (origin salah di host non-pemilik)"
fi

# ─── 0q. minio EKSTERNAL ⇒ `minio-init` TIDAK ditempatkan ────────────────────
# Regresi: `TIER_SERVICES['state']` memuat minio + minio-init. Bila `minio` dijadikan
# eksternal (P3), hanya `minio` yang di-exclude → `minio-init` nyasar ke bundle `state`
# (padahal `minio` absen) → `install.sh --host state` menjalankannya tanpa guna.
_qr="$(python3 - "${DEPLOY_DIR}" <<'PY'
import importlib.util, sys
d = sys.argv[1]
s = importlib.util.spec_from_file_location("inv", f"{d}/lib/cluster/inventory.py")
inv = importlib.util.module_from_spec(s); s.loader.exec_module(inv)
doc = {"version": 1, "tls": {"mode": "none"},
       "hosts": {"control": {"address": "10.20.0.10", "tiers": ["identity", "policy", "pipeline", "governance"]},
                 "compute": {"address": "10.20.0.11", "tiers": ["coord", "engine"]},
                 "state":   {"address": "10.20.0.12", "tiers": ["state"]}},
       "components": {"minio": {"address": "https://minio.corp:9000"}}}
errs = inv.validate(doc)
svcs = set(inv.plan(doc)["state"]["services"])
ok = (not errs) and "minio" not in svcs and "minio-init" not in svcs
print("1" if ok else f"0 err={errs} state={sorted(svcs)}")
PY
)"
if [[ "${_qr}" == "1" ]]; then
  ok "minio eksternal: minio-init tak ditempatkan (bundle host bebas minio-init)"
else
  bad "minio eksternal: minio-init masih ditempatkan → ${_qr}"
fi

# ─── 0r. Nessie host-facing: port benar lewat sicabe_host_url ─────────────────
# Regresi: verify-phase2/bootstrap membentuk `http://${NESSIE_HOST}:${NESSIE_PORT}` →
# untuk Nessie EKSTERNAL (NESSIE_URL = http://<ip>:<NESSIE_INTERNAL_PORT>, mis. 19120)
# port jadi SALAH (12120 = port TERBIT lokal) → gate FAIL walau layanan sehat.
_nr=1
grep -q 'sicabe_host_url()' "${DEPLOY_DIR}/lib/common.sh" || _nr=0
grep -q 'sicabe_host_url NESSIE_URL NESSIE_HOST' "${DEPLOY_DIR}/scripts/verify-phase2-storage.sh" || _nr=0
grep -q 'sicabe_host_url NESSIE_URL NESSIE_HOST' "${DEPLOY_DIR}/bootstrap.sh" || _nr=0
grep -q 'NESSIE_HOST_API="http://${NESSIE_HOST}:${NESSIE_PORT}"' "${DEPLOY_DIR}/scripts/verify-phase2-storage.sh" && _nr=0
_nr_beh="$(bash -c '
  # shellcheck disable=SC1091
  source "'"${DEPLOY_DIR}"'/lib/common.sh" >/dev/null 2>&1 || exit 9
  NESSIE_HOST=localhost NESSIE_PORT=12120
  NESSIE_URL=http://nessie:19120; a="$(sicabe_host_url NESSIE_URL NESSIE_HOST "$NESSIE_PORT")"
  NESSIE_URL=http://100.93.54.108:19120 NESSIE_HOST=100.93.54.108
  b="$(sicabe_host_url NESSIE_URL NESSIE_HOST "$NESSIE_PORT")"
  printf "%s|%s" "$a" "$b"' 2>/dev/null)"
[[ "${_nr_beh}" == "http://localhost:12120|http://100.93.54.108:19120" ]] || _nr=0
if [[ "${_nr}" == "1" ]]; then
  ok "Nessie host-facing: pakai sicabe_host_url (alamat → NESSIE_URL apa adanya; lokal → host:port terbit)"
else
  bad "Nessie host-facing: port bisa salah untuk Nessie eksternal (beh='${_nr_beh}')"
fi

# ─── 0s. Realm Keycloak Trino terparameter (SICABE_REALM, bukan literal `sicabe`) ─
# Regresi: issuer/auth/token/jwks Trino dipatok `/realms/sicabe` → client ber-realm lain
# (SICABE_REALM != sicabe) tetap diarahkan ke realm `sicabe` → OIDC gagal.
_sr_tc="${DEPLOY_DIR}/config/trino/config-coordinator.properties"
_sr=1
grep -q 'realms/sicabe' "${_sr_tc}" && _sr=0
grep -q 'realms/${ENV:SICABE_REALM}' "${_sr_tc}" || _sr=0
grep -q 'SICABE_REALM: ${SICABE_REALM:-sicabe}' "${DEPLOY_DIR}/docker-compose.yml" || _sr=0
if [[ "${_sr}" == "1" ]]; then
  ok "Trino OAuth2: realm pakai \${ENV:SICABE_REALM} + SICABE_REALM diteruskan ke container"
else
  bad "Trino OAuth2: realm masih literal 'sicabe' atau SICABE_REALM tak diteruskan"
fi

# ─── 1. generate ────────────────────────────────────────────────────────────
if ! python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \
      "${INVENTORY}" "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" \
      "${OUT}" "${DEPLOY_DIR}/scripts/render-config.sh" > "${OUT}/generate.log" 2>&1; then
  bad "generate gagal:"; sed 's/^/      /' "${OUT}/generate.log" | head -10; exit 1
fi
ok "generate berhasil ($(grep -c '^host-' "${OUT}/generate.log") host)"
HOSTS="$(grep '^host-' "${OUT}/generate.log" | awk '{print $1}' | sed 's/^host-//')"

# ─── 1b. target node_exporter (job `node`, file_sd) di bundle Prometheus ─────
# node_exporter = PRA-SYARAT HOST (di luar installer), jadi generate.py menulis
# daftar target SELURUH host dari cluster.yml ke bundle host pemilik Prometheus.
nodeyml="${OUT}/host-control/config/prometheus/targets/node.yml"
if [[ -f "${nodeyml}" ]]; then
  nsum="$(python3 - "${nodeyml}" <<'PY'
import sys, yaml
data = yaml.safe_load(open(sys.argv[1], encoding="utf-8")) or []
hosts = sorted((e.get("labels") or {}).get("host", "") for e in data)
addrs = sorted(t for e in data for t in (e.get("targets") or []))
print(f"{len(hosts)}|{','.join(hosts)}|{','.join(addrs)}")
PY
)"
  ncount="${nsum%%|*}"; rest="${nsum#*|}"; nhosts="${rest%%|*}"; naddrs="${rest##*|}"
  exp_hosts="$(echo ${HOSTS} | tr ' ' '\n' | sort | paste -sd, -)"
  nhost_total="$(echo ${HOSTS} | wc -w | tr -d ' ')"
  if [[ "${ncount}" == "${nhost_total}" && "${nhosts}" == "${exp_hosts}" ]]; then
    ok "node targets: ${ncount} host digenerate di bundle Prometheus (${nhosts})"
  else
    bad "node targets: ${ncount} host (harus ${nhost_total}); host=${nhosts} (harus ${exp_hosts})"
  fi
  if echo "${naddrs}" | tr ',' '\n' | grep -qx '10.20.0.12:9100'; then
    ok 'node targets: format "<alamat>:9100" per host (contoh 10.20.0.12:9100)'
  else
    bad "node targets: format alamat:9100 salah (${naddrs})"
  fi
else
  bad "node targets: ${nodeyml} tidak digenerate"
fi

# ─── 2. compose valid per host + berkas runtime bind-mount berupa FILE ──────
for h in ${HOSTS}; do
  d="${OUT}/host-${h}"
  if docker compose -f "${d}/docker-compose.yml" --env-file "${d}/.env" --profile all config -q 2>"${OUT}/err-${h}"; then
    ok "host-${h}: docker compose config valid"
  else
    bad "host-${h}: compose TIDAK valid"; head -3 "${OUT}/err-${h}" | sed 's/^/      /'
  fi
  # WAJIB FILE (bukan direktori): Docker membuat DIRECTORY bila sumber bind-mount
  # hilang → Airflow SimpleAuthManager IsADirectoryError / Trino gagal auth.
  for rf in config/airflow/simple_auth_manager_passwords.json config/trino/password.db; do
    [[ -f "${d}/${rf}" ]] && ok "host-${h}: ${rf} berupa file" \
                         || bad "host-${h}: ${rf} BUKAN file (bind-mount akan jadi DIRECTORY)"
  done
done

# ─── 3. .env: tidak ada variabel topologi ganda ─────────────────────────────
TOPO='EXTERNAL_SERVICES|LOCAL_SERVICES|POSTGRES_HOST|MINIO_HOST|KEYCLOAK_HOST|OPENMETADATA_HOST|AIRFLOW_WEBSERVER_HOST|STARROCKS_FE_HOST|STARROCKS_CN_HOST|OPENSEARCH_HOST|S3_ENDPOINT|NESSIE_URL|LDAP_URL|OPA_URL'
for h in ${HOSTS}; do
  dup="$(grep -E "^(${TOPO})=" "${OUT}/host-${h}/.env" | cut -d= -f1 | sort | uniq -d | paste -sd' ' -)"
  [[ -z "${dup}" ]] && ok "host-${h}: .env tanpa variabel topologi ganda" \
                    || bad "host-${h}: .env punya variabel ganda: ${dup}"
done

# ─── 4 & 5. perbandingan semantik dengan render single-host ─────────────────
docker compose -f "${DEPLOY_DIR}/docker-compose.yml" --env-file "${DEPLOY_DIR}/.env" \
  --profile all config --format json > "${OUT}/base.json" 2>/dev/null
if [[ ! -s "${OUT}/base.json" ]]; then
  bad "tidak bisa render single-host (base) sebagai pembanding"; exit 1
fi

for h in ${HOSTS}; do
  d="${OUT}/host-${h}"
  docker compose -f "${d}/docker-compose.yml" --env-file "${d}/.env" \
    --profile all config --format json > "${OUT}/host-${h}.json" 2>/dev/null
done

PLAN_JSON="$(python3 "${DEPLOY_DIR}/lib/cluster/inventory.py" plan "${INVENTORY}")"
COMP_JSON="$(python3 "${DEPLOY_DIR}/lib/cluster/inventory.py" components "${INVENTORY}" | python3 -c "
import sys, json
d = {}
for line in sys.stdin:
    name, _, hosts = line.strip().partition(' -> ')
    name = name.strip()          # keluaran CLI mem-pad nama (kolom 24)
    if name: d[name] = hosts.strip().split(',')
print(json.dumps(d))")"

# ─── 3b. target Prometheus lintas-host → portnya HARUS diterbitkan ───────────
# Regresi nyata: prometheus.yml memakai `trino-jmx-exporter:9404` (→ IP host coord
# saat generate) tetapi 9404 TIDAK diterbitkan → job `trino` DOWN di multi-host.
# Guard: tiap target `static_configs` yang menunjuk ALAMAT HOST harus punya binding
# `<addr>:<port>:<port>` di bundle host pemilik. Target berupa NAMA container
# (mis. `statsd-exporter:9102`) harus SE-HOST dengan Prometheus (host-control).
pym="${OUT}/host-control/config/prometheus/prometheus.yml"
if [[ -f "${pym}" ]]; then
  prom_issues="$(python3 - "${OUT}" "${PLAN_JSON}" <<'PY'
import json, sys, yaml
from pathlib import Path
out = Path(sys.argv[1]); plan = json.loads(sys.argv[2])
addr2host = {v.get("address"): h for h, v in plan.items()}
doc = yaml.safe_load((out / "host-control" / "config" / "prometheus" / "prometheus.yml").read_text()) or {}

def ports_of(hostdir):
    comp = yaml.safe_load((out / hostdir / "docker-compose.yml").read_text()) or {}
    found = set()
    for svc in (comp.get("services") or {}).values():
        for e in svc.get("ports") or []:
            if isinstance(e, dict):
                found.add(f"{e.get('host_ip','')}:{e.get('published')}:{e.get('target')}")
            else:
                found.add(str(e))
    return found

issues = []
for job in doc.get("scrape_configs", []):
    if "file_sd_configs" in job:
        continue  # node_exporter = proses di HOST (:9100), bukan container
    for sc in job.get("static_configs", []) or []:
        for t in sc.get("targets", []) or []:
            if ":" not in t:
                continue
            a, p = t.rsplit(":", 1)
            if a in ("localhost", "127.0.0.1"):
                continue
            if a in addr2host:
                owner = addr2host[a]
                if f"{a}:{p}:{p}" not in ports_of(f"host-{owner}"):
                    issues.append(f"job {job.get('job_name')}: {t} lintas-host tetapi {a}:{p} TIDAK diterbitkan di host-{owner}")
            else:
                ctrl = yaml.safe_load((out / "host-control" / "docker-compose.yml").read_text()) or {}
                if a not in (ctrl.get("services") or {}):
                    issues.append(f"job {job.get('job_name')}: {t} bukan alamat host & bukan container di host-control")
print("\n".join(issues))
PY
)"
  if [[ -z "${prom_issues}" ]]; then
    ok "prometheus.yml: target lintas-host menunjuk port yang diterbitkan"
  else
    bad "prometheus.yml target bermasalah:"
    echo "${prom_issues}" | sed 's/^/      /'
  fi
else
  bad "prometheus.yml tidak ada di bundle control: ${pym}"
fi

# ─── 3c. config TIDAK boleh menunjuk ALAMAT HOST SENDIRI pada port tak-terbit ─
# Kelas bug: komponen SE-HOST ditulis ke alamat host (mis. grafana→prometheus jadi
# `http://<ip>:9090`) padahal port container itu TIDAK diterbitkan di host → connection
# refused. Komponen se-host seharusnya tetap memakai NAMA container.
coloc_issues="$(python3 - "${OUT}" "${PLAN_JSON}" <<'PY'
import json, re, sys, yaml
from pathlib import Path
out = Path(sys.argv[1]); plan = json.loads(sys.argv[2])
issues = []
for host, spec in plan.items():
    addr = spec.get("address")
    hd = out / f"host-{host}"
    compf = hd / "docker-compose.yml"
    if not addr or not compf.is_file():
        continue
    comp = yaml.safe_load(compf.read_text()) or {}
    published = set()
    for svc in (comp.get("services") or {}).values():
        for e in svc.get("ports") or []:
            if isinstance(e, dict):
                published.add(str(e.get("published")))
            elif isinstance(e, str):
                parts = e.split(":")
                if len(parts) == 2:
                    published.add(parts[0])
                elif len(parts) == 3:
                    published.add(parts[1])
    cfg = hd / "config"
    if not cfg.is_dir():
        continue
    for f in cfg.rglob("*"):
        if not f.is_file():
            continue
        try:
            text = f.read_text(encoding="utf-8", errors="ignore")
        except OSError:
            continue
        # 9100 = node_exporter: proses di HOST (pra-syarat), BUKAN container → wajar tak
        # ada di `ports:` compose. Port host-level lain bisa ditambah ke sini.
        host_level_ports = {"9100"}
        for m in re.finditer(re.escape(addr) + r":(\d+)", text):
            port = m.group(1)
            if port in host_level_ports:
                continue
            if port not in published:
                issues.append(f"host-{host}: {f.relative_to(hd)} menunjuk alamat HOST SENDIRI {addr}:{port} (port tak diterbitkan)")
print("\n".join(sorted(set(issues))))
PY
)"
if [[ -z "${coloc_issues}" ]]; then
  ok "config: tidak ada rujukan ke alamat host sendiri pada port tak-terbit"
else
  bad "config menunjuk alamat host sendiri pada port tak-terbit:"
  echo "${coloc_issues}" | sed 's/^/      /'
fi

sem_out="$(python3 - "${OUT}" "${HOSTS}" "${DEPLOY_DIR}" "${PLAN_JSON}" "${COMP_JSON}" <<'PY'
import json, re, sys
from pathlib import Path

out = Path(sys.argv[1]); hosts = sys.argv[2].split(); deploy = Path(sys.argv[3])
plan = json.loads(sys.argv[4]); comps = json.loads(sys.argv[5])
base = json.loads((out / "base.json").read_text())["services"]

COMPARE = ("image", "entrypoint", "user", "restart", "profiles",
           "deploy", "networks", "ulimits", "cap_add")

def split_ports(ports, addr):
    """Pisahkan binding port internal multi-host (host_ip == alamat host) dari dasar.

    Fase 2 menambahkan `alamat:cport:cport` HANYA di artefak multi-host (port container
    yang harus terjangkau lintas host, mis. postgres 5432). Binding itu tidak ada di
    compose dasar, jadi harus dipisahkan sebelum membandingkan — sekaligus diperiksa
    bentuknya (host port == container port, dikunci ke alamat host).
    """
    base, internal = [], []
    for p in ports or []:
        if isinstance(p, dict):
            if p.get("host_ip") == addr:
                internal.append(p)
            else:
                base.append(p)
        elif isinstance(p, str) and p.startswith(addr + ":"):
            internal.append(p)
        else:
            base.append(p)
    return base, internal

def addr_of(host: str) -> str:
    return plan.get(host, {}).get("address", "")

def name_addr(host: str) -> dict:
    """hostname container → alamat, dengan aturan yang sama seperti generate.py."""
    out_map = {}
    for name, owners in comps.items():
        if not owners:
            continue
        owner = host if host in owners else owners[0]
        if owner in plan:          # `plan` di-key oleh NAMA HOST
            out_map[name] = plan[owner]["address"]
    return out_map

def normalize(value, mapping: dict):
    """Samakan nama container DAN alamat host menjadi satu placeholder.

    Yang diuji: satu-satunya perbedaan nilai dasar vs per-host adalah **identitas
    host** yang dipakai (nama container ↔ alamat host). Pendekatan "rekonstruksi
    nilai yang diharapkan" dicoba lebih dulu dan rapuh — gagal pada bentuk
    berkutip (`"starrocks-cn"`) dan salah menandai nilai `postgres` yang memang
    dialek DB, bukan endpoint. Normalisasi dua arah menghindari keduanya.
    """
    if isinstance(value, str):
        s = value
        for _name, addr in mapping.items():
            s = s.replace(addr, "<HOST>")
        for name in mapping:
            s = re.sub(r"(?<![\w.-])" + re.escape(name) + r"(?![\w-])", "<HOST>", s)
        return s
    if isinstance(value, list):
        return [normalize(v, mapping) for v in value]
    if isinstance(value, dict):
        return {k: normalize(v, mapping) for k, v in value.items()}
    return value

# Kunci yang nilainya URL BROWSER-FACING. Nilainya memang berbeda antara single-host
# (localhost) dan multi-host (alamat host yang menjalankan komponen) — itu DISENGAJA,
# justru itu inti perbaikannya. Untuk kunci ini `localhost` diperlakukan sama seperti
# alamat host, sehingga yang masih terdeteksi sebagai masalah hanyalah perbedaan
# sebenarnya (skema, port, atau path berubah).
PUBLIC_URL_KEYS = (
    "KC_HOSTNAME", "KC_HOSTNAME_URL",
    "GF_SERVER_ROOT_URL", "AIRFLOW__WEBSERVER__BASE_URL",
    "KEYCLOAK_PUBLIC_URL", "GRAFANA_PUBLIC_URL", "OPENMETADATA_PUBLIC_URL",
    "AIRFLOW_PUBLIC_URL", "TRINO_PUBLIC_URL", "MINIO_CONSOLE_PUBLIC_URL",
    # Konsumen browser-facing turunan URL publik (T-21). Disertakan supaya beda
    # localhost (single-host) vs alamat host (multi-host) tidak dianggap regresi,
    # sementara beda port/skema tetap tertangkap.
    "OPENMETADATA_SERVER_URL", "MINIO_BROWSER_REDIRECT_URL", "MINIO_SERVER_URL",
    "OIDC_SERVER_URL", "OIDC_CALLBACK", "AUTHENTICATION_CALLBACK_URL",
)

def normalize_env_key(key: str, value, mapping: dict):
    s = normalize(value, mapping)
    if isinstance(s, str) and key in PUBLIC_URL_KEYS:
        s = re.sub(r"(?<![\w.-])localhost(?![\w-])", "<HOST>", s)
    return s

def norm_volumes(vols, root: Path):
    result = []
    for v in vols or []:
        item = dict(v)
        src = item.get("source", "") or ""
        prefix = str(root) + "/"
        if src.startswith(prefix): src = src[len(prefix):]
        item["source"] = src
        result.append(item)
    return result

issues, checked, changed_env, internal_count = [], 0, 0, 0
warn_skip = 0
for h in hosts:
    host_root = out / f"host-{h}"
    doc = json.loads((out / f"host-{h}.json").read_text())
    svcs = doc["services"]; names = set(svcs); mapping = name_addr(h)
    for name, spec in svcs.items():
        ref = base.get(name)
        if ref is None:
            issues.append(f"host-{h}: service '{name}' TIDAK ada di compose dasar"); continue
        for key in COMPARE:
            if spec.get(key) != ref.get(key):
                issues.append(f"host-{h}/{name}: field '{key}' berbeda dari single-host")
        # ports: binding internal multi-host dipisahkan dulu, lalu bentuknya diperiksa
        addr = addr_of(h)
        hb, hi = split_ports(spec.get("ports"), addr)
        rb, ri = split_ports(ref.get("ports"), addr)
        if hb != rb:
            issues.append(f"host-{h}/{name}: ports dasar berbeda dari single-host")
        if ri:
            issues.append(f"host-{h}/{name}: compose DASAR sudah punya binding internal? {ri}")
        for entry in hi:
            if isinstance(entry, str):
                parts = entry.split(":")
                ok = len(parts) == 3 and parts[0] == addr and parts[1] == parts[2]
                if not ok:
                    issues.append(f"host-{h}/{name}: binding internal salah bentuk: {entry}")
            elif str(entry.get("target")) != str(entry.get("published")) or entry.get("host_ip") != addr:
                issues.append(f"host-{h}/{name}: binding internal salah bentuk: {entry}")
        internal_count += len(hi)
        if norm_volumes(spec.get("volumes"), host_root) != norm_volumes(ref.get("volumes"), deploy):
            issues.append(f"host-{h}/{name}: 'volumes' berbeda (di luar prefix direktori)")
        for key in ("command", "healthcheck", "entrypoint"):
            if normalize(spec.get(key), mapping) != normalize(ref.get(key), mapping):
                issues.append(f"host-{h}/{name}: '{key}' berbeda di luar identitas host")
        dd = spec.get("depends_on", {}) or {}; db = ref.get("depends_on", {}) or {}
        if set(dd) - set(db):
            issues.append(f"host-{h}/{name}: depends_on menambah {sorted(set(dd)-set(db))}")
        if set(dd) - names:
            issues.append(f"host-{h}/{name}: depends_on masih lintas host {sorted(set(dd)-names)}")
        ea = spec.get("environment", {}) or {}; eb = ref.get("environment", {}) or {}
        if set(ea) != set(eb):
            issues.append(f"host-{h}/{name}: kunci environment berbeda "
                          f"(hilang={sorted(set(eb)-set(ea))}, tambahan={sorted(set(ea)-set(eb))})")
        for k in set(ea) & set(eb):
            if normalize_env_key(k, ea[k], mapping) != normalize_env_key(k, eb[k], mapping):
                issues.append(f"host-{h}/{name}: env '{k}' berbeda di luar identitas host\n"
                              f"        dasar={eb[k]!r}\n        host ={ea[k]!r}")
            elif eb[k] != ea[k]:
                changed_env += 1
        checked += 1
print(f"checked={checked}")
print(f"env_diarahkan={changed_env}")
print(f"binding_internal={internal_count}")
for i in issues:
    print(f"issue={i}")
PY
)"
checked="$(echo "${sem_out}" | sed -n 's/^checked=//p')"
internal="$(echo "${sem_out}" | sed -n 's/^binding_internal=//p')"
issues="$(echo "${sem_out}" | grep -c '^issue=' || true)"
if [[ "${issues}" -eq 0 ]]; then
  ok "semantik: ${checked} service identik dengan single-host (kecuali variabel topologi)"
  ok "port internal multi-host: ${internal} binding (dikunci ke alamat host, host port == container port)"
else
  bad "semantik: ${issues} masalah"
  echo "${sem_out}" | grep '^issue=' | sed 's/^issue=/      /' | head -12
fi

# ─── 6. URL browser-facing menurut tls.mode ────────────────────────────────
# Mode `none` (contoh) tidak boleh menghasilkan URL publik sama sekali; mode `edge`
# harus menghasilkan https://<label>.<domain> untuk enam UI, TANPA membocorkan nama
# domain ke variabel internal (container tetap memakai alamat).
echo
echo "── URL browser-facing (tls.mode) ──"
# Jebakan bash: `grep -c` mencetak "0" SEKALIGUS keluar status 1, sehingga
# `|| echo 0` menambah baris kedua ("0\n0") dan [[ ]] jadi error sintaks.
# Pakai `|| true` lalu default-nya.
none_urls="$(grep -c '_PUBLIC_URL=' "${OUT}/host-control/.env" 2>/dev/null || true)"
none_urls="${none_urls:-0}"
if [[ "${none_urls}" -eq 6 ]]; then
  ok "mode none: 6 URL publik di .env host"
else
  bad "mode none: ${none_urls} URL publik (harus 6)"
fi

# Mode none = tanpa TLS/proxy → browser memakai alamat HOST, jadi URL publik:
#   (a) harus menunjuk alamat host, BUKAN localhost atau nama service internal
#       (dulu defaultnya localhost / airflow-webserver:8080 → browser host lain
#        dibuang ke localhost, dan issuer OIDC ikut salah);
#   (b) harus SAMA di setiap bundle — issuer OIDC hanya boleh satu nilai.
none_bad=""
while IFS= read -r line; do
  [[ -z "${line}" ]] && continue
  val="${line#*=}"
  [[ "${val}" =~ ^https?://([0-9]{1,3}\.){3}[0-9]{1,3}:[0-9]+$ ]] || none_bad+=" ${line}"
done < <(grep '_PUBLIC_URL=' "${OUT}/host-control/.env" 2>/dev/null || true)
if [[ -z "${none_bad}" ]]; then
  ok "mode none: URL publik memakai alamat host (bukan localhost/nama internal)"
else
  bad "mode none: URL publik bukan alamat host:${none_bad}"
fi

for f in "${OUT}"/host-*/.env; do
  grep '_PUBLIC_URL=' "${f}" 2>/dev/null | sort > "${f}.urls" || true
done
if diff -q "${OUT}/host-control/.env.urls" "${OUT}/host-state/.env.urls" >/dev/null 2>&1 \
   && diff -q "${OUT}/host-control/.env.urls" "${OUT}/host-compute/.env.urls" >/dev/null 2>&1; then
  ok "mode none: URL publik identik di semua bundle (issuer satu nilai)"
else
  bad "mode none: URL publik BEDA antar bundle"
fi

# KC_HOSTNAME harus berisi URL publik itu — opsi inilah yang menentukan issuer.
kc_pub="$(grep '^KEYCLOAK_PUBLIC_URL=' "${OUT}/host-control/.env" 2>/dev/null | cut -d= -f2- || true)"
kc_resolved="$(docker compose -f "${OUT}/host-control/docker-compose.yml" \
  --env-file "${OUT}/host-control/.env" --profile all config 2>/dev/null \
  | python3 -c "
import sys, yaml
d = yaml.safe_load(sys.stdin) or {}
print(((d.get('services') or {}).get('keycloak') or {}).get('environment', {}).get('KC_HOSTNAME', ''))" 2>/dev/null || true)"
if [[ -n "${kc_pub}" && "${kc_resolved}" == "${kc_pub}" ]]; then
  ok "mode none: KC_HOSTNAME = URL publik Keycloak (${kc_pub})"
else
  bad "mode none: KC_HOSTNAME='${kc_resolved}' != KEYCLOAK_PUBLIC_URL='${kc_pub}'"
fi
# T-21 di mode none: console MinIO memakai URL host pemilik (state), bukan localhost.
rmc="$(docker compose -f "${OUT}/host-state/docker-compose.yml" --env-file "${OUT}/host-state/.env" \
  --profile all config 2>/dev/null | python3 -c "
import sys, yaml
d = yaml.safe_load(sys.stdin) or {}
e = ((d.get('services') or {}).get('minio') or {}).get('environment', {}) or {}
print(e.get('MINIO_BROWSER_REDIRECT_URL', ''))" 2>/dev/null || true)"
if [[ "${rmc}" == "http://10.20.0.12:12901" ]]; then
  ok "mode none: MINIO_BROWSER_REDIRECT_URL = alamat host state"
else
  bad "mode none: MINIO_BROWSER_REDIRECT_URL='${rmc}' (harus http://10.20.0.12:12901)"
fi
# T-20: tanpa TLS/ingress, service traefik TIDAK ikut (single-host tetap utuh)
if grep -q '^  traefik:' "${OUT}/host-control/docker-compose.yml" 2>/dev/null; then
  bad "mode none: service traefik ikut padahal tidak ada ingress"
else
  ok "mode none: service traefik tidak ikut (single-host tidak berubah)"
fi
if grep -qx 'TRINO_PROCESS_FORWARDED=false' "${OUT}/host-control/.env" 2>/dev/null; then
  ok "mode none: TRINO_PROCESS_FORWARDED=false (menolak header X-Forwarded-* dari klien)"
else
  bad "mode none: TRINO_PROCESS_FORWARDED bukan false"
fi
# sslRequired: mode none = HTTP polos -> realm Keycloak tidak boleh menuntut HTTPS.
# Alamat Tailscale 100.64.0.0/10 dianggap NON-lokal oleh Keycloak, sehingga default
# `external` membuat admin console gagal dengan "HTTPS required".
if grep -qx 'KEYCLOAK_SSL_REQUIRED=none' "${OUT}/host-control/.env" 2>/dev/null; then
  ok "mode none: KEYCLOAK_SSL_REQUIRED=none (bukan external)"
else
  bad "mode none: KEYCLOAK_SSL_REQUIRED bukan none"
fi
if grep -q 'KC_HOSTNAME_URL' "${OUT}/host-control/docker-compose.yml" 2>/dev/null; then
  bad "compose masih memuat KC_HOSTNAME_URL (opsi dihapus di Keycloak 26, tidak berefek)"
else
  ok "compose tidak memuat KC_HOSTNAME_URL (benar: dihapus di KC 26)"
fi

EDGE_INV="${OUT}/edge.yml"
cat > "${EDGE_INV}" <<'YAML'
version: 1
cluster:
  name: sicabe-prod
  env: production
  domain: sicabe.com
preset: a3
tls:
  mode: edge
hosts:
  control: { address: 10.20.0.10, tiers: [identity, policy, pipeline, governance] }
  compute: { address: 10.20.0.11, tiers: [coord, engine] }
  state:   { address: 10.20.0.12, tiers: [state] }
components:
  dremio: { enabled: false }
YAML
EDGE_OUT="${OUT}/edge"
if python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render "${EDGE_INV}" \
     "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" "${EDGE_OUT}" \
     "${DEPLOY_DIR}/scripts/render-config.sh" >/dev/null 2>&1; then
  edge_env="${EDGE_OUT}/host-control/.env"
  edge_n="$(grep -c '_PUBLIC_URL=' "${edge_env}" 2>/dev/null || true)"
  edge_n="${edge_n:-0}"
  [[ "${edge_n}" -eq 6 ]] && ok "mode edge: 6 URL publik di .env" \
                          || bad "mode edge: ${edge_n} URL publik (harus 6)"
  for pair in "KEYCLOAK_PUBLIC_URL=https://kc.sicabe.com" \
              "GRAFANA_PUBLIC_URL=https://grafana.sicabe.com" \
              "OPENMETADATA_PUBLIC_URL=https://om.sicabe.com" \
              "AIRFLOW_PUBLIC_URL=https://airflow.sicabe.com" \
              "TRINO_PUBLIC_URL=https://trino.sicabe.com" \
              "MINIO_CONSOLE_PUBLIC_URL=https://minio.sicabe.com"; do
    grep -qx "${pair}" "${edge_env}" && ok "mode edge: ${pair%%=*}" \
      || bad "mode edge: ${pair} tidak ada"
  done
  # ── T-20 (ingress/TLS): service Traefik + rute + kebijakan proxy ────────────
  svc_labels() { # <host> <service>
    docker compose -f "${EDGE_OUT}/host-$1/docker-compose.yml" --env-file "${EDGE_OUT}/host-$1/.env" \
      --profile all --profile edge config 2>/dev/null | python3 -c "
import sys, yaml
d = yaml.safe_load(sys.stdin) or {}
lab = (((d.get('services') or {}).get('$2') or {}).get('labels')) or []
if isinstance(lab, dict):
    for k, v in lab.items():
        print(k + '=' + str(v))
else:
    for x in lab:
        print(x)" 2>/dev/null || true
  }
  host_port_of() { # <host> <service> <container_port>
    docker compose -f "${EDGE_OUT}/host-$1/docker-compose.yml" --env-file "${EDGE_OUT}/host-$1/.env" \
      --profile all --profile edge config 2>/dev/null | python3 -c "
import sys, yaml
d = yaml.safe_load(sys.stdin) or {}
for e in (((d.get('services') or {}).get('$2') or {}).get('ports') or []):
    if isinstance(e, dict) and str(e.get('target')) == '$3':
        print(e.get('host_ip')); break" 2>/dev/null || true
  }

  # Ingress SATU host (default: pemilik keycloak = control). Ini yang menghilangkan
  # perebutan record TXT DNS-01 (dulu tiap host menerbitkan wildcard yang sama) dan
  # menekan pemakaian batas sertifikat Let's Encrypt.
  grep -q '^  traefik:' "${EDGE_OUT}/host-control/docker-compose.yml" 2>/dev/null \
    && ok "mode edge: service traefik ada di host ingress (control)" \
    || bad "mode edge: service traefik TIDAK ada di host ingress"
  for h in compute state; do
    grep -q '^  traefik:' "${EDGE_OUT}/host-${h}/docker-compose.yml" 2>/dev/null \
      && bad "mode edge: service traefik ikut di host ${h} (harus hanya ingress)" \
      || ok "mode edge: host ${h} tanpa Traefik (rute jarak jauh ditangani ingress)"
  done
  for pt in 80 443; do
    got="$(host_port_of control traefik "${pt}")"
    [[ "${got}" == "10.20.0.10" ]] && ok "mode edge: traefik :${pt} terikat ke 10.20.0.10" \
                                  || bad "mode edge: traefik :${pt} terikat ke '${got}' (harus 10.20.0.10)"
  done
  # CLUSTER_DOMAIN hanya perlu di host ingress (label rute)
  grep -qx 'CLUSTER_DOMAIN=sicabe.com' "${EDGE_OUT}/host-control/.env" 2>/dev/null \
    && ok "mode edge: CLUSTER_DOMAIN ada di host ingress" \
    || bad "mode edge: CLUSTER_DOMAIN tidak ada di host ingress"
  grep -q '^CLUSTER_DOMAIN=' "${EDGE_OUT}/host-compute/.env" 2>/dev/null \
    && bad "mode edge: CLUSTER_DOMAIN ikut di host non-ingress" \
    || ok "mode edge: host non-ingress tanpa CLUSTER_DOMAIN"

  # Rute untuk komponen di HOST LAIN ditulis di berkas dinamis (file provider),
  # karena Traefik hanya melihat container di host-nya sendiri.
  dyn="${EDGE_OUT}/host-control/config/traefik/dynamic.yml"
  if [[ -f "${dyn}" ]]; then
    ok "mode edge: berkas dinamis ingress dibuat"
    grep -q 'https://10.20.0.11:12443' "${dyn}" \
      && ok "mode edge: rute trino -> compute:12443 (https/backend)" \
      || bad "mode edge: rute trino tidak menunjuk compute:12443"
    grep -q 'http://10.20.0.12:12901' "${dyn}" \
      && ok "mode edge: rute minio -> state:12901 (console)" \
      || bad "mode edge: rute minio tidak menunjuk state:12901"
    grep -q 'serversTransport: trino-insecure' "${dyn}" \
      && ok "mode edge: rute trino memakai serversTransport insecure (sertifikat internal)" \
      || bad "mode edge: rute trino tanpa serversTransport"
    grep -qE 'kc\.sicabe\.com|grafana\.sicabe\.com' "${dyn}" \
      && bad "mode edge: komponen lokal ikut ditulis di berkas dinamis (harus lewat label)" \
      || ok "mode edge: berkas dinamis hanya memuat rute jarak jauh"
  else
    bad "mode edge: berkas dinamis ingress tidak dibuat"
  fi

  # Rute Host() per layanan memakai domain dari bundle (bukan jatuh ke `localhost`).
  for spec in "control:keycloak:kc" "control:grafana:grafana" "control:openmetadata-server:om" \
              "control:airflow-webserver:airflow"; do
    h="${spec%%:*}"; rest="${spec#*:}"; svc="${rest%%:*}"; label="${rest##*:}"
    svc_labels "${h}" "${svc}" | grep -qF "Host(\`${label}.sicabe.com\`)" \
      && ok "mode edge: rute $svc = Host(\`${label}.sicabe.com\`)" \
      || bad "mode edge: rute $svc tidak memakai domain bundle"
  done

  # Trino internal HTTPS -> backend https + serversTransport dari provider file
  # (transport TIDAK bisa lewat label; lewat label muncul error runtime).
  svc_labels compute trino-coordinator | grep -qF "loadbalancer.server.scheme=https" \
    && ok "mode edge: backend Trino memakai https" \
    || bad "mode edge: backend Trino tidak diset https"
  svc_labels compute trino-coordinator | grep -qF "serverstransport=trino-insecure@file" \
    && ok "mode edge: Trino memakai serversTransport dari provider file" \
    || bad "mode edge: serversTransport Trino salah"
  # Jeda propagasi DNS-01: apex + wildcard menulis nama TXT yang SAMA
  # (_acme-challenge.<domain>) -> tanpa jeda, LE bisa membaca record lama.
  if docker compose -f "${EDGE_OUT}/host-control/docker-compose.yml" \
       --env-file "${EDGE_OUT}/host-control/.env" --profile all --profile edge config 2>/dev/null \
       | grep -q 'dnschallenge.propagation.delayBeforeChecks='; then
    ok "mode edge: ACME memakai propagation.delayBeforeChecks (jeda propagasi DNS)"
  else
    bad "mode edge: ACME tanpa jeda propagasi DNS"
  fi

  grep -q 'insecureSkipVerify: true' "${DEPLOY_DIR}/config/traefik/dynamic.yml" 2>/dev/null \
    && ok "mode edge: berkas dinamis mendefinisikan insecureSkipVerify untuk Trino" \
    || bad "mode edge: config/traefik/dynamic.yml tidak berisi insecureSkipVerify"

  if grep -qx 'KEYCLOAK_SSL_REQUIRED=external' "${edge_env}" 2>/dev/null; then
    ok "mode edge: KEYCLOAK_SSL_REQUIRED=external (ada TLS/proxy di depan)"
  else
    bad "mode edge: KEYCLOAK_SSL_REQUIRED bukan external"
  fi
  # Kelas bug "hostname bare": config yang memakai nilai bare untuk key host
  # (mis. Sling `host: postgres`) tidak punya ':' atau '/', sehingga sempat TIDAK
  # tersubstitusi -> di host lain mencari nama container yang tidak ada. Sekaligus
  # pastikan nilai non-host yang kebetulan sama (dialek DB) TIDAK ikut berubah.
  sling="${EDGE_OUT}/host-control/config/airflow/sling/env.yaml"
  if [[ -f "${sling}" ]]; then
    grep -q "host: 10.20.0.12" "${sling}" \
      && ok "config: nilai bare key 'host' tersubstitusi (sling 5432 -> alamat state)" \
      || bad "config: 'host: postgres' di sling TIDAK tersubstitusi"
    grep -q "type: postgres" "${sling}" \
      && ok "config: 'type: postgres' (dialek) TIDAK ikut berubah" \
      || bad "config: nilai dialek 'type: postgres' ikut berubah (berbahaya)"
  else
    warn_skip=$((warn_skip + 1))
  fi
  # DB_HOST OpenMetadata: sempat literal "postgres" dan menyebabkan
  # java.net.UnknownHostException di container openmetadata-migrate.
  # CATATAN: compose hasil generate SENGAJA tetap memuat ${POSTGRES_HOST:-postgres}
  # (resolusi terjadi saat `up` lewat .env host) — berbeda dari config/** yang
  # di-render menjadi nilai literal. Jadi yang diuji ada DUA:
  #   (a) nilainya sudah terparameterisasi (bukan literal "postgres"), dan
  #   (b) hasil RESOLUSI-nya benar-benar alamat host state.
  om_cfg="${EDGE_OUT}/host-control/docker-compose.yml"
  grep -q 'DB_HOST: ${POSTGRES_HOST:-postgres}' "${om_cfg}" \
    && ok "compose: DB_HOST OpenMetadata terparameterisasi (bukan literal)" \
    || bad "compose: DB_HOST OpenMetadata masih literal 'postgres'"
  resolved_om="$(docker compose -f "${om_cfg}" --env-file "${EDGE_OUT}/host-control/.env" \
      --profile all config 2>/dev/null | grep -m1 'DB_HOST:' | awk '{print $2}')"
  [[ "${resolved_om}" == "10.20.0.12" ]] \
    && ok "compose: DB_HOST ter-resolve ke alamat state (${resolved_om})" \
    || bad "compose: DB_HOST ter-resolve '${resolved_om}' (harusnya alamat state 10.20.0.12)"

  # Precedence nilai: Compose memprioritaskan variabel SHELL di atas --env-file,
  # sedangkan install.sh mengekspor default (POSTGRES_HOST=postgres) sebelum memanggil
  # compose. Bundle host HARUS tetap menang setelah .env-nya di-source -- inilah yang
  # run_host lakukan. Tanpa itu container mencari hostname `postgres` yang hanya ada di
  # host lain: java.net.UnknownHostException (bug yang dilaporkan dari server).
  resolved_prec="$(set -a; . "${DEPLOY_DIR}/config/environment.defaults.sh"; set +a
                   set -a; . "${EDGE_OUT}/host-control/.env"; set +a
                   docker compose -f "${EDGE_OUT}/host-control/docker-compose.yml" \
                     --env-file "${EDGE_OUT}/host-control/.env" --profile all config 2>/dev/null \
                     | grep -m1 'DB_HOST:' | awk '{print $2}')"
  if [[ "${resolved_prec}" == "10.20.0.12" ]]; then
    ok "precedence: nilai bundle host menang atas default yang diekspor (DB_HOST=${resolved_prec})"
  else
    bad "precedence: DB_HOST ter-resolve '${resolved_prec}' — default menimpa bundle host"
  fi

  # variabel internal TIDAK boleh memakai nama domain (container pakai alamat)
  # Helper: nilai env satu service pada compose host ter-resolve.
  pub_env() { # <host> <service> <key>
    docker compose -f "${EDGE_OUT}/host-$1/docker-compose.yml" --env-file "${EDGE_OUT}/host-$1/.env" \
      --profile all config 2>/dev/null | python3 -c "
import sys, yaml
d = yaml.safe_load(sys.stdin) or {}
e = ((d.get('services') or {}).get('$2') or {}).get('environment', {}) or {}
print(e.get('$3', ''))" 2>/dev/null || true
  }

  # T-21: URL publik tersambung ke konsumen browser-facing-nya (bukan hanya ditulis di .env).
  om_srv="$(pub_env control openmetadata-server OPENMETADATA_SERVER_URL)"
  if [[ "${om_srv}" == "https://om.sicabe.com" ]]; then
    ok "mode edge: OPENMETADATA_SERVER_URL = URL publik (link email/notifikasi)"
  else
    bad "mode edge: OPENMETADATA_SERVER_URL='${om_srv}' (harus https://om.sicabe.com)"
  fi
  mc_redir="$(pub_env state minio MINIO_BROWSER_REDIRECT_URL)"
  if [[ "${mc_redir}" == "https://minio.sicabe.com" ]]; then
    ok "mode edge: MINIO_BROWSER_REDIRECT_URL = URL publik (console)"
  else
    bad "mode edge: MINIO_BROWSER_REDIRECT_URL='${mc_redir}' (harus https://minio.sicabe.com)"
  fi

  # INVRASIAN: SERVER_HOST_API_URL dipakai sisi SERVER (Airflow/ingestion memanggil balik
  # OM) -> wajib tetap alamat internal; kalau ikut URL publik, ingestion rusak.
  om_api="$(pub_env control openmetadata-server SERVER_HOST_API_URL)"
  if [[ -n "${om_api}" && "${om_api}" != *"sicabe.com"* ]]; then
    ok "mode edge: SERVER_HOST_API_URL tetap internal (${om_api})"
  else
    bad "mode edge: SERVER_HOST_API_URL='${om_api}' — harus internal, bukan URL publik"
  fi

  if grep -E '^(KEYCLOAK_HOST|NESSIE_URL|S3_ENDPOINT|OPA_URL|LDAP_URL|POSTGRES_HOST)=' "${edge_env}" | grep -q 'sicabe\.com'; then
    bad "mode edge: nama domain bocor ke variabel internal"
  else
    ok "mode edge: variabel internal tetap pakai alamat (bukan domain)"
  fi
else
  bad "mode edge: generate gagal"
fi

echo
if [[ ${FAIL_N} -eq 0 ]]; then
  echo -e "  ${GRN}${BLD}SEMUA LULUS${NC} (${PASS_N} grup cek)"
  exit 0
fi
echo -e "  ${RED}${BLD}GAGAL${NC}: ${FAIL_N} masalah · ${PASS_N} lulus"
exit 1