#!/usr/bin/env bash
# =============================================================================
# eBdesk SiCABE — Test & Verification Script
# =============================================================================
# Sesuai Bab 6 dokumen Deployment Guide:
#   6.1 Daftar Uji Verifikasi per Lapisan
#   6.2 End-to-End Smoke Test (Lintas Lapisan)
#   6.3 Performance Baseline Test
#
# Test Categories:
#   L1 = Ingestion Layer    (Airflow, Sling)
#   L2 = Storage Layer      (MinIO, Iceberg, Nessie)
#   L3 = Processing Layer   (Trino, OPA, dbt)
#   L4 = Serving Layer      (StarRocks shared-data, Dremio)
#   L5 = Governance Layer   (Keycloak, OpenMetadata, Great Expectations, Grafana)
#   E2E = End-to-End Smoke Test (Lintas Lapisan)
#   PERF = Performance Baseline
#
# Usage:
#   ./test-verification.sh                # Jalankan semua test
#   ./test-verification.sh --layer L1     # Test layer tertentu (L1-L5)
#   ./test-verification.sh --layer E2E    # End-to-End smoke test
#   ./test-verification.sh --layer PERF   # Performance baseline
#   ./test-verification.sh --list         # Daftar semua test
#   ./test-verification.sh --report       # Tampilkan laporan test terakhir
# =============================================================================

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "${SCRIPT_DIR}"

# ─── Shared Library ──────────────────────────────────────────────────────────
source "${SCRIPT_DIR}/lib/common.sh"

source "${SCRIPT_DIR}/config/environment.defaults.sh"

# Multi-host (Fase 2c): --env-file <berkas> memakai env milik SATU HOST
# (generated/host-<x>/.env) sehingga pemeriksaan berjalan sebagaimana host itu
# melihatnya — endpoint container menunjuk alamat host, bukan localhost/12xxx.
# Dipindai lebih dulu karena pemuatan env terjadi sebelum parsing argumen.
ENV_FILE_OVERRIDE=""
_prev=""
for _arg in "$@"; do
  [[ "${_prev}" == "--env-file" ]] && ENV_FILE_OVERRIDE="${_arg}"
  _prev="${_arg}"
done
ENV_FILE="${ENV_FILE_OVERRIDE:-${SCRIPT_DIR}/.env}"
[[ "${ENV_FILE}" != /* ]] && ENV_FILE="${SCRIPT_DIR}/${ENV_FILE}"
if [[ -n "${ENV_FILE_OVERRIDE}" && ! -f "${ENV_FILE}" ]]; then
  echo -e "${ERR} Berkas env tidak ditemukan: ${ENV_FILE}"
  exit 1
fi
if [[ -f "${ENV_FILE}" ]]; then
  set -a; source "${ENV_FILE}"; set +a
  [[ -n "${ENV_FILE_OVERRIDE}" ]] && echo -e "${INFO} env-file: ${ENV_FILE} (mode host)"
fi

# ─── Endpoint E2E (sadar host) ──────────────────────────────────────────────
# `sicabe_local_url`/`sicabe_local_host` (lib/common.sh): single-host → localhost;
# multi-host (bundle memuat SICABE_HOST_NAME) → alamat host PEMILIK. Sebelumnya blok
# E2E memakai `localhost` hardcode → GAGAL di multi-host karena Trino/StarRocks/Airflow
# berada di host lain.
E2E_TRINO_URL="$(sicabe_local_url https TRINO_HOST "${TRINO_HTTPS_PORT:-12443}" TRINO_URL)"
E2E_AIRFLOW_URL="$(sicabe_local_url http AIRFLOW_WEBSERVER_HOST "${AIRFLOW_PORT:-12082}" AIRFLOW_URL)"
E2E_OM_URL="$(sicabe_local_url http OPENMETADATA_HOST "${OPENMETADATA_PORT:-12085}" OPENMETADATA_URL)"
E2E_GRAFANA_URL="$(sicabe_local_url http GRAFANA_HOST "${GRAFANA_PORT:-12300}" GRAFANA_URL)"
# StarRocks via MySQL/TCP: `localhost` memicu unix-socket → pakai 127.0.0.1 di single-host.
E2E_SR_HOST="$(sicabe_local_host STARROCKS_FE_HOST)"
[[ "${E2E_SR_HOST}" == "localhost" ]] && E2E_SR_HOST="127.0.0.1"

# ─── Report File ─────────────────────────────────────────────────────────────
REPORT_FILE="${SCRIPT_DIR}/output/test-report-$(date +%Y%m%d-%H%M%S).log"
mkdir -p "${SCRIPT_DIR}/output"
PASS_COUNT=0
FAIL_COUNT=0
SKIP_COUNT=0
TEST_LOG=""

# ─── Test Helpers ────────────────────────────────────────────────────────────
print_section() {
  local title="$1"
  echo "" | tee -a "${TEST_LOG}"
  echo -e "${BOLD}${BLUE}═══════════════════════════════════════════════════════${NC}" | tee -a "${TEST_LOG}"
  echo -e "${BOLD}  ${title}${NC}" | tee -a "${TEST_LOG}"
  echo -e "${BOLD}${BLUE}═══════════════════════════════════════════════════════${NC}" | tee -a "${TEST_LOG}"
  echo "" | tee -a "${TEST_LOG}"
}

test_pass() {
  local test_name="$1"
  local detail="${2:-}"
  PASS_COUNT=$((PASS_COUNT + 1))
  echo -e "  ${PASS} ${test_name}" | tee -a "${TEST_LOG}"
  if [[ -n "${detail}" ]]; then
    echo "       ${detail}" | tee -a "${TEST_LOG}"
  fi
  return 0
}

test_fail() {
  local test_name="$1"
  local detail="${2:-}"
  FAIL_COUNT=$((FAIL_COUNT + 1))
  echo -e "  ${FAIL} ${test_name}" | tee -a "${TEST_LOG}"
  if [[ -n "${detail}" ]]; then
    echo "       ${detail}" | tee -a "${TEST_LOG}"
  fi
  return 0
}

test_skip() {
  local test_name="$1"
  local reason="${2:-}"
  SKIP_COUNT=$((SKIP_COUNT + 1))
  echo -e "  ${YELLOW}[SKIP]${NC} ${test_name}" | tee -a "${TEST_LOG}"
  if [[ -n "${reason}" ]]; then
    echo "       ${reason}" | tee -a "${TEST_LOG}"
  fi
  return 0
}

check_url() {
  # <name> <url> [budget-detik] — POLL sampai budget habis. Verifikasi dijalankan tepat
  # setelah fresh install, saat banyak service masih warmup (mis. container
  # sicabe-airflow-webserver menjalankan `airflow api-server` dgn healthcheck
  # start_period 30s + interval 15s). Sebelumnya HANYA 1 percobaan (arg ke-3 dipakai
  # sebagai --max-time curl) → service yang belum siap dilaporkan FAIL padahal sehat.
  local name="$1"
  local url="$2"
  local budget="${3:-10}"
  local deadline=$(( $(date +%s) + budget ))
  while :; do
    if curl -skf --max-time 5 "${url}" >/dev/null 2>&1; then
      test_pass "${name}" "${url}"
      return 0
    fi
    if (( $(date +%s) >= deadline )); then
      test_fail "${name}" "Tidak dapat mengakses ${url} (setelah ${budget}s; masih warmup?)"
      return 1
    fi
    sleep 2
  done
}

# ─── Komponen existing (dikelola sistem lain) ────────────────────────────────
# Bila endpoint komponen diarahkan ke luar (.env), container-nya memang TIDAK ada
# di host ini → uji container jadi SKIP, bukan FAIL. Logika sama dengan install.sh.
external_services() {
  local out=" ${EXTERNAL_SERVICES:-} "
  if [[ -n "${KEYCLOAK_URL:-}" && "${KEYCLOAK_URL}" != "http://localhost:${KEYCLOAK_PORT:-12080}" ]]; then out+=" keycloak "; fi
  if [[ -n "${LDAP_URL:-}" && "${LDAP_URL}" != "ldap://openldap:389" ]]; then out+=" openldap "; fi
  if [[ -n "${S3_ENDPOINT:-}" && "${S3_ENDPOINT}" != "http://minio:9000" ]]; then out+=" minio minio-init "; fi
  if [[ -n "${NESSIE_URL:-}" && "${NESSIE_URL}" != "http://nessie:19120" ]]; then out+=" nessie "; fi
  if [[ -n "${TRINO_URL:-}" && "${TRINO_URL}" != "https://localhost:${TRINO_HTTPS_PORT:-12443}" ]]; then out+=" trino "; fi
  if [[ -n "${OPA_URL:-}" && "${OPA_URL}" != "http://localhost:${OPA_PORT:-12181}" ]]; then out+=" opa "; fi
  if [[ -n "${STARROCKS_HOST:-}" && "${STARROCKS_HOST}" != "localhost" && "${STARROCKS_HOST}" != "127.0.0.1" ]]; then out+=" starrocks-fe starrocks-cn "; fi
  if [[ -n "${AIRFLOW_HOST:-}" && "${AIRFLOW_HOST}" != "localhost" && "${AIRFLOW_HOST}" != "127.0.0.1" ]]; then out+=" airflow-init airflow-webserver airflow-dag-processor airflow-scheduler "; fi
  if [[ -n "${OPENMETADATA_URL:-}" && "${OPENMETADATA_URL}" != "http://localhost:${OPENMETADATA_PORT:-12085}" ]]; then out+=" openmetadata-opensearch openmetadata-migrate openmetadata-server "; fi
  if [[ -n "${GRAFANA_URL:-}" && "${GRAFANA_URL}" != "http://localhost:${GRAFANA_PORT:-12300}" ]]; then out+=" grafana "; fi
  # PENTING: `grep -v '^$'` exit 1 saat tak ada baris cocok (tanpa service eksternal)
  # → dengan `set -euo pipefail` bisa mematikan skrip. `awk 'NF'` setara, selalu exit 0.
  echo "${out}" | tr ' ' '\n' | awk 'NF' | sort -u | paste -sd' ' -
}

# sicabe-trino → trino ; sicabe-starrocks-cn → starrocks-cn
container_external() {
  local svc="${1#sicabe-}"
  [[ " $(external_services) " == *" ${svc} "* ]]
}

HOST_MODE=0
HOST_CONTAINERS=""
init_host_mode() {
  [[ -z "${ENV_FILE_OVERRIDE:-}" ]] && return 0
  local dir; dir="$(dirname "${ENV_FILE}")"
  [[ -f "${dir}/docker-compose.yml" ]] || return 0
  HOST_MODE=1
  # `|| true` PENTING: di bawah `set -euo pipefail`, kegagalan pipeline ini MEMATIKAN
  # skrip DIAM-DIAM (gejala: berhenti tepat setelah baris "[INFO] env-file:"). Penyebab
  # umum: `docker compose` tak tersedia/daemon mati, atau PyYAML (python3-yaml) belum ada.
  HOST_CONTAINERS="$(docker compose -f "${dir}/docker-compose.yml" --env-file "${ENV_FILE}" \
      --profile all config 2>/dev/null | python3 -c "
import sys, yaml
d = yaml.safe_load(sys.stdin) or {}
for s in (d.get('services') or {}).values():
    n = s.get('container_name')
    if n: print(n)" 2>/dev/null || true)"
  if [[ -z "${HOST_CONTAINERS}" ]]; then
    echo -e "${ERR} Gagal membaca daftar container dari bundle host." >&2
    echo "    docker compose -f ${dir}/docker-compose.yml --env-file ${ENV_FILE} --profile all config" >&2
    echo "  Cek: 'docker compose' tersedia & daemon jalan, dan PyYAML terpasang" >&2
    echo "       (mis. sudo ./scripts/prepare-host.sh). Lalu ulangi perintah ini." >&2
    exit 1
  fi
  [[ -n "${SICABE_HOST_NAME:-}" ]] && echo -e "${INFO} Mode host '${SICABE_HOST_NAME}' (tiers=${SICABE_HOST_TIERS:-?}) — komponen host lain di-SKIP"
}

host_expects_container() {
  [[ ${HOST_MODE} -eq 0 ]] && return 0
  printf '%s\n' "${HOST_CONTAINERS}" | grep -qx "$1"
}

# Cek endpoint untuk komponen TERTENTU. Di mode host, komponen milik host lain
# di-SKIP (bukan FAIL): `localhost:12xxx` memang tidak melayani komponen yang ada di
# host lain. Keterjangkauan lintas host diverifikasi `install.sh --preflight`.
check_url_scoped() { # <label> <container> <url> [timeout]
  local label="$1" container="$2"; shift 2
  if host_expects_container "${container}"; then
    check_url "${label}" "$@"
  else
    test_skip "${label}" "Dijalankan di host lain (mode --env-file)"
  fi
}

# Jalur compose yang sama dengan init_host_mode (berkas compose ada di sebelah .env).
compose_ref() {
  local dir; dir="$(dirname "${ENV_FILE:-.env}")"
  printf ' -f %s/docker-compose.yml --env-file %s' "${dir}" "${ENV_FILE:-.env}"
}

# Citra klien untuk cek "dari dalam jaringan Docker". Diambil dari daftar citra stack
# sendiri (itulah yang di-prefetch), BUKAN tag :latest hardcode: compose memakai tag
# pinned, dan cek pernah FAIL hanya karena meminta citra yang memang tidak pernah ada
# di server (tarikan quay.io di sana gagal).
resolve_stack_image() { # <pola-egrep> [kandidat-cadangan...]
  local pat="$1"; shift
  local cand images
  images="$(docker compose $(compose_ref) config --images 2>/dev/null | grep -iE "${pat}" || true)"
  for cand in ${images} "$@"; do
    if docker image inspect "${cand}" >/dev/null 2>&1; then printf '%s' "${cand}"; return 0; fi
  done
  return 1
}

# Nama jaringan Docker milik sebuah container — jangan hardcode nama jaringan.
container_network() { # <container>
  docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{end}}' "$1" 2>/dev/null \
    | awk '{print $1}'
}

# Cek fungsional komponen bermakna bila container-nya ADA DI HOST INI, atau endpoint-nya
# memang diarahkan ke luar (.env) oleh operator.
component_checkable() { # <container>
  if docker ps --format '{{.Names}}' 2>/dev/null | grep -qx "$1"; then return 0; fi
  if container_external "$1"; then return 0; fi
  return 1
}

# Bila komponen dikelola host ini -> kembali 0 (cek boleh dijalankan). Selain itu catat
# SKIP untuk setiap label dan kembali 1. Dipakai sebagai `... || return 0`.
# Sebabnya: cek fungsional di skrip ini memakai URL localhost, jadi kalau komponennya ada
# di host lain hasilnya FAIL palsu, padahal yang benar SKIP.
require_component() { # <container> <label>...
  local container="$1"; shift
  if component_checkable "${container}"; then return 0; fi
  local label
  for label in "$@"; do
    test_skip "${label}" "Dijalankan di host lain (mode --env-file)"
  done
  return 1
}

check_docker() {
  local container="$1"
  local name="$2"
  if ! host_expects_container "${container}"; then
    test_skip "${name}" "Container ${container} dijalankan di host lain (mode --env-file)"
    return 0
  fi
  if docker ps --format "{{.Names}}" | grep -q "${container}"; then
    test_pass "${name}" "Container ${container} running"
    return 0
  fi
  if container_external "${container}"; then
    test_skip "${name}" "Container ${container} dikelola eksternal (endpoint di .env)"
    return 0
  fi
  test_fail "${name}" "Container ${container} tidak running"
  return 1
}

# Helper: query Trino via SERVICE ACCOUNT (HTTP Basic) lewat nginx proxy.
# Returns: seluruh response JSON yang mengandung 'data'
trino_query() {
  local sql="$1"
  local user="${2:-svc_seed}"
  local pass="${3:-${TRINO_SVC_SEED_PASSWORD}}"
  local next resp

  resp=$(curl -skf -X POST "${E2E_TRINO_URL}/v1/statement" \
    -u "${user}:${pass}" \
    -H "Content-Type: text/plain" \
    -d "${sql}" 2>/dev/null || true)
  if [[ -z "${resp}" ]]; then
    return 1
  fi
  next=$(echo "${resp}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('nextUri',''))" 2>/dev/null || true)

  local i=0
  while [[ -n "${next}" && ${i} -lt 15 ]]; do
    sleep 2
    resp=$(curl -skf -u "${user}:${pass}" "${next}" 2>/dev/null || true)
    if [[ -z "${resp}" ]]; then
      return 1
    fi
    next=$(echo "${resp}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('nextUri',''))" 2>/dev/null || true)
    echo "${resp}" | python3 -c "import sys,json; d=json.load(sys.stdin); [print(r) for r in d.get('data',[])]" 2>/dev/null || true
    i=$((i + 1))
  done
  return 0
}

# =============================================================================
# L5: Governance Layer — Keycloak (bab 4.1.1)
# =============================================================================
test_governance_keycloak() {
  print_section "L5.1 — Governance: Keycloak (IAM)"

  # 4.1.1.9 Verifikasi: test login SSO
  check_docker "sicabe-keycloak" "Keycloak container running"

  local kc_url="http://localhost:${KEYCLOAK_PORT:-12080}"
  check_url_scoped "Keycloak health endpoint" sicabe-keycloak "${kc_url}/health/ready" 60

  # Cek di bawah memakai URL localhost -> hanya bermakna bila Keycloak di host ini.
  # Cek di bawah memakai URL localhost + kredensial ADMIN → hanya bermakna bila
  # Keycloak di host ini. Bila Keycloak EKSTERNAL (host lain, tanpa akses admin),
  # verifikasi admin didelegasikan ke pengelola IdP → SKIP (bukan FAIL).
  if ! host_expects_container sicabe-keycloak; then
    test_skip "Keycloak admin login" "Keycloak eksternal/milik host lain — didelegasikan ke admin IdP"
    test_skip "Keycloak realm '${SICABE_REALM:-sicabe}'" "eksternal"
    test_skip "Keycloak OIDC configuration" "eksternal"
    return 0
  fi

  # Test login dengan admin credentials
  local token
  token=$(curl -sf -X POST "${kc_url}/realms/master/protocol/openid-connect/token" \
    -H "Content-Type: application/x-www-form-urlencoded" \
    --data-urlencode "client_id=admin-cli" \
    --data-urlencode "username=${KEYCLOAK_ADMIN:-admin}" \
    --data-urlencode "password=${KEYCLOAK_ADMIN_PASSWORD}" \
    --data-urlencode "grant_type=password" \
    2>/dev/null) || true

  local access_token=""
  if [[ -n "${token}" ]]; then
    access_token=$(echo "${token}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('access_token',''))" 2>/dev/null)
  fi
  if [[ -n "${access_token}" ]]; then
    test_pass "Keycloak admin login" "Token diperoleh"
  else
    test_fail "Keycloak admin login" "Gagal login sebagai ${KEYCLOAK_ADMIN:-admin} (periksa KEYCLOAK_ADMIN_PASSWORD)"
  fi

  # Test realm sicabe ada (Keycloak Admin API butuh Bearer token, bukan Basic Auth)
  local realm_check
  realm_check=$(curl -sf -H "Authorization: Bearer ${access_token}" \
    "${kc_url}/admin/realms/${SICABE_REALM:-sicabe}" 2>/dev/null) || true
  if echo "${realm_check}" | grep -q "enabled"; then
    test_pass "Keycloak realm '${SICABE_REALM:-sicabe}' ada" "Realm aktif"
  else
    test_fail "Keycloak realm '${SICABE_REALM:-sicabe}'" "Realm tidak ditemukan"
  fi

  # Test OpenID configuration
  local oidc_config
  oidc_config=$(curl -sf "${kc_url}/realms/${SICABE_REALM:-sicabe}/.well-known/openid-configuration" 2>/dev/null) || true
  if echo "${oidc_config}" | grep -q "authorization_endpoint"; then
    test_pass "Keycloak OIDC configuration" "OpenID discovery berfungsi"
  else
    test_fail "Keycloak OIDC configuration" "OpenID discovery gagal"
  fi
}

# =============================================================================
# L2: Storage Layer — MinIO (bab 4.2.1)
# =============================================================================
test_storage_minio() {
  print_section "L2.1 — Storage: MinIO (Object Storage)"

  check_docker "sicabe-minio" "MinIO container running"

  local minio_url="http://localhost:${MINIO_PORT:-12900}"
  check_url_scoped "MinIO API" sicabe-minio "${minio_url}/minio/health/live" 45
  check_url_scoped "MinIO Cluster health" sicabe-minio "${minio_url}/minio/health/cluster" 45

  # Test S3 API — buat bucket test via AWS CLI atau mc
  if command -v mc &>/dev/null; then
    # MinIO milik host lain -> mc ke URL localhost akan FAIL palsu; catat SKIP.
    if require_component sicabe-minio \
         "MinIO bucket '${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}' ada" \
         "MinIO upload/download"; then
      mc alias set "sicabe-test" "${minio_url}" \
        "${MINIO_ROOT_USER:-minioadmin}" "${MINIO_ROOT_PASSWORD}" >/dev/null 2>&1

      # List buckets
      local buckets
      buckets=$(mc ls "sicabe-test" 2>/dev/null)
      if echo "${buckets}" | grep -q "${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}"; then
        test_pass "MinIO bucket '${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}' ada"
      else
        test_fail "MinIO bucket '${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}'" "Tidak ditemukan"
      fi

      # Test upload/download (4.2.1.9). Catatan: `mc cp` dites langsung sebagai kondisi
      # (dulu lewat $? — di bawah `set -e` kegagalan justru menghentikan skrip).
      echo "test-data-${RANDOM}" > /tmp/sicabe-verify.txt
      if mc cp /tmp/sicabe-verify.txt "sicabe-test/${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/.verify/test-${RANDOM}.txt" >/dev/null 2>&1; then
        test_pass "MinIO upload/download" "S3 API berfungsi (upload)"
        mc cp "sicabe-test/${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/.verify/"* /tmp/sicabe-download.txt >/dev/null 2>&1 || true
      else
        test_fail "MinIO upload/download" "Gagal upload file via S3 API"
      fi
      rm -f /tmp/sicabe-verify.txt /tmp/sicabe-download.txt
    fi
  else
    test_skip "MinIO S3 API test" "mc client tidak tersedia"
    # Fallback: jalankan klien mc dari dalam jaringan Docker. Hanya bermakna bila
    # MinIO memang dikelola host ini — kalau tidak, cek ini PASS palsu.
    # Penjaga utama: container benar-benar JALAN di host ini. Urutan ini penting —
    # di bundle host, S3_ENDPOINT memang berbeda (IP lintas host) sehingga
    # container_external bisa salah menganggap MinIO "eksternal" padahal lokal.
    if ! docker ps --format '{{.Names}}' | grep -qx 'sicabe-minio'; then
      if container_external sicabe-minio; then
        test_skip "MinIO accessible via container" "Container sicabe-minio dikelola eksternal (endpoint di .env)"
      else
        test_skip "MinIO accessible via container" "Container sicabe-minio dijalankan di host lain (mode --env-file)"
      fi
    else
      local mc_image="" mc_net="" mc_err=""
      mc_image="$(resolve_stack_image '/mc[:@]' 'minio/mc:latest')" || mc_image=""
      mc_net="$(container_network sicabe-minio)" || mc_net=""
      if [[ -z "${mc_image}" ]]; then
        test_skip "MinIO accessible via container" "citra mc tidak ada di host — jalankan scripts/prefetch-images.sh"
      elif [[ -z "${mc_net}" ]]; then
        test_skip "MinIO accessible via container" "jaringan container MinIO tidak terbaca"
      elif mc_err="$(docker run --rm --pull=never --network="${mc_net}" "${mc_image}" \
             alias set test http://minio:9000 "${MINIO_ROOT_USER:-minioadmin}" "${MINIO_ROOT_PASSWORD:-}" 2>&1)"; then
        test_pass "MinIO accessible via container" "Docker mc dapat konek (${mc_image})"
      else
        test_fail "MinIO accessible via container" "Docker mc gagal konek (${mc_image}, network ${mc_net})"
        printf '%s\n' "${mc_err}" | grep -v '^[[:space:]]*$' | head -3 | sed 's/^/    /' || true
      fi
    fi
  fi
}

# =============================================================================
# L2: Storage Layer — Nessie (bab 4.2.3)
# =============================================================================
test_storage_nessie() {
  print_section "L2.2 — Storage: Project Nessie (Metadata Catalog)"

  check_docker "sicabe-nessie" "Nessie container running"

  local nessie_url="http://localhost:${NESSIE_PORT:-12120}"

  # Health via API port (management 9000 tidak dipublikasikan)
  check_url_scoped "Nessie API config" sicabe-nessie "${nessie_url}/api/v2/config" 60
    check_url_scoped "Nessie Iceberg REST config" sicabe-nessie "${nessie_url}${NESSIE_ICEBERG_PREFIX:-/iceberg}/v1/config" 60

  require_component sicabe-nessie \
    "Nessie branch 'main'" \
    "Nessie warehouse" || return 0

  # Cek reference 'main' via Nessie REST API v2
  local tree
  tree=$(curl -sf "${nessie_url}/api/v2/trees" 2>/dev/null || true)
  if echo "${tree}" | grep -q "main"; then
    test_pass "Nessie branch 'main'" "Reference utama tersedia"
  else
    test_skip "Nessie branch 'main'" "Belum ada reference (katalog kosong)"
  fi

  # Warehouse terhubung MinIO (indikatif via config)
  if curl -sf "${nessie_url}/api/v2/config" 2>/dev/null | grep -qi "warehouse"; then
    test_pass "Nessie warehouse" "Konfigurasi warehouse tersedia"
  else
    test_skip "Nessie warehouse" "Info warehouse tidak terlihat di /api/v2/config"
  fi
}

# =============================================================================
# L3: Processing Layer — Trino (bab 4.3.1)
# =============================================================================
test_processing_trino() {
  print_section "L3.1 — Processing: Trino (Query Engine)"

  check_docker "sicabe-trino-coordinator" "Trino coordinator running"

  local trino_url="https://localhost:${TRINO_HTTPS_PORT:-12443}"
  check_url_scoped "Trino health" sicabe-trino-coordinator "${trino_url}/v1/info" 60

  require_component sicabe-trino-coordinator \
    "Trino node status" \
    "Trino Iceberg catalog" \
    "Trino system catalog" \
    "Trino Iceberg schemas" || return 0

  # Get Trino node info
  local node_info
  node_info=$(curl -skf "${trino_url}/v1/info" 2>/dev/null)
  if echo "${node_info}" | grep -qE '"state":"ACTIVE"|"state" : "ACTIVE"'; then
    test_pass "Trino node status" "Trino dalam keadaan ACTIVE"
  else
    test_fail "Trino node status" "Trino tidak active: ${node_info}"
  fi

  # Test query — SHOW CATALOGS (4.3.1.8)
  # Trino 483 via service account (HTTP Basic) + polling nextUri
  local query_result
  query_result=$(trino_query "SHOW CATALOGS") || true

  if echo "${query_result}" | grep -q "iceberg"; then
    test_pass "Trino Iceberg catalog" "Catalog 'iceberg' terdeteksi"
  else
    test_fail "Trino Iceberg catalog" "Catalog 'iceberg' tidak muncul. Cek konektor Nessie."
  fi

  if echo "${query_result}" | grep -q "system"; then
    test_pass "Trino system catalog" "Catalog 'system' terdeteksi"
  fi

  # Test query — SHOW SCHEMAS di iceberg catalog
  local schemas
  schemas=$(trino_query "SHOW SCHEMAS FROM iceberg") || true

  if echo "${schemas}" | grep -q "bronze"; then
    test_pass "Trino Iceberg schemas" "Schema 'bronze' terdeteksi"
  elif echo "${schemas}" | grep -q "information_schema"; then
    test_pass "Trino Iceberg schemas" "Minimal information_schema ada"
  else
    test_skip "Trino Iceberg schemas" "Schema belum dibuat. Jalankan bootstrap.sh step 4."
  fi
}

# =============================================================================
# L4: Serving Layer — StarRocks (bab 4.4.1)
# =============================================================================
test_serving_starrocks() {
  print_section "L4.1 — Serving: StarRocks (shared-data: FE + CN)"

  check_docker "sicabe-starrocks-fe" "StarRocks FE container running"
  check_docker "sicabe-starrocks-cn" "StarRocks CN container running"

  # Test koneksi MySQL ke StarRocks FE
  if command -v mysql &>/dev/null; then
    # StarRocks milik host lain -> mysql ke URL localhost akan FAIL palsu; catat SKIP.
    if require_component sicabe-starrocks-fe \
             "StarRocks FE status" \
             "StarRocks CN status" \
             "StarRocks storage volume" \
             "StarRocks External Catalog" \
             "StarRocks LDAP group provider"; then
        sr_mysql() {
          mysql -h 127.0.0.1 -P "${STARROCKS_FE_PORT:-12930}" -u root -p"${STARROCKS_ROOT_PASSWORD}" -e "$1" 2>/dev/null
        }

        local sr_status
        sr_status=$(sr_mysql "SHOW PROC '/frontends';") || true
        if echo "${sr_status}" | grep -q "Alive"; then
          test_pass "StarRocks FE status" "Frontend leader aktif"
        else
          test_fail "StarRocks FE status" "Tidak bisa konek. mysql -h localhost -P ${STARROCKS_FE_PORT:-12930} -u root -p"
        fi

        # Compute node (shared-data) — bukan BACKEND
        local cn_status
        cn_status=$(sr_mysql "SHOW COMPUTE NODES;") || true
        if echo "${cn_status}" | grep -qi "Alive"; then
          test_pass "StarRocks CN status" "Compute node aktif"
        else
          test_fail "StarRocks CN status" "Tidak ada compute node aktif"
        fi

        # Storage volume S3 (shared-data)
        local sv
        sv=$(sr_mysql "SHOW STORAGE VOLUMES;") || true
        if echo "${sv}" | grep -q "s3_volume"; then
          test_pass "StarRocks storage volume" "s3_volume (S3/MinIO) terdaftar"
        else
          test_skip "StarRocks storage volume" "s3_volume belum dibuat. Jalankan bootstrap.sh step 5."
        fi

        # External Catalog via Nessie
        local catalogs
        catalogs=$(sr_mysql "SHOW CATALOGS;") || true
        if echo "${catalogs}" | grep -q "iceberg_catalog"; then
          test_pass "StarRocks External Catalog" "Catalog 'iceberg_catalog' terdaftar"
        else
          test_skip "StarRocks External Catalog" "Catalog 'iceberg_catalog' belum dibuat. Jalankan bootstrap.sh step 5."
        fi

        # LDAP group provider
        local gp
        gp=$(sr_mysql "SHOW CREATE GROUP PROVIDER sr_ldap;") || true
        if echo "${gp}" | grep -qi "ldap"; then
          test_pass "StarRocks LDAP group provider" "sr_ldap terdaftar"
        else
          test_skip "StarRocks LDAP group provider" "sr_ldap belum dibuat."
        fi
    fi
  else
    test_skip "StarRocks FE connection" "MySQL client tidak tersedia"
    # Fallback tanpa citra tambahan: klien mysql sudah ADA di dalam container
    # starrocks-fe (dipakai healthcheck-nya sendiri) -> cukup docker exec.
    # mysql:8 tidak dipakai stack, jadi tidak pernah ikut di-prefetch (dulu FAIL).
    if ! docker ps --format '{{.Names}}' | grep -qx 'sicabe-starrocks-fe'; then
      test_skip "StarRocks accessible via container" "Container sicabe-starrocks-fe tidak jalan di host ini"
    elif docker exec sicabe-starrocks-fe \
         mysql --connect-timeout 5 -h127.0.0.1 -P9030 -uroot -p"${STARROCKS_ROOT_PASSWORD:-}" \
         -e "SELECT 1;" >/dev/null 2>&1; then
      test_pass "StarRocks accessible via container" "Docker MySQL dapat konek (exec starrocks-fe)"
    else
      test_fail "StarRocks accessible via container" "Docker MySQL gagal konek (exec starrocks-fe)"
    fi
  fi
}

# =============================================================================
# L1: Ingestion Layer — Airflow (bab 4.5.1)
# =============================================================================
test_ingestion_airflow() {
  print_section "L1.1 — Ingestion: Airflow (Orchestrator)"

  check_docker "sicabe-airflow-webserver" "Airflow webserver container running"
  check_docker "sicabe-airflow-scheduler" "Airflow scheduler container running"

  local airflow_url="http://localhost:${AIRFLOW_PORT:-12082}"
  check_url_scoped "Airflow webserver" sicabe-airflow-webserver "${airflow_url}/api/v2/monitor/health" 90

  require_component sicabe-airflow-webserver \
    "Airflow health" \
    "Airflow REST API" \
    "Airflow DAG 'sicabe_batch_daily'" || return 0

  # Cek health endpoint (Airflow 3.x: /api/v2/monitor/health)
  local health
  health=$(curl -sf "${airflow_url}/api/v2/monitor/health" 2>/dev/null)
  if echo "${health}" | grep -q "healthy"; then
    test_pass "Airflow health" "Semua komponen healthy"
  elif echo "${health}" | grep -q "scheduler.*status"; then
    test_pass "Airflow health endpoint" "Health endpoint berfungsi"
  else
    test_fail "Airflow health" "Health check gagal: ${health}"
  fi

  # Cek airflow info via REST API (4.5.1.9) — Airflow 3.x pakai /api/v2
  local af_info
  af_info=$(curl -sf "${airflow_url}/api/v2/monitor/health" 2>/dev/null || true)
  if [[ -n "${af_info}" ]]; then
    test_pass "Airflow REST API" "API v2 berfungsi"
  else
    test_skip "Airflow REST API" "API tidak merespon. Mungkin auth diperlukan."
  fi

  # Cek DAG example via Airflow CLI (Airflow 3.x: REST /api/v1/dags tidak ada)
  local dags
  if ! host_expects_container sicabe-airflow-webserver; then
    test_skip "Airflow DAG 'sicabe_batch_daily'" "Airflow dijalankan di host lain (mode --env-file)"
    dags=""
  else
    dags=$(docker exec sicabe-airflow-webserver airflow dags list 2>/dev/null || true)
  fi
  if echo "${dags}" | grep -q "sicabe_batch_daily"; then
    test_pass "Airflow DAG 'sicabe_batch_daily'" "DAG terdaftar"
  else
    test_skip "Airflow DAG 'sicabe_batch_daily'" "DAG belum didaftarkan. Jalankan bootstrap.sh step 6."
  fi
  if echo "${dags}" | grep -q "sicabe_cdc_realtime"; then
    test_pass "Airflow DAG 'sicabe_cdc_realtime'" "DAG terdaftar"
  fi
}

# =============================================================================
# L5: Governance Layer — OpenMetadata (bab 4.6.1)
# =============================================================================
test_governance_openmetadata() {
  print_section "L5.2 — Governance: OpenMetadata (Metadata Platform)"

  check_docker "sicabe-openmetadata" "OpenMetadata container running"

  local om_url="http://localhost:${OPENMETADATA_PORT:-12085}"
  check_url_scoped "OpenMetadata health" sicabe-openmetadata "${om_url}/health" 90

  require_component sicabe-openmetadata \
    "OpenMetadata API" \
    "OpenMetadata UI" \
    "OpenMetadata login" || return 0

  # Cek API version
  local api_info
  api_info=$(curl -sf "${om_url}/health" 2>/dev/null)
  if [[ -n "${api_info}" ]]; then
    test_pass "OpenMetadata API" "API v1 berfungsi"
  fi

  # Cek UI
  local ui_check
  ui_check=$(curl -sf "${om_url}" 2>/dev/null)
  if echo "${ui_check}" | grep -qi "openmetadata"; then
    test_pass "OpenMetadata UI" "UI dapat diakses"
  elif [[ -n "${ui_check}" ]]; then
    test_pass "OpenMetadata UI" "UI merespon"
  else
    test_fail "OpenMetadata UI" "UI tidak merespon"
  fi

  # Test login (OpenMetadata 2.0.x: email admin@open-metadata.org, password Base64.
  # Catatan: OM 2.0.x tidak lagi membaca OPENMETADATA_ADMIN_PASSWORD saat bootstrap
  # — default-nya 'admin'; password dari .env dipakai bila sudah diganti lewat UI.)
  local login_test
  local om_pass_b64
  om_pass_b64=$(printf '%s' "${OPENMETADATA_ADMIN_PASSWORD:-admin}" | base64)
  login_test=$(curl -sf -X POST "${om_url}/api/v1/users/login" \
    -H "Content-Type: application/json" \
    -d "{\"email\":\"admin@open-metadata.org\",\"password\":\"${om_pass_b64}\"}" \
    2>/dev/null) || true
  if echo "${login_test}" | grep -q "accessToken"; then
    test_pass "OpenMetadata login" "Admin login berhasil"
  else
    test_skip "OpenMetadata login" "Login gagal — mungkin password berbeda"
  fi
}

# =============================================================================
# L5: Governance Layer — Grafana (bab 4.6.3)
# =============================================================================
test_governance_grafana() {
  print_section "L5.3 — Governance: Grafana (Observability)"

  check_docker "sicabe-grafana" "Grafana container running"

  local grafana_url="http://localhost:${GRAFANA_PORT:-12300}"
  check_url_scoped "Grafana health" sicabe-grafana "${grafana_url}/api/health" 60

  require_component sicabe-grafana \
    "Grafana login" \
    "Grafana datasource Prometheus" \
    "Grafana datasource MinIO Metrics" || return 0

  # Cek login (Grafana 13.x: /api/auth/login deprecated — pakai Basic Auth)
  local login
  login=$(curl -sf -u "admin:${GRAFANA_ADMIN_PASSWORD:-grafana123}" \
    "${grafana_url}/api/user" 2>/dev/null) || true
  if echo "${login}" | grep -q "\"login\""; then
    test_pass "Grafana login" "Admin Basic Auth berhasil"
  else
    test_fail "Grafana login" "Gagal autentikasi"
  fi

  # Cek datasource provisioning (4.6.3.6)
  # Catatan (review Fase 6): datasource 'OpenMetadata' DIHAPUS karena plugin
  # 'grafana-opendatasource-http' tidak ada di katalog Grafana. 'MinIO Metrics'
  # juga menunjuk Prometheus — metrik MinIO diambil lewat scrape, bukan dibaca
  # langsung dari API S3 MinIO.
  local ds
  ds=$(curl -sf -u "admin:${GRAFANA_ADMIN_PASSWORD:-grafana123}" \
    "${grafana_url}/api/datasources" 2>/dev/null || true)
  if [[ -n "${ds}" ]]; then
    if echo "${ds}" | grep -q "Prometheus"; then
      test_pass "Grafana datasource Prometheus" "Terprovisioning"
    fi
    if echo "${ds}" | grep -q "MinIO Metrics"; then
      test_pass "Grafana datasource MinIO Metrics" "Terprovisioning (metrik MinIO via Prometheus)"
    fi
  fi
}

# =============================================================================
# E2E: End-to-End Smoke Test (Bab 6.2)
# =============================================================================
test_e2e_smoke() {
  print_section "E2E — End-to-End Smoke Test (Lintas Lapisan)"
  echo -e "${INFO} Skenario: Alur data dari Ingestion → Storage → Processing → Serving → Governance${NC}"
  echo ""

  # Prasyarat: semua komponen running
  echo -e "${INFO} 1/5 Memeriksa semua komponen running..."
  local all_running=true
  # dremio OPT-IN (profile khusus, tidak ikut `--profile all`) → tidak wajib.
  local optional_svcs="dremio"
  for svc in keycloak openldap minio nessie opa trino starrocks-fe starrocks-cn dremio airflow-webserver openmetadata grafana; do
      # Mode host: komponen host lain tidak dihitung (keterjangkauannya
      # sudah diverifikasi `install.sh --preflight`).
      if ! host_expects_container "sicabe-${svc}"; then
        echo -e "  [ .. ] sicabe-${svc} — di host lain, dilewati"
        continue
      fi
    if ! docker ps --format "{{.Names}}" | grep -q "sicabe-${svc}"; then
      if [[ " ${optional_svcs} " == *" ${svc} "* ]]; then
        echo -e "  ${WARN} sicabe-${svc} — tidak running (opsional, dilewati)"
      else
        echo -e "  ${FAIL} sicabe-${svc} — TIDAK RUNNING"
        all_running=false
      fi
    fi
  done
  if ${all_running}; then
    if [[ ${HOST_MODE} -eq 1 ]]; then
      test_pass "E2E Prasyarat" "Semua container MILIK HOST INI running (host lain di-SKIP; keterjangkauan dicek --preflight)"
    else
      test_pass "E2E Prasyarat" "Semua container running"
    fi
  else
    test_fail "E2E Prasyarat" "Beberapa container tidak running"
    echo -e "${ERR} E2E test dihentikan — tidak semua komponen tersedia."
    return 1
  fi

  # Step 1: [Ingestion] Test koneksi ke sumber data & MinIO
  echo -e "${INFO} 2/5 [Ingestion] Verifikasi pipeline integration..."
  local airflow_ok=false
  if curl -sf "${E2E_AIRFLOW_URL}/api/v2/monitor/health" >/dev/null 2>&1; then
    airflow_ok=true
    test_pass "E2E Ingestion: Airflow siap"
  else
    test_fail "E2E Ingestion: Airflow" "Airflow tidak siap (${E2E_AIRFLOW_URL})"
  fi

  # Step 2: [Storage] Data tersimpan sebagai Iceberg, tercatat di Nessie
  echo -e "${INFO} 3/5 [Storage] Verifikasi Iceberg + Nessie..."
  local tq_result
  tq_result=$(trino_query "SELECT 'Storage Layer OK' AS status") || true
  if [[ -n "${tq_result}" ]]; then
    test_pass "E2E Storage: Trino query execution"
    # Coba query Iceberg catalog
    local iceberg_query
    iceberg_query=$(trino_query "SHOW TABLES FROM iceberg.bronze") || true
    if [[ -n "${iceberg_query}" ]]; then
      test_pass "E2E Storage: Iceberg catalog queryable via Trino"
    fi
  else
    test_fail "E2E Storage: Trino" "Query execution gagal"
  fi

  # Step 3: [Processing] dbt via Trino transformasi
  echo -e "${INFO} 4/5 [Processing] Verifikasi data processing..."
  if [[ "${airflow_ok}" == true ]]; then
    # `docker exec` hanya berlaku untuk container LOKAL. Di host lain → SKIP (bukan FAIL):
    # keterjangkauan lintas host sudah diverifikasi `--preflight`, dan health API di atas
    # membuktikan Airflow siap dari sini.
    if [[ ${HOST_MODE} -eq 1 ]] && ! host_expects_container sicabe-airflow-webserver; then
      test_skip "E2E Processing: DAG" "Airflow di host lain (mode --env-file) — cek DAG via UI/API di sana"
    elif docker exec sicabe-airflow-webserver airflow dags list 2>/dev/null | grep -q "sicabe"; then
      test_pass "E2E Processing: DAG siap dijalankan"
    else
      test_skip "E2E Processing: DAG" "DAG contoh belum di-deploy. Jalankan bootstrap.sh step 6."
    fi
  fi

  # Step 4: [Serving] StarRocks siap melayani query
  echo -e "${INFO} 5/5 [Serving] Verifikasi StarRocks..."
  if command -v mysql &>/dev/null; then
    if mysql --protocol=TCP -h "${E2E_SR_HOST}" -P "${STARROCKS_FE_PORT:-12930}" -u root -p"${STARROCKS_ROOT_PASSWORD}" -e "SELECT 1 AS test;" >/dev/null 2>&1; then
      test_pass "E2E Serving: StarRocks query execution"
    else
      test_fail "E2E Serving: StarRocks" "Query execution gagal (${E2E_SR_HOST}:${STARROCKS_FE_PORT:-12930})"
    fi
  else
    test_skip "E2E Serving: StarRocks" "MySQL client tidak tersedia"
  fi

  # Step 5: [Governance] Verifikasi governance
  echo -e "${INFO} 6/5 [Governance] Verifikasi metadata & observability..."
  if curl -sf "${E2E_OM_URL}/health" >/dev/null 2>&1; then
    test_pass "E2E Governance: OpenMetadata siap"
  fi
  if curl -sf "${E2E_GRAFANA_URL}/api/health" >/dev/null 2>&1; then
    test_pass "E2E Governance: Grafana siap"
  fi
}

# =============================================================================
# PERF: Performance Baseline Test (Bab 6.3)
# =============================================================================
test_performance_baseline() {
  print_section "PERF — Performance Baseline Test"

  # Test latency endpoint setiap komponen
  echo -e "${INFO} Mengukur latency endpoint setiap lapisan...${NC}"

  local endpoints=(
    "Keycloak:http://localhost:${KEYCLOAK_PORT:-12080}/health/ready"
    "MinIO:http://localhost:${MINIO_PORT:-12900}/minio/health/live"
    "Nessie:http://localhost:${NESSIE_PORT:-12120}/api/v2/config"
    "Trino:https://localhost:${TRINO_HTTPS_PORT:-12443}/v1/info"
    "OPA:http://localhost:${OPA_PORT:-12181}/v1/data/trino/allow"
    "Airflow:http://localhost:${AIRFLOW_PORT:-12082}/api/v2/monitor/health"
    "OpenMetadata:http://localhost:${OPENMETADATA_PORT:-12085}/health"
    "Grafana:http://localhost:${GRAFANA_PORT:-12300}/api/health"
  )

  for ep in "${endpoints[@]}"; do
    local name="${ep%%:*}"
    local url="${ep#*:}"
    # Mode host: ukur hanya komponen milik host ini — latency ke
    # localhost komponen host lain tidak bermakna.
    local ep_container=""
    case "${name}" in
      Keycloak)     ep_container="sicabe-keycloak" ;;
      MinIO)        ep_container="sicabe-minio" ;;
      Nessie)       ep_container="sicabe-nessie" ;;
      Trino)        ep_container="sicabe-trino-coordinator" ;;
      OPA)          ep_container="sicabe-opa" ;;
      Airflow)      ep_container="sicabe-airflow-webserver" ;;
      OpenMetadata) ep_container="sicabe-openmetadata" ;;
      Grafana)      ep_container="sicabe-grafana" ;;
    esac
    if [[ -n "${ep_container}" ]] && ! host_expects_container "${ep_container}"; then
      echo -e "  [ .. ] ${name} — di host lain, dilewati"
      continue
    fi
    local total_time=0
    local success=true

    for run in 1 2 3; do
      local time_val
      time_val=$(curl -kso /dev/null -w "%{time_total}" --max-time 5 "${url}" 2>/dev/null || echo "0")
      total_time=$(echo "${total_time} + ${time_val}" | bc -l 2>/dev/null || echo "0")
      if [[ "${time_val}" == "0" ]]; then
        success=false
      fi
    done

    if [[ "${success}" == true ]]; then
      local avg
      avg=$(echo "scale=3; ${total_time} / 3" | bc -l 2>/dev/null || echo "N/A")
      test_pass "PERF Latency: ${name}" "Rata-rata ${avg}s"
    else
      test_skip "PERF Latency: ${name}" "Endpoint tidak merespon"
    fi
  done

  # Test query latency Trino
  echo -e "${INFO} Mengukur query latency Trino...${NC}"
  if curl -skf "https://localhost:${TRINO_HTTPS_PORT:-12443}/v1/info" >/dev/null 2>&1; then
    local query_times=()
    for run in 1 2; do
      local qtime
      qtime=$(curl -skf -X POST "https://localhost:${TRINO_HTTPS_PORT:-12443}/v1/statement" \
        -u "svc_seed:${TRINO_SVC_SEED_PASSWORD}" \
        -H "Content-Type: text/plain" \
        -d "SELECT 1" -w "%{time_total}" -o /dev/null 2>/dev/null || echo "0")
      query_times+=("${qtime}")
    done
    local qavg
    qavg=$(echo "scale=3; (${query_times[0]} + ${query_times[1]}) / 2" | bc -l 2>/dev/null || echo "N/A")
    test_pass "PERF Trino query" "Rata-rata ${qavg}s"
  else
    test_skip "PERF Trino query" "Trino tidak tersedia"
  fi
}

# =============================================================================
# REPORT
# =============================================================================
show_report() {
  echo ""
  echo -e "${BOLD}${BLUE}═══════════════════════════════════════════════════════${NC}"
  echo -e "${BOLD}  LAPORAN VERIFIKASI${NC}"
  echo -e "${BOLD}${BLUE}═══════════════════════════════════════════════════════${NC}"
  echo ""
  echo -e "  Total Pass: ${PASS_COUNT}"
  echo -e "  Total Fail: ${FAIL_COUNT}"
  echo -e "  Total Skip: ${SKIP_COUNT}"
  echo -e "  Total:      $((PASS_COUNT + FAIL_COUNT + SKIP_COUNT))"
  echo ""
  echo -e "  Report file: ${REPORT_FILE}"
  echo ""

  if [[ ${FAIL_COUNT} -eq 0 ]]; then
    echo -e "${GREEN}${BOLD}  ✅ SEMUA TEST PASS${NC}"
  else
    echo -e "${RED}${BOLD}  ❌ ${FAIL_COUNT} TEST FAILED${NC}"
    echo -e "  Periksa ${REPORT_FILE} untuk detail."
  fi
  echo ""
}

# =============================================================================
# MAIN
# =============================================================================
main() {
  # Mode per-host ditentukan setelah env dimuat (ENV_FILE) dan fungsi tersedia.
  init_host_mode

  local TEST_LAYER="ALL"

  if [[ $# -gt 0 ]]; then
    case "$1" in
      --env-file)
        # Sudah ditangani sebelum pemuatan env (lihat di atas banner).
        shift 2 2>/dev/null || shift
        ;;
      --help|-h)
        echo "Usage: $0 [--layer L1|L2|L3|L4|L5|E2E|PERF] [--list] [--report]"
        echo "  Default: semua test"
        echo "  --layer L1    Test Ingestion Layer (Airflow)"
        echo "  --layer L2    Test Storage Layer (MinIO, Nessie)"
        echo "  --layer L3    Test Processing Layer (Trino)"
        echo "  --layer L4    Test Serving Layer (StarRocks)"
        echo "  --layer L5    Test Governance Layer (Keycloak, OpenMetadata, Grafana)"
        echo "  --layer E2E   End-to-End Smoke Test"
        echo "  --layer PERF  Performance Baseline Test"
        echo "  --list        Daftar test yang tersedia"
        echo "  --report      Tampilkan hasil test terakhir"
        exit 0
        ;;
      --list)
        echo "Test yang tersedia:"
        echo "  L1   - Ingestion Layer: Airflow"
        echo "  L2   - Storage Layer: MinIO, Nessie"
        echo "  L3   - Processing Layer: Trino"
        echo "  L4   - Serving Layer: StarRocks"
        echo "  L5   - Governance Layer: Keycloak, OpenMetadata, Grafana"
        echo "  E2E  - End-to-End Smoke Test"
        echo "  PERF - Performance Baseline Test"
        exit 0
        ;;
      --layer)
        TEST_LAYER="${2:-ALL}"
        ;;
      --report)
        local latest_report
        latest_report=$(ls -t "${SCRIPT_DIR}/output"/test-report-*.log 2>/dev/null | head -1)
        if [[ -n "${latest_report}" ]]; then
          cat "${latest_report}"
        else
          echo "Belum ada laporan test. Jalankan $0 terlebih dahulu."
        fi
        exit 0
        ;;
      *)
        echo -e "${ERR} Argumen tidak dikenal: $1"
        echo "Gunakan --help untuk bantuan."
        exit 1
        ;;
    esac
  fi

  # Setup test log
  TEST_LOG="${REPORT_FILE}"
  exec > >(tee -a "${TEST_LOG}") 2>&1

  echo -e "${BOLD}${BLUE}"
  echo "╔═══════════════════════════════════════════════════════╗"
  echo "║   eBdesk SiCABE — Test & Verification                ║"
  echo "║   Bab 6: Verifikasi & Smoke Test                     ║"
  echo "╚═══════════════════════════════════════════════════════╝"
  echo -e "${NC}"
  echo "Timestamp: $(date)"
  echo ""

  # Pastikan Docker berjalan
  if ! docker ps >/dev/null 2>&1; then
    echo -e "${ERR} Docker tidak berjalan. Test dihentikan."
    exit 1
  fi

  case "${TEST_LAYER}" in
    L1) test_ingestion_airflow ;;
    L2) test_storage_minio; test_storage_nessie ;;
    L3) test_processing_trino ;;
    L4) test_serving_starrocks ;;
    L5) test_governance_keycloak; test_governance_openmetadata; test_governance_grafana ;;
    E2E) test_e2e_smoke ;;
    PERF) test_performance_baseline ;;
    ALL)
      test_governance_keycloak
      test_storage_minio
      test_storage_nessie
      test_processing_trino
      test_serving_starrocks
      test_ingestion_airflow
      test_governance_openmetadata
      test_governance_grafana
      test_e2e_smoke
      test_performance_baseline
      ;;
  esac

  show_report
}

main "$@"