sicabe-core/deploy/scripts/verify-phase4-serving.sh

#!/usr/bin/env bash
# =============================================================================
# eBdesk SiCABE — Verifikasi Phase 4 (Serving): StarRocks Shared-Data
#
# Gate "kontrak serving": memastikan StarRocks (FE + Compute Node) benar-benar
# siap dipakai — termasuk saat FE/CN EXISTING (dikelola sistem lain).
#
# Yang diperiksa:
#   A. Kontrak service   : FE LEADER + Alive, CN Alive, protokol MySQL terjangkau
#   B. Storage volume    : s3_volume — ada, DEFAULT, enabled, location & endpoint S3
#                          sesuai konfigurasi deploy (DESC STORAGE VOLUME)
#   C. Keamanan          : sr_oidc (permitted_groups, redirect_url, issuer, audience,
#                          principal_field) + sr_ldap (ldap_conn_url == LDAP_URL),
#                          role sr_*, activate_all_roles_on_login, dan
#                          authentication_chain yang MEMUAT sr_oidc
#   D. External catalog  : iceberg_catalog (Iceberg via Nessie) + database terbaca
#   E. Antarmuka konsumen: OpenMetadata/Grafana + konsistensi antar-fase → WARN/SKIP
#
# Catatan desain (mengikuti §2.10 dokumen fase): hanya memverifikasi kontrak yang
# dimiliki fase ini. Bagian E bersifat informatif karena artefaknya milik fase 6.
#
# ⚠️ `authentication_chain` adalah dynamic config yang HILANG saat FE restart;
#    gate ini sengaja memeriksanya sebagai FAIL karena tanpa itu login OIDC mati.
#
# Exit code: 0 = semua gate lolos (WARN/SKIP boleh), 1 = ada FAIL.
#
# Usage:
#   ./scripts/verify-phase4-serving.sh
#   # menunjuk StarRocks existing:
#   STARROCKS_HOST=starrocks.corp STARROCKS_PORT=9030 \
#     STARROCKS_ROOT_PASSWORD=... ./scripts/verify-phase4-serving.sh
# =============================================================================
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
# shellcheck source=../lib/common.sh
source "${SCRIPT_DIR}/lib/common.sh"
# shellcheck source=../config/environment.defaults.sh
source "${SCRIPT_DIR}/config/environment.defaults.sh"
sicabe_load_env "${SCRIPT_DIR}" "$@"

# Config RELEVAN: mode host (--env-file) = BUNDLE host (ter-render), bukan deploy/config/ sumber.
if [[ -n "${SICABE_BUNDLE_DIR:-}" ]]; then CONFIG_DIR="${SICABE_BUNDLE_DIR}/config"; else CONFIG_DIR="${SCRIPT_DIR}/config"; fi

# STARROCKS_HOST tetap = penanda EXTERNAL (nilai operator; default localhost).
STARROCKS_HOST="${STARROCKS_HOST:-localhost}"
STARROCKS_PORT="${STARROCKS_PORT:-${STARROCKS_FE_PORT:-12930}}"
# Host EFEKTIF untuk koneksi: alamat host pemilik (multi-host) atau override operator.
SR_CONNECT="$(sicabe_local_host STARROCKS_FE_HOST STARROCKS_HOST)"
SR_USER="${STARROCKS_USER:-root}"
SR_PW="${STARROCKS_ROOT_PASSWORD:-}"
BUCKET="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}"
S3_ENDPOINT="${S3_ENDPOINT:-http://minio:9000}"
S3_REGION="${S3_REGION:-us-east-1}"
LDAP_URL="${LDAP_URL:-ldap://openldap:389}"
KC_BASE="$(sicabe_keycloak_url)"
REALM="${SICABE_REALM:-sicabe}"
GROUP_PROVIDER_NAME="${STARROCKS_GROUP_PROVIDER:-sr_ldap}"
OIDC_NAME="${STARROCKS_OIDC_INTEGRATION:-sr_oidc}"
CATALOG="${STARROCKS_ICEBERG_CATALOG:-iceberg_catalog}"
EXPECTED_GROUPS="data-admin,data-engineer,data-analyst,data-viewer"

# Klien MySQL: pakai host bila ada, jika tidak fallback ke container (tanpa pull
# mengejutkan — hanya bila image-nya sudah ada).
SR_MYSQL_IMAGE="${SR_MYSQL_IMAGE:-mysql:8}"
if command -v mysql >/dev/null 2>&1; then
  SR_MODE="host"
elif docker image inspect "${SR_MYSQL_IMAGE}" >/dev/null 2>&1; then
  SR_MODE="container"
else
  SR_MODE="none"
fi

PASS_N=0
FAIL_N=0
WARN_N=0
SKIP_N=0
pass() { echo -e "  ${PASS} $1"; PASS_N=$((PASS_N + 1)); }
FAIL_MSGS=()
fail() { echo -e "  ${FAIL} $1"; FAIL_N=$((FAIL_N + 1)); FAIL_MSGS+=("$1"); }
warn() { echo -e "  ${WARN} $1"; WARN_N=$((WARN_N + 1)); }
skip() { echo -e "  ${YELLOW}[SKIP]${NC} $1"; SKIP_N=$((SKIP_N + 1)); }

container_running() {
  docker ps --format "{{.Names}}" 2>/dev/null | grep -q "^$1$"
}

# ─────────────────────────────────────────────────────────────────────────────
# Helper: query StarRocks
# ─────────────────────────────────────────────────────────────────────────────
# sr_sql <sql>      → tab-separated, tanpa header (untuk parsing)
# sr_sql_vertical   → format \G (label: value) untuk FE/CN
sr_sql() {
  case "${SR_MODE}" in
    host)
      # --protocol=TCP: tanpa ini, -h localhost akan memakai unix socket, bukan port.
      MYSQL_PWD="${SR_PW}" mysql --protocol=TCP -h "${SR_CONNECT}" -P "${STARROCKS_PORT}" \
        -u "${SR_USER}" -N -B -e "$1" 2>/dev/null
      ;;
    container)
      docker run --rm -i --network "${SICABE_NETWORK:-sicabe-network}" \
        -e MYSQL_PWD="${SR_PW}" "${SR_MYSQL_IMAGE}" \
        mysql --protocol=TCP -h "${SR_CONNECT}" -P "${STARROCKS_PORT}" -u "${SR_USER}" \
        -N -B -e "$1" 2>/dev/null
      ;;
    *) return 1 ;;
  esac
}

sr_sql_vertical() {
  case "${SR_MODE}" in
    host)
      MYSQL_PWD="${SR_PW}" mysql --protocol=TCP -h "${SR_CONNECT}" -P "${STARROCKS_PORT}" \
        -u "${SR_USER}" -e "$1\G" 2>/dev/null
      ;;
    container)
      docker run --rm -i --network "${SICABE_NETWORK:-sicabe-network}" \
        -e MYSQL_PWD="${SR_PW}" "${SR_MYSQL_IMAGE}" \
        mysql --protocol=TCP -h "${SR_CONNECT}" -P "${STARROCKS_PORT}" -u "${SR_USER}" \
        -e "$1\G" 2>/dev/null
      ;;
    *) return 1 ;;
  esac
}

# ─────────────────────────────────────────────────────────────────────────────
# A. Kontrak service
# ─────────────────────────────────────────────────────────────────────────────
check_service() {
  echo -e "${BOLD}A. Kontrak service StarRocks${NC}"

  local external_expected=0
  [[ "${STARROCKS_HOST}" != "localhost" && "${STARROCKS_HOST}" != "127.0.0.1" ]] && external_expected=1

  if ! sicabe_host_expects_container sicabe-starrocks-fe; then
    skip "Container sicabe-starrocks-fe di host lain (mode host) - dilewati"
  elif container_running sicabe-starrocks-fe; then
    pass "Container sicabe-starrocks-fe berjalan"
  elif [[ ${external_expected} -eq 1 ]]; then
    skip "FE dikelola eksternal (${STARROCKS_HOST}:${STARROCKS_PORT}) — lewati cek container"
  else
    fail "Container sicabe-starrocks-fe tidak berjalan"
  fi
  if ! sicabe_host_expects_container sicabe-starrocks-cn; then
    skip "Container sicabe-starrocks-cn di host lain (mode host) - dilewati"
  elif container_running sicabe-starrocks-cn; then
    pass "Container sicabe-starrocks-cn berjalan"
  elif [[ ${external_expected} -eq 1 ]]; then
    skip "Compute Node dikelola eksternal — lewati cek container"
  else
    fail "Container sicabe-starrocks-cn tidak berjalan"
  fi

  if [[ "${SR_MODE}" == "none" ]]; then
    fail "Klien MySQL tidak tersedia (host) & image ${SR_MYSQL_IMAGE} belum ada lokal — semua cek SQL dilewati"
    return 1
  fi
  if [[ -z "${SR_PW}" ]]; then
    fail "STARROCKS_ROOT_PASSWORD kosong — tidak bisa terhubung"
    return 1
  fi

  if [[ "$(sr_sql 'SELECT 1')" != "1" ]]; then
    fail "Tidak bisa terhubung ke StarRocks di ${STARROCKS_HOST}:${STARROCKS_PORT}"
    return 1
  fi
  pass "Protokol MySQL terjangkau (${STARROCKS_HOST}:${STARROCKS_PORT})"

  local fe
  fe="$(sr_sql_vertical 'SHOW FRONTENDS')"
  if [[ "${fe}" == *"Alive: true"* ]]; then
    pass "FE Alive"
  else
    fail "FE tidak Alive"
  fi
  if [[ "${fe}" == *"Role: LEADER"* ]]; then
    pass "FE berperan LEADER"
  else
    warn "FE bukan LEADER (cluster multi-FE?)"
  fi

  local cn
  cn="$(sr_sql_vertical 'SHOW COMPUTE NODES')"
  if [[ "${cn}" == *"Alive: true"* ]]; then
    pass "Compute Node Alive (shared-data)"
  else
    fail "Tidak ada Compute Node yang Alive"
  fi
}

# ─────────────────────────────────────────────────────────────────────────────
# B. Storage volume (shared-data) — inti fase ini
# ─────────────────────────────────────────────────────────────────────────────
check_storage_volume() {
  echo -e "${BOLD}B. Storage volume S3 (shared-data)${NC}"

  if ! sr_sql "SHOW STORAGE VOLUMES;" | grep -q "s3_volume"; then
    fail "Storage volume 's3_volume' tidak ada (jalankan bootstrap.sh --step 5)"
    return 1
  fi
  pass "Storage volume 's3_volume' terdaftar"

  local desc
  desc="$(sr_sql "DESC STORAGE VOLUME s3_volume;")"
  # Kolom: Name · Type · IsDefault · Location · Params · Enabled · Comment
  local is_default location params enabled
  is_default="$(echo "${desc}" | awk -F'\t' '{print $3}')"
  location="$(echo "${desc}" | awk -F'\t' '{print $4}')"
  params="$(echo "${desc}" | awk -F'\t' '{print $5}')"
  enabled="$(echo "${desc}" | awk -F'\t' '{print $6}')"

  if [[ "${enabled}" == "true" ]]; then
    pass "s3_volume enabled"
  else
    fail "s3_volume tidak enabled ('${enabled}')"
  fi
  if [[ "${is_default}" == "true" ]]; then
    pass "s3_volume = DEFAULT STORAGE VOLUME"
  else
    fail "s3_volume bukan default storage volume ('${is_default}')"
  fi
  if [[ "${location}" == "s3://${BUCKET}/starrocks/" ]]; then
    pass "Location → ${location}"
  else
    fail "Location '${location}' ≠ 's3://${BUCKET}/starrocks/'"
  fi

  local ep region path_style
  ep="$(echo "${params}" | grep -oE '"aws\.s3\.endpoint":"[^"]*"' | sed 's/.*:"//; s/"//')"
  region="$(echo "${params}" | grep -oE '"aws\.s3\.region":"[^"]*"' | sed 's/.*:"//; s/"//')"
  path_style="$(echo "${params}" | grep -oE '"aws\.s3\.enable_path_style_access":"[^"]*"' | sed 's/.*:"//; s/"//')"
  if [[ "${ep}" == "${S3_ENDPOINT}" ]]; then
    pass "Params aws.s3.endpoint → ${ep}"
  else
    fail "Params aws.s3.endpoint '${ep:-tak terbaca}' ≠ S3_ENDPOINT '${S3_ENDPOINT}'"
  fi
  if [[ "${region}" == "${S3_REGION}" ]]; then
    pass "Params aws.s3.region → ${region}"
  else
    fail "Params aws.s3.region '${region:-tak terbaca}' ≠ S3_REGION '${S3_REGION}'"
  fi
  if [[ "${path_style}" == "true" ]]; then
    pass "Path-style access aktif (wajib untuk MinIO/S3 non-AWS)"
  else
    warn "Path-style access '${path_style:-tak terbaca}' — bisa gagal di MinIO"
  fi
}

# ─────────────────────────────────────────────────────────────────────────────
# C. Keamanan: OIDC + LDAP group provider + role
# ─────────────────────────────────────────────────────────────────────────────
check_security() {
  echo -e "${BOLD}C. Keamanan — OIDC Keycloak + group provider LDAP${NC}"

  local oidc
  oidc="$(sr_sql "SHOW CREATE SECURITY INTEGRATION ${OIDC_NAME};")"
  if [[ -z "${oidc}" ]]; then
    fail "Security integration '${OIDC_NAME}' tidak ada"
  else
    pass "Security integration '${OIDC_NAME}' ada"
    local pat
    for pat in "client_id\" = \"starrocks" "principal_field\" = \"preferred_username" \
               "group_provider\" = \"${GROUP_PROVIDER_NAME}\"" "required_audience\" = \"starrocks" \
               "required_issuer\" = \"${KC_BASE}/realms/${REALM}\""; do
      if [[ "${oidc}" == *"${pat}"* ]]; then
        pass "  ${OIDC_NAME}: $(echo "${pat}" | sed 's/\\"//g')"
      else
        fail "  ${OIDC_NAME}: tidak memuat ${pat} (cek bootstrap.sh --step 5)"
      fi
    done
    # redirect_url harus = alamat BROWSER (host FE) — bukan localhost/nama container.
    local _sraddr="${STARROCKS_FE_HOST:-}"
    [[ "${_sraddr}" == *.* && "${_sraddr}" != "127.0.0.1" ]] || _sraddr="${SICABE_HOST_ADDRESS:-localhost}"
    local _srexp="http://${_sraddr}:${STARROCKS_HTTP_PORT:-12030}/api/oauth2"
    if [[ "${oidc}" == *"redirect_url\" = \"${_srexp}\""* ]]; then
      pass "  ${OIDC_NAME}: redirect_url → ${_srexp}"
    else
      warn "  ${OIDC_NAME}: redirect_url bukan '${_srexp}' (cek bootstrap.sh --step 5)"
    fi
    local g miss="" groups
    IFS=',' read -ra groups <<< "${EXPECTED_GROUPS}"
    for g in "${groups[@]}"; do
      [[ "${oidc}" == *"${g}"* ]] || miss+=" ${g}"
    done
    if [[ -z "${miss}" ]]; then
      pass "  permitted_groups memuat 4 grup (${EXPECTED_GROUPS})"
    else
      fail "  permitted_groups kehilangan:${miss}"
    fi
  fi

  local ldap
  ldap="$(sr_sql "SHOW CREATE GROUP PROVIDER ${GROUP_PROVIDER_NAME};")"
  if [[ -z "${ldap}" ]]; then
    fail "Group provider '${GROUP_PROVIDER_NAME}' tidak ada"
  else
    pass "Group provider '${GROUP_PROVIDER_NAME}' ada"
    if [[ "${ldap}" == *"\"ldap_conn_url\" = \"${LDAP_URL}\""* ]]; then
      pass "  ldap_conn_url → ${LDAP_URL}"
    else
      fail "  ldap_conn_url tidak menunjuk LDAP_URL (${LDAP_URL}) — cek bootstrap.sh --step 5"
    fi
    if [[ "${ldap}" == *"\"ldap_group_identifier_attr\" = \"cn\""* ]]; then
      pass "  group identifier = cn (cocok dengan grup LDAP data-*)"
    else
      fail "  ldap_group_identifier_attr bukan 'cn'"
    fi
    if [[ "${ldap}" == *"***"* ]]; then
      pass "  bind password tersimpan (disamarkan oleh StarRocks)"
    fi
  fi

  local roles missing=""
  roles="$(sr_sql "SHOW ROLES;")"
  local r
  for r in sr_admin sr_engineer sr_analyst sr_viewer; do
    grep -q "^${r}	" <<< "${roles}" || missing+=" ${r}"
  done
  if [[ -z "${missing}" ]]; then
    pass "Role sr_admin/sr_engineer/sr_analyst/sr_viewer ada"
  else
    fail "Role hilang:${missing}"
  fi

  if [[ "$(sr_sql "SHOW VARIABLES LIKE 'activate_all_roles_on_login';" | awk -F'\t' '{print $2}')" == "true" ]]; then
    pass "activate_all_roles_on_login = true"
  else
    fail "activate_all_roles_on_login bukan true"
  fi

  # ⚠️ Dynamic config — hilang saat FE restart. Tanpa ini login OIDC mati meski
  # integration-nya ada.
  local chain
  chain="$(sr_sql "ADMIN SHOW FRONTEND CONFIG LIKE 'authentication_chain';" | awk -F'\t' '{print $3}')"
  if [[ "${chain}" == *"${OIDC_NAME}"* ]]; then
    pass "authentication_chain memuat ${OIDC_NAME} (${chain})"
  else
    fail "authentication_chain = ${chain:-tak terbaca} TIDAK memuat ${OIDC_NAME} → login OIDC mati"
    warn "  perbaiki: ./bootstrap.sh --step 5 (dynamic config hilang setelah FE restart)"
  fi

  # Pemetaan role → EXTERNAL GROUP: tidak dapat diintrospeksi tanpa login OIDC.
  local app
  app="$(sr_sql "SELECT GRANTEE FROM information_schema.applicable_roles WHERE GRANTEE LIKE 'data-%';")"
  if [[ -n "${app}" ]]; then
    pass "GRANT role → EXTERNAL GROUP terlihat untuk grup: $(echo "${app}" | tr '\n' ' ')"
  else
    skip "Pemetaan role→EXTERNAL GROUP tidak terlihat (baru tampil setelah login OIDC) — diterapkan bootstrap step 5"
  fi
}

# ─────────────────────────────────────────────────────────────────────────────
# D. External catalog Iceberg via Nessie
# ─────────────────────────────────────────────────────────────────────────────
check_catalog() {
  echo -e "${BOLD}D. External catalog Iceberg → Nessie${NC}"

  if sr_sql "SHOW CATALOGS;" | grep -q "^${CATALOG}"; then
    pass "Catalog '${CATALOG}' terdaftar"
  else
    fail "Catalog '${CATALOG}' tidak ada (jalankan bootstrap.sh --step 5)"
    return 1
  fi

  local dbs
  dbs="$(sr_sql "SHOW DATABASES FROM ${CATALOG};")"
  if [[ -z "${dbs}" ]]; then
    fail "Tidak bisa membaca database dari ${CATALOG} (Nessie tidak terjangkau?)"
    return 1
  fi
  pass "Database terbaca dari ${CATALOG}: $(echo "${dbs}" | tr '\n' ' ')"

  # Query nyata — tabel berasal dari seed fase 5, jadi SKIP (bukan FAIL) bila belum ada.
  local t state n
  for t in bronze.orders silver.orders_silver gold.orders_gold; do
    n="$(sr_sql "SELECT count(*) FROM ${CATALOG}.${t};" 2>/dev/null | head -1)"
    if [[ "${n}" =~ ^[0-9]+$ ]]; then
      pass "SELECT count(*) FROM ${CATALOG}.${t} → ${n} baris"
    elif [[ -z "${n}" ]]; then
      skip "Tabel ${CATALOG}.${t} belum ada (di-seed fase 5)"
    else
      warn "Query ${CATALOG}.${t} → '$(echo "${n}" | cut -c1-70)'"
    fi
  done
}

# ─────────────────────────────────────────────────────────────────────────────
# E. Antarmuka konsumen — milik fase 6 (WARN/SKIP)
# ─────────────────────────────────────────────────────────────────────────────
check_consumer_interfaces() {
  echo -e "${BOLD}E. Antarmuka konsumen & konsistensi antar-fase (informatif)${NC}"

  # Konsistensi statis: SQL s3_volume di bootstrap harus memakai S3_* params.
  local bs="${SCRIPT_DIR}/bootstrap.sh"
  if grep -q "aws.s3.endpoint' = '\${S3_ENDPOINT" "${bs}" 2>/dev/null; then
    pass "bootstrap.sh: s3_volume memakai \${S3_ENDPOINT} (siap S3 existing)"
  else
    warn "bootstrap.sh: 'aws.s3.endpoint' tidak memakai \${S3_ENDPOINT} — sulit diarahkan ke S3 existing"
  fi

  # permitted_groups (fase 4) ⊆ roles di ACL OPA (fase 3) — artefak lintas fase.
  local opa_roles="" g
  if [[ -f "${CONFIG_DIR}/opa/data.json" ]]; then
    opa_roles="$(grep -oE '"(data-[a-z]+)"' "${CONFIG_DIR}/opa/data.json" | tr -d '"' | sort -u | tr '\n' ' ')"
    local miss="" groups
    IFS=',' read -ra groups <<< "${EXPECTED_GROUPS}"
    for g in "${groups[@]}"; do
      [[ " ${opa_roles} " == *" ${g} "* ]] || miss+=" ${g}"
    done
    if [[ -z "${miss}" ]]; then
      pass "permitted_groups StarRocks ⊇ kunci roles di config/opa/data.json"
    else
      warn "Grup StarRocks tanpa padanan di data.json:${miss}"
    fi
  else
    skip "config/opa/data.json tidak ada — lewati konsistensi antar-fase"
  fi

  if ! sicabe_host_expects_container sicabe-openmetadata; then
    skip "Container sicabe-openmetadata di host lain (mode host) - dilewati"
  elif container_running sicabe-openmetadata; then
    pass "OpenMetadata berjalan — konektor StarRocks diuji di gate Phase 6"
  else
    skip "OpenMetadata belum berjalan (fase 6) — lewati pemeriksaan antarmukanya"
  fi
  if ! sicabe_host_expects_container sicabe-grafana; then
    skip "Container sicabe-grafana di host lain (mode host) - dilewati"
  elif container_running sicabe-grafana; then
    pass "Grafana berjalan — datasource StarRocks diuji di gate Phase 6"
  else
    skip "Grafana belum berjalan (fase 6) — lewati pemeriksaan antarmukanya"
  fi
}

# ─────────────────────────────────────────────────────────────────────────────
# Main
# ─────────────────────────────────────────────────────────────────────────────
main() {
  print_section "Verifikasi Phase 4 — Kontrak Serving (StarRocks shared-data)"
  echo -e "  FE     : ${CYAN}${STARROCKS_HOST}:${STARROCKS_PORT}${NC} (root) · klien MySQL: ${SR_MODE}"
  echo -e "  Volume : ${CYAN}s3_volume${NC} → s3://${BUCKET}/starrocks/ @ ${S3_ENDPOINT} (${S3_REGION})"
  echo -e "  Catalog: ${CYAN}${CATALOG}${NC} (Iceberg via Nessie) · OIDC + ${GROUP_PROVIDER_NAME}"
  echo ""

  check_service
  check_storage_volume
  check_security
  check_catalog
  check_consumer_interfaces

  echo ""
  echo -e "${BOLD}Ringkasan:${NC} ${GREEN}${PASS_N} PASS${NC} · ${RED}${FAIL_N} FAIL${NC} · ${YELLOW}${WARN_N} WARN${NC} · ${YELLOW}${SKIP_N} SKIP${NC}"
  if [[ ${FAIL_N} -gt 0 ]]; then
    echo ""
    echo -e "${BOLD}Yang gagal (${FAIL_N}):${NC}"
    local _i=1 _m
    for _m in "${FAIL_MSGS[@]}"; do
      echo -e "  ${RED}${_i}.${NC} ${_m}"
      _i=$((_i + 1))
    done
    echo -e "${INFO} Perbaiki item di atas, lalu jalankan ulang: ./scripts/${0##*/}"
  fi
  if [[ ${FAIL_N} -eq 0 ]]; then
    echo -e "${OK} ${GREEN}Kontrak serving Phase 4 terpenuhi — aman lanjut ke fase berikutnya.${NC}"
    exit 0
  fi
  echo -e "${ERR} Ada ${FAIL_N} gate yang gagal — perbaiki sebelum melanjutkan."
  exit 1
}

main "$@"