From 50a9b415c36ec046ff6a2cc45748c069f67dcd34 Mon Sep 17 00:00:00 2001 From: Subroto Date: Mon, 12 Oct 2026 21:53:26 +0700 Subject: [PATCH] sync @ 3f3e69b --- deploy/scripts/test-generate.sh | 11 ++ deploy/scripts/verify-nessie.sh | 201 ++++++++++++++++++++++++ deploy/scripts/verify-phase2-storage.sh | 2 +- 3 files changed, 213 insertions(+), 1 deletion(-) create mode 100755 deploy/scripts/verify-nessie.sh diff --git a/deploy/scripts/test-generate.sh b/deploy/scripts/test-generate.sh index 713305a..fa7a1c6 100755 --- a/deploy/scripts/test-generate.sh +++ b/deploy/scripts/test-generate.sh @@ -512,6 +512,17 @@ else bad "bootstrap: tak ada uji mendalam katalog StarRocks (bisa false-green saat Nessie tanpa REST)" fi +# ─── 0x. verify-nessie.sh: versi + dukungan API (native/REST) + contoh schema ─ +_vn="${DEPLOY_DIR}/scripts/verify-nessie.sh" +if [[ -x "${_vn}" ]] && bash -n "${_vn}" 2>/dev/null \ + && grep -q 'api/v2/config' "${_vn}" && grep -q 'v1/namespaces' "${_vn}" \ + && grep -q 'Iceberg REST' "${_vn}" && grep -q 'code_hint' "${_vn}" \ + && grep -q 'namespace' "${_vn}"; then + ok "verify-nessie.sh: versi + dukungan API (native v2 + Iceberg REST) + contoh namespace" +else + bad "verify-nessie.sh hilang/tidak lengkap" +fi + # ─── 1. generate ──────────────────────────────────────────────────────────── if ! python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \ "${INVENTORY}" "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" \ diff --git a/deploy/scripts/verify-nessie.sh b/deploy/scripts/verify-nessie.sh new file mode 100755 index 0000000..d450891 --- /dev/null +++ b/deploy/scripts/verify-nessie.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# ============================================================================= +# eBdesk SiCABE — Verifikasi Nessie: versi + dukungan API (native v2 & Iceberg REST) +# + contoh schema pada catalog (memastikan konfigurasi benar). +# +# PENTING saat Nessie EXISTING: Trino/Dremio memakai API NATIVE v2 (/api/v2), +# sedangkan StarRocks & Sling memakai Iceberg REST (/iceberg). Bila Nessie existing +# HANYA punya API native (tanpa Iceberg REST), StarRocks/Sling GAGAL — skrip ini +# membuktikannya lebih awal (bukan menunggu verify-phase4/5). +# +# Yang diperiksa: +# A. Konektivitas & versi : /api/v2/config (specVersion, defaultBranch, …) + +# versi server (env NESSIE_VERSION / image container bila lokal). +# B. Dukungan API : native v2 + Iceberg REST (/v1/config) — wajib ada. +# C. Catalog (contoh) : ≤5 namespace (schema) via Iceberg REST + branch/refs. +# D. Ringkasan +# +# Pemakaian: +# ./scripts/verify-nessie.sh +# ./scripts/verify-nessie.sh --env-file generated/host-compute/.env +# NESSIE_URL=http://nessie.corp:19120 ./scripts/verify-nessie.sh +# Keluar: 0 = OK (WARN boleh) · 1 = ada FAIL · 2 = argumen salah +# ============================================================================= +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=../lib/common.sh +source "${SCRIPT_DIR}/lib/common.sh" 2>/dev/null || true +# shellcheck source=../config/environment.defaults.sh +source "${SCRIPT_DIR}/config/environment.defaults.sh" 2>/dev/null || true +sicabe_load_env "${SCRIPT_DIR}" "$@" || exit 2 + +usage() { awk 'NR>1 && /^#/ {sub(/^# ?/,""); print; next} NR>1 {exit}' "$0"; } +while [[ $# -gt 0 ]]; do + case "$1" in + --env-file) shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "Argumen tidak dikenal: $1" >&2; exit 2 ;; + esac +done + +BOLD=$'\e[1m'; GREEN=$'\e[32m'; RED=$'\e[31m'; YELLOW=$'\e[33m'; CYAN=$'\e[36m'; NC=$'\e[0m' +PASSC="${GREEN}[PASS]${NC}"; FAILC="${RED}[FAIL]${NC}"; WARNC="${YELLOW}[WARN]${NC}"; INFOC="${CYAN}[INFO]${NC}" +PASS_N=0; FAIL_N=0; WARN_N=0 +pass() { echo -e " ${PASSC} $1"; PASS_N=$((PASS_N + 1)); } +fail() { echo -e " ${FAILC} $1"; FAIL_N=$((FAIL_N + 1)); } +warn() { echo -e " ${WARNC} $1"; WARN_N=$((WARN_N + 1)); } +info() { echo -e " ${INFOC} $1"; } + +# Dasar URL host-facing (sadar NESSIE_URL eksternal/lintas-host) + prefix REST + konteks. +BASE="$(sicabe_host_url NESSIE_URL NESSIE_HOST "${NESSIE_PORT:-12120}")" +PFX="${NESSIE_ICEBERG_PREFIX:-/iceberg}" +WAREHOUSE="${ICEBERG_WAREHOUSE:-warehouse}" +BUCKET="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}" + +# Kode HTTP + penyebab (agar FAIL informatif, bukan "tidak merespon" saja). +http_code() { local c; c="$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$1" 2>/dev/null)"; printf '%s' "${c:-000}"; } +code_hint() { + case "$1" in + 000) echo "tidak terjangkau (koneksi/DNS/timeout) — cek port/alamat/jaringan";; + 401|403) echo "butuh autentikasi (Nessie existing memakai bearer?)";; + 404) echo "path tidak ada — versi Nessie lama / Iceberg REST belum aktif (butuh warehouse+object store) / proxy tak merutekan path ini";; + 429) echo "rate-limited oleh proxy/edge";; + 500|502|503|504) echo "error di server/proxy — cek konfigurasi Nessie (warehouse/object store)";; + *) echo "HTTP $1";; + esac +} + +echo -e "${BOLD}Verifikasi Nessie — ${BASE} (Iceberg REST prefix '${PFX}')${NC}" +echo "" + +# ───────────────────────────────────────────────────────────────────────────── +# A. Konektivitas & versi +# ───────────────────────────────────────────────────────────────────────────── +echo -e "${BOLD}A. Konektivitas & versi${NC}" +cfg="$(curl -sf --max-time 10 "${BASE}/api/v2/config" 2>/dev/null || echo "")" +if [[ -z "${cfg}" ]]; then + code="$(http_code "${BASE}/api/v2/config")" + fail "Nessie API native v2 tidak merespon di ${BASE}/api/v2/config [HTTP ${code} — $(code_hint "${code}")]" + echo "" + echo -e "${BOLD}Ringkasan: ${PASS_N} PASS · ${FAIL_N} FAIL · ${WARN_N} WARN${NC}" + echo -e "${RED}✗ Nessie tak terjangkau / API v2 tak ada.${NC}" + exit 1 +fi +pass "Nessie API native v2 aktif" +printf '%s' "${cfg}" | python3 -c ' +import sys, json +try: + d = json.load(sys.stdin) +except Exception: + sys.exit(0) +for k in ("specVersion", "defaultBranch", "maxSupportedApiVersion", "minSupportedApiVersion", "upstreamCommitId"): + if d.get(k) is not None: + print(f" {k} = {d.get(k)}") +' 2>/dev/null || true +ver_env="${NESSIE_VERSION:-}" +img="$(docker inspect --format '{{.Config.Image}}' sicabe-nessie 2>/dev/null || true)" +[[ -n "${ver_env}" ]] && info "NESSIE_VERSION(.env) = ${ver_env}" +[[ -n "${img}" ]] && info "image container (lokal) = ${img}" +hdr="$(curl -sD - -o /dev/null --max-time 10 "${BASE}/api/v2/config" 2>/dev/null | tr -d '\r' | grep -iE '^(server|via|x-nessie)' | paste -sd' ' - || true)" +[[ -n "${hdr}" ]] && info "header: ${hdr}" +if [[ -z "${ver_env}" && -z "${img}" ]]; then + warn "Versi SERVER Nessie tak terdeteksi (bukan container lokal) — set NESSIE_VERSION bila perlu" +fi +echo "" + +# ───────────────────────────────────────────────────────────────────────────── +# B. Dukungan API (native v2 + Iceberg REST) +# ───────────────────────────────────────────────────────────────────────────── +echo -e "${BOLD}B. Dukungan API${NC}" +pass "native v2 (/api/v2) — dipakai Trino & Dremio" +ic_cfg="$(curl -sf --max-time 10 "${BASE}${PFX}/v1/config" 2>/dev/null || echo "")" +if [[ -n "${ic_cfg}" ]]; then + pass "Iceberg REST (${PFX}/v1/config) aktif — dipakai Sling & StarRocks" + printf '%s' "${ic_cfg}" | python3 -c ' +import sys, json +try: + d = json.load(sys.stdin) +except Exception: + sys.exit(0) +ov = d.get("overrides") or {} +df = d.get("defaults") or {} +for k in ("warehouse", "prefix"): + v = ov.get(k) or df.get(k) + if v: + print(f" {k} = {v}") +' 2>/dev/null || true +else + code="$(http_code "${BASE}${PFX}/v1/config")" + fail "Iceberg REST ${BASE}${PFX} TIDAK tersedia [HTTP ${code} — $(code_hint "${code}")]" + warn "→ StarRocks (katalog 'iceberg_catalog') & Sling (catalog_type=rest) akan GAGAL." + warn " Trino/Dremio tetap OK (mereka memakai API native v2)." +fi +v1="$(http_code "${BASE}/api/v1/config")" +[[ "${v1}" == "200" ]] && info "API native v1 MASIH tersedia (HTTP 200) — versi lama, tak dipakai" +echo "" + +# ───────────────────────────────────────────────────────────────────────────── +# C. Catalog — contoh schema (namespace) & refs +# ───────────────────────────────────────────────────────────────────────────── +echo -e "${BOLD}C. Catalog — contoh schema (≤5) & refs${NC}" +ns_json="$(curl -sf --max-time 10 "${BASE}${PFX}/v1/namespaces" 2>/dev/null || echo "")" +if [[ -z "${ns_json}" ]]; then + code="$(http_code "${BASE}${PFX}/v1/namespaces")" + if [[ -z "${ic_cfg}" ]]; then + warn "Tak bisa membaca namespace karena Iceberg REST tak tersedia [HTTP ${code}]" + else + fail "Iceberg REST aktif tetapi namespace tak terbaca [HTTP ${code} — $(code_hint "${code}")]" + fi +else + ns_out="$(printf '%s' "${ns_json}" | python3 -c ' +import sys, json +try: + d = json.load(sys.stdin) +except Exception: + print("0|"); sys.exit() +ns = d.get("namespaces") or [] +def flat(n): return ".".join(n) if isinstance(n, list) else str(n) +print(f"{len(ns)}|" + ", ".join(flat(n) for n in ns[:5])) +' 2>/dev/null)" + ns_count="${ns_out%%|*}"; ns_list="${ns_out#*|}" + if [[ "${ns_count:-0}" -gt 0 ]]; then + pass "Catalog berisi ${ns_count} namespace — contoh (≤5): ${ns_list}" + else + warn "Iceberg REST aktif tetapi catalog MASIH KOSONG (0 namespace) — normal sebelum ingest" + info "Konteks konfigurasi: bucket='${BUCKET}' warehouse='${WAREHOUSE}' prefix='${PFX}'" + fi +fi +main_code="$(http_code "${BASE}/api/v2/trees/main")" +[[ "${main_code}" == "200" ]] && pass "Branch 'main' tersedia" \ + || warn "Branch 'main' tak merespon 200 [HTTP ${main_code} — $(code_hint "${main_code}")]" +refs_json="$(curl -sf --max-time 10 "${BASE}/api/v2/trees" 2>/dev/null || echo "")" +if [[ -n "${refs_json}" ]]; then + refs_list="$(printf '%s' "${refs_json}" | python3 -c ' +import sys, json +try: + d = json.load(sys.stdin) +except Exception: + sys.exit(0) +rs = d.get("references") or d.get("trees") or [] +out = [] +for r in rs[:5]: + if isinstance(r, dict): + n, t = r.get("name"), r.get("type", "") + if n: out.append(f"{n}({t})" if t else n) +print(", ".join(out)) +' 2>/dev/null)" + [[ -n "${refs_list}" ]] && info "refs (≤5): ${refs_list}" +fi +echo "" + +# ───────────────────────────────────────────────────────────────────────────── +# D. Ringkasan +# ───────────────────────────────────────────────────────────────────────────── +echo -e "${BOLD}Ringkasan: ${PASS_N} PASS · ${FAIL_N} FAIL · ${WARN_N} WARN${NC}" +if [[ ${FAIL_N} -eq 0 ]]; then + echo -e "${GREEN}✓ Nessie siap (API native v2 + Iceberg REST).${NC}" + exit 0 +fi +echo -e "${RED}✗ Ada ${FAIL_N} kegagalan — Nessie belum memenuhi kebutuhan SiCABE.${NC}" +exit 1 diff --git a/deploy/scripts/verify-phase2-storage.sh b/deploy/scripts/verify-phase2-storage.sh index 47053b4..c24fdec 100755 --- a/deploy/scripts/verify-phase2-storage.sh +++ b/deploy/scripts/verify-phase2-storage.sh @@ -85,7 +85,7 @@ skip() { echo -e " ${YELLOW}[SKIP]${NC} $1"; SKIP_N=$((SKIP_N + 1)); } # Kode HTTP untuk (000 = koneksi gagal/timeout). Dipakai agar pesan FAIL # menampilkan PENYEBAB (404/401/500/000), bukan hanya "tidak merespon". -http_code() { curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$1" 2>/dev/null || echo 000; } +http_code() { local c; c="$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$1" 2>/dev/null)"; printf '%s' "${c:-000}"; } code_hint() { case "$1" in 000) echo "tidak terjangkau (koneksi/DNS/timeout) — cek port, alamat, jaringan";;