From 1da75121504a6d06b39c934cf750f8d299d5b67d Mon Sep 17 00:00:00 2001 From: Subroto Date: Tue, 13 Oct 2026 07:07:05 +0700 Subject: [PATCH] sync @ 94e1ad6 --- deploy/scripts/test-generate.sh | 16 +- deploy/scripts/verify-minio.sh | 262 ++++++++++++++++++++++++++++++++ deploy/scripts/verify-nessie.sh | 27 +++- 3 files changed, 300 insertions(+), 5 deletions(-) create mode 100755 deploy/scripts/verify-minio.sh diff --git a/deploy/scripts/test-generate.sh b/deploy/scripts/test-generate.sh index 37e3861..068ac97 100755 --- a/deploy/scripts/test-generate.sh +++ b/deploy/scripts/test-generate.sh @@ -517,12 +517,26 @@ _vn="${DEPLOY_DIR}/scripts/verify-nessie.sh" if [[ -x "${_vn}" ]] && bash -n "${_vn}" 2>/dev/null \ && grep -q 'api/v2/config' "${_vn}" && grep -q 'v1/namespaces' "${_vn}" \ && grep -q 'ns_cands' "${_vn}" && grep -q 'Iceberg REST' "${_vn}" \ - && grep -q 'code_hint' "${_vn}" && grep -q 'namespace' "${_vn}"; then + && grep -q 'code_hint' "${_vn}" && grep -q 'q/info' "${_vn}" \ + && grep -q 'namespace' "${_vn}"; then ok "verify-nessie.sh: versi + dukungan API (native v2 + Iceberg REST) + contoh namespace (multi-kandidat path)" else bad "verify-nessie.sh hilang/tidak lengkap" fi +# ─── 0y. verify-minio.sh: bukti S3_ENDPOINT via Python SigV4 + contoh ≤5 object ─ +# TANPA mc: akses S3 lewat Python stdlib (SigV4) — mensyaratkan endpoint & kredensial +# benar, lalu membuktikannya dengan ListObjectsV2 (≤5 objek). +_vm="${DEPLOY_DIR}/scripts/verify-minio.sh" +if [[ -x "${_vm}" ]] && bash -n "${_vm}" 2>/dev/null \ + && grep -q 'AWS4-HMAC-SHA256' "${_vm}" && grep -q 'list-type' "${_vm}" \ + && grep -q 'ListBuckets' "${_vm}" && grep -q '/minio/health/live' "${_vm}" \ + && ! grep -q 'minio/mc' "${_vm}"; then + ok "verify-minio.sh: bukti S3_ENDPOINT (Python SigV4) + contoh ≤5 object (tanpa mc)" +else + bad "verify-minio.sh hilang/tidak lengkap (atau masih memakai mc)" +fi + # ─── 1. generate ──────────────────────────────────────────────────────────── if ! python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \ "${INVENTORY}" "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" \ diff --git a/deploy/scripts/verify-minio.sh b/deploy/scripts/verify-minio.sh new file mode 100755 index 0000000..cd35fb6 --- /dev/null +++ b/deploy/scripts/verify-minio.sh @@ -0,0 +1,262 @@ +#!/usr/bin/env bash +# ============================================================================= +# eBdesk SiCABE — Verifikasi MinIO/S3: memastikan S3_ENDPOINT (url) di config +# BENAR & bisa dipakai — dibuktikan dengan listing object nyata (contoh ≤5). +# +# TANPA mc: seluruh akses S3 memakai Python stdlib (SigV4 + urllib). +# +# Yang diperiksa: +# A. Konfigurasi & endpoint : S3_ENDPOINT, endpoint host-facing, bucket, region, kredensial. +# B. Kesehatan : GET /minio/health/live (dari host). +# C. Akses S3 : ListBuckets (uji ENDPOINT + KREDENSIAL, SigV4). +# D. Bucket & object : bucket ada? + contoh ≤5 object (Key+Size) + jumlah (maks 1000). +# E. Ringkasan +# +# Pemakaian: +# ./scripts/verify-minio.sh +# ./scripts/verify-minio.sh --env-file generated/host-state/.env +# S3_ENDPOINT=https://minio.corp:9000 S3_INSECURE=1 ./scripts/verify-minio.sh +# Dukungan self-signed: set S3_INSECURE=1 (lewati verifikasi TLS). +# Keluar: 0 = OK (WARN boleh) · 1 = ada FAIL · 2 = argumen salah +# ============================================================================= +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=../lib/common.sh +source "${SCRIPT_DIR}/lib/common.sh" 2>/dev/null || true +# shellcheck source=../config/environment.defaults.sh +source "${SCRIPT_DIR}/config/environment.defaults.sh" 2>/dev/null || true +sicabe_load_env "${SCRIPT_DIR}" "$@" || exit 2 + +usage() { awk 'NR>1 && /^#/ {sub(/^# ?/,""); print; next} NR>1 {exit}' "$0"; } +while [[ $# -gt 0 ]]; do + case "$1" in + --env-file) shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "Argumen tidak dikenal: $1" >&2; exit 2 ;; + esac +done + +BOLD=$'\e[1m'; GREEN=$'\e[32m'; RED=$'\e[31m'; YELLOW=$'\e[33m'; CYAN=$'\e[36m'; NC=$'\e[0m' +PASSC="${GREEN}[PASS]${NC}"; FAILC="${RED}[FAIL]${NC}"; WARNC="${YELLOW}[WARN]${NC}"; INFOC="${CYAN}[INFO]${NC}" +PASS_N=0; FAIL_N=0; WARN_N=0 +pass() { echo -e " ${PASSC} $1"; PASS_N=$((PASS_N + 1)); } +fail() { echo -e " ${FAILC} $1"; FAIL_N=$((FAIL_N + 1)); } +warn() { echo -e " ${WARNC} $1"; WARN_N=$((WARN_N + 1)); } +info() { echo -e " ${INFOC} $1"; } + +# ── Konfigurasi (S3_ENDPOINT dilihat CONTAINER; skrip ini di HOST) ─────────── +S3_ENDPOINT="${S3_ENDPOINT:-http://minio:9000}" +S3_REGION="${S3_REGION:-us-east-1}" +S3_ACCESS_KEY="${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}" +S3_SECRET_KEY="${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-}}" +BUCKET="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}" + +# Endpoint HOST-facing: pakai S3_ENDPOINT bila menunjuk ALAMAT (IP/FQDN); +# selain itu (nama service `minio`) → ://localhost: (port terbit). +SCHEME="${S3_ENDPOINT%%://*}" +_s3host="${S3_ENDPOINT#*://}"; _s3host="${_s3host%%/*}"; _s3host="${_s3host%%:*}" +if [[ "${_s3host}" == *.* ]]; then + S3_HOST_EP="${S3_ENDPOINT%/}" +elif [[ "${MINIO_HOST:-}" == *.* ]]; then + S3_HOST_EP="${SCHEME}://${MINIO_HOST}:${MINIO_PORT:-12900}" +else + S3_HOST_EP="${SCHEME}://localhost:${MINIO_PORT:-12900}" +fi + +# ── Python stdlib: SigV4 + S3 (health / listbuckets / listobjects) ─────────── +_PY="$(mktemp)"; trap 'rm -f "${_PY}"' EXIT +cat > "${_PY}" <<'PYEOF' +import os, sys, re, hashlib, hmac, datetime, ssl, urllib.request, urllib.parse, urllib.error + +mode = os.environ.get("MODE", "") +ep = (os.environ.get("S3E") or "").rstrip("/") +ak = os.environ.get("S3U") or "" +sk = os.environ.get("S3P") or "" +region = os.environ.get("S3R") or "us-east-1" +bucket = os.environ.get("BUCKET") or "" +insecure = os.environ.get("S3_INSECURE") == "1" +u = urllib.parse.urlparse(ep) +scheme = u.scheme or "http" +host = u.netloc +base_path = u.path.rstrip("/") + +def ctx(): + return ssl._create_unverified_context() if insecure else None + +def tags(xml, tag): + return re.findall(rf"<{tag}>(.*?)", xml, re.S) + +def get(path, query=None): + query = query or [] + t = datetime.datetime.now(datetime.timezone.utc) + amzdate = t.strftime("%Y%m%dT%H%M%SZ"); datestamp = t.strftime("%Y%m%d") + curi = path or "/" + cqs = "&".join(f"{urllib.parse.quote(k, safe='-_.~')}={urllib.parse.quote(str(v), safe='-_.~')}" + for k, v in sorted(query)) + phash = hashlib.sha256(b"").hexdigest() + ch = f"host:{host}\nx-amz-content-sha256:{phash}\nx-amz-date:{amzdate}\n" + sh = "host;x-amz-content-sha256;x-amz-date" + cr = "\n".join(["GET", curi, cqs, ch, sh, phash]) + scope = f"{datestamp}/{region}/s3/aws4_request" + sts = "\n".join(["AWS4-HMAC-SHA256", amzdate, scope, hashlib.sha256(cr.encode()).hexdigest()]) + def hm(k, m): return hmac.new(k, m.encode(), hashlib.sha256).digest() + kk = hm(("AWS4" + sk).encode(), datestamp); kk = hm(kk, region); kk = hm(kk, "s3"); kk = hm(kk, "aws4_request") + sig = hmac.new(kk, sts.encode(), hashlib.sha256).hexdigest() + auth = f"AWS4-HMAC-SHA256 Credential={ak}/{scope}, SignedHeaders={sh}, Signature={sig}" + url = f"{scheme}://{host}{curi}" + (f"?{cqs}" if cqs else "") + req = urllib.request.Request(url, method="GET") + for h, v in (("Host", host), ("x-amz-date", amzdate), + ("x-amz-content-sha256", phash), ("Authorization", auth)): + req.add_header(h, v) + try: + with urllib.request.urlopen(req, timeout=15, context=ctx()) as r: + return r.status, r.read().decode("utf-8", "replace") + except urllib.error.HTTPError as e: + return e.code, e.read().decode("utf-8", "replace") + except Exception as e: + return 0, str(e) + +def errline(st, body): + c = tags(body, "Code"); m = tags(body, "Message") + code = (c[0] if c else "").replace("|", " ") + msg = (m[0] if m else body[:140]).replace("|", " ").strip() + print(f"ERR|{st}|{code}|{msg}") + +if mode == "health": + try: + with urllib.request.urlopen(f"{scheme}://{host}/minio/health/live", timeout=8, context=ctx()) as r: + print(f"HEALTH|{r.status}") + except urllib.error.HTTPError as e: + print(f"HEALTH|{e.code}") + except Exception: + print("HEALTH|000") +elif mode == "buckets": + st, body = get(base_path or "/") + if st == 200: + names = tags(body, "Name") + print(f"BUCKETS|{len(names)}") + for n in names: + print(f"BUCKET|{n}") + else: + errline(st, body) +elif mode == "objects": + maxk = os.environ.get("MAXKEY", "5") + st, body = get(f"{base_path}/{bucket}", [("list-type", "2"), ("max-keys", maxk)]) + if st == 404 or "NoSuchBucket" in body: + print("ERR|404|NoSuchBucket|bucket tak ada") + elif st != 200: + errline(st, body) + else: + keys = tags(body, "Key"); sizes = tags(body, "Size") + trunc = "true" if "true" in body else "false" + print(f"OK|keycount={len(keys)}|truncated={trunc}") + for i, kk in enumerate(keys): + # TAB-separated: key S3 boleh memuat '|' → jangan pakai '|' sebagai pemisah. + print(f"OBJ\t{kk}\t{sizes[i] if i < len(sizes) else ''}") +PYEOF + +s3py() { # [maxkey] → stdout + MODE="$1" MAXKEY="${2:-}" S3E="${S3_HOST_EP}" S3U="${S3_ACCESS_KEY}" S3P="${S3_SECRET_KEY}" \ + S3R="${S3_REGION}" BUCKET="${BUCKET}" S3_INSECURE="${S3_INSECURE:-0}" python3 "${_PY}" 2>/dev/null +} + +echo -e "${BOLD}Verifikasi MinIO/S3 — bucket '${BUCKET}'${NC}" +echo "" + +# ───────────────────────────────────────────────────────────────────────────── +# A. Konfigurasi & endpoint +# ───────────────────────────────────────────────────────────────────────────── +echo -e "${BOLD}A. Konfigurasi & endpoint${NC}" +info "S3_ENDPOINT (container) = ${S3_ENDPOINT}" +info "endpoint (host, dipakai) = ${S3_HOST_EP}" +info "bucket='${BUCKET}' region='${S3_REGION}' akses='${S3_ACCESS_KEY}'" +if [[ -z "${S3_SECRET_KEY}" ]]; then + fail "S3_SECRET_KEY / MINIO_ROOT_PASSWORD kosong — kredensial tak lengkap" +fi +echo "" + +# ───────────────────────────────────────────────────────────────────────────── +# B. Kesehatan (host) +# ───────────────────────────────────────────────────────────────────────────── +echo -e "${BOLD}B. Kesehatan${NC}" +hc="$(s3py health)" +hc="${hc#HEALTH|}" +if [[ "${hc}" == "200" ]]; then + pass "MinIO sehat: ${S3_HOST_EP}/minio/health/live (HTTP 200)" +elif [[ "${hc}" == "000" ]]; then + warn "MinIO tak terjangkau dari host (${S3_HOST_EP}) — normal bila S3 eksternal / beda jaringan" +else + warn "Health MinIO HTTP ${hc} di ${S3_HOST_EP} — cek endpoint/port" +fi +echo "" + +# ───────────────────────────────────────────────────────────────────────────── +# C. Akses S3 (ListBuckets) — uji ENDPOINT + KREDENSIAL (SigV4) +# ───────────────────────────────────────────────────────────────────────────── +echo -e "${BOLD}C. Akses S3 (ListBuckets)${NC}" +bout="$(s3py buckets)" +if [[ "${bout}" == BUCKETS\|* ]]; then + nb="${bout#BUCKETS|}"; nb="${nb%%$'\n'*}" + pass "ListBuckets OK via ${S3_HOST_EP} — ${nb} bucket" + printf '%s\n' "${bout}" | awk -F'|' '$1=="BUCKET"{printf " • %s\n", $2}' +elif [[ "${bout}" == ERR\|* ]]; then + IFS='|' read -r _ st code msg <<< "${bout}" + hint="" + case "${st}" in + 0) hint="tidak terjangkau (koneksi/DNS/TLS) — cek endpoint/port; self-signed? set S3_INSECURE=1";; + 403) hint="kredensial/diakses ditolak (cek S3_ACCESS_KEY / S3_SECRET_KEY)";; + 401) hint="butuh autentikasi";; + *) hint="${code:-HTTP ${st}}";; + esac + fail "ListBuckets GAGAL [HTTP ${st} — ${hint}]${msg:+ · ${msg}}" +else + fail "ListBuckets tak menghasilkan keluaran yang diharapkan" +fi +echo "" + +# ───────────────────────────────────────────────────────────────────────────── +# D. Bucket & object (contoh ≤5) +# ───────────────────────────────────────────────────────────────────────────── +echo -e "${BOLD}D. Bucket & object (contoh ≤5)${NC}" +o5="$(s3py objects 5)" +if [[ "${o5}" == OK\|* ]]; then + pass "bucket '${BUCKET}' ada & bisa dibaca" + meta="${o5%%$'\n'*}"; rest="${meta#*|}"; kc="${rest%%|*}" + if [[ "${kc#keycount=}" -gt 0 ]]; then + pass "menampilkan ≤5 object (bukti S3_ENDPOINT benar)" + printf '%s\n' "${o5}" | awk -F'\t' '$1=="OBJ"{printf " • %s (%s bytes)\n", $2, $3}' + else + warn "bucket ada tetapi KOSONG (0 object) — normal bila belum ada ingest" + fi + # jumlah (maks 1000) — info + o1000="$(s3py objects 1000)" + if [[ "${o1000}" == OK\|* ]]; then + meta="${o1000%%$'\n'*}"; r="${meta#*|}"; k1="${r%%|*}"; tr="${r##*|}" + info "jumlah object (maks 1000): ${k1#keycount=}${tr:+ · ${tr}}" + fi +elif [[ "${o5}" == ERR\|* ]]; then + IFS='|' read -r _ st code msg <<< "${o5}" + if [[ "${st}" == "404" ]]; then + fail "bucket '${BUCKET}' TIDAK ADA [HTTP 404 — ${msg:-NoSuchBucket}]" + info "Buat bucket di sisi pemilik/MinIO (bootstrap step 2 / config/minio/buckets.sh)." + else + hint="cek endpoint/kredensial" + [[ "${st}" == "0" ]] && hint="tidak terjangkau (koneksi/DNS/TLS) — self-signed? set S3_INSECURE=1" + fail "baca bucket '${BUCKET}' GAGAL [HTTP ${st} — ${hint}]${msg:+ · ${msg}}" + fi +else + fail "baca bucket tak menghasilkan keluaran yang diharapkan" +fi +echo "" + +# ───────────────────────────────────────────────────────────────────────────── +# E. Ringkasan +# ───────────────────────────────────────────────────────────────────────────── +echo -e "${BOLD}Ringkasan: ${PASS_N} PASS · ${FAIL_N} FAIL · ${WARN_N} WARN${NC}" +if [[ ${FAIL_N} -eq 0 ]]; then + echo -e "${GREEN}✓ S3_ENDPOINT valid & bisa dipakai (bucket '${BUCKET}').${NC}" + exit 0 +fi +echo -e "${RED}✗ Ada ${FAIL_N} kegagalan — cek S3_ENDPOINT / kredensial / bucket.${NC}" +exit 1 diff --git a/deploy/scripts/verify-nessie.sh b/deploy/scripts/verify-nessie.sh index 21f4385..15be6c1 100755 --- a/deploy/scripts/verify-nessie.sh +++ b/deploy/scripts/verify-nessie.sh @@ -9,8 +9,8 @@ # membuktikannya lebih awal (bukan menunggu verify-phase4/5). # # Yang diperiksa: -# A. Konektivitas & versi : /api/v2/config (specVersion, defaultBranch, …) + -# versi server (env NESSIE_VERSION / image container bila lokal). +# A. Konektivitas & versi : /api/v2/config (specVersion, defaultBranch, …) + versi +# server: /q/info (build.version) · NESSIE_VERSION(.env) · image container. # B. Dukungan API : native v2 + Iceberg REST (/v1/config) — wajib ada. # C. Catalog (contoh) : ≤5 namespace (schema) via Iceberg REST + branch/refs. # D. Ringkasan @@ -95,12 +95,31 @@ for k in ("specVersion", "defaultBranch", "maxSupportedApiVersion", "minSupporte ' 2>/dev/null || true ver_env="${NESSIE_VERSION:-}" img="$(docker inspect --format '{{.Config.Image}}' sicabe-nessie 2>/dev/null || true)" +# Versi APLIKASI via Quarkus info (/q/info) — bila ekstensi quarkus-info aktif. +qi="$(curl -sf --max-time 10 "${BASE}/q/info" 2>/dev/null || echo "")" +qi_ver="" +if [[ -n "${qi}" ]]; then + qi_ver="$(printf '%s' "${qi}" | python3 -c ' +import sys, json +try: + d = json.load(sys.stdin) +except Exception: + sys.exit() +b = d.get("build") or {} +print(b.get("version") or d.get("version") or "") +' 2>/dev/null)" +fi [[ -n "${ver_env}" ]] && info "NESSIE_VERSION(.env) = ${ver_env}" [[ -n "${img}" ]] && info "image container (lokal) = ${img}" +if [[ -n "${qi_ver}" ]]; then + info "versi server (/q/info build.version) = ${qi_ver}" +elif [[ -n "${qi}" ]]; then + info "/q/info tersedia tetapi build.version tak terlihat" +fi hdr="$(curl -sD - -o /dev/null --max-time 10 "${BASE}/api/v2/config" 2>/dev/null | tr -d '\r' | grep -iE '^(server|via|x-nessie)' | paste -sd' ' - || true)" [[ -n "${hdr}" ]] && info "header: ${hdr}" -if [[ -z "${ver_env}" && -z "${img}" ]]; then - warn "Versi SERVER Nessie tak terdeteksi (bukan container lokal) — set NESSIE_VERSION bila perlu" +if [[ -z "${ver_env}" && -z "${img}" && -z "${qi_ver}" ]]; then + warn "Versi SERVER Nessie tak terdeteksi (bukan container lokal; /q/info & header kosong) — set NESSIE_VERSION bila perlu" fi echo ""