platform/sicabe-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync @ 94e1ad6

Browse Source
This commit is contained in:
subroto
2026-10-13 07:07:05 +07:00
parent 82c6514b25
commit 1da7512150
3 changed files with 300 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/scripts/test-generate.sh
+15 -1
View File
@@ -517,12 +517,26 @@ _vn="${DEPLOY_DIR}/scripts/verify-nessie.sh"
if [[ -x "${_vn}" ]] && bash -n "${_vn}" 2>/dev/null \ if [[ -x "${_vn}" ]] && bash -n "${_vn}" 2>/dev/null \
&& grep -q 'api/v2/config' "${_vn}" && grep -q 'v1/namespaces' "${_vn}" \ && grep -q 'api/v2/config' "${_vn}" && grep -q 'v1/namespaces' "${_vn}" \
&& grep -q 'ns_cands' "${_vn}" && grep -q 'Iceberg REST' "${_vn}" \ && grep -q 'ns_cands' "${_vn}" && grep -q 'Iceberg REST' "${_vn}" \
&& grep -q 'code_hint' "${_vn}" && grep -q 'namespace' "${_vn}"; then && grep -q 'code_hint' "${_vn}" && grep -q 'q/info' "${_vn}" \
&& grep -q 'namespace' "${_vn}"; then
ok "verify-nessie.sh: versi + dukungan API (native v2 + Iceberg REST) + contoh namespace (multi-kandidat path)" ok "verify-nessie.sh: versi + dukungan API (native v2 + Iceberg REST) + contoh namespace (multi-kandidat path)"
else else
bad "verify-nessie.sh hilang/tidak lengkap" bad "verify-nessie.sh hilang/tidak lengkap"
fi fi
# ─── 0y. verify-minio.sh: bukti S3_ENDPOINT via Python SigV4 + contoh ≤5 object ─
# TANPA mc: akses S3 lewat Python stdlib (SigV4) — mensyaratkan endpoint & kredensial
# benar, lalu membuktikannya dengan ListObjectsV2 (≤5 objek).
_vm="${DEPLOY_DIR}/scripts/verify-minio.sh"
if [[ -x "${_vm}" ]] && bash -n "${_vm}" 2>/dev/null \
&& grep -q 'AWS4-HMAC-SHA256' "${_vm}" && grep -q 'list-type' "${_vm}" \
&& grep -q 'ListBuckets' "${_vm}" && grep -q '/minio/health/live' "${_vm}" \
&& ! grep -q 'minio/mc' "${_vm}"; then
ok "verify-minio.sh: bukti S3_ENDPOINT (Python SigV4) + contoh ≤5 object (tanpa mc)"
else
bad "verify-minio.sh hilang/tidak lengkap (atau masih memakai mc)"
fi
# ─── 1. generate ──────────────────────────────────────────────────────────── # ─── 1. generate ────────────────────────────────────────────────────────────
if ! python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \ if ! python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \
"${INVENTORY}" "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" \ "${INVENTORY}" "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" \
deploy/scripts/verify-minio.sh
Executable
+262
View File
@@ -0,0 +1,262 @@
#!/usr/bin/env bash
# =============================================================================
# eBdesk SiCABE — Verifikasi MinIO/S3: memastikan S3_ENDPOINT (url) di config
# BENAR & bisa dipakai — dibuktikan dengan listing object nyata (contoh ≤5).
#
# TANPA mc: seluruh akses S3 memakai Python stdlib (SigV4 + urllib).
#
# Yang diperiksa:
# A. Konfigurasi & endpoint : S3_ENDPOINT, endpoint host-facing, bucket, region, kredensial.
# B. Kesehatan : GET /minio/health/live (dari host).
# C. Akses S3 : ListBuckets (uji ENDPOINT + KREDENSIAL, SigV4).
# D. Bucket & object : bucket ada? + contoh ≤5 object (Key+Size) + jumlah (maks 1000).
# E. Ringkasan
#
# Pemakaian:
# ./scripts/verify-minio.sh
# ./scripts/verify-minio.sh --env-file generated/host-state/.env
# S3_ENDPOINT=https://minio.corp:9000 S3_INSECURE=1 ./scripts/verify-minio.sh
# Dukungan self-signed: set S3_INSECURE=1 (lewati verifikasi TLS).
# Keluar: 0 = OK (WARN boleh) · 1 = ada FAIL · 2 = argumen salah
# =============================================================================
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
# shellcheck source=../lib/common.sh
source "${SCRIPT_DIR}/lib/common.sh" 2>/dev/null || true
# shellcheck source=../config/environment.defaults.sh
source "${SCRIPT_DIR}/config/environment.defaults.sh" 2>/dev/null || true
sicabe_load_env "${SCRIPT_DIR}" "$@" || exit 2
usage() { awk 'NR>1 && /^#/ {sub(/^# ?/,""); print; next} NR>1 {exit}' "$0"; }
while [[ $# -gt 0 ]]; do
case "$1" in
--env-file) shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "Argumen tidak dikenal: $1" >&2; exit 2 ;;
esac
done
BOLD=$'\e[1m'; GREEN=$'\e[32m'; RED=$'\e[31m'; YELLOW=$'\e[33m'; CYAN=$'\e[36m'; NC=$'\e[0m'
PASSC="${GREEN}[PASS]${NC}"; FAILC="${RED}[FAIL]${NC}"; WARNC="${YELLOW}[WARN]${NC}"; INFOC="${CYAN}[INFO]${NC}"
PASS_N=0; FAIL_N=0; WARN_N=0
pass() { echo -e " ${PASSC} $1"; PASS_N=$((PASS_N + 1)); }
fail() { echo -e " ${FAILC} $1"; FAIL_N=$((FAIL_N + 1)); }
warn() { echo -e " ${WARNC} $1"; WARN_N=$((WARN_N + 1)); }
info() { echo -e " ${INFOC} $1"; }
# ── Konfigurasi (S3_ENDPOINT dilihat CONTAINER; skrip ini di HOST) ───────────
S3_ENDPOINT="${S3_ENDPOINT:-http://minio:9000}"
S3_REGION="${S3_REGION:-us-east-1}"
S3_ACCESS_KEY="${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}"
S3_SECRET_KEY="${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-}}"
BUCKET="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}"
# Endpoint HOST-facing: pakai S3_ENDPOINT bila menunjuk ALAMAT (IP/FQDN);
# selain itu (nama service `minio`) → <scheme>://localhost:<MINIO_PORT> (port terbit).
SCHEME="${S3_ENDPOINT%%://*}"
_s3host="${S3_ENDPOINT#*://}"; _s3host="${_s3host%%/*}"; _s3host="${_s3host%%:*}"
if [[ "${_s3host}" == *.* ]]; then
S3_HOST_EP="${S3_ENDPOINT%/}"
elif [[ "${MINIO_HOST:-}" == *.* ]]; then
S3_HOST_EP="${SCHEME}://${MINIO_HOST}:${MINIO_PORT:-12900}"
else
S3_HOST_EP="${SCHEME}://localhost:${MINIO_PORT:-12900}"
fi
# ── Python stdlib: SigV4 + S3 (health / listbuckets / listobjects) ───────────
_PY="$(mktemp)"; trap 'rm -f "${_PY}"' EXIT
cat > "${_PY}" <<'PYEOF'
import os, sys, re, hashlib, hmac, datetime, ssl, urllib.request, urllib.parse, urllib.error
mode = os.environ.get("MODE", "")
ep = (os.environ.get("S3E") or "").rstrip("/")
ak = os.environ.get("S3U") or ""
sk = os.environ.get("S3P") or ""
region = os.environ.get("S3R") or "us-east-1"
bucket = os.environ.get("BUCKET") or ""
insecure = os.environ.get("S3_INSECURE") == "1"
u = urllib.parse.urlparse(ep)
scheme = u.scheme or "http"
host = u.netloc
base_path = u.path.rstrip("/")
def ctx():
return ssl._create_unverified_context() if insecure else None
def tags(xml, tag):
return re.findall(rf"<{tag}>(.*?)</{tag}>", xml, re.S)
def get(path, query=None):
query = query or []
t = datetime.datetime.now(datetime.timezone.utc)
amzdate = t.strftime("%Y%m%dT%H%M%SZ"); datestamp = t.strftime("%Y%m%d")
curi = path or "/"
cqs = "&".join(f"{urllib.parse.quote(k, safe='-_.~')}={urllib.parse.quote(str(v), safe='-_.~')}"
for k, v in sorted(query))
phash = hashlib.sha256(b"").hexdigest()
ch = f"host:{host}\nx-amz-content-sha256:{phash}\nx-amz-date:{amzdate}\n"
sh = "host;x-amz-content-sha256;x-amz-date"
cr = "\n".join(["GET", curi, cqs, ch, sh, phash])
scope = f"{datestamp}/{region}/s3/aws4_request"
sts = "\n".join(["AWS4-HMAC-SHA256", amzdate, scope, hashlib.sha256(cr.encode()).hexdigest()])
def hm(k, m): return hmac.new(k, m.encode(), hashlib.sha256).digest()
kk = hm(("AWS4" + sk).encode(), datestamp); kk = hm(kk, region); kk = hm(kk, "s3"); kk = hm(kk, "aws4_request")
sig = hmac.new(kk, sts.encode(), hashlib.sha256).hexdigest()
auth = f"AWS4-HMAC-SHA256 Credential={ak}/{scope}, SignedHeaders={sh}, Signature={sig}"
url = f"{scheme}://{host}{curi}" + (f"?{cqs}" if cqs else "")
req = urllib.request.Request(url, method="GET")
for h, v in (("Host", host), ("x-amz-date", amzdate),
("x-amz-content-sha256", phash), ("Authorization", auth)):
req.add_header(h, v)
try:
with urllib.request.urlopen(req, timeout=15, context=ctx()) as r:
return r.status, r.read().decode("utf-8", "replace")
except urllib.error.HTTPError as e:
return e.code, e.read().decode("utf-8", "replace")
except Exception as e:
return 0, str(e)
def errline(st, body):
c = tags(body, "Code"); m = tags(body, "Message")
code = (c[0] if c else "").replace("|", " ")
msg = (m[0] if m else body[:140]).replace("|", " ").strip()
print(f"ERR|{st}|{code}|{msg}")
if mode == "health":
try:
with urllib.request.urlopen(f"{scheme}://{host}/minio/health/live", timeout=8, context=ctx()) as r:
print(f"HEALTH|{r.status}")
except urllib.error.HTTPError as e:
print(f"HEALTH|{e.code}")
except Exception:
print("HEALTH|000")
elif mode == "buckets":
st, body = get(base_path or "/")
if st == 200:
names = tags(body, "Name")
print(f"BUCKETS|{len(names)}")
for n in names:
print(f"BUCKET|{n}")
else:
errline(st, body)
elif mode == "objects":
maxk = os.environ.get("MAXKEY", "5")
st, body = get(f"{base_path}/{bucket}", [("list-type", "2"), ("max-keys", maxk)])
if st == 404 or "NoSuchBucket" in body:
print("ERR|404|NoSuchBucket|bucket tak ada")
elif st != 200:
errline(st, body)
else:
keys = tags(body, "Key"); sizes = tags(body, "Size")
trunc = "true" if "<IsTruncated>true</IsTruncated>" in body else "false"
print(f"OK|keycount={len(keys)}|truncated={trunc}")
for i, kk in enumerate(keys):
# TAB-separated: key S3 boleh memuat '|' → jangan pakai '|' sebagai pemisah.
print(f"OBJ\t{kk}\t{sizes[i] if i < len(sizes) else ''}")
PYEOF
s3py() { # <mode> [maxkey] → stdout
MODE="$1" MAXKEY="${2:-}" S3E="${S3_HOST_EP}" S3U="${S3_ACCESS_KEY}" S3P="${S3_SECRET_KEY}" \
S3R="${S3_REGION}" BUCKET="${BUCKET}" S3_INSECURE="${S3_INSECURE:-0}" python3 "${_PY}" 2>/dev/null
}
echo -e "${BOLD}Verifikasi MinIO/S3 — bucket '${BUCKET}'${NC}"
echo ""
# ─────────────────────────────────────────────────────────────────────────────
# A. Konfigurasi & endpoint
# ─────────────────────────────────────────────────────────────────────────────
echo -e "${BOLD}A. Konfigurasi & endpoint${NC}"
info "S3_ENDPOINT (container) = ${S3_ENDPOINT}"
info "endpoint (host, dipakai) = ${S3_HOST_EP}"
info "bucket='${BUCKET}' region='${S3_REGION}' akses='${S3_ACCESS_KEY}'"
if [[ -z "${S3_SECRET_KEY}" ]]; then
fail "S3_SECRET_KEY / MINIO_ROOT_PASSWORD kosong — kredensial tak lengkap"
fi
echo ""
# ─────────────────────────────────────────────────────────────────────────────
# B. Kesehatan (host)
# ─────────────────────────────────────────────────────────────────────────────
echo -e "${BOLD}B. Kesehatan${NC}"
hc="$(s3py health)"
hc="${hc#HEALTH|}"
if [[ "${hc}" == "200" ]]; then
pass "MinIO sehat: ${S3_HOST_EP}/minio/health/live (HTTP 200)"
elif [[ "${hc}" == "000" ]]; then
warn "MinIO tak terjangkau dari host (${S3_HOST_EP}) — normal bila S3 eksternal / beda jaringan"
else
warn "Health MinIO HTTP ${hc} di ${S3_HOST_EP} — cek endpoint/port"
fi
echo ""
# ─────────────────────────────────────────────────────────────────────────────
# C. Akses S3 (ListBuckets) — uji ENDPOINT + KREDENSIAL (SigV4)
# ─────────────────────────────────────────────────────────────────────────────
echo -e "${BOLD}C. Akses S3 (ListBuckets)${NC}"
bout="$(s3py buckets)"
if [[ "${bout}" == BUCKETS\|* ]]; then
nb="${bout#BUCKETS|}"; nb="${nb%%$'\n'*}"
pass "ListBuckets OK via ${S3_HOST_EP} — ${nb} bucket"
printf '%s\n' "${bout}" | awk -F'|' '$1=="BUCKET"{printf " • %s\n", $2}'
elif [[ "${bout}" == ERR\|* ]]; then
IFS='|' read -r _ st code msg <<< "${bout}"
hint=""
case "${st}" in
0) hint="tidak terjangkau (koneksi/DNS/TLS) — cek endpoint/port; self-signed? set S3_INSECURE=1";;
403) hint="kredensial/diakses ditolak (cek S3_ACCESS_KEY / S3_SECRET_KEY)";;
401) hint="butuh autentikasi";;
*) hint="${code:-HTTP ${st}}";;
esac
fail "ListBuckets GAGAL [HTTP ${st} — ${hint}]${msg:+ · ${msg}}"
else
fail "ListBuckets tak menghasilkan keluaran yang diharapkan"
fi
echo ""
# ─────────────────────────────────────────────────────────────────────────────
# D. Bucket & object (contoh ≤5)
# ─────────────────────────────────────────────────────────────────────────────
echo -e "${BOLD}D. Bucket & object (contoh ≤5)${NC}"
o5="$(s3py objects 5)"
if [[ "${o5}" == OK\|* ]]; then
pass "bucket '${BUCKET}' ada & bisa dibaca"
meta="${o5%%$'\n'*}"; rest="${meta#*|}"; kc="${rest%%|*}"
if [[ "${kc#keycount=}" -gt 0 ]]; then
pass "menampilkan ≤5 object (bukti S3_ENDPOINT benar)"
printf '%s\n' "${o5}" | awk -F'\t' '$1=="OBJ"{printf " • %s (%s bytes)\n", $2, $3}'
else
warn "bucket ada tetapi KOSONG (0 object) — normal bila belum ada ingest"
fi
# jumlah (maks 1000) — info
o1000="$(s3py objects 1000)"
if [[ "${o1000}" == OK\|* ]]; then
meta="${o1000%%$'\n'*}"; r="${meta#*|}"; k1="${r%%|*}"; tr="${r##*|}"
info "jumlah object (maks 1000): ${k1#keycount=}${tr:+ · ${tr}}"
fi
elif [[ "${o5}" == ERR\|* ]]; then
IFS='|' read -r _ st code msg <<< "${o5}"
if [[ "${st}" == "404" ]]; then
fail "bucket '${BUCKET}' TIDAK ADA [HTTP 404 — ${msg:-NoSuchBucket}]"
info "Buat bucket di sisi pemilik/MinIO (bootstrap step 2 / config/minio/buckets.sh)."
else
hint="cek endpoint/kredensial"
[[ "${st}" == "0" ]] && hint="tidak terjangkau (koneksi/DNS/TLS) — self-signed? set S3_INSECURE=1"
fail "baca bucket '${BUCKET}' GAGAL [HTTP ${st} — ${hint}]${msg:+ · ${msg}}"
fi
else
fail "baca bucket tak menghasilkan keluaran yang diharapkan"
fi
echo ""
# ─────────────────────────────────────────────────────────────────────────────
# E. Ringkasan
# ─────────────────────────────────────────────────────────────────────────────
echo -e "${BOLD}Ringkasan: ${PASS_N} PASS · ${FAIL_N} FAIL · ${WARN_N} WARN${NC}"
if [[ ${FAIL_N} -eq 0 ]]; then
echo -e "${GREEN}✓ S3_ENDPOINT valid & bisa dipakai (bucket '${BUCKET}').${NC}"
exit 0
fi
echo -e "${RED}✗ Ada ${FAIL_N} kegagalan — cek S3_ENDPOINT / kredensial / bucket.${NC}"
exit 1
deploy/scripts/verify-nessie.sh
+23 -4
View File
@@ -9,8 +9,8 @@
# membuktikannya lebih awal (bukan menunggu verify-phase4/5). # membuktikannya lebih awal (bukan menunggu verify-phase4/5).
# #
# Yang diperiksa: # Yang diperiksa:
# A. Konektivitas & versi : /api/v2/config (specVersion, defaultBranch, …) + # A. Konektivitas & versi : /api/v2/config (specVersion, defaultBranch, …) + versi
# versi server (env NESSIE_VERSION / image container bila lokal). # server: /q/info (build.version) · NESSIE_VERSION(.env) · image container.
# B. Dukungan API : native v2 + Iceberg REST (<prefix>/v1/config) — wajib ada. # B. Dukungan API : native v2 + Iceberg REST (<prefix>/v1/config) — wajib ada.
# C. Catalog (contoh) : ≤5 namespace (schema) via Iceberg REST + branch/refs. # C. Catalog (contoh) : ≤5 namespace (schema) via Iceberg REST + branch/refs.
# D. Ringkasan # D. Ringkasan
@@ -95,12 +95,31 @@ for k in ("specVersion", "defaultBranch", "maxSupportedApiVersion", "minSupporte
' 2>/dev/null || true ' 2>/dev/null || true
ver_env="${NESSIE_VERSION:-}" ver_env="${NESSIE_VERSION:-}"
img="$(docker inspect --format '{{.Config.Image}}' sicabe-nessie 2>/dev/null || true)" img="$(docker inspect --format '{{.Config.Image}}' sicabe-nessie 2>/dev/null || true)"
# Versi APLIKASI via Quarkus info (/q/info) — bila ekstensi quarkus-info aktif.
qi="$(curl -sf --max-time 10 "${BASE}/q/info" 2>/dev/null || echo "")"
qi_ver=""
if [[ -n "${qi}" ]]; then
qi_ver="$(printf '%s' "${qi}" | python3 -c '
import sys, json
try:
d = json.load(sys.stdin)
except Exception:
sys.exit()
b = d.get("build") or {}
print(b.get("version") or d.get("version") or "")
' 2>/dev/null)"
fi
[[ -n "${ver_env}" ]] && info "NESSIE_VERSION(.env) = ${ver_env}" [[ -n "${ver_env}" ]] && info "NESSIE_VERSION(.env) = ${ver_env}"
[[ -n "${img}" ]] && info "image container (lokal) = ${img}" [[ -n "${img}" ]] && info "image container (lokal) = ${img}"
if [[ -n "${qi_ver}" ]]; then
info "versi server (/q/info build.version) = ${qi_ver}"
elif [[ -n "${qi}" ]]; then
info "/q/info tersedia tetapi build.version tak terlihat"
fi
hdr="$(curl -sD - -o /dev/null --max-time 10 "${BASE}/api/v2/config" 2>/dev/null | tr -d '\r' | grep -iE '^(server|via|x-nessie)' | paste -sd' ' - || true)" hdr="$(curl -sD - -o /dev/null --max-time 10 "${BASE}/api/v2/config" 2>/dev/null | tr -d '\r' | grep -iE '^(server|via|x-nessie)' | paste -sd' ' - || true)"
[[ -n "${hdr}" ]] && info "header: ${hdr}" [[ -n "${hdr}" ]] && info "header: ${hdr}"
if [[ -z "${ver_env}" && -z "${img}" ]]; then if [[ -z "${ver_env}" && -z "${img}" && -z "${qi_ver}" ]]; then
warn "Versi SERVER Nessie tak terdeteksi (bukan container lokal) — set NESSIE_VERSION bila perlu" warn "Versi SERVER Nessie tak terdeteksi (bukan container lokal; /q/info & header kosong) — set NESSIE_VERSION bila perlu"
fi fi
echo "" echo ""