109 lines
4.5 KiB
Bash
Executable File
109 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# Dremio — Register Nessie catalog source via REST API
|
|
# =============================================================================
|
|
# Membuat source "Nessie" di Dremio OSS lewat API (bukan UI).
|
|
#
|
|
# ⚠️ Catatan: dokumentasi Dremio menyatakan Catalog API "unable to retrieve or
|
|
# manage Nessie sources" (DX-64994). Namun TERBUKTI (Dremio OSS 25.2.0, diuji
|
|
# 2026-09-10) bahwa POST /api/v3/catalog dengan type=NESSIE BERHASIL, dan
|
|
# GET /api/v3/catalog/by-path/<name> juga bekerja. Kunci config:
|
|
# - JANGAN pakai "storageType" (ditolak: "Invalid value found at: storageType")
|
|
# - awsRootPath format "/bucket/path" (TANPA prefix s3://)
|
|
#
|
|
# Pemakaian:
|
|
# export DREMIO_USER=admin DREMIO_PASSWORD='<password>'
|
|
# ./scripts/dremio/register-nessie-source.sh
|
|
#
|
|
# Env (opsional, ada default):
|
|
# DREMIO_URL default http://localhost:12947
|
|
# DREMIO_SOURCE_NAME default nessie
|
|
# NESSIE_ENDPOINT default ${NESSIE_URL}/api/v2 (http://nessie:19120/api/v2)
|
|
# MINIO_ENDPOINT default host dari S3_ENDPOINT (tanpa skema)
|
|
# AWS_ROOT_PATH default /<bucket>/warehouse
|
|
# MINIO_ROOT_USER / MINIO_ROOT_PASSWORD (fallback: S3_ACCESS_KEY / S3_SECRET_KEY)
|
|
# =============================================================================
|
|
set -euo pipefail
|
|
|
|
DREMIO_URL="${DREMIO_URL:-http://localhost:12947}"
|
|
DREMIO_USER="${DREMIO_USER:-admin}"
|
|
DREMIO_PASSWORD="${DREMIO_PASSWORD:?set DREMIO_PASSWORD}"
|
|
DREMIO_SOURCE_NAME="${DREMIO_SOURCE_NAME:-nessie}"
|
|
NESSIE_ENDPOINT="${NESSIE_ENDPOINT:-${NESSIE_URL:-http://nessie:19120}/api/v2}"
|
|
# MINIO_ENDPOINT dipakai tanpa skema (Dremio memisahkan host:port) → buang skema.
|
|
_minio_host="${MINIO_ENDPOINT:-${S3_ENDPOINT:-http://minio:9000}}"
|
|
MINIO_ENDPOINT="${_minio_host#*://}"
|
|
AWS_ROOT_PATH="${AWS_ROOT_PATH:-/${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/warehouse}"
|
|
MINIO_ROOT_USER="${MINIO_ROOT_USER:-${S3_ACCESS_KEY:-minioadmin}}"
|
|
MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:-${S3_SECRET_KEY:?set MINIO_ROOT_PASSWORD atau S3_SECRET_KEY}}"
|
|
|
|
echo "==> Login Dremio (${DREMIO_URL}) sebagai '${DREMIO_USER}'..."
|
|
LOGIN_RESP=$(curl -s -X POST "${DREMIO_URL}/apiv2/login" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"userName\":\"${DREMIO_USER}\",\"password\":\"${DREMIO_PASSWORD}\"}")
|
|
TOKEN=$(echo "${LOGIN_RESP}" | python3 -c "import sys,json;print(json.load(sys.stdin).get('token',''))" 2>/dev/null || echo "")
|
|
if [[ -z "${TOKEN}" ]]; then
|
|
echo "ERR: login gagal. Respons: ${LOGIN_RESP}"
|
|
exit 1
|
|
fi
|
|
AUTH="Authorization: _dremio${TOKEN}"
|
|
echo " OK: token diperoleh."
|
|
|
|
# Cek apakah source sudah ada (best-effort; GET Nessie mungkin tidak didukung)
|
|
EXIST_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "${AUTH}" \
|
|
"${DREMIO_URL}/api/v3/catalog/by-path/${DREMIO_SOURCE_NAME}" || echo "000")
|
|
if [[ "${EXIST_CODE}" == "200" ]]; then
|
|
echo "OK: source '${DREMIO_SOURCE_NAME}' sudah ada."
|
|
exit 0
|
|
fi
|
|
echo " Source belum ada (GET by-path -> HTTP ${EXIST_CODE}). Membuat..."
|
|
|
|
BODY=$(cat <<JSON
|
|
{
|
|
"entityType": "source",
|
|
"name": "${DREMIO_SOURCE_NAME}",
|
|
"type": "NESSIE",
|
|
"config": {
|
|
"nessieEndpoint": "${NESSIE_ENDPOINT}",
|
|
"nessieAuthType": "NONE",
|
|
"awsRootPath": "${AWS_ROOT_PATH}",
|
|
"credentialType": "ACCESS_KEY",
|
|
"awsAccessKey": "${MINIO_ROOT_USER}",
|
|
"awsAccessSecret": "${MINIO_ROOT_PASSWORD}",
|
|
"secure": false,
|
|
"asyncEnabled": true,
|
|
"isCachingEnabled": false,
|
|
"propertyList": [
|
|
{ "name": "fs.s3a.path.style.access", "value": "true" },
|
|
{ "name": "fs.s3a.endpoint", "value": "${MINIO_ENDPOINT}" },
|
|
{ "name": "dremio.s3.compat", "value": "true" }
|
|
]
|
|
}
|
|
}
|
|
JSON
|
|
)
|
|
|
|
echo "==> POST /api/v3/catalog (type=NESSIE)..."
|
|
CREATE_CODE=$(curl -s -o /tmp/dremio_create_resp.json -w "%{http_code}" \
|
|
-X POST "${DREMIO_URL}/api/v3/catalog" \
|
|
-H "${AUTH}" -H "Content-Type: application/json" -d "${BODY}")
|
|
echo " HTTP ${CREATE_CODE}"
|
|
cat /tmp/dremio_create_resp.json 2>/dev/null; echo
|
|
|
|
case "${CREATE_CODE}" in
|
|
200|201)
|
|
echo "✅ Source '${DREMIO_SOURCE_NAME}' dibuat via API."
|
|
;;
|
|
409)
|
|
echo "✅ Source '${DREMIO_SOURCE_NAME}' sudah ada (409 Conflict)."
|
|
;;
|
|
*)
|
|
echo "⚠️ API menolak pembuatan source Nessie (HTTP ${CREATE_CODE})."
|
|
echo " Sesuai docs (DX-64994), Catalog API tidak mengelola Nessie source."
|
|
echo " Buat manual via UI: http://localhost:12947 → Add Source → Nessie"
|
|
echo " endpoint: ${NESSIE_ENDPOINT}"
|
|
echo " storage : AWS S3 / MinIO (path-style), endpoint ${MINIO_ENDPOINT}"
|
|
exit 2
|
|
;;
|
|
esac
|