sicabe-core/deploy/scripts/dremio/register-nessie-source.sh

#!/usr/bin/env bash
# =============================================================================
# Dremio — Register Nessie catalog source via REST API
# =============================================================================
# Membuat source "Nessie" di Dremio OSS lewat API (bukan UI).
#
# ⚠️ Catatan: dokumentasi Dremio menyatakan Catalog API "unable to retrieve or
# manage Nessie sources" (DX-64994). Namun TERBUKTI (Dremio OSS 25.2.0, diuji
# 2026-09-10) bahwa POST /api/v3/catalog dengan type=NESSIE BERHASIL, dan
# GET /api/v3/catalog/by-path/<name> juga bekerja. Kunci config:
#   - JANGAN pakai "storageType" (ditolak: "Invalid value found at: storageType")
#   - awsRootPath format "/bucket/path" (TANPA prefix s3://)
#
# Pemakaian:
#   export DREMIO_USER=admin DREMIO_PASSWORD='<password>'
#   ./scripts/dremio/register-nessie-source.sh
#
# Env (opsional, ada default):
#   DREMIO_URL          default http://localhost:12947
#   DREMIO_SOURCE_NAME  default nessie
#   NESSIE_ENDPOINT     default ${NESSIE_URL}/api/v2  (http://nessie:19120/api/v2)
#   MINIO_ENDPOINT      default host dari S3_ENDPOINT (tanpa skema)
#   AWS_ROOT_PATH       default /<bucket>/warehouse
#   MINIO_ROOT_USER / MINIO_ROOT_PASSWORD  (fallback: S3_ACCESS_KEY / S3_SECRET_KEY)
# =============================================================================
set -euo pipefail

DREMIO_URL="${DREMIO_URL:-http://localhost:12947}"
DREMIO_USER="${DREMIO_USER:-admin}"
DREMIO_PASSWORD="${DREMIO_PASSWORD:?set DREMIO_PASSWORD}"
DREMIO_SOURCE_NAME="${DREMIO_SOURCE_NAME:-nessie}"
NESSIE_ENDPOINT="${NESSIE_ENDPOINT:-${NESSIE_URL:-http://nessie:19120}/api/v2}"
# MINIO_ENDPOINT dipakai tanpa skema (Dremio memisahkan host:port) → buang skema.
_minio_host="${MINIO_ENDPOINT:-${S3_ENDPOINT:-http://minio:9000}}"
MINIO_ENDPOINT="${_minio_host#*://}"
AWS_ROOT_PATH="${AWS_ROOT_PATH:-/${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/${ICEBERG_WAREHOUSE:-warehouse}}"
MINIO_ROOT_USER="${MINIO_ROOT_USER:-${S3_ACCESS_KEY:-minioadmin}}"
MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:-${S3_SECRET_KEY:?set MINIO_ROOT_PASSWORD atau S3_SECRET_KEY}}"

echo "==> Login Dremio (${DREMIO_URL}) sebagai '${DREMIO_USER}'..."
LOGIN_RESP=$(curl -s -X POST "${DREMIO_URL}/apiv2/login" \
  -H "Content-Type: application/json" \
  -d "{\"userName\":\"${DREMIO_USER}\",\"password\":\"${DREMIO_PASSWORD}\"}")
TOKEN=$(echo "${LOGIN_RESP}" | python3 -c "import sys,json;print(json.load(sys.stdin).get('token',''))" 2>/dev/null || echo "")
if [[ -z "${TOKEN}" ]]; then
  echo "ERR: login gagal. Respons: ${LOGIN_RESP}"
  exit 1
fi
AUTH="Authorization: _dremio${TOKEN}"
echo "    OK: token diperoleh."

# Cek apakah source sudah ada (best-effort; GET Nessie mungkin tidak didukung)
EXIST_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "${AUTH}" \
  "${DREMIO_URL}/api/v3/catalog/by-path/${DREMIO_SOURCE_NAME}" || echo "000")
if [[ "${EXIST_CODE}" == "200" ]]; then
  echo "OK: source '${DREMIO_SOURCE_NAME}' sudah ada."
  exit 0
fi
echo "    Source belum ada (GET by-path -> HTTP ${EXIST_CODE}). Membuat..."

BODY=$(cat <<JSON
{
  "entityType": "source",
  "name": "${DREMIO_SOURCE_NAME}",
  "type": "NESSIE",
  "config": {
    "nessieEndpoint": "${NESSIE_ENDPOINT}",
    "nessieAuthType": "NONE",
    "awsRootPath": "${AWS_ROOT_PATH}",
    "credentialType": "ACCESS_KEY",
    "awsAccessKey": "${MINIO_ROOT_USER}",
    "awsAccessSecret": "${MINIO_ROOT_PASSWORD}",
    "secure": false,
    "asyncEnabled": true,
    "isCachingEnabled": false,
    "propertyList": [
      { "name": "fs.s3a.path.style.access", "value": "true" },
      { "name": "fs.s3a.endpoint", "value": "${MINIO_ENDPOINT}" },
      { "name": "dremio.s3.compat", "value": "true" }
    ]
  }
}
JSON
)

echo "==> POST /api/v3/catalog (type=NESSIE)..."
CREATE_CODE=$(curl -s -o /tmp/dremio_create_resp.json -w "%{http_code}" \
  -X POST "${DREMIO_URL}/api/v3/catalog" \
  -H "${AUTH}" -H "Content-Type: application/json" -d "${BODY}")
echo "    HTTP ${CREATE_CODE}"
cat /tmp/dremio_create_resp.json 2>/dev/null; echo

case "${CREATE_CODE}" in
  200|201)
    echo "✅ Source '${DREMIO_SOURCE_NAME}' dibuat via API."
    ;;
  409)
    echo "✅ Source '${DREMIO_SOURCE_NAME}' sudah ada (409 Conflict)."
    ;;
  *)
    echo "⚠️  API menolak pembuatan source Nessie (HTTP ${CREATE_CODE})."
    echo "    Sesuai docs (DX-64994), Catalog API tidak mengelola Nessie source."
    echo "    Buat manual via UI: http://localhost:12947 → Add Source → Nessie"
    echo "      endpoint: ${NESSIE_ENDPOINT}"
    echo "      storage : AWS S3 / MinIO (path-style), endpoint ${MINIO_ENDPOINT}"
    exit 2
    ;;
esac