diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 86bf21b..94ed9e1 100755 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -868,7 +868,7 @@ bootstrap_starrocks() { 'type' = 'iceberg', 'iceberg.catalog.type' = 'rest', 'iceberg.catalog.uri' = '${NESSIE_URL:-http://nessie:19120}/iceberg/', - 'iceberg.catalog.warehouse' = 'warehouse', + 'iceberg.catalog.warehouse' = '${ICEBERG_WAREHOUSE:-warehouse}', 'iceberg.catalog.rest.nested-namespace-enabled' = 'true', 'aws.s3.endpoint' = '${S3_ENDPOINT:-http://minio:9000}', 'aws.s3.access_key' = '${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}', @@ -911,7 +911,7 @@ print_starrocks_sql() { CREATE EXTERNAL CATALOG iceberg_catalog PROPERTIES ( 'type'='iceberg','iceberg.catalog.type'='rest', 'iceberg.catalog.uri'='${NESSIE_URL:-http://nessie:19120}/iceberg/', - 'iceberg.catalog.warehouse'='warehouse', + 'iceberg.catalog.warehouse'='${ICEBERG_WAREHOUSE:-warehouse}', 'aws.s3.endpoint'='${S3_ENDPOINT:-http://minio:9000}', ...); EOF } @@ -1120,7 +1120,7 @@ connections: catalog_name: iceberg schema: main rest_uri: ${NESSIE_URL:-http://nessie:19120}/iceberg/ - rest_warehouse: warehouse + rest_warehouse: ${ICEBERG_WAREHOUSE:-warehouse} s3_endpoint: ${S3_ENDPOINT:-http://minio:9000} s3_access_key_id: ${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}} s3_secret_access_key: ${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD}} diff --git a/deploy/config/environment.defaults.sh b/deploy/config/environment.defaults.sh index 05c8a07..e555f04 100644 --- a/deploy/config/environment.defaults.sh +++ b/deploy/config/environment.defaults.sh @@ -86,6 +86,11 @@ export MINIO_PORT="${MINIO_PORT:-12900}" export MINIO_CONSOLE_PORT="${MINIO_CONSOLE_PORT:-12901}" export MINIO_ROOT_USER="${MINIO_ROOT_USER:-minioadmin}" export MINIO_BUCKET_ICEBERG_DATA="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}" +# Nama + prefix path warehouse Iceberg. Dipakai IDENTIK di: Nessie +# (NESSIE_CATALOG_DEFAULT_WAREHOUSE + location S3), Trino default-warehouse-dir, +# StarRocks/OM/Sling external catalog. Ganti SATU nilai ini bila client memakai +# nama/path berbeda (mis. `wh`) — lalu `--generate --sync`. +export ICEBERG_WAREHOUSE="${ICEBERG_WAREHOUSE:-warehouse}" # Apache Nessie (Phase 2 — Storage: Metadata Catalog) export NESSIE_VERSION="${NESSIE_VERSION:-0.108.4-java}" diff --git a/deploy/config/trino/catalog/iceberg.properties b/deploy/config/trino/catalog/iceberg.properties index 6667073..4c2abb9 100644 --- a/deploy/config/trino/catalog/iceberg.properties +++ b/deploy/config/trino/catalog/iceberg.properties @@ -15,7 +15,7 @@ connector.name=iceberg iceberg.catalog.type=nessie iceberg.nessie-catalog.uri=${ENV:NESSIE_URL}/api/v2 iceberg.nessie-catalog.ref=main -iceberg.nessie-catalog.default-warehouse-dir=s3://${ENV:MINIO_BUCKET_ICEBERG_DATA}/warehouse +iceberg.nessie-catalog.default-warehouse-dir=s3://${ENV:MINIO_BUCKET_ICEBERG_DATA}/${ENV:ICEBERG_WAREHOUSE} # --- S3 Native Filesystem (MinIO / S3 existing) --- fs.s3.enabled=true diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 5599559..0242530 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -334,11 +334,12 @@ services: QUARKUS_DATASOURCE_POSTGRESQL_PASSWORD: "${NESSIE_DB_PASSWORD:?err}" # ── Iceberg REST catalog + warehouse di MinIO (S3 server-side) ── - # Warehouse = bucket iceberg-data (konvensi base) + prefix warehouse/. - NESSIE_CATALOG_DEFAULT_WAREHOUSE: "warehouse" - NESSIE_CATALOG_WAREHOUSES_WAREHOUSE_LOCATION: "s3://${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/warehouse" + # Warehouse = bucket (MINIO_BUCKET_ICEBERG_DATA) + prefix (ICEBERG_WAREHOUSE). + # Nama & prefix samakan nilai ICEBERG_WAREHOUSE (lihat environment.defaults.sh). + NESSIE_CATALOG_DEFAULT_WAREHOUSE: "${ICEBERG_WAREHOUSE:-warehouse}" + NESSIE_CATALOG_WAREHOUSES_WAREHOUSE_LOCATION: "s3://${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/${ICEBERG_WAREHOUSE:-warehouse}" NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_ENDPOINT: "${S3_ENDPOINT:-http://minio:9000}" - NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_REGION: "us-east-1" + NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_REGION: "${S3_REGION:-us-east-1}" NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_AUTH_TYPE: "STATIC" NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_PATH_STYLE_ACCESS: "true" # Kredensial S3 via Quarkus secret store. Nama secret memakai TITIK @@ -459,6 +460,7 @@ services: MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?err} MINIO_BUCKET_ICEBERG_DATA: ${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data} + ICEBERG_WAREHOUSE: ${ICEBERG_WAREHOUSE:-warehouse} S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000} S3_REGION: ${S3_REGION:-us-east-1} S3_ACCESS_KEY: ${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}} @@ -561,6 +563,7 @@ services: MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?err} MINIO_BUCKET_ICEBERG_DATA: ${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data} + ICEBERG_WAREHOUSE: ${ICEBERG_WAREHOUSE:-warehouse} S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000} S3_REGION: ${S3_REGION:-us-east-1} S3_ACCESS_KEY: ${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}} diff --git a/deploy/install.sh b/deploy/install.sh index 83d8ef3..5dcb874 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -81,6 +81,11 @@ OPENMETADATA_ADMIN_PASSWORD="$(openssl rand -hex 12)Aa1!" LDAP_ADMIN_PASSWORD=$(openssl rand -hex 8) LDAP_CONFIG_PASSWORD=$(openssl rand -hex 8) +# Storage & katalog (NON-rahasia) — sesuaikan bila client memakai nama berbeda, +# lalu jalankan: ./install.sh --generate [--sync] +MINIO_BUCKET_ICEBERG_DATA=iceberg-data +ICEBERG_WAREHOUSE=warehouse + # OIDC Client Secrets (harus sinkron dengan config .properties/.yaml files) TRINO_CLIENT_SECRET=$(openssl rand -hex 16) STARROCKS_CLIENT_SECRET=$(openssl rand -hex 16) diff --git a/deploy/scripts/dremio/register-nessie-source.sh b/deploy/scripts/dremio/register-nessie-source.sh index f5c4e66..02e038f 100755 --- a/deploy/scripts/dremio/register-nessie-source.sh +++ b/deploy/scripts/dremio/register-nessie-source.sh @@ -33,7 +33,7 @@ NESSIE_ENDPOINT="${NESSIE_ENDPOINT:-${NESSIE_URL:-http://nessie:19120}/api/v2}" # MINIO_ENDPOINT dipakai tanpa skema (Dremio memisahkan host:port) → buang skema. _minio_host="${MINIO_ENDPOINT:-${S3_ENDPOINT:-http://minio:9000}}" MINIO_ENDPOINT="${_minio_host#*://}" -AWS_ROOT_PATH="${AWS_ROOT_PATH:-/${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/warehouse}" +AWS_ROOT_PATH="${AWS_ROOT_PATH:-/${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/${ICEBERG_WAREHOUSE:-warehouse}}" MINIO_ROOT_USER="${MINIO_ROOT_USER:-${S3_ACCESS_KEY:-minioadmin}}" MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:-${S3_SECRET_KEY:?set MINIO_ROOT_PASSWORD atau S3_SECRET_KEY}}" diff --git a/deploy/scripts/seed-starrocks.sh b/deploy/scripts/seed-starrocks.sh index 96d5c32..e5ecbee 100755 --- a/deploy/scripts/seed-starrocks.sh +++ b/deploy/scripts/seed-starrocks.sh @@ -52,7 +52,7 @@ ensure_external_catalog() { 'type' = 'iceberg', 'iceberg.catalog.type' = 'rest', 'iceberg.catalog.uri' = '${NESSIE_URL:-http://nessie:19120}/iceberg/', - 'iceberg.catalog.warehouse' = 'warehouse', + 'iceberg.catalog.warehouse' = '${ICEBERG_WAREHOUSE:-warehouse}', 'iceberg.catalog.rest.nested-namespace-enabled' = 'true', 'aws.s3.endpoint' = '${S3_ENDPOINT:-http://minio:9000}', 'aws.s3.access_key' = '${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}', diff --git a/deploy/scripts/test-generate.sh b/deploy/scripts/test-generate.sh index ee47c80..88ac9ee 100755 --- a/deploy/scripts/test-generate.sh +++ b/deploy/scripts/test-generate.sh @@ -273,6 +273,40 @@ else bad "--sync: masih ada path repo ter-hardcode (~/sicabe-doc/deploy)" fi +# ─── 0n. Bucket & warehouse Iceberg terkonfigurasi (bukan literal) ─────────── +# Bucket = MINIO_BUCKET_ICEBERG_DATA; warehouse = ICEBERG_WAREHOUSE (nama REST +# Nessie + prefix path S3, IDENTIK). Client boleh punya nama berbeda → tak boleh +# ada literal 'warehouse' / region Nessie hardcoded. +_wh_dc="${DEPLOY_DIR}/docker-compose.yml"; _wh_ed="${DEPLOY_DIR}/config/environment.defaults.sh" +_wh_ip="${DEPLOY_DIR}/config/trino/catalog/iceberg.properties"; _wh_bs="${DEPLOY_DIR}/bootstrap.sh" +_wh_se="${DEPLOY_DIR}/scripts/seed-starrocks.sh"; _wh_v2="${DEPLOY_DIR}/scripts/verify-phase2-storage.sh" +_wh_ins="${DEPLOY_DIR}/install.sh" +wh_bad="" +have() { grep -qF "$2" "$1" 2>/dev/null || wh_bad="${wh_bad} $(basename "$1"):$3"; } +lack() { grep -qF "$2" "$1" 2>/dev/null && wh_bad="${wh_bad} $(basename "$1"):$3"; } +have "${_wh_ed}" 'export ICEBERG_WAREHOUSE=' "env-default" +have "${_wh_dc}" 'NESSIE_CATALOG_DEFAULT_WAREHOUSE: "${ICEBERG_WAREHOUSE:-warehouse}"' "nessie-name" +have "${_wh_dc}" 's3://${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/${ICEBERG_WAREHOUSE:-warehouse}' "nessie-loc" +have "${_wh_dc}" 'DEFAULT_OPTIONS_REGION: "${S3_REGION:-us-east-1}"' "nessie-region" +have "${_wh_dc}" 'ICEBERG_WAREHOUSE: ${ICEBERG_WAREHOUSE:-warehouse}' "trino-env" +have "${_wh_ip}" '${ENV:ICEBERG_WAREHOUSE}' "trino-catalog" +have "${_wh_bs}" "'iceberg.catalog.warehouse' = '\${ICEBERG_WAREHOUSE:-warehouse}'" "sr-catalog" +have "${_wh_bs}" "'iceberg.catalog.warehouse'='\${ICEBERG_WAREHOUSE:-warehouse}'" "sr-catalog2" +have "${_wh_bs}" 'rest_warehouse: ${ICEBERG_WAREHOUSE:-warehouse}' "sling" +have "${_wh_v2}" 'WAREHOUSE="${ICEBERG_WAREHOUSE:-warehouse}"' "gate-def" +have "${_wh_v2}" 's3://${BUCKET}/${WAREHOUSE}' "gate-expect" +have "${_wh_ins}" 'MINIO_BUCKET_ICEBERG_DATA=iceberg-data' "env-bucket" +have "${_wh_ins}" 'ICEBERG_WAREHOUSE=warehouse' "env-warehouse" +lack "${_wh_bs}" "'iceberg.catalog.warehouse' = 'warehouse'" "sr-literal" +lack "${_wh_bs}" "'iceberg.catalog.warehouse'='warehouse'" "sr-literal2" +lack "${_wh_bs}" 'rest_warehouse: warehouse' "sling-literal" +lack "${_wh_se}" "'iceberg.catalog.warehouse' = 'warehouse'" "seed-literal" +if [[ -z "${wh_bad}" ]]; then + ok "storage: bucket+warehouse terkonfigurasi (nama & prefix), region Nessie = S3_REGION" +else + bad "storage: konfigurasi bucket/warehouse belum lengkap →${wh_bad}" +fi + # ─── 1. generate ──────────────────────────────────────────────────────────── if ! python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \ "${INVENTORY}" "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" \ diff --git a/deploy/scripts/verify-phase2-storage.sh b/deploy/scripts/verify-phase2-storage.sh index 5af0c4a..8731b40 100755 --- a/deploy/scripts/verify-phase2-storage.sh +++ b/deploy/scripts/verify-phase2-storage.sh @@ -46,6 +46,7 @@ S3_REGION="${S3_REGION:-us-east-1}" S3_ACCESS_KEY="${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}" S3_SECRET_KEY="${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-}}" BUCKET="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}" +WAREHOUSE="${ICEBERG_WAREHOUSE:-warehouse}" NESSIE_URL="${NESSIE_URL:-http://nessie:19120}" NESSIE_HOST="${NESSIE_HOST:-localhost}" NESSIE_PORT="${NESSIE_PORT:-12120}" @@ -236,10 +237,10 @@ check_storage_contract() { wh=$(echo "${cfg}" | sed -n 's/.*WAREHOUSE_LOCATION: *//p' | head -1 | tr -d '"') ep=$(echo "${cfg}" | sed -n 's/.*DEFAULT_OPTIONS_ENDPOINT: *//p' | head -1 | tr -d '"') - if [[ "${wh}" == "s3://${BUCKET}/warehouse" ]]; then - pass "Warehouse Nessie → ${wh} (sesuai bucket '${BUCKET}')" + if [[ "${wh}" == "s3://${BUCKET}/${WAREHOUSE}" ]]; then + pass "Warehouse Nessie → ${wh} (bucket '${BUCKET}' + warehouse '${WAREHOUSE}')" elif [[ -n "${wh}" ]]; then - fail "Warehouse Nessie '${wh}' ≠ 's3://${BUCKET}/warehouse' — bucket tidak sinkron" + fail "Warehouse Nessie '${wh}' ≠ 's3://${BUCKET}/${WAREHOUSE}' — bucket/warehouse tidak sinkron" else fail "WAREHOUSE_LOCATION Nessie tidak ditemukan di compose" fi @@ -372,7 +373,7 @@ except Exception: print('')" 2>/dev/null) # ───────────────────────────────────────────────────────────────────────────── main() { print_section "Verifikasi Phase 2 — Kontrak Storage & Catalog (MinIO/S3 + Nessie)" - echo -e " Object storage : ${CYAN}${S3_ENDPOINT}${NC} (bucket ${BUCKET}, region ${S3_REGION})" + echo -e " Object storage : ${CYAN}${S3_ENDPOINT}${NC} (bucket ${BUCKET}, warehouse ${WAREHOUSE}, region ${S3_REGION})" echo -e " Nessie : ${CYAN}${NESSIE_URL}${NC} (API host ${NESSIE_HOST_API})" echo ""