sync dari sicabe-doc/deploy

deploy/bootstrap.sh

@@ -868,7 +868,7 @@ bootstrap_starrocks() {
       'type' = 'iceberg',
       'iceberg.catalog.type' = 'rest',
       'iceberg.catalog.uri' = '${NESSIE_URL:-http://nessie:19120}/iceberg/',
-      'iceberg.catalog.warehouse' = 'warehouse',
+      'iceberg.catalog.warehouse' = '${ICEBERG_WAREHOUSE:-warehouse}',
       'iceberg.catalog.rest.nested-namespace-enabled' = 'true',
       'aws.s3.endpoint' = '${S3_ENDPOINT:-http://minio:9000}',
       'aws.s3.access_key' = '${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}',
@@ -911,7 +911,7 @@ print_starrocks_sql() {
   CREATE EXTERNAL CATALOG iceberg_catalog PROPERTIES (
     'type'='iceberg','iceberg.catalog.type'='rest',
     'iceberg.catalog.uri'='${NESSIE_URL:-http://nessie:19120}/iceberg/',
-    'iceberg.catalog.warehouse'='warehouse',
+    'iceberg.catalog.warehouse'='${ICEBERG_WAREHOUSE:-warehouse}',
     'aws.s3.endpoint'='${S3_ENDPOINT:-http://minio:9000}', ...);
 EOF
 }
@@ -1120,7 +1120,7 @@ connections:
     catalog_name: iceberg
     schema: main
     rest_uri: ${NESSIE_URL:-http://nessie:19120}/iceberg/
-    rest_warehouse: warehouse
+    rest_warehouse: ${ICEBERG_WAREHOUSE:-warehouse}
     s3_endpoint: ${S3_ENDPOINT:-http://minio:9000}
     s3_access_key_id: ${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}
     s3_secret_access_key: ${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD}}

deploy/config/environment.defaults.sh

@@ -86,6 +86,11 @@ export MINIO_PORT="${MINIO_PORT:-12900}"
 export MINIO_CONSOLE_PORT="${MINIO_CONSOLE_PORT:-12901}"
 export MINIO_ROOT_USER="${MINIO_ROOT_USER:-minioadmin}"
 export MINIO_BUCKET_ICEBERG_DATA="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}"
+# Nama + prefix path warehouse Iceberg. Dipakai IDENTIK di: Nessie
+# (NESSIE_CATALOG_DEFAULT_WAREHOUSE + location S3), Trino default-warehouse-dir,
+# StarRocks/OM/Sling external catalog. Ganti SATU nilai ini bila client memakai
+# nama/path berbeda (mis. `wh`) — lalu `--generate --sync`.
+export ICEBERG_WAREHOUSE="${ICEBERG_WAREHOUSE:-warehouse}"
 
 # Apache Nessie (Phase 2 — Storage: Metadata Catalog)
 export NESSIE_VERSION="${NESSIE_VERSION:-0.108.4-java}"

deploy/config/trino/catalog/iceberg.properties

@@ -15,7 +15,7 @@ connector.name=iceberg
 iceberg.catalog.type=nessie
 iceberg.nessie-catalog.uri=${ENV:NESSIE_URL}/api/v2
 iceberg.nessie-catalog.ref=main
-iceberg.nessie-catalog.default-warehouse-dir=s3://${ENV:MINIO_BUCKET_ICEBERG_DATA}/warehouse
+iceberg.nessie-catalog.default-warehouse-dir=s3://${ENV:MINIO_BUCKET_ICEBERG_DATA}/${ENV:ICEBERG_WAREHOUSE}
 
 # --- S3 Native Filesystem (MinIO / S3 existing) ---
 fs.s3.enabled=true

deploy/docker-compose.yml

@@ -334,11 +334,12 @@ services:
       QUARKUS_DATASOURCE_POSTGRESQL_PASSWORD: "${NESSIE_DB_PASSWORD:?err}"
 
       # ── Iceberg REST catalog + warehouse di MinIO (S3 server-side) ──
-      # Warehouse = bucket iceberg-data (konvensi base) + prefix warehouse/.
+      # Warehouse = bucket (MINIO_BUCKET_ICEBERG_DATA) + prefix (ICEBERG_WAREHOUSE).
-      NESSIE_CATALOG_DEFAULT_WAREHOUSE: "warehouse"
+      # Nama & prefix samakan nilai ICEBERG_WAREHOUSE (lihat environment.defaults.sh).
-      NESSIE_CATALOG_WAREHOUSES_WAREHOUSE_LOCATION: "s3://${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/warehouse"
+      NESSIE_CATALOG_DEFAULT_WAREHOUSE: "${ICEBERG_WAREHOUSE:-warehouse}"
+      NESSIE_CATALOG_WAREHOUSES_WAREHOUSE_LOCATION: "s3://${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/${ICEBERG_WAREHOUSE:-warehouse}"
       NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_ENDPOINT: "${S3_ENDPOINT:-http://minio:9000}"
-      NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_REGION: "us-east-1"
+      NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_REGION: "${S3_REGION:-us-east-1}"
       NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_AUTH_TYPE: "STATIC"
       NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_PATH_STYLE_ACCESS: "true"
       # Kredensial S3 via Quarkus secret store. Nama secret memakai TITIK
@@ -459,6 +460,7 @@ services:
       MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
       MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?err}
       MINIO_BUCKET_ICEBERG_DATA: ${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}
+      ICEBERG_WAREHOUSE: ${ICEBERG_WAREHOUSE:-warehouse}
       S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
       S3_REGION: ${S3_REGION:-us-east-1}
       S3_ACCESS_KEY: ${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}
@@ -561,6 +563,7 @@ services:
       MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
       MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?err}
       MINIO_BUCKET_ICEBERG_DATA: ${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}
+      ICEBERG_WAREHOUSE: ${ICEBERG_WAREHOUSE:-warehouse}
       S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
       S3_REGION: ${S3_REGION:-us-east-1}
       S3_ACCESS_KEY: ${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}

deploy/install.sh

@@ -81,6 +81,11 @@ OPENMETADATA_ADMIN_PASSWORD="$(openssl rand -hex 12)Aa1!"
 LDAP_ADMIN_PASSWORD=$(openssl rand -hex 8)
 LDAP_CONFIG_PASSWORD=$(openssl rand -hex 8)
 
+# Storage & katalog (NON-rahasia) — sesuaikan bila client memakai nama berbeda,
+# lalu jalankan: ./install.sh --generate [--sync]
+MINIO_BUCKET_ICEBERG_DATA=iceberg-data
+ICEBERG_WAREHOUSE=warehouse
+
 # OIDC Client Secrets (harus sinkron dengan config .properties/.yaml files)
 TRINO_CLIENT_SECRET=$(openssl rand -hex 16)
 STARROCKS_CLIENT_SECRET=$(openssl rand -hex 16)

deploy/scripts/dremio/register-nessie-source.sh

@@ -33,7 +33,7 @@ NESSIE_ENDPOINT="${NESSIE_ENDPOINT:-${NESSIE_URL:-http://nessie:19120}/api/v2}"
 # MINIO_ENDPOINT dipakai tanpa skema (Dremio memisahkan host:port) → buang skema.
 _minio_host="${MINIO_ENDPOINT:-${S3_ENDPOINT:-http://minio:9000}}"
 MINIO_ENDPOINT="${_minio_host#*://}"
-AWS_ROOT_PATH="${AWS_ROOT_PATH:-/${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/warehouse}"
+AWS_ROOT_PATH="${AWS_ROOT_PATH:-/${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/${ICEBERG_WAREHOUSE:-warehouse}}"
 MINIO_ROOT_USER="${MINIO_ROOT_USER:-${S3_ACCESS_KEY:-minioadmin}}"
 MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:-${S3_SECRET_KEY:?set MINIO_ROOT_PASSWORD atau S3_SECRET_KEY}}"
 

deploy/scripts/seed-starrocks.sh

@@ -52,7 +52,7 @@ ensure_external_catalog() {
     'type' = 'iceberg',
     'iceberg.catalog.type' = 'rest',
     'iceberg.catalog.uri' = '${NESSIE_URL:-http://nessie:19120}/iceberg/',
-    'iceberg.catalog.warehouse' = 'warehouse',
+    'iceberg.catalog.warehouse' = '${ICEBERG_WAREHOUSE:-warehouse}',
     'iceberg.catalog.rest.nested-namespace-enabled' = 'true',
     'aws.s3.endpoint' = '${S3_ENDPOINT:-http://minio:9000}',
     'aws.s3.access_key' = '${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}',

deploy/scripts/test-generate.sh

@@ -273,6 +273,40 @@ else
   bad "--sync: masih ada path repo ter-hardcode (~/sicabe-doc/deploy)"
 fi
 
+# ─── 0n. Bucket & warehouse Iceberg terkonfigurasi (bukan literal) ───────────
+# Bucket = MINIO_BUCKET_ICEBERG_DATA; warehouse = ICEBERG_WAREHOUSE (nama REST
+# Nessie + prefix path S3, IDENTIK). Client boleh punya nama berbeda → tak boleh
+# ada literal 'warehouse' / region Nessie hardcoded.
+_wh_dc="${DEPLOY_DIR}/docker-compose.yml"; _wh_ed="${DEPLOY_DIR}/config/environment.defaults.sh"
+_wh_ip="${DEPLOY_DIR}/config/trino/catalog/iceberg.properties"; _wh_bs="${DEPLOY_DIR}/bootstrap.sh"
+_wh_se="${DEPLOY_DIR}/scripts/seed-starrocks.sh"; _wh_v2="${DEPLOY_DIR}/scripts/verify-phase2-storage.sh"
+_wh_ins="${DEPLOY_DIR}/install.sh"
+wh_bad=""
+have() { grep -qF "$2" "$1" 2>/dev/null || wh_bad="${wh_bad} $(basename "$1"):$3"; }
+lack() { grep -qF "$2" "$1" 2>/dev/null && wh_bad="${wh_bad} $(basename "$1"):$3"; }
+have "${_wh_ed}" 'export ICEBERG_WAREHOUSE=' "env-default"
+have "${_wh_dc}" 'NESSIE_CATALOG_DEFAULT_WAREHOUSE: "${ICEBERG_WAREHOUSE:-warehouse}"' "nessie-name"
+have "${_wh_dc}" 's3://${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/${ICEBERG_WAREHOUSE:-warehouse}' "nessie-loc"
+have "${_wh_dc}" 'DEFAULT_OPTIONS_REGION: "${S3_REGION:-us-east-1}"' "nessie-region"
+have "${_wh_dc}" 'ICEBERG_WAREHOUSE: ${ICEBERG_WAREHOUSE:-warehouse}' "trino-env"
+have "${_wh_ip}" '${ENV:ICEBERG_WAREHOUSE}' "trino-catalog"
+have "${_wh_bs}" "'iceberg.catalog.warehouse' = '\${ICEBERG_WAREHOUSE:-warehouse}'" "sr-catalog"
+have "${_wh_bs}" "'iceberg.catalog.warehouse'='\${ICEBERG_WAREHOUSE:-warehouse}'" "sr-catalog2"
+have "${_wh_bs}" 'rest_warehouse: ${ICEBERG_WAREHOUSE:-warehouse}' "sling"
+have "${_wh_v2}" 'WAREHOUSE="${ICEBERG_WAREHOUSE:-warehouse}"' "gate-def"
+have "${_wh_v2}" 's3://${BUCKET}/${WAREHOUSE}' "gate-expect"
+have "${_wh_ins}" 'MINIO_BUCKET_ICEBERG_DATA=iceberg-data' "env-bucket"
+have "${_wh_ins}" 'ICEBERG_WAREHOUSE=warehouse' "env-warehouse"
+lack "${_wh_bs}" "'iceberg.catalog.warehouse' = 'warehouse'" "sr-literal"
+lack "${_wh_bs}" "'iceberg.catalog.warehouse'='warehouse'" "sr-literal2"
+lack "${_wh_bs}" 'rest_warehouse: warehouse' "sling-literal"
+lack "${_wh_se}" "'iceberg.catalog.warehouse' = 'warehouse'" "seed-literal"
+if [[ -z "${wh_bad}" ]]; then
+  ok "storage: bucket+warehouse terkonfigurasi (nama & prefix), region Nessie = S3_REGION"
+else
+  bad "storage: konfigurasi bucket/warehouse belum lengkap →${wh_bad}"
+fi
+
 # ─── 1. generate ────────────────────────────────────────────────────────────
 if ! python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \
       "${INVENTORY}" "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" \

deploy/scripts/verify-phase2-storage.sh

@@ -46,6 +46,7 @@ S3_REGION="${S3_REGION:-us-east-1}"
 S3_ACCESS_KEY="${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}"
 S3_SECRET_KEY="${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-}}"
 BUCKET="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}"
+WAREHOUSE="${ICEBERG_WAREHOUSE:-warehouse}"
 NESSIE_URL="${NESSIE_URL:-http://nessie:19120}"
 NESSIE_HOST="${NESSIE_HOST:-localhost}"
 NESSIE_PORT="${NESSIE_PORT:-12120}"
@@ -236,10 +237,10 @@ check_storage_contract() {
   wh=$(echo "${cfg}" | sed -n 's/.*WAREHOUSE_LOCATION: *//p' | head -1 | tr -d '"')
   ep=$(echo "${cfg}" | sed -n 's/.*DEFAULT_OPTIONS_ENDPOINT: *//p' | head -1 | tr -d '"')
 
-  if [[ "${wh}" == "s3://${BUCKET}/warehouse" ]]; then
+  if [[ "${wh}" == "s3://${BUCKET}/${WAREHOUSE}" ]]; then
-    pass "Warehouse Nessie → ${wh} (sesuai bucket '${BUCKET}')"
+    pass "Warehouse Nessie → ${wh} (bucket '${BUCKET}' + warehouse '${WAREHOUSE}')"
   elif [[ -n "${wh}" ]]; then
-    fail "Warehouse Nessie '${wh}' ≠ 's3://${BUCKET}/warehouse' — bucket tidak sinkron"
+    fail "Warehouse Nessie '${wh}' ≠ 's3://${BUCKET}/${WAREHOUSE}' — bucket/warehouse tidak sinkron"
   else
     fail "WAREHOUSE_LOCATION Nessie tidak ditemukan di compose"
   fi
@@ -372,7 +373,7 @@ except Exception: print('')" 2>/dev/null)
 # ─────────────────────────────────────────────────────────────────────────────
 main() {
   print_section "Verifikasi Phase 2 — Kontrak Storage & Catalog (MinIO/S3 + Nessie)"
-  echo -e "  Object storage : ${CYAN}${S3_ENDPOINT}${NC} (bucket ${BUCKET}, region ${S3_REGION})"
+  echo -e "  Object storage : ${CYAN}${S3_ENDPOINT}${NC} (bucket ${BUCKET}, warehouse ${WAREHOUSE}, region ${S3_REGION})"
   echo -e "  Nessie         : ${CYAN}${NESSIE_URL}${NC} (API host ${NESSIE_HOST_API})"
   echo ""