platform/sicabe-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync dari sicabe-doc/deploy

Browse Source
This commit is contained in:
subroto
2026-10-11 15:09:43 +07:00
parent cbabe96061
commit 6e4dfb69ef
9 changed files with 62 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/scripts/dremio/register-nessie-source.sh
+1 -1
View File
@@ -33,7 +33,7 @@ NESSIE_ENDPOINT="${NESSIE_ENDPOINT:-${NESSIE_URL:-http://nessie:19120}/api/v2}"
# MINIO_ENDPOINT dipakai tanpa skema (Dremio memisahkan host:port) → buang skema.
_minio_host="${MINIO_ENDPOINT:-${S3_ENDPOINT:-http://minio:9000}}"
MINIO_ENDPOINT="${_minio_host#*://}"
AWS_ROOT_PATH="${AWS_ROOT_PATH:-/${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/warehouse}"
AWS_ROOT_PATH="${AWS_ROOT_PATH:-/${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/${ICEBERG_WAREHOUSE:-warehouse}}"
MINIO_ROOT_USER="${MINIO_ROOT_USER:-${S3_ACCESS_KEY:-minioadmin}}"
MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:-${S3_SECRET_KEY:?set MINIO_ROOT_PASSWORD atau S3_SECRET_KEY}}"
deploy/scripts/seed-starrocks.sh
+1 -1
View File
@@ -52,7 +52,7 @@ ensure_external_catalog() {
'type' = 'iceberg',
'iceberg.catalog.type' = 'rest',
'iceberg.catalog.uri' = '${NESSIE_URL:-http://nessie:19120}/iceberg/',
'iceberg.catalog.warehouse' = 'warehouse',
'iceberg.catalog.warehouse' = '${ICEBERG_WAREHOUSE:-warehouse}',
'iceberg.catalog.rest.nested-namespace-enabled' = 'true',
'aws.s3.endpoint' = '${S3_ENDPOINT:-http://minio:9000}',
'aws.s3.access_key' = '${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}',
deploy/scripts/test-generate.sh
+34
View File
@@ -273,6 +273,40 @@ else
bad "--sync: masih ada path repo ter-hardcode (~/sicabe-doc/deploy)"
fi
# ─── 0n. Bucket & warehouse Iceberg terkonfigurasi (bukan literal) ───────────
# Bucket = MINIO_BUCKET_ICEBERG_DATA; warehouse = ICEBERG_WAREHOUSE (nama REST
# Nessie + prefix path S3, IDENTIK). Client boleh punya nama berbeda → tak boleh
# ada literal 'warehouse' / region Nessie hardcoded.
_wh_dc="${DEPLOY_DIR}/docker-compose.yml"; _wh_ed="${DEPLOY_DIR}/config/environment.defaults.sh"
_wh_ip="${DEPLOY_DIR}/config/trino/catalog/iceberg.properties"; _wh_bs="${DEPLOY_DIR}/bootstrap.sh"
_wh_se="${DEPLOY_DIR}/scripts/seed-starrocks.sh"; _wh_v2="${DEPLOY_DIR}/scripts/verify-phase2-storage.sh"
_wh_ins="${DEPLOY_DIR}/install.sh"
wh_bad=""
have() { grep -qF "$2" "$1" 2>/dev/null || wh_bad="${wh_bad} $(basename "$1"):$3"; }
lack() { grep -qF "$2" "$1" 2>/dev/null && wh_bad="${wh_bad} $(basename "$1"):$3"; }
have "${_wh_ed}" 'export ICEBERG_WAREHOUSE=' "env-default"
have "${_wh_dc}" 'NESSIE_CATALOG_DEFAULT_WAREHOUSE: "${ICEBERG_WAREHOUSE:-warehouse}"' "nessie-name"
have "${_wh_dc}" 's3://${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/${ICEBERG_WAREHOUSE:-warehouse}' "nessie-loc"
have "${_wh_dc}" 'DEFAULT_OPTIONS_REGION: "${S3_REGION:-us-east-1}"' "nessie-region"
have "${_wh_dc}" 'ICEBERG_WAREHOUSE: ${ICEBERG_WAREHOUSE:-warehouse}' "trino-env"
have "${_wh_ip}" '${ENV:ICEBERG_WAREHOUSE}' "trino-catalog"
have "${_wh_bs}" "'iceberg.catalog.warehouse' = '\${ICEBERG_WAREHOUSE:-warehouse}'" "sr-catalog"
have "${_wh_bs}" "'iceberg.catalog.warehouse'='\${ICEBERG_WAREHOUSE:-warehouse}'" "sr-catalog2"
have "${_wh_bs}" 'rest_warehouse: ${ICEBERG_WAREHOUSE:-warehouse}' "sling"
have "${_wh_v2}" 'WAREHOUSE="${ICEBERG_WAREHOUSE:-warehouse}"' "gate-def"
have "${_wh_v2}" 's3://${BUCKET}/${WAREHOUSE}' "gate-expect"
have "${_wh_ins}" 'MINIO_BUCKET_ICEBERG_DATA=iceberg-data' "env-bucket"
have "${_wh_ins}" 'ICEBERG_WAREHOUSE=warehouse' "env-warehouse"
lack "${_wh_bs}" "'iceberg.catalog.warehouse' = 'warehouse'" "sr-literal"
lack "${_wh_bs}" "'iceberg.catalog.warehouse'='warehouse'" "sr-literal2"
lack "${_wh_bs}" 'rest_warehouse: warehouse' "sling-literal"
lack "${_wh_se}" "'iceberg.catalog.warehouse' = 'warehouse'" "seed-literal"
if [[ -z "${wh_bad}" ]]; then
ok "storage: bucket+warehouse terkonfigurasi (nama & prefix), region Nessie = S3_REGION"
else
bad "storage: konfigurasi bucket/warehouse belum lengkap →${wh_bad}"
fi
# ─── 1. generate ────────────────────────────────────────────────────────────
if ! python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \
"${INVENTORY}" "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" \
deploy/scripts/verify-phase2-storage.sh
+5 -4
View File
@@ -46,6 +46,7 @@ S3_REGION="${S3_REGION:-us-east-1}"
S3_ACCESS_KEY="${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}"
S3_SECRET_KEY="${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-}}"
BUCKET="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}"
WAREHOUSE="${ICEBERG_WAREHOUSE:-warehouse}"
NESSIE_URL="${NESSIE_URL:-http://nessie:19120}"
NESSIE_HOST="${NESSIE_HOST:-localhost}"
NESSIE_PORT="${NESSIE_PORT:-12120}"
@@ -236,10 +237,10 @@ check_storage_contract() {
wh=$(echo "${cfg}" | sed -n 's/.*WAREHOUSE_LOCATION: *//p' | head -1 | tr -d '"')
ep=$(echo "${cfg}" | sed -n 's/.*DEFAULT_OPTIONS_ENDPOINT: *//p' | head -1 | tr -d '"')
if [[ "${wh}" == "s3://${BUCKET}/warehouse" ]]; then
pass "Warehouse Nessie → ${wh} (sesuai bucket '${BUCKET}')"
if [[ "${wh}" == "s3://${BUCKET}/${WAREHOUSE}" ]]; then
pass "Warehouse Nessie → ${wh} (bucket '${BUCKET}' + warehouse '${WAREHOUSE}')"
elif [[ -n "${wh}" ]]; then
fail "Warehouse Nessie '${wh}' ≠ 's3://${BUCKET}/warehouse' — bucket tidak sinkron"
fail "Warehouse Nessie '${wh}' ≠ 's3://${BUCKET}/${WAREHOUSE}' — bucket/warehouse tidak sinkron"
else
fail "WAREHOUSE_LOCATION Nessie tidak ditemukan di compose"
fi
@@ -372,7 +373,7 @@ except Exception: print('')" 2>/dev/null)
# ─────────────────────────────────────────────────────────────────────────────
main() {
print_section "Verifikasi Phase 2 — Kontrak Storage & Catalog (MinIO/S3 + Nessie)"
echo -e " Object storage : ${CYAN}${S3_ENDPOINT}${NC} (bucket ${BUCKET}, region ${S3_REGION})"
echo -e " Object storage : ${CYAN}${S3_ENDPOINT}${NC} (bucket ${BUCKET}, warehouse ${WAREHOUSE}, region ${S3_REGION})"
echo -e " Nessie : ${CYAN}${NESSIE_URL}${NC} (API host ${NESSIE_HOST_API})"
echo ""