platform/sicabe-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync dari sicabe-doc/deploy

Browse Source
This commit is contained in:
subroto
2026-10-11 15:09:43 +07:00
parent cbabe96061
commit 6e4dfb69ef
9 changed files with 62 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/bootstrap.sh
+3 -3
View File
@@ -868,7 +868,7 @@ bootstrap_starrocks() {
'type' = 'iceberg',
'iceberg.catalog.type' = 'rest',
'iceberg.catalog.uri' = '${NESSIE_URL:-http://nessie:19120}/iceberg/',
'iceberg.catalog.warehouse' = 'warehouse',
'iceberg.catalog.warehouse' = '${ICEBERG_WAREHOUSE:-warehouse}',
'iceberg.catalog.rest.nested-namespace-enabled' = 'true',
'aws.s3.endpoint' = '${S3_ENDPOINT:-http://minio:9000}',
'aws.s3.access_key' = '${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}',
@@ -911,7 +911,7 @@ print_starrocks_sql() {
CREATE EXTERNAL CATALOG iceberg_catalog PROPERTIES (
'type'='iceberg','iceberg.catalog.type'='rest',
'iceberg.catalog.uri'='${NESSIE_URL:-http://nessie:19120}/iceberg/',
'iceberg.catalog.warehouse'='warehouse',
'iceberg.catalog.warehouse'='${ICEBERG_WAREHOUSE:-warehouse}',
'aws.s3.endpoint'='${S3_ENDPOINT:-http://minio:9000}', ...);
EOF
}
@@ -1120,7 +1120,7 @@ connections:
catalog_name: iceberg
schema: main
rest_uri: ${NESSIE_URL:-http://nessie:19120}/iceberg/
rest_warehouse: warehouse
rest_warehouse: ${ICEBERG_WAREHOUSE:-warehouse}
s3_endpoint: ${S3_ENDPOINT:-http://minio:9000}
s3_access_key_id: ${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}
s3_secret_access_key: ${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD}}
deploy/config/environment.defaults.sh
+5
View File
@@ -86,6 +86,11 @@ export MINIO_PORT="${MINIO_PORT:-12900}"
export MINIO_CONSOLE_PORT="${MINIO_CONSOLE_PORT:-12901}"
export MINIO_ROOT_USER="${MINIO_ROOT_USER:-minioadmin}"
export MINIO_BUCKET_ICEBERG_DATA="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}"
# Nama + prefix path warehouse Iceberg. Dipakai IDENTIK di: Nessie
# (NESSIE_CATALOG_DEFAULT_WAREHOUSE + location S3), Trino default-warehouse-dir,
# StarRocks/OM/Sling external catalog. Ganti SATU nilai ini bila client memakai
# nama/path berbeda (mis. `wh`) — lalu `--generate --sync`.
export ICEBERG_WAREHOUSE="${ICEBERG_WAREHOUSE:-warehouse}"
# Apache Nessie (Phase 2 — Storage: Metadata Catalog)
export NESSIE_VERSION="${NESSIE_VERSION:-0.108.4-java}"
deploy/config/trino/catalog/iceberg.properties
+1 -1
View File
@@ -15,7 +15,7 @@ connector.name=iceberg
iceberg.catalog.type=nessie
iceberg.nessie-catalog.uri=${ENV:NESSIE_URL}/api/v2
iceberg.nessie-catalog.ref=main
iceberg.nessie-catalog.default-warehouse-dir=s3://${ENV:MINIO_BUCKET_ICEBERG_DATA}/warehouse
iceberg.nessie-catalog.default-warehouse-dir=s3://${ENV:MINIO_BUCKET_ICEBERG_DATA}/${ENV:ICEBERG_WAREHOUSE}
# --- S3 Native Filesystem (MinIO / S3 existing) ---
fs.s3.enabled=true
deploy/docker-compose.yml
+7 -4
View File
@@ -334,11 +334,12 @@ services:
QUARKUS_DATASOURCE_POSTGRESQL_PASSWORD: "${NESSIE_DB_PASSWORD:?err}"
# ── Iceberg REST catalog + warehouse di MinIO (S3 server-side) ──
# Warehouse = bucket iceberg-data (konvensi base) + prefix warehouse/.
NESSIE_CATALOG_DEFAULT_WAREHOUSE: "warehouse"
NESSIE_CATALOG_WAREHOUSES_WAREHOUSE_LOCATION: "s3://${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/warehouse"
# Warehouse = bucket (MINIO_BUCKET_ICEBERG_DATA) + prefix (ICEBERG_WAREHOUSE).
# Nama & prefix samakan nilai ICEBERG_WAREHOUSE (lihat environment.defaults.sh).
NESSIE_CATALOG_DEFAULT_WAREHOUSE: "${ICEBERG_WAREHOUSE:-warehouse}"
NESSIE_CATALOG_WAREHOUSES_WAREHOUSE_LOCATION: "s3://${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/${ICEBERG_WAREHOUSE:-warehouse}"
NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_ENDPOINT: "${S3_ENDPOINT:-http://minio:9000}"
NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_REGION: "us-east-1"
NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_REGION: "${S3_REGION:-us-east-1}"
NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_AUTH_TYPE: "STATIC"
NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_PATH_STYLE_ACCESS: "true"
# Kredensial S3 via Quarkus secret store. Nama secret memakai TITIK
@@ -459,6 +460,7 @@ services:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?err}
MINIO_BUCKET_ICEBERG_DATA: ${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}
ICEBERG_WAREHOUSE: ${ICEBERG_WAREHOUSE:-warehouse}
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_ACCESS_KEY: ${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}
@@ -561,6 +563,7 @@ services:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?err}
MINIO_BUCKET_ICEBERG_DATA: ${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}
ICEBERG_WAREHOUSE: ${ICEBERG_WAREHOUSE:-warehouse}
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_ACCESS_KEY: ${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}
deploy/install.sh
+5
View File
@@ -81,6 +81,11 @@ OPENMETADATA_ADMIN_PASSWORD="$(openssl rand -hex 12)Aa1!"
LDAP_ADMIN_PASSWORD=$(openssl rand -hex 8)
LDAP_CONFIG_PASSWORD=$(openssl rand -hex 8)
# Storage & katalog (NON-rahasia) — sesuaikan bila client memakai nama berbeda,
# lalu jalankan: ./install.sh --generate [--sync]
MINIO_BUCKET_ICEBERG_DATA=iceberg-data
ICEBERG_WAREHOUSE=warehouse
# OIDC Client Secrets (harus sinkron dengan config .properties/.yaml files)
TRINO_CLIENT_SECRET=$(openssl rand -hex 16)
STARROCKS_CLIENT_SECRET=$(openssl rand -hex 16)
deploy/scripts/dremio/register-nessie-source.sh
+1 -1
View File
@@ -33,7 +33,7 @@ NESSIE_ENDPOINT="${NESSIE_ENDPOINT:-${NESSIE_URL:-http://nessie:19120}/api/v2}"
# MINIO_ENDPOINT dipakai tanpa skema (Dremio memisahkan host:port) → buang skema.
_minio_host="${MINIO_ENDPOINT:-${S3_ENDPOINT:-http://minio:9000}}"
MINIO_ENDPOINT="${_minio_host#*://}"
AWS_ROOT_PATH="${AWS_ROOT_PATH:-/${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/warehouse}"
AWS_ROOT_PATH="${AWS_ROOT_PATH:-/${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/${ICEBERG_WAREHOUSE:-warehouse}}"
MINIO_ROOT_USER="${MINIO_ROOT_USER:-${S3_ACCESS_KEY:-minioadmin}}"
MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:-${S3_SECRET_KEY:?set MINIO_ROOT_PASSWORD atau S3_SECRET_KEY}}"
deploy/scripts/seed-starrocks.sh
+1 -1
View File
@@ -52,7 +52,7 @@ ensure_external_catalog() {
'type' = 'iceberg',
'iceberg.catalog.type' = 'rest',
'iceberg.catalog.uri' = '${NESSIE_URL:-http://nessie:19120}/iceberg/',
'iceberg.catalog.warehouse' = 'warehouse',
'iceberg.catalog.warehouse' = '${ICEBERG_WAREHOUSE:-warehouse}',
'iceberg.catalog.rest.nested-namespace-enabled' = 'true',
'aws.s3.endpoint' = '${S3_ENDPOINT:-http://minio:9000}',
'aws.s3.access_key' = '${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}',
deploy/scripts/test-generate.sh
+34
View File
@@ -273,6 +273,40 @@ else
bad "--sync: masih ada path repo ter-hardcode (~/sicabe-doc/deploy)"
fi
# ─── 0n. Bucket & warehouse Iceberg terkonfigurasi (bukan literal) ───────────
# Bucket = MINIO_BUCKET_ICEBERG_DATA; warehouse = ICEBERG_WAREHOUSE (nama REST
# Nessie + prefix path S3, IDENTIK). Client boleh punya nama berbeda → tak boleh
# ada literal 'warehouse' / region Nessie hardcoded.
_wh_dc="${DEPLOY_DIR}/docker-compose.yml"; _wh_ed="${DEPLOY_DIR}/config/environment.defaults.sh"
_wh_ip="${DEPLOY_DIR}/config/trino/catalog/iceberg.properties"; _wh_bs="${DEPLOY_DIR}/bootstrap.sh"
_wh_se="${DEPLOY_DIR}/scripts/seed-starrocks.sh"; _wh_v2="${DEPLOY_DIR}/scripts/verify-phase2-storage.sh"
_wh_ins="${DEPLOY_DIR}/install.sh"
wh_bad=""
have() { grep -qF "$2" "$1" 2>/dev/null || wh_bad="${wh_bad} $(basename "$1"):$3"; }
lack() { grep -qF "$2" "$1" 2>/dev/null && wh_bad="${wh_bad} $(basename "$1"):$3"; }
have "${_wh_ed}" 'export ICEBERG_WAREHOUSE=' "env-default"
have "${_wh_dc}" 'NESSIE_CATALOG_DEFAULT_WAREHOUSE: "${ICEBERG_WAREHOUSE:-warehouse}"' "nessie-name"
have "${_wh_dc}" 's3://${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/${ICEBERG_WAREHOUSE:-warehouse}' "nessie-loc"
have "${_wh_dc}" 'DEFAULT_OPTIONS_REGION: "${S3_REGION:-us-east-1}"' "nessie-region"
have "${_wh_dc}" 'ICEBERG_WAREHOUSE: ${ICEBERG_WAREHOUSE:-warehouse}' "trino-env"
have "${_wh_ip}" '${ENV:ICEBERG_WAREHOUSE}' "trino-catalog"
have "${_wh_bs}" "'iceberg.catalog.warehouse' = '\${ICEBERG_WAREHOUSE:-warehouse}'" "sr-catalog"
have "${_wh_bs}" "'iceberg.catalog.warehouse'='\${ICEBERG_WAREHOUSE:-warehouse}'" "sr-catalog2"
have "${_wh_bs}" 'rest_warehouse: ${ICEBERG_WAREHOUSE:-warehouse}' "sling"
have "${_wh_v2}" 'WAREHOUSE="${ICEBERG_WAREHOUSE:-warehouse}"' "gate-def"
have "${_wh_v2}" 's3://${BUCKET}/${WAREHOUSE}' "gate-expect"
have "${_wh_ins}" 'MINIO_BUCKET_ICEBERG_DATA=iceberg-data' "env-bucket"
have "${_wh_ins}" 'ICEBERG_WAREHOUSE=warehouse' "env-warehouse"
lack "${_wh_bs}" "'iceberg.catalog.warehouse' = 'warehouse'" "sr-literal"
lack "${_wh_bs}" "'iceberg.catalog.warehouse'='warehouse'" "sr-literal2"
lack "${_wh_bs}" 'rest_warehouse: warehouse' "sling-literal"
lack "${_wh_se}" "'iceberg.catalog.warehouse' = 'warehouse'" "seed-literal"
if [[ -z "${wh_bad}" ]]; then
ok "storage: bucket+warehouse terkonfigurasi (nama & prefix), region Nessie = S3_REGION"
else
bad "storage: konfigurasi bucket/warehouse belum lengkap →${wh_bad}"
fi
# ─── 1. generate ────────────────────────────────────────────────────────────
if ! python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \
"${INVENTORY}" "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" \
deploy/scripts/verify-phase2-storage.sh
+5 -4
View File
@@ -46,6 +46,7 @@ S3_REGION="${S3_REGION:-us-east-1}"
S3_ACCESS_KEY="${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}"
S3_SECRET_KEY="${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-}}"
BUCKET="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}"
WAREHOUSE="${ICEBERG_WAREHOUSE:-warehouse}"
NESSIE_URL="${NESSIE_URL:-http://nessie:19120}"
NESSIE_HOST="${NESSIE_HOST:-localhost}"
NESSIE_PORT="${NESSIE_PORT:-12120}"
@@ -236,10 +237,10 @@ check_storage_contract() {
wh=$(echo "${cfg}" | sed -n 's/.*WAREHOUSE_LOCATION: *//p' | head -1 | tr -d '"')
ep=$(echo "${cfg}" | sed -n 's/.*DEFAULT_OPTIONS_ENDPOINT: *//p' | head -1 | tr -d '"')
if [[ "${wh}" == "s3://${BUCKET}/warehouse" ]]; then
pass "Warehouse Nessie → ${wh} (sesuai bucket '${BUCKET}')"
if [[ "${wh}" == "s3://${BUCKET}/${WAREHOUSE}" ]]; then
pass "Warehouse Nessie → ${wh} (bucket '${BUCKET}' + warehouse '${WAREHOUSE}')"
elif [[ -n "${wh}" ]]; then
fail "Warehouse Nessie '${wh}' ≠ 's3://${BUCKET}/warehouse' — bucket tidak sinkron"
fail "Warehouse Nessie '${wh}' ≠ 's3://${BUCKET}/${WAREHOUSE}' — bucket/warehouse tidak sinkron"
else
fail "WAREHOUSE_LOCATION Nessie tidak ditemukan di compose"
fi
@@ -372,7 +373,7 @@ except Exception: print('')" 2>/dev/null)
# ─────────────────────────────────────────────────────────────────────────────
main() {
print_section "Verifikasi Phase 2 — Kontrak Storage & Catalog (MinIO/S3 + Nessie)"
echo -e " Object storage : ${CYAN}${S3_ENDPOINT}${NC} (bucket ${BUCKET}, region ${S3_REGION})"
echo -e " Object storage : ${CYAN}${S3_ENDPOINT}${NC} (bucket ${BUCKET}, warehouse ${WAREHOUSE}, region ${S3_REGION})"
echo -e " Nessie : ${CYAN}${NESSIE_URL}${NC} (API host ${NESSIE_HOST_API})"
echo ""