first commit
This commit is contained in:
Executable
+169
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Seed Sample Data: Iceberg bronze.orders
|
||||
# =============================================================================
|
||||
# Membuat namespace + tabel contoh `iceberg.bronze.orders` (8 kolom, sesuai
|
||||
# dbt sources.yml) lalu mengisinya dengan 6 baris contoh. Idempoten: aman
|
||||
# dijalankan ulang (schema/table IF NOT EXISTS, INSERT hanya bila kosong).
|
||||
#
|
||||
# Autentikasi: OIDC password-grant (client 'trino') ke Keycloak → Bearer JWT.
|
||||
# ⚠️ Password-grant bersifat dev-only (bukan untuk produksi).
|
||||
#
|
||||
# Pemakaian:
|
||||
# cd deploy
|
||||
# set -a; source config/environment.defaults.sh; source .env; set +a
|
||||
# ./scripts/seed-sample-data.sh
|
||||
#
|
||||
# Env (opsional, ada default):
|
||||
# TRINO_HTTPS_PORT (12443), KEYCLOAK_PORT (12080), SICABE_REALM (sicabe),
|
||||
# TRINO_CLIENT_SECRET, KEYCLOAK_ADMIN_PASSWORD, SEED_USER (admin),
|
||||
# SEED_CATALOG (iceberg), SEED_SCHEMA (bronze), SEED_TABLE (orders)
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "${SCRIPT_DIR}"
|
||||
|
||||
# ─── Shared Library & Environment ───────────────────────────────────────────
|
||||
source "${SCRIPT_DIR}/lib/common.sh"
|
||||
source "${SCRIPT_DIR}/config/environment.defaults.sh"
|
||||
sicabe_load_env "${SCRIPT_DIR}" "$@"
|
||||
|
||||
# Service account (HTTP Basic) — langsung ke Trino HTTPS (self-signed).
|
||||
# Multi-host: Trino ada di host `compute` → endpoint host-aware (single-host tetap localhost).
|
||||
TRINO_URL="$(sicabe_local_url https TRINO_HOST "${TRINO_HTTPS_PORT:-12443}" TRINO_URL)"
|
||||
SEED_USER="${SEED_USER:-svc_seed}"
|
||||
SEED_PASSWORD="${SEED_PASSWORD:-${TRINO_SVC_SEED_PASSWORD:-}}"
|
||||
SEED_CATALOG="${SEED_CATALOG:-iceberg}"
|
||||
SEED_SCHEMA="${SEED_SCHEMA:-bronze}"
|
||||
SEED_TABLE="${SEED_TABLE:-orders}"
|
||||
FQN="${SEED_CATALOG}.${SEED_SCHEMA}.${SEED_TABLE}"
|
||||
|
||||
usage_seed() {
|
||||
echo "Usage: $0 [--user <trino-user>] [--help]"
|
||||
echo " Seed tabel contoh '${FQN}' (idempoten)."
|
||||
}
|
||||
|
||||
# ─── Auth: service account (HTTP Basic) ─────────────────────────────────────
|
||||
check_auth() {
|
||||
[[ -n "${SEED_USER}" && -n "${SEED_PASSWORD}" ]]
|
||||
}
|
||||
|
||||
# ─── Submit SQL ke Trino + polling nextUri (Trino 483) ──────────────────────
|
||||
# Cetak baris data (jika ada); gagal (return 1) bila Trino melaporkan error.
|
||||
trino_exec() {
|
||||
local sql="$1" resp next err i=0
|
||||
resp=$(curl -skf -X POST "${TRINO_URL}/v1/statement" \
|
||||
-u "${SEED_USER}:${SEED_PASSWORD}" \
|
||||
-H "Content-Type: text/plain" \
|
||||
-d "${sql}" 2>/dev/null) || { echo -e "${ERR} submit ke Trino gagal."; return 1; }
|
||||
|
||||
while :; do
|
||||
err=$(echo "${resp}" | python3 -c "import sys,json;d=json.load(sys.stdin);e=d.get('error');print(e.get('message','') if e else '')" 2>/dev/null || true)
|
||||
if [[ -n "${err}" ]]; then
|
||||
echo -e "${ERR} ${err}"
|
||||
return 1
|
||||
fi
|
||||
echo "${resp}" | python3 -c "import sys,json;d=json.load(sys.stdin);[print('\t',r) for r in (d.get('data') or [])]" 2>/dev/null || true
|
||||
next=$(echo "${resp}" | python3 -c "import sys,json;print(json.load(sys.stdin).get('nextUri',''))" 2>/dev/null || true)
|
||||
[[ -z "${next}" ]] && break
|
||||
i=$((i + 1))
|
||||
if [[ ${i} -gt 60 ]]; then
|
||||
echo -e "${WARN} Polling Trino melebihi batas waktu."
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
resp=$(curl -skf -u "${SEED_USER}:${SEED_PASSWORD}" "${next}" 2>/dev/null) || return 1
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# ─── Ambil satu nilai skalar dari Trino ─────────────────────────────────────
|
||||
trino_value() {
|
||||
local sql="$1" resp next val i=0
|
||||
resp=$(curl -skf -X POST "${TRINO_URL}/v1/statement" \
|
||||
-u "${SEED_USER}:${SEED_PASSWORD}" \
|
||||
-H "Content-Type: text/plain" \
|
||||
-d "${sql}" 2>/dev/null) || return 1
|
||||
|
||||
while :; do
|
||||
val=$(echo "${resp}" | python3 -c "import sys,json;d=json.load(sys.stdin);r=d.get('data') or [];print(r[0][0] if r else '')" 2>/dev/null || true)
|
||||
[[ -n "${val}" ]] && { echo "${val}"; return 0; }
|
||||
next=$(echo "${resp}" | python3 -c "import sys,json;print(json.load(sys.stdin).get('nextUri',''))" 2>/dev/null || true)
|
||||
[[ -z "${next}" ]] && { echo "${val}"; return 0; }
|
||||
i=$((i + 1)); [[ ${i} -gt 60 ]] && return 1
|
||||
sleep 1
|
||||
resp=$(curl -skf -u "${SEED_USER}:${SEED_PASSWORD}" "${next}" 2>/dev/null) || return 1
|
||||
done
|
||||
}
|
||||
|
||||
# ─── DDL + seed (idempoten) ─────────────────────────────────────────────────
|
||||
seed_table() {
|
||||
echo -e "${INFO} [1/3] Buat namespace '${SEED_CATALOG}.${SEED_SCHEMA}'..."
|
||||
trino_exec "CREATE SCHEMA IF NOT EXISTS ${SEED_CATALOG}.${SEED_SCHEMA}" \
|
||||
&& echo -e "${OK} Namespace '${SEED_SCHEMA}' siap."
|
||||
|
||||
echo -e "${INFO} [2/3] Buat tabel '${FQN}'..."
|
||||
trino_exec "CREATE TABLE IF NOT EXISTS ${FQN} (
|
||||
id bigint,
|
||||
order_date date,
|
||||
komoditas varchar,
|
||||
harga_per_kg double,
|
||||
stok_kg bigint,
|
||||
region varchar,
|
||||
created_at timestamp(6),
|
||||
updated_at timestamp(6)
|
||||
) WITH (format = 'PARQUET')" \
|
||||
&& echo -e "${OK} Tabel '${FQN}' siap."
|
||||
|
||||
echo -e "${INFO} [3/3] Isi data contoh (bila kosong)..."
|
||||
local count
|
||||
count=$(trino_value "SELECT count(*) FROM ${FQN}") || count="0"
|
||||
if [[ "${count}" -gt 0 ]]; then
|
||||
echo -e "${OK} Tabel sudah berisi ${count} baris (skip seed)."
|
||||
return 0
|
||||
fi
|
||||
|
||||
trino_exec "INSERT INTO ${FQN} VALUES
|
||||
(1, DATE '2026-09-01', 'beras', 12000.0, 500, 'jakarta', TIMESTAMP '2026-09-01 08:00:00', TIMESTAMP '2026-09-01 08:00:00'),
|
||||
(2, DATE '2026-09-01', 'gula', 15000.0, 300, 'bandung', TIMESTAMP '2026-09-01 09:00:00', TIMESTAMP '2026-09-01 09:00:00'),
|
||||
(3, DATE '2026-09-02', 'minyak', 18000.0, 200, 'surabaya', TIMESTAMP '2026-09-02 08:00:00', TIMESTAMP '2026-09-02 08:00:00'),
|
||||
(4, DATE '2026-09-02', 'beras', 12500.0, 450, 'jakarta', TIMESTAMP '2026-09-02 10:00:00', TIMESTAMP '2026-09-02 10:00:00'),
|
||||
(5, DATE '2026-09-03', 'tepung', 11000.0, 350, 'medan', TIMESTAMP '2026-09-03 08:00:00', TIMESTAMP '2026-09-03 08:00:00'),
|
||||
(6, DATE '2026-09-03', 'gula', 15500.0, 250, 'bandung', TIMESTAMP '2026-09-03 11:00:00', TIMESTAMP '2026-09-03 11:00:00')" \
|
||||
&& echo -e "${OK} 6 baris contoh dimasukkan."
|
||||
}
|
||||
|
||||
# ─── Verifikasi akhir ───────────────────────────────────────────────────────
|
||||
verify_seed() {
|
||||
echo -e "${INFO} Verifikasi:"
|
||||
trino_exec "SELECT id, komoditas, harga_per_kg, stok_kg, region FROM ${FQN} ORDER BY id"
|
||||
}
|
||||
|
||||
# ─── MAIN ───────────────────────────────────────────────────────────────────
|
||||
main() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env-file) shift 2 ;; # sudah dipakai sicabe_load_env() di atas
|
||||
--user) SEED_USER="$2"; shift 2 ;;
|
||||
--help|-h) usage_seed; exit 0 ;;
|
||||
*) echo -e "${ERR} Argumen tidak dikenal: $1"; usage_seed; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo -e "${BOLD}${BLUE}═══ Seed Sample Data — Iceberg '${FQN}' ═══${NC}"
|
||||
|
||||
echo -e "${INFO} Autentikasi Trino (service account '${SEED_USER}')..."
|
||||
if ! check_auth; then
|
||||
echo -e "${ERR} SEED_PASSWORD / TRINO_SVC_SEED_PASSWORD belum diset."
|
||||
exit 1
|
||||
fi
|
||||
echo -e "${OK} Kredensial service account siap."
|
||||
|
||||
seed_table
|
||||
verify_seed
|
||||
|
||||
echo -e "${OK} ${GREEN}Seed selesai — '${FQN}' siap dipakai.${NC}"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user