170 lines
7.6 KiB
Bash
Executable File
170 lines
7.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# eBdesk SiCABE — Seed Sample Data: Iceberg bronze.orders
|
|
# =============================================================================
|
|
# Membuat namespace + tabel contoh `iceberg.bronze.orders` (8 kolom, sesuai
|
|
# dbt sources.yml) lalu mengisinya dengan 6 baris contoh. Idempoten: aman
|
|
# dijalankan ulang (schema/table IF NOT EXISTS, INSERT hanya bila kosong).
|
|
#
|
|
# Autentikasi: OIDC password-grant (client 'trino') ke Keycloak → Bearer JWT.
|
|
# ⚠️ Password-grant bersifat dev-only (bukan untuk produksi).
|
|
#
|
|
# Pemakaian:
|
|
# cd deploy
|
|
# set -a; source config/environment.defaults.sh; source .env; set +a
|
|
# ./scripts/seed-sample-data.sh
|
|
#
|
|
# Env (opsional, ada default):
|
|
# TRINO_HTTPS_PORT (12443), KEYCLOAK_PORT (12080), SICABE_REALM (sicabe),
|
|
# TRINO_CLIENT_SECRET, KEYCLOAK_ADMIN_PASSWORD, SEED_USER (admin),
|
|
# SEED_CATALOG (iceberg), SEED_SCHEMA (bronze), SEED_TABLE (orders)
|
|
# =============================================================================
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "${SCRIPT_DIR}"
|
|
|
|
# ─── Shared Library & Environment ───────────────────────────────────────────
|
|
source "${SCRIPT_DIR}/lib/common.sh"
|
|
source "${SCRIPT_DIR}/config/environment.defaults.sh"
|
|
sicabe_load_env "${SCRIPT_DIR}" "$@"
|
|
|
|
# Service account (HTTP Basic) — langsung ke Trino HTTPS (self-signed).
|
|
# Multi-host: Trino ada di host `compute` → endpoint host-aware (single-host tetap localhost).
|
|
TRINO_URL="$(sicabe_local_url https TRINO_HOST "${TRINO_HTTPS_PORT:-12443}" TRINO_URL)"
|
|
SEED_USER="${SEED_USER:-svc_seed}"
|
|
SEED_PASSWORD="${SEED_PASSWORD:-${TRINO_SVC_SEED_PASSWORD:-}}"
|
|
SEED_CATALOG="${SEED_CATALOG:-iceberg}"
|
|
SEED_SCHEMA="${SEED_SCHEMA:-bronze}"
|
|
SEED_TABLE="${SEED_TABLE:-orders}"
|
|
FQN="${SEED_CATALOG}.${SEED_SCHEMA}.${SEED_TABLE}"
|
|
|
|
usage_seed() {
|
|
echo "Usage: $0 [--user <trino-user>] [--help]"
|
|
echo " Seed tabel contoh '${FQN}' (idempoten)."
|
|
}
|
|
|
|
# ─── Auth: service account (HTTP Basic) ─────────────────────────────────────
|
|
check_auth() {
|
|
[[ -n "${SEED_USER}" && -n "${SEED_PASSWORD}" ]]
|
|
}
|
|
|
|
# ─── Submit SQL ke Trino + polling nextUri (Trino 483) ──────────────────────
|
|
# Cetak baris data (jika ada); gagal (return 1) bila Trino melaporkan error.
|
|
trino_exec() {
|
|
local sql="$1" resp next err i=0
|
|
resp=$(curl -skf -X POST "${TRINO_URL}/v1/statement" \
|
|
-u "${SEED_USER}:${SEED_PASSWORD}" \
|
|
-H "Content-Type: text/plain" \
|
|
-d "${sql}" 2>/dev/null) || { echo -e "${ERR} submit ke Trino gagal."; return 1; }
|
|
|
|
while :; do
|
|
err=$(echo "${resp}" | python3 -c "import sys,json;d=json.load(sys.stdin);e=d.get('error');print(e.get('message','') if e else '')" 2>/dev/null || true)
|
|
if [[ -n "${err}" ]]; then
|
|
echo -e "${ERR} ${err}"
|
|
return 1
|
|
fi
|
|
echo "${resp}" | python3 -c "import sys,json;d=json.load(sys.stdin);[print('\t',r) for r in (d.get('data') or [])]" 2>/dev/null || true
|
|
next=$(echo "${resp}" | python3 -c "import sys,json;print(json.load(sys.stdin).get('nextUri',''))" 2>/dev/null || true)
|
|
[[ -z "${next}" ]] && break
|
|
i=$((i + 1))
|
|
if [[ ${i} -gt 60 ]]; then
|
|
echo -e "${WARN} Polling Trino melebihi batas waktu."
|
|
return 1
|
|
fi
|
|
sleep 1
|
|
resp=$(curl -skf -u "${SEED_USER}:${SEED_PASSWORD}" "${next}" 2>/dev/null) || return 1
|
|
done
|
|
return 0
|
|
}
|
|
|
|
# ─── Ambil satu nilai skalar dari Trino ─────────────────────────────────────
|
|
trino_value() {
|
|
local sql="$1" resp next val i=0
|
|
resp=$(curl -skf -X POST "${TRINO_URL}/v1/statement" \
|
|
-u "${SEED_USER}:${SEED_PASSWORD}" \
|
|
-H "Content-Type: text/plain" \
|
|
-d "${sql}" 2>/dev/null) || return 1
|
|
|
|
while :; do
|
|
val=$(echo "${resp}" | python3 -c "import sys,json;d=json.load(sys.stdin);r=d.get('data') or [];print(r[0][0] if r else '')" 2>/dev/null || true)
|
|
[[ -n "${val}" ]] && { echo "${val}"; return 0; }
|
|
next=$(echo "${resp}" | python3 -c "import sys,json;print(json.load(sys.stdin).get('nextUri',''))" 2>/dev/null || true)
|
|
[[ -z "${next}" ]] && { echo "${val}"; return 0; }
|
|
i=$((i + 1)); [[ ${i} -gt 60 ]] && return 1
|
|
sleep 1
|
|
resp=$(curl -skf -u "${SEED_USER}:${SEED_PASSWORD}" "${next}" 2>/dev/null) || return 1
|
|
done
|
|
}
|
|
|
|
# ─── DDL + seed (idempoten) ─────────────────────────────────────────────────
|
|
seed_table() {
|
|
echo -e "${INFO} [1/3] Buat namespace '${SEED_CATALOG}.${SEED_SCHEMA}'..."
|
|
trino_exec "CREATE SCHEMA IF NOT EXISTS ${SEED_CATALOG}.${SEED_SCHEMA}" \
|
|
&& echo -e "${OK} Namespace '${SEED_SCHEMA}' siap."
|
|
|
|
echo -e "${INFO} [2/3] Buat tabel '${FQN}'..."
|
|
trino_exec "CREATE TABLE IF NOT EXISTS ${FQN} (
|
|
id bigint,
|
|
order_date date,
|
|
komoditas varchar,
|
|
harga_per_kg double,
|
|
stok_kg bigint,
|
|
region varchar,
|
|
created_at timestamp(6),
|
|
updated_at timestamp(6)
|
|
) WITH (format = 'PARQUET')" \
|
|
&& echo -e "${OK} Tabel '${FQN}' siap."
|
|
|
|
echo -e "${INFO} [3/3] Isi data contoh (bila kosong)..."
|
|
local count
|
|
count=$(trino_value "SELECT count(*) FROM ${FQN}") || count="0"
|
|
if [[ "${count}" -gt 0 ]]; then
|
|
echo -e "${OK} Tabel sudah berisi ${count} baris (skip seed)."
|
|
return 0
|
|
fi
|
|
|
|
trino_exec "INSERT INTO ${FQN} VALUES
|
|
(1, DATE '2026-09-01', 'beras', 12000.0, 500, 'jakarta', TIMESTAMP '2026-09-01 08:00:00', TIMESTAMP '2026-09-01 08:00:00'),
|
|
(2, DATE '2026-09-01', 'gula', 15000.0, 300, 'bandung', TIMESTAMP '2026-09-01 09:00:00', TIMESTAMP '2026-09-01 09:00:00'),
|
|
(3, DATE '2026-09-02', 'minyak', 18000.0, 200, 'surabaya', TIMESTAMP '2026-09-02 08:00:00', TIMESTAMP '2026-09-02 08:00:00'),
|
|
(4, DATE '2026-09-02', 'beras', 12500.0, 450, 'jakarta', TIMESTAMP '2026-09-02 10:00:00', TIMESTAMP '2026-09-02 10:00:00'),
|
|
(5, DATE '2026-09-03', 'tepung', 11000.0, 350, 'medan', TIMESTAMP '2026-09-03 08:00:00', TIMESTAMP '2026-09-03 08:00:00'),
|
|
(6, DATE '2026-09-03', 'gula', 15500.0, 250, 'bandung', TIMESTAMP '2026-09-03 11:00:00', TIMESTAMP '2026-09-03 11:00:00')" \
|
|
&& echo -e "${OK} 6 baris contoh dimasukkan."
|
|
}
|
|
|
|
# ─── Verifikasi akhir ───────────────────────────────────────────────────────
|
|
verify_seed() {
|
|
echo -e "${INFO} Verifikasi:"
|
|
trino_exec "SELECT id, komoditas, harga_per_kg, stok_kg, region FROM ${FQN} ORDER BY id"
|
|
}
|
|
|
|
# ─── MAIN ───────────────────────────────────────────────────────────────────
|
|
main() {
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--env-file) shift 2 ;; # sudah dipakai sicabe_load_env() di atas
|
|
--user) SEED_USER="$2"; shift 2 ;;
|
|
--help|-h) usage_seed; exit 0 ;;
|
|
*) echo -e "${ERR} Argumen tidak dikenal: $1"; usage_seed; exit 1 ;;
|
|
esac
|
|
done
|
|
|
|
echo -e "${BOLD}${BLUE}═══ Seed Sample Data — Iceberg '${FQN}' ═══${NC}"
|
|
|
|
echo -e "${INFO} Autentikasi Trino (service account '${SEED_USER}')..."
|
|
if ! check_auth; then
|
|
echo -e "${ERR} SEED_PASSWORD / TRINO_SVC_SEED_PASSWORD belum diset."
|
|
exit 1
|
|
fi
|
|
echo -e "${OK} Kredensial service account siap."
|
|
|
|
seed_table
|
|
verify_seed
|
|
|
|
echo -e "${OK} ${GREEN}Seed selesai — '${FQN}' siap dipakai.${NC}"
|
|
}
|
|
|
|
main "$@"
|