1432 lines
70 KiB
YAML
1432 lines
70 KiB
YAML
# =============================================================================
|
|
# eBdesk SiCABE — Docker Compose Deployment
|
|
# Arsitektur 5 Lapisan: Ingestion | Storage | Processing | Serving | Governance
|
|
# Urutan Deployment sesuai Dependency Graph (bab 3.1 & 12.2):
|
|
# 1. Governance (IAM): Keycloak + OpenLDAP
|
|
# 2. Storage: MinIO → Nessie
|
|
# 3. Processing: Trino
|
|
# 4. Serving: StarRocks
|
|
# 5. Ingestion: Airflow
|
|
# 6. Governance (Lanjutan): OpenMetadata → Grafana
|
|
# =============================================================================
|
|
|
|
name: sicabe
|
|
|
|
networks:
|
|
sicabe-network:
|
|
name: sicabe-network
|
|
driver: bridge
|
|
ipam:
|
|
config:
|
|
- subnet: 172.28.0.0/16
|
|
|
|
volumes:
|
|
traefik-acme:
|
|
# ─── Database (Shared PostgreSQL — semua service) ─────────────────────
|
|
postgres-data:
|
|
|
|
# ─── Governance Layer ──────────────────────────────────────────────────
|
|
keycloak-data:
|
|
|
|
# ─── Identity (LDAP group provider — sumber grup untuk Trino/StarRocks) ─
|
|
openldap-data:
|
|
openldap-config:
|
|
|
|
# ─── Storage Layer ─────────────────────────────────────────────────────
|
|
minio-data:
|
|
|
|
# ─── Processing Layer (coordinator & worker dipisah) ───────────────────
|
|
trino-coordinator-data:
|
|
trino-worker-data:
|
|
|
|
# ─── Serving Layer (shared-data: FE + CN) ──────────────────────────────
|
|
starrocks-fe-data:
|
|
cn-cache:
|
|
# ─── Serving Layer (Paralel: Dremio) ───────────────────────────────────
|
|
dremio-data:
|
|
|
|
# ─── Ingestion Layer ───────────────────────────────────────────────────
|
|
airflow-logs:
|
|
airflow-dags:
|
|
# Config pipeline yang DIBANGKITKAN plugin OpenMetadata (DAG TestSuite/DQ).
|
|
# Tanpa volume ini, config hilang saat container Airflow di-recreate sehingga
|
|
# DAG yang tertinggal di airflow-dags gagal diimpor:
|
|
# ConfigurationError: Cannot open config file .../dag_generated_configs/<name>.json
|
|
airflow-dags-configs:
|
|
|
|
# ─── Governance Layer (Lanjutan) ───────────────────────────────────────
|
|
openmetadata-data:
|
|
# Index OpenSearch 3.x. Bersifat DERIVED — dibangun ulang OpenMetadata saat
|
|
# re-index; index 2.x tidak dimigrasikan karena formatnya berbeda antar
|
|
# versi mayor (2.x -> 3.x tidak kompatibel).
|
|
opensearch-data:
|
|
grafana-data:
|
|
prometheus-data:
|
|
|
|
# ============================================================================
|
|
# PHASE 1 — Governance Layer (IAM): Keycloak
|
|
# Dependency: tidak ada (deploy pertama)
|
|
# ============================================================================
|
|
services:
|
|
|
|
postgres:
|
|
image: postgres:15
|
|
container_name: sicabe-postgres
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
volumes:
|
|
- postgres-data:/var/lib/postgresql/data
|
|
# Init script: create databases & users untuk Keycloak, Nessie, Airflow, OpenMetadata
|
|
- ./config/postgres/init:/docker-entrypoint-initdb.d:ro
|
|
environment:
|
|
# Superuser (digunakan init script untuk membuat database lain)
|
|
POSTGRES_DB: keycloak
|
|
POSTGRES_USER: sicabe_admin
|
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?err}
|
|
# Password untuk masing-masing service database (dibaca oleh init script)
|
|
KEYCLOAK_DB_PASSWORD: ${KEYCLOAK_DB_PASSWORD:?err}
|
|
NESSIE_DB_PASSWORD: ${NESSIE_DB_PASSWORD:?err}
|
|
AIRFLOW_DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:?err}
|
|
OPENMETADATA_DB_PASSWORD: ${OPENMETADATA_DB_PASSWORD:?err}
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U sicabe_admin -d keycloak"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 5
|
|
start_period: 30s
|
|
# ⚠️ Port 5432 TIDAK di-expose ke host — hanya diakses internal (postgres:5432)
|
|
# Untuk debugging dari host, gunakan: docker exec -it sicabe-postgres psql -U sicabe_admin -d keycloak
|
|
profiles:
|
|
- phase1
|
|
- phase2
|
|
- phase3
|
|
- phase4
|
|
- phase5
|
|
- phase6
|
|
- all
|
|
|
|
keycloak:
|
|
image: quay.io/keycloak/keycloak:${KEYCLOAK_VERSION:-26.7}
|
|
container_name: sicabe-keycloak
|
|
labels:
|
|
- "traefik.enable=true"
|
|
- "traefik.http.routers.kc.rule=Host(`kc.${CLUSTER_DOMAIN:-localhost}`)"
|
|
- "traefik.http.routers.kc.entrypoints=websecure"
|
|
- "traefik.http.routers.kc.tls=true"
|
|
- "traefik.http.routers.kc.tls.certresolver=le"
|
|
- "traefik.http.routers.kc.tls.domains[0].main=${CLUSTER_DOMAIN:-localhost}"
|
|
- "traefik.http.routers.kc.tls.domains[0].sans=*.${CLUSTER_DOMAIN:-localhost}"
|
|
- "traefik.http.services.kc.loadbalancer.server.port=8080"
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
volumes:
|
|
- keycloak-data:/opt/keycloak/data
|
|
# Realm import akan dilakukan oleh bootstrap.sh setelah Keycloak siap
|
|
environment:
|
|
KC_DB: postgres
|
|
KC_DB_URL: jdbc:postgresql://${POSTGRES_HOST:-postgres}:${POSTGRES_INTERNAL_PORT:-5432}/keycloak
|
|
KC_DB_USERNAME: keycloak
|
|
KC_DB_PASSWORD: ${KEYCLOAK_DB_PASSWORD:?err}
|
|
# Keycloak 26.x (hostname v2): KC_HOSTNAME = URL PUBLIK LENGKAP (skema+host+port).
|
|
# Inilah yang menentukan issuer OIDC, dokumen discovery, dan redirect login.
|
|
#
|
|
# JANGAN pakai KC_HOSTNAME_URL: opsi hostname v1 itu DIHAPUS di Keycloak 26.0.0,
|
|
# jadi memasangnya tidak berefek apa pun (dulu sempat dipasang di sini sehingga
|
|
# KEYCLOAK_PUBLIC_URL seolah berfungsi padahal tidak). KC_HOSTNAME_BACKEND_URL
|
|
# juga bukan opsi Keycloak sama sekali.
|
|
KC_HOSTNAME: ${KEYCLOAK_PUBLIC_URL:-http://localhost:${KEYCLOAK_PORT:-12080}}
|
|
# Endpoint server-ke-server (token/jwks) mengikuti Host request, bukan URL publik.
|
|
# Dengan ini container lain tetap bisa memakai nama internal keycloak:8080
|
|
# sementara browser memakai alamat publik — tanpa perlu dua nilai issuer.
|
|
KC_HOSTNAME_BACKCHANNEL_DYNAMIC: "true"
|
|
KC_HTTP_ENABLED: "true"
|
|
# Cache/klaster Infinispan. Default `ispn` memakai registry DI DATABASE, dan
|
|
# pemeriksaan "Keycloak cluster health check" akan DOWN bila tabel `jgroups` masih
|
|
# memuat nama cluster dari deployment lama di DB yang sama.
|
|
# PENTING: container yang unhealthy DIBUANG dari routing oleh Traefik (v3:
|
|
# "Filtering unhealthy or starting container"), sehingga Keycloak bisa hilang dari
|
|
# ingress tanpa error apa pun. Untuk instalasi SATU Keycloak, `local` mematikan
|
|
# klaster sekaligus menghapus pemeriksaan itu.
|
|
KC_CACHE: ${KEYCLOAK_CACHE:-ispn}
|
|
# KC_PROXY_HEADERS hanya bila ADA reverse proxy di depan (Traefik, tls.mode != none).
|
|
# Tanpa proxy, memasangnya membuat Keycloak memercayai header X-Forwarded-* dari
|
|
# siapa pun (host bisa dipalsukan klien). Generator mengisinya hanya saat ada proxy.
|
|
KC_PROXY_HEADERS: ${KC_PROXY_HEADERS:-}
|
|
# Keycloak 26.x: bootstrap admin. KEYCLOAK_ADMIN* deprecated (KC-SERVICES0110)
|
|
# dan dihapus pada rilis berikutnya → pakai KC_BOOTSTRAP_ADMIN_*.
|
|
# Nilai tetap dari env kita (KEYCLOAK_ADMIN / KEYCLOAK_ADMIN_PASSWORD di .env).
|
|
KC_BOOTSTRAP_ADMIN_USERNAME: ${KEYCLOAK_ADMIN:-admin}
|
|
KC_BOOTSTRAP_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD:?err}
|
|
KC_METRICS_ENABLED: "true"
|
|
KC_HEALTH_ENABLED: "true"
|
|
KC_LOG_LEVEL: INFO
|
|
# Mode dev: batasi heap JVM (tanpa ini Quarkus mengikuti RAM host → ±3.8 G)
|
|
JAVA_OPTS_APPEND: "-Xmx768m"
|
|
# Keycloak 27.x: health/metrics pindah ke management port 9000.
|
|
# Disable management health agar tetap di port 8080 (backward compatible).
|
|
KC_HTTP_MANAGEMENT_HEALTH_ENABLED: "false"
|
|
# Database connection pool — pool kecil untuk menghindari timeout
|
|
KC_DB_POOL_INITIAL_SIZE: 1
|
|
KC_DB_POOL_MAX_SIZE: 5
|
|
KC_DB_POOL_MIN_SIZE: 1
|
|
# Mode dev: batasi memori container — Quarkus memakai MaxRAMPercentage=70
|
|
# (bawaan) yang diukur dari RAM host bila tanpa limit. Heap efektif ≤700 MB.
|
|
mem_limit: 1g
|
|
# Build-time flags sebagai CLI arg
|
|
command: ["start", "--db=postgres", "--health-enabled=true", "--metrics-enabled=true", "--db-pool-initial-size=1", "--db-pool-max-size=5", "--http-enabled=true", "--proxy-headers=xforwarded", "--http-management-health-enabled=false"]
|
|
healthcheck:
|
|
# Keycloak 24.x+ image tidak menyertakan curl/wget, gunakan bash /dev/tcp.
|
|
# Cek endpoint /health/ready (bukan hanya port) agar sesuai Keycloak 27 behavior.
|
|
test: ["CMD-SHELL", "{ printf 'GET /health/ready HTTP/1.0\\r\\n\\r\\n' >&0; grep -q 'HTTP/1.0 200'; } 0<>/dev/tcp/127.0.0.1/8080"]
|
|
interval: 15s
|
|
timeout: 10s
|
|
retries: 10
|
|
start_period: 60s
|
|
ports:
|
|
- "${KEYCLOAK_PORT:-12080}:8080"
|
|
profiles:
|
|
- phase1
|
|
- phase2
|
|
- phase3
|
|
- phase4
|
|
- phase5
|
|
- phase6
|
|
- all
|
|
|
|
# ============================================================================
|
|
# PHASE 1b — Identity: OpenLDAP (sumber grup untuk Trino & StarRocks)
|
|
# Dependency: tidak ada. LDAP = source of truth grup (user tetap di Keycloak).
|
|
# ============================================================================
|
|
|
|
openldap:
|
|
image: osixia/openldap:${OPENLDAP_VERSION:-1.5.0}
|
|
container_name: sicabe-openldap
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
volumes:
|
|
- openldap-data:/var/lib/ldap
|
|
- openldap-config:/etc/ldap/slapd.d
|
|
environment:
|
|
LDAP_ORGANISATION: "SiCABE"
|
|
LDAP_DOMAIN: "sicabe.local"
|
|
LDAP_ADMIN_PASSWORD: ${LDAP_ADMIN_PASSWORD:?err}
|
|
LDAP_CONFIG_PASSWORD: ${LDAP_CONFIG_PASSWORD:?err}
|
|
LDAP_TLS: "false"
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "ldapsearch -x -H ldap://localhost -b dc=sicabe,dc=local -D cn=admin,dc=sicabe,dc=local -w $${LDAP_ADMIN_PASSWORD} '(objectClass=*)' | grep -q 'dc=sicabe'"]
|
|
interval: 20s
|
|
timeout: 10s
|
|
retries: 10
|
|
start_period: 30s
|
|
ports:
|
|
- "${OPENLDAP_PORT:-12389}:389"
|
|
profiles:
|
|
- phase1
|
|
- phase2
|
|
- phase3
|
|
- phase4
|
|
- all
|
|
|
|
# ============================================================================
|
|
# PHASE 2 — Storage Layer: MinIO → Nessie
|
|
# Dependency: Keycloak (Phase 1) sudah running
|
|
# ============================================================================
|
|
|
|
minio:
|
|
image: ${MINIO_IMAGE:-quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z}
|
|
container_name: sicabe-minio
|
|
labels:
|
|
- "traefik.enable=true"
|
|
- "traefik.http.routers.minio.rule=Host(`minio.${CLUSTER_DOMAIN:-localhost}`)"
|
|
- "traefik.http.routers.minio.entrypoints=websecure"
|
|
- "traefik.http.routers.minio.tls=true"
|
|
- "traefik.http.routers.minio.tls.certresolver=le"
|
|
- "traefik.http.routers.minio.tls.domains[0].main=${CLUSTER_DOMAIN:-localhost}"
|
|
- "traefik.http.routers.minio.tls.domains[0].sans=*.${CLUSTER_DOMAIN:-localhost}"
|
|
- "traefik.http.services.minio.loadbalancer.server.port=9001"
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
volumes:
|
|
- minio-data:/data
|
|
environment:
|
|
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
|
|
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?err}
|
|
MINIO_BROWSER: "on"
|
|
# URL yang dilihat BROWSER untuk console (SSO/redirect lewat reverse proxy).
|
|
# Ini "current" (bukan deprecated; jangan tertukar dengan MINIO_BROWSER).
|
|
# Dibiarkan kosong dulu = console memakai host internal → redirect tak terjangkau
|
|
# saat ada proxy. Tanpa proxy (tls.mode none) nilainya sama dengan akses langsung.
|
|
MINIO_BROWSER_REDIRECT_URL: ${MINIO_CONSOLE_PUBLIC_URL:-http://localhost:${MINIO_CONSOLE_PORT:-12901}}
|
|
MINIO_PROMETHEUS_AUTH_TYPE: "public"
|
|
command: server /data --console-address ":9001"
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
|
interval: 15s
|
|
timeout: 10s
|
|
retries: 5
|
|
start_period: 30s
|
|
ports:
|
|
- "${MINIO_PORT:-12900}:9000"
|
|
- "${MINIO_CONSOLE_PORT:-12901}:9001"
|
|
profiles:
|
|
- phase2
|
|
- phase3
|
|
- phase4
|
|
- all
|
|
|
|
# ─── One-shot: buat bucket MinIO sebelum Nessie start (idempotent) ────────
|
|
minio-init:
|
|
image: ${MINIO_MC_IMAGE:-quay.io/minio/mc:RELEASE.2025-08-13T08-35-41Z}
|
|
container_name: sicabe-minio-init
|
|
restart: "no"
|
|
networks:
|
|
- sicabe-network
|
|
depends_on:
|
|
minio:
|
|
condition: service_healthy
|
|
entrypoint: ["/bin/sh", "-c"]
|
|
command:
|
|
- |
|
|
mc alias set local http://${MINIO_HOST:-minio}:${MINIO_INTERNAL_PORT:-9000} "$${MINIO_ROOT_USER}" "$${MINIO_ROOT_PASSWORD}"
|
|
for b in "$${MINIO_BUCKET_ICEBERG_DATA}"; do
|
|
mc mb --ignore-existing "local/$${b}"
|
|
done
|
|
echo "[SiCABE] Bucket MinIO siap."
|
|
environment:
|
|
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
|
|
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?err}
|
|
MINIO_BUCKET_ICEBERG_DATA: ${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}
|
|
profiles:
|
|
- phase2
|
|
- phase3
|
|
- phase4
|
|
- all
|
|
|
|
nessie:
|
|
image: ${NESSIE_IMAGE:-ghcr.io/projectnessie/nessie:0.108.4-java}
|
|
container_name: sicabe-nessie
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
minio:
|
|
condition: service_healthy
|
|
minio-init:
|
|
condition: service_completed_successfully
|
|
environment:
|
|
# ── Version store: JDBC2 di PostgreSQL (ikut konvensi shared Postgres) ──
|
|
NESSIE_VERSION_STORE_TYPE: "JDBC2"
|
|
NESSIE_VERSION_STORE_PERSIST_JDBC_DATASOURCE: "postgresql"
|
|
QUARKUS_DATASOURCE_POSTGRESQL_DB_KIND: "postgresql"
|
|
QUARKUS_DATASOURCE_POSTGRESQL_ACTIVE: "true"
|
|
QUARKUS_DATASOURCE_POSTGRESQL_DEVSERVICES_ENABLED: "false"
|
|
QUARKUS_DATASOURCE_POSTGRESQL_JDBC_URL: "jdbc:postgresql://${POSTGRES_HOST:-postgres}:${POSTGRES_INTERNAL_PORT:-5432}/nessie?currentSchema=nessie"
|
|
QUARKUS_DATASOURCE_POSTGRESQL_USERNAME: "nessie"
|
|
QUARKUS_DATASOURCE_POSTGRESQL_PASSWORD: "${NESSIE_DB_PASSWORD:?err}"
|
|
|
|
# ── Iceberg REST catalog + warehouse di MinIO (S3 server-side) ──
|
|
# Warehouse = bucket (MINIO_BUCKET_ICEBERG_DATA) + prefix (ICEBERG_WAREHOUSE).
|
|
# Nama & prefix samakan nilai ICEBERG_WAREHOUSE (lihat environment.defaults.sh).
|
|
NESSIE_CATALOG_DEFAULT_WAREHOUSE: "${ICEBERG_WAREHOUSE:-warehouse}"
|
|
NESSIE_CATALOG_WAREHOUSES_WAREHOUSE_LOCATION: "s3://${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/${ICEBERG_WAREHOUSE:-warehouse}"
|
|
NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_ENDPOINT: "${S3_ENDPOINT:-http://minio:9000}"
|
|
NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_REGION: "${S3_REGION:-us-east-1}"
|
|
NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_AUTH_TYPE: "STATIC"
|
|
NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_PATH_STYLE_ACCESS: "true"
|
|
# Kredensial S3 via Quarkus secret store. Nama secret memakai TITIK
|
|
# (bukan hyphen) — hyphen tidak ter-decode benar oleh Quarkus env mapping
|
|
# (temuan eksperimen nessie-dremio).
|
|
NESSIE_CATALOG_SERVICE_S3_DEFAULT_OPTIONS_ACCESS_KEY: "urn:nessie-secret:quarkus:minio.creds"
|
|
MINIO_CREDS_NAME: "${MINIO_ROOT_USER:-minioadmin}"
|
|
MINIO_CREDS_SECRET: "${MINIO_ROOT_PASSWORD:?err}"
|
|
# Mode dev: batasi memori container. Image Nessie memakai
|
|
# -XX:MaxRAMPercentage=80 (bawaan) yang diukur dari RAM host → tanpa limit
|
|
# heap-nya bisa ±12 GB. Dengan limit 1g, heap efektif ±800 MB.
|
|
mem_limit: 1g
|
|
healthcheck:
|
|
# Image Nessie tanpa curl/wget → probe /dev/tcp ke management port 9000.
|
|
test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/localhost/9000 && printf 'GET /q/health/ready HTTP/1.0\\r\\n\\r\\n' >&3 && cat <&3 | grep -q '200 OK'"]
|
|
interval: 15s
|
|
timeout: 10s
|
|
retries: 15
|
|
start_period: 60s
|
|
ports:
|
|
- "${NESSIE_PORT:-12120}:19120"
|
|
profiles:
|
|
- phase2
|
|
- phase3
|
|
- phase4
|
|
- all
|
|
|
|
|
|
# ============================================================================
|
|
# PHASE 3 — Processing Layer: Trino + OPA (Authorization)
|
|
# Dependency: Storage (MinIO, Nessie) + Keycloak sudah running
|
|
# ============================================================================
|
|
|
|
opa:
|
|
image: openpolicyagent/opa:${OPA_VERSION:-1.20.2}
|
|
container_name: sicabe-opa
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
# Decision point authz Trino: policy Rego + data ACL lokal (deny-by-default).
|
|
# TODO Fase 7: pin versi OPA stabil.
|
|
command: ["run", "--server", "--log-level=info", "--addr=:8181", "/policies", "/data/data.json"]
|
|
volumes:
|
|
- ./config/opa/policies:/policies:ro
|
|
- ./config/opa/data.json:/data/data.json:ro
|
|
# Tanpa healthcheck: image OPA tidak menyediakan shell/curl.
|
|
# Kesiapan diverifikasi dari host (bootstrap/verify) via decision endpoint.
|
|
ports:
|
|
- "${OPA_PORT:-12181}:8181"
|
|
profiles:
|
|
- phase3
|
|
- phase4
|
|
- all
|
|
|
|
# ── Trino — coordinator & worker DIPISAH sejak awal ───────────────────────
|
|
# Agar tier R11 bersih: `trino-coordinator` = tier `coord` (koordinasi, auth,
|
|
# client-facing), `trino-worker` = tier `engine` (eksekusi task, scale-out).
|
|
# Single-host: keduanya di host yang sama. Multi-host: coordinator → host
|
|
# `coord`, worker → host `engine`. Klien hanya melihat HTTPS coordinator.
|
|
trino-coordinator:
|
|
image: trinodb/trino:${TRINO_VERSION:-483}
|
|
container_name: sicabe-trino-coordinator
|
|
labels:
|
|
- "traefik.enable=true"
|
|
- "traefik.http.routers.trino.rule=Host(`trino.${CLUSTER_DOMAIN:-localhost}`)"
|
|
- "traefik.http.routers.trino.entrypoints=websecure"
|
|
- "traefik.http.routers.trino.tls=true"
|
|
- "traefik.http.routers.trino.tls.certresolver=le"
|
|
- "traefik.http.routers.trino.tls.domains[0].main=${CLUSTER_DOMAIN:-localhost}"
|
|
- "traefik.http.routers.trino.tls.domains[0].sans=*.${CLUSTER_DOMAIN:-localhost}"
|
|
- "traefik.http.services.trino.loadbalancer.server.port=8443"
|
|
- "traefik.http.services.trino.loadbalancer.server.scheme=https"
|
|
- "traefik.http.services.trino.loadbalancer.serverstransport=trino-insecure@file"
|
|
hostname: trino-coordinator
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
depends_on:
|
|
nessie:
|
|
condition: service_healthy
|
|
minio:
|
|
condition: service_healthy
|
|
opa:
|
|
condition: service_started
|
|
volumes:
|
|
- trino-coordinator-data:/data/trino
|
|
# Konfigurasi coordinator (HTTPS + OAuth2/Basic, discovery.uri)
|
|
- ./config/trino/config-coordinator.properties:/etc/trino/config.properties:ro
|
|
# Access Control via OPA (bab 4.3.1.6)
|
|
- ./config/trino/access-control.properties:/etc/trino/access-control.properties:ro
|
|
# Konfigurasi konektor Iceberg dengan Nessie (bab 4.3.1.3)
|
|
- ./config/trino/catalog/iceberg.properties:/etc/trino/catalog/iceberg.properties:ro
|
|
# Konfigurasi resource groups (bab 4.3.1.5)
|
|
- ./config/trino/resource-groups.properties:/etc/trino/resource-groups.properties:ro
|
|
- ./config/trino/resource-groups.json:/etc/trino/resource-groups.json:ro
|
|
# Konfigurasi JVM tuning — coordinator (heap 512m, tidak mengeksekusi task)
|
|
- ./config/trino/jvm-coordinator.config:/etc/trino/jvm.config:ro
|
|
# Konfigurasi log
|
|
- ./config/trino/log.properties:/etc/trino/log.properties:ro
|
|
# LDAP GroupProvider config (grup dari OpenLDAP)
|
|
- ./config/trino/group-provider.properties:/etc/trino/group-provider.properties:ro
|
|
# Password authenticator (HTTP Basic) untuk service account
|
|
- ./config/trino/password-authenticator.properties:/etc/trino/password-authenticator.properties:ro
|
|
- ./config/trino/password.db:/etc/trino/password.db:ro
|
|
# Keystore TLS untuk Web UI OAuth2 (OAuth2 Web UI wajib HTTPS)
|
|
- ./config/trino/keystore.p12:/etc/trino/keystore.p12:ro
|
|
environment:
|
|
TRINO_HOME: /etc/trino
|
|
# Static catalog management — load dari etc/catalog/*.properties
|
|
CATALOG_MANAGEMENT: static
|
|
|
|
# Native S3/MinIO credentials (native FS, Trino 481+)
|
|
AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER:-minioadmin}
|
|
AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD:?err}
|
|
AWS_REGION: ${S3_REGION:-us-east-1}
|
|
AWS_ENDPOINT_URL: ${S3_ENDPOINT:-http://minio:9000}
|
|
# Dipakai oleh ${ENV:...} di catalog/iceberg.properties (bisa diarahkan ke S3 existing)
|
|
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
|
|
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?err}
|
|
MINIO_BUCKET_ICEBERG_DATA: ${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}
|
|
ICEBERG_WAREHOUSE: ${ICEBERG_WAREHOUSE:-warehouse}
|
|
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
|
|
S3_REGION: ${S3_REGION:-us-east-1}
|
|
S3_ACCESS_KEY: ${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}
|
|
S3_SECRET_KEY: ${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:?err}}
|
|
NESSIE_URL: ${NESSIE_URL:-http://nessie:19120}
|
|
# OAuth2 client credential untuk Keycloak
|
|
TRINO_CLIENT_SECRET: ${TRINO_CLIENT_SECRET:?err}
|
|
# URL publik Keycloak — dipakai config.properties lewat ${ENV:KEYCLOAK_PUBLIC_URL}
|
|
# untuk issuer + auth-url. Trino GAGAL START bila variabel ini tidak ada (bukan
|
|
# dianggap kosong), jadi sengaja diberi default eksplisit di sini.
|
|
KEYCLOAK_PUBLIC_URL: ${KEYCLOAK_PUBLIC_URL:-http://localhost:${KEYCLOAK_PORT:-12080}}
|
|
# Realm Keycloak (issuer/auth-url/token/jwks di config-coordinator.properties lewat
|
|
# ${ENV:SICABE_REALM}). WAJIB ada: Trino gagal start bila variabel yang dirujuk tak ada.
|
|
SICABE_REALM: ${SICABE_REALM:-sicabe}
|
|
# Di belakang Traefik (tls.mode edge/full) nilai ini `true`. Wajib ada: Trino gagal
|
|
# start bila variabel yang dirujuk config.properties tidak tersedia.
|
|
TRINO_PROCESS_FORWARDED: ${TRINO_PROCESS_FORWARDED:-false}
|
|
# LDAP GroupProvider: DN + password akun BIND (OpenLDAP / IAM existing).
|
|
# LDAP_ADMIN_DN = DN akun bind — boleh akun READ-ONLY, bukan harus admin. Di-expand
|
|
# oleh group-provider.properties lewat ${ENV:LDAP_ADMIN_DN}. Wajib ada: Trino gagal
|
|
# start bila variabel yang dirujuk config tidak tersedia.
|
|
LDAP_ADMIN_DN: ${LDAP_ADMIN_DN:-cn=admin,dc=sicabe,dc=local}
|
|
LDAP_ADMIN_PASSWORD: ${LDAP_ADMIN_PASSWORD:?err}
|
|
LDAP_URL: ${LDAP_URL:-ldap://openldap:389}
|
|
# Port internal Keycloak untuk config Trino (config-coordinator.properties
|
|
# memakai ${ENV:KEYCLOAK_INTERNAL_PORT}) — wajib ada di container.
|
|
KEYCLOAK_INTERNAL_PORT: ${KEYCLOAK_INTERNAL_PORT:-8080}
|
|
# Host Keycloak untuk token-url/jwks-url (server-ke-server). Multi-host dgn Keycloak
|
|
# EKSTERNAL: ini = ALAMAT host pemilik (mis. 100.93.54.108) — bukan nama container
|
|
# `keycloak` yang tak bisa di-resolve → penukaran token gagal ("could not be verified").
|
|
KEYCLOAK_HOST: ${KEYCLOAK_HOST:-keycloak}
|
|
# URL server→server LENGKAP (skema+host+port) untuk token/jwks. Default dibangun
|
|
# dari HOST:PORT; generator menimpanya bila Keycloak EKSTERNAL ber-URL penuh
|
|
# (mis. https://sso.example.com) — lihat config-coordinator.properties.
|
|
KEYCLOAK_INTERNAL_URL: ${KEYCLOAK_INTERNAL_URL:-http://${KEYCLOAK_HOST:-keycloak}:${KEYCLOAK_INTERNAL_PORT:-8080}}
|
|
# Kunci HMAC tetap untuk state OAuth2 Trino (default Trino acak tiap start).
|
|
TRINO_OAUTH_STATE_KEY: ${TRINO_OAUTH_STATE_KEY:-trino-oauth-state-dev-only}
|
|
healthcheck:
|
|
# Trino internal port adalah 8080 (http-server.http.port)
|
|
test: ["CMD", "curl", "-f", "http://localhost:8080/v1/info"]
|
|
interval: 15s
|
|
timeout: 10s
|
|
retries: 10
|
|
start_period: 60s
|
|
ports:
|
|
# Host 12443 → Container 8443 (HTTPS: OAuth2 Web UI + Basic service account)
|
|
- "${TRINO_HTTPS_PORT:-12443}:8443"
|
|
# HTTP polos TIDAK dilayani (http-server.authentication.allow-insecure-over-http=false)
|
|
# dan tidak ada port HTTP yang di-expose — hanya HTTPS di atas.
|
|
profiles:
|
|
- phase3
|
|
- all
|
|
|
|
# ─── trino-jmx-exporter: saturasi Trino (JMX) → Prometheus (M3 observabilitas) ─
|
|
# Sidecar yang membaca JMX coordinator via RMI (lihat jvm-coordinator.config) dan
|
|
# menyajikannya sebagai metrik Prometheus di :9404. Satu tier dengan trino-coordinator
|
|
# (`coord`) agar koneksi JMX tetap lokal; lihat lib/cluster/inventory.py.
|
|
trino-jmx-exporter:
|
|
image: bitnami/jmx-exporter:${JMX_EXPORTER_VERSION:-latest}
|
|
container_name: sicabe-trino-jmx-exporter
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
depends_on:
|
|
trino-coordinator:
|
|
condition: service_started
|
|
volumes:
|
|
- ./config/trino/jmx-exporter.yml:/config/config.yml:ro
|
|
# Entrypoint image: `java -jar jmx_prometheus_standalone.jar` → arg = <port> <config>.
|
|
command: ["9404", "/config/config.yml"]
|
|
profiles:
|
|
- phase3
|
|
- all
|
|
|
|
# Worker: TIDAK ada `ports:` (sengaja) — klien tidak boleh menyentuhnya.
|
|
# Ia mendaftar ke coordinator via discovery.uri (config-worker.properties).
|
|
trino-worker:
|
|
image: trinodb/trino:${TRINO_VERSION:-483}
|
|
container_name: sicabe-trino-worker
|
|
hostname: trino-worker
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
depends_on:
|
|
trino-coordinator:
|
|
condition: service_healthy
|
|
volumes:
|
|
- trino-worker-data:/data/trino
|
|
# Konfigurasi worker (coordinator=false; tanpa HTTPS/auth klien)
|
|
- ./config/trino/config-worker.properties:/etc/trino/config.properties:ro
|
|
# Worker juga membaca katalog yang sama (data di Snowflake-like remote FS)
|
|
- ./config/trino/catalog/iceberg.properties:/etc/trino/catalog/iceberg.properties:ro
|
|
# Konfigurasi JVM tuning — worker (heap 1G, mengeksekusi task)
|
|
- ./config/trino/jvm-worker.config:/etc/trino/jvm.config:ro
|
|
# Konfigurasi log
|
|
- ./config/trino/log.properties:/etc/trino/log.properties:ro
|
|
environment:
|
|
TRINO_HOME: /etc/trino
|
|
# Static catalog management — load dari etc/catalog/*.properties
|
|
CATALOG_MANAGEMENT: static
|
|
|
|
# Env yang dipakai ${ENV:...} di catalog/iceberg.properties → wajib di worker juga
|
|
AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER:-minioadmin}
|
|
AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD:?err}
|
|
AWS_REGION: ${S3_REGION:-us-east-1}
|
|
AWS_ENDPOINT_URL: ${S3_ENDPOINT:-http://minio:9000}
|
|
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
|
|
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?err}
|
|
MINIO_BUCKET_ICEBERG_DATA: ${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}
|
|
ICEBERG_WAREHOUSE: ${ICEBERG_WAREHOUSE:-warehouse}
|
|
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
|
|
S3_REGION: ${S3_REGION:-us-east-1}
|
|
S3_ACCESS_KEY: ${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}
|
|
S3_SECRET_KEY: ${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:?err}}
|
|
NESSIE_URL: ${NESSIE_URL:-http://nessie:19120}
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8080/v1/info"]
|
|
interval: 15s
|
|
timeout: 10s
|
|
retries: 10
|
|
start_period: 60s
|
|
profiles:
|
|
- phase3
|
|
- all
|
|
|
|
|
|
# ============================================================================
|
|
# PHASE 4 — Serving Layer: StarRocks (shared-data: FE + Compute Node)
|
|
# Dependency: Storage (MinIO, Nessie) + Keycloak + OpenLDAP sudah running
|
|
# ============================================================================
|
|
|
|
starrocks-fe:
|
|
image: starrocks/fe-ubuntu:${STARROCKS_VERSION:-4.0.13}
|
|
container_name: sicabe-starrocks-fe
|
|
# Hostname tetap: identitas replika BDB FE memakai hostname; recreate
|
|
# container TANPA hostname tetap → identitas stale → FE stuck INIT/UNKNOWN.
|
|
hostname: sicabe-starrocks-fe
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
depends_on:
|
|
nessie:
|
|
condition: service_healthy
|
|
minio:
|
|
condition: service_healthy
|
|
keycloak:
|
|
condition: service_healthy
|
|
openldap:
|
|
condition: service_healthy
|
|
volumes:
|
|
- starrocks-fe-data:/opt/starrocks/fe
|
|
# fe.conf: run_mode=shared_data + cloud_native_storage_type=S3 (WAJIB sebelum first start)
|
|
- ./config/starrocks/fe.conf:/opt/starrocks/fe/conf/fe.conf:ro
|
|
environment:
|
|
STARROCKS_ROOT_PASSWORD: ${STARROCKS_ROOT_PASSWORD:?err}
|
|
AWS_EC2_METADATA_DISABLED: "true"
|
|
# Kredensial S3/MinIO untuk external catalog (Iceberg via Nessie).
|
|
# Pakai S3_* dengan fallback saat-pakai ke MINIO_ROOT_* — konsisten dengan
|
|
# Trino & Nessie, dan benar saat memakai S3 existing (§2.9/§2.13).
|
|
AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}
|
|
AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:?err}}
|
|
AWS_REGION: ${S3_REGION:-us-east-1}
|
|
AWS_ENDPOINT_URL: ${S3_ENDPOINT:-http://minio:9000}
|
|
command: ["bash", "/opt/starrocks/fe/bin/start_fe.sh", "--host_type", "FQDN"]
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "mysql -u root -p$${STARROCKS_ROOT_PASSWORD} -h 127.0.0.1 -P 9030 -e \"SHOW FRONTENDS\" 2>/dev/null | grep -q \"Alive\" | head -1"]
|
|
interval: 20s
|
|
timeout: 15s
|
|
retries: 15
|
|
start_period: 120s
|
|
ports:
|
|
- "${STARROCKS_FE_PORT:-12930}:9030"
|
|
- "${STARROCKS_HTTP_PORT:-12030}:8030"
|
|
profiles:
|
|
- phase4
|
|
- all
|
|
|
|
starrocks-cn:
|
|
image: starrocks/cn-ubuntu:${STARROCKS_VERSION:-4.0.13}
|
|
container_name: sicabe-starrocks-cn
|
|
hostname: sicabe-starrocks-cn
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
depends_on:
|
|
starrocks-fe:
|
|
condition: service_started
|
|
volumes:
|
|
- cn-cache:/opt/starrocks/cn/cache
|
|
- ./config/starrocks/cn.conf:/opt/starrocks/cn/conf/cn.conf:ro
|
|
environment:
|
|
HOST_TYPE: FQDN
|
|
STARROCKS_ROOT_PASSWORD: ${STARROCKS_ROOT_PASSWORD:?err}
|
|
AWS_EC2_METADATA_DISABLED: "true"
|
|
# Sama dengan FE: pakai S3_* dengan fallback saat-pakai (benar untuk S3 existing)
|
|
AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}
|
|
AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:?err}}
|
|
AWS_REGION: ${S3_REGION:-us-east-1}
|
|
AWS_ENDPOINT_URL: ${S3_ENDPOINT:-http://minio:9000}
|
|
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
|
|
S3_USE_PATH_STYLE_ENDPOINT: "true"
|
|
command:
|
|
- /bin/bash
|
|
- -c
|
|
- |
|
|
ulimit -n 65535
|
|
# 1) Tunggu FE siap, lalu pastikan root FE berpassword.
|
|
for i in $$(seq 1 30); do
|
|
if mysql --connect-timeout 3 -h ${STARROCKS_FE_HOST:-starrocks-fe} -P 9030 -u root -e "SELECT 1" >/dev/null 2>&1; then
|
|
mysql --connect-timeout 3 -h ${STARROCKS_FE_HOST:-starrocks-fe} -P 9030 -u root \
|
|
-e "SET PASSWORD FOR 'root' = PASSWORD('$${STARROCKS_ROOT_PASSWORD}')" >/dev/null 2>&1 \
|
|
|| mysql --connect-timeout 3 -h ${STARROCKS_FE_HOST:-starrocks-fe} -P 9030 -u root \
|
|
-e "ALTER USER 'root'@'%' IDENTIFIED BY '$${STARROCKS_ROOT_PASSWORD}'" >/dev/null 2>&1 || true
|
|
echo "[SiCABE] FE siap; root password diset (attempt $$i)."
|
|
break
|
|
fi
|
|
echo "[SiCABE] Menunggu FE siap... (attempt $$i/30)"
|
|
sleep 5
|
|
done
|
|
# 2) Daftarkan CN ini ke FE (shared-data → ADD COMPUTE NODE, bukan BACKEND).
|
|
if mysql --connect-timeout 3 -h ${STARROCKS_FE_HOST:-starrocks-fe} -P 9030 -u root -p"$${STARROCKS_ROOT_PASSWORD}" \
|
|
-e 'SHOW COMPUTE NODES;' 2>/dev/null | grep -q "${STARROCKS_CN_HOST:-starrocks-cn}"; then
|
|
echo "[SiCABE] CN sudah terdaftar."
|
|
else
|
|
for i in $$(seq 1 20); do
|
|
if mysql --connect-timeout 3 -h ${STARROCKS_FE_HOST:-starrocks-fe} -P 9030 -u root -p"$${STARROCKS_ROOT_PASSWORD}" \
|
|
-e 'ALTER SYSTEM ADD COMPUTE NODE "${STARROCKS_CN_HOST:-starrocks-cn}:9050";' >/dev/null 2>&1; then
|
|
echo "[SiCABE] CN terdaftar ke FE (attempt $$i)."
|
|
break
|
|
fi
|
|
echo "[SiCABE] Menunggu FE menerima registrasi... (attempt $$i/20)"
|
|
sleep 5
|
|
done
|
|
fi
|
|
bash /opt/starrocks/cn/bin/start_cn.sh
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "mysql -u root -p$${STARROCKS_ROOT_PASSWORD} -h ${STARROCKS_FE_HOST:-starrocks-fe} -P 9030 -e \"SHOW COMPUTE NODES\" 2>/dev/null | grep -q \"Alive\" | head -1"]
|
|
interval: 20s
|
|
timeout: 15s
|
|
retries: 15
|
|
start_period: 120s
|
|
profiles:
|
|
- phase4
|
|
- all
|
|
|
|
|
|
# ============================================================================
|
|
# PHASE 4b — Serving (Paralel): Dremio OSS (query engine di atas Nessie)
|
|
# Dependency: Storage (MinIO, Nessie). Auth lokal (first-run admin via UI).
|
|
# ============================================================================
|
|
|
|
dremio:
|
|
image: dremio/dremio-oss:${DREMIO_VERSION:-25.2.0}
|
|
container_name: sicabe-dremio
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
# Tanpa depends_on: Dremio opt-in (profile `dremio`) sedangkan minio/nessie
|
|
# berada di profile `all`. `depends_on` ke service di luar profile aktif
|
|
# membuat Compose menolak project ("depends on undefined service"). Saat
|
|
# diaktifkan, stack utama sudah berjalan sehingga urutan tak diperlukan.
|
|
environment:
|
|
# Catatan: Dremio MENOLAK start bila heap < 4 GB ("Dremio requires at
|
|
# least 4 GB memory to run") → heap-nya TIDAK bisa diturunkan seperti
|
|
# komponen JVM lain. Biarkan default: heap 4096 MB dan
|
|
# MaxDirectMemorySize 8192 MB (yang terakhir hanya plafon, bukan alokasi).
|
|
DREMIO_JAVA_SERVER_EXTRA_OPTS: "-Dpaths.dist=file:///opt/dremio/data/dist"
|
|
volumes:
|
|
- dremio-data:/opt/dremio/data
|
|
ports:
|
|
- "${DREMIO_UI_PORT:-12947}:9047" # Web UI
|
|
- "${DREMIO_JDBC_PORT:-12010}:31010" # JDBC (legacy)
|
|
- "${DREMIO_FLIGHT_PORT:-12011}:32010" # Arrow Flight
|
|
# OPT-IN: Dremio tidak ikut `--profile all` / `phaseN`. Aktifkan khusus:
|
|
# docker compose --profile dremio up -d dremio
|
|
# Alasan: Dremio menolak heap < 4 GB (hard requirement installer-nya)
|
|
# sehingga memakan RAM besar; di mode dev umumnya tidak diperlukan.
|
|
profiles:
|
|
- dremio
|
|
|
|
# ============================================================================
|
|
# PHASE 5 — Ingestion Layer: Airflow
|
|
# Dependency: Storage (MinIO, Nessie) + Processing (Trino) + Keycloak
|
|
# ============================================================================
|
|
|
|
# ─── Init volume: izin tulis untuk user airflow (uid 50000) ────────────────
|
|
# Volume bernama dibuat Docker sebagai root:root. Plugin OpenMetadata menulis
|
|
# config pipeline sebagai user airflow sehingga GAGAL SENYAP bila izin belum
|
|
# diperbaiki — gejalanya DAG hasil plugin tidak bisa diimpor:
|
|
# ConfigurationError: Cannot open config file .../dag_generated_configs/<n>.json
|
|
# Container ini berjalan sekali sebagai root, membetulkan izin, lalu keluar.
|
|
airflow-volume-init:
|
|
image: ${AIRFLOW_IMAGE:-sicabe/airflow:3.3.2-om203-ol}
|
|
container_name: sicabe-airflow-volume-init
|
|
restart: "no"
|
|
user: root
|
|
networks:
|
|
- sicabe-network
|
|
entrypoint: ["/bin/bash", "-c"]
|
|
command:
|
|
- "mkdir -p /cfg /dags /logs && chown -R 50000:0 /cfg /dags /logs && chmod 775 /cfg && echo 'Izin volume Airflow siap (uid 50000)'"
|
|
volumes:
|
|
- airflow-dags-configs:/cfg
|
|
- airflow-dags:/dags
|
|
- airflow-logs:/logs
|
|
profiles:
|
|
- phase5
|
|
- all
|
|
|
|
airflow-init:
|
|
image: ${AIRFLOW_IMAGE:-sicabe/airflow:3.3.2-om203-ol}
|
|
container_name: sicabe-airflow-init
|
|
# Hostname HARUS bisa diresolusi. Secara default Docker memberi hostname = ID
|
|
# container, dan task log server Airflow memakai socket.gethostname() sebagai
|
|
# alamatnya -> muncul "Could not read served logs: HTTPConnectionPool(host=
|
|
# '<id-container>', port=8793) ... Connection refused" (kosmetik, tapi
|
|
# membingungkan). Dengan hostname = nama container, alamat itu resolvable.
|
|
hostname: sicabe-airflow-init
|
|
restart: "no"
|
|
networks:
|
|
- sicabe-network
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
# Perbaiki izin volume (root) SEBELUM penulisan DAG/config oleh plugin.
|
|
airflow-volume-init:
|
|
condition: service_completed_successfully
|
|
volumes:
|
|
- ./config/airflow/airflow.cfg:/opt/airflow/airflow.cfg:ro
|
|
- ./config/airflow/simple_auth_manager_passwords.json:/opt/airflow/simple_auth_manager_passwords.json
|
|
- airflow-dags:/opt/airflow/dags
|
|
- airflow-dags-configs:/opt/airflow/dag_generated_configs
|
|
- airflow-logs:/opt/airflow/logs
|
|
- ./scripts/airflow:/opt/airflow/scripts:ro
|
|
- ./config/airflow/dbt:/opt/airflow/dbt
|
|
- ./config/airflow/great_expectations:/opt/airflow/great_expectations:ro
|
|
- ./config/airflow/sling:/opt/airflow/sling:ro
|
|
# Sertifikat Trino (PEM) untuk klien yang MEMVERIFIKASI TLS — dipakai
|
|
# workflow ingestion OpenMetadata lewat field `verify` (dbt memakai cert:false).
|
|
- ./config/trino/certs:/etc/ssl/trino:ro
|
|
environment:
|
|
# dbt menulis artefak ke <project>/target dan log ke <project>/logs. Direktori
|
|
# dbt di-mount dari repo (pemilik uid host) sementara container berjalan sebagai
|
|
# uid 50000 (airflow) -> TIDAK bisa menulis, dan dbt gagal dengan
|
|
# [Errno 13] Permission denied: '/opt/airflow/dbt/sicabe/target'
|
|
# Bahkan pesannya pun tidak terlihat karena log-nya juga tidak bisa ditulis
|
|
# (gejalanya: "exit code 2" tanpa keluaran sama sekali). Arahkan ke /tmp.
|
|
DBT_LOG_PATH: ${DBT_LOG_PATH:-/tmp/dbt/logs}
|
|
DBT_TARGET_PATH: ${DBT_TARGET_PATH:-/tmp/dbt/target}
|
|
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:${AIRFLOW_DB_PASSWORD:?err}@${POSTGRES_HOST:-postgres}:${POSTGRES_INTERNAL_PORT:-5432}/airflow
|
|
AIRFLOW__CORE__EXECUTOR: ${AIRFLOW_EXECUTOR:-LocalExecutor}
|
|
# Secret key konsisten untuk JWT internal (task runner auth)
|
|
AIRFLOW__API_AUTH__JWT_SECRET: ${AIRFLOW_SECRET_KEY:?err}
|
|
# SimpleAuthManager users (format: username:password[:team]).
|
|
# ⚠️ Field ketiga adalah TEAM, bukan role — mengisinya memicu crash
|
|
# "user ... is associated to at least one team and multi-team mode is not
|
|
# configured" (dibuktikan di experiments/om20x). Jadi cukup dua field.
|
|
AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: ${AIRFLOW_SIMPLE_AUTH_USERS:-admin:Admin}
|
|
# Password superuser PostgreSQL (untuk DAG akses sicabe_data)
|
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?err}
|
|
# ─── Metrik → statsd-exporter (M2 observabilitas, Review 19) ───
|
|
AIRFLOW__METRICS__STATSD_ON: "true"
|
|
AIRFLOW__METRICS__STATSD_HOST: ${STATSD_HOST:-statsd-exporter}
|
|
AIRFLOW__METRICS__STATSD_PORT: ${STATSD_PORT:-9125}
|
|
AIRFLOW__METRICS__STATSD_PREFIX: airflow
|
|
# Note: Airflow 3.x menggunakan 'airflow db migrate' (bukan 'db init')
|
|
# Admin user dibuat via 'airflow users create' (SimpleAuthManager — bukan lagi _AIRFLOW_WWW_USER_*)
|
|
command:
|
|
- bash
|
|
- -c
|
|
- airflow db migrate && airflow users create --username admin --password ${AIRFLOW_ADMIN_PASSWORD:?err} --role Admin --email admin@sicabe.local --firstname Admin --lastname User 2>/dev/null || true && echo 'Airflow DB initialized + admin user'
|
|
profiles:
|
|
- phase5
|
|
- all
|
|
|
|
airflow-webserver:
|
|
image: ${AIRFLOW_IMAGE:-sicabe/airflow:3.3.2-om203-ol}
|
|
container_name: sicabe-airflow-webserver
|
|
# Hostname HARUS bisa diresolusi. Secara default Docker memberi hostname = ID
|
|
# container, dan task log server Airflow memakai socket.gethostname() sebagai
|
|
# alamatnya -> muncul "Could not read served logs: HTTPConnectionPool(host=
|
|
# '<id-container>', port=8793) ... Connection refused" (kosmetik, tapi
|
|
# membingungkan). Dengan hostname = nama container, alamat itu resolvable.
|
|
hostname: sicabe-airflow-webserver
|
|
labels:
|
|
- "traefik.enable=true"
|
|
- "traefik.http.routers.airflow.rule=Host(`airflow.${CLUSTER_DOMAIN:-localhost}`)"
|
|
- "traefik.http.routers.airflow.entrypoints=websecure"
|
|
- "traefik.http.routers.airflow.tls=true"
|
|
- "traefik.http.routers.airflow.tls.certresolver=le"
|
|
- "traefik.http.routers.airflow.tls.domains[0].main=${CLUSTER_DOMAIN:-localhost}"
|
|
- "traefik.http.routers.airflow.tls.domains[0].sans=*.${CLUSTER_DOMAIN:-localhost}"
|
|
- "traefik.http.services.airflow.loadbalancer.server.port=8080"
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
depends_on:
|
|
airflow-init:
|
|
condition: service_completed_successfully
|
|
postgres:
|
|
condition: service_healthy
|
|
volumes:
|
|
- ./config/airflow/airflow.cfg:/opt/airflow/airflow.cfg:ro
|
|
- ./config/airflow/simple_auth_manager_passwords.json:/opt/airflow/simple_auth_manager_passwords.json
|
|
- airflow-dags:/opt/airflow/dags
|
|
- airflow-dags-configs:/opt/airflow/dag_generated_configs
|
|
- airflow-logs:/opt/airflow/logs
|
|
- ./scripts/airflow:/opt/airflow/scripts:ro
|
|
- ./config/airflow/dbt:/opt/airflow/dbt
|
|
- ./config/airflow/great_expectations:/opt/airflow/great_expectations:ro
|
|
- ./config/airflow/sling:/opt/airflow/sling:ro
|
|
# Sertifikat Trino (PEM) untuk klien yang MEMVERIFIKASI TLS — dipakai
|
|
# workflow ingestion OpenMetadata lewat field `verify` (dbt memakai cert:false).
|
|
- ./config/trino/certs:/etc/ssl/trino:ro
|
|
environment:
|
|
# dbt menulis artefak ke <project>/target dan log ke <project>/logs. Direktori
|
|
# dbt di-mount dari repo (pemilik uid host) sementara container berjalan sebagai
|
|
# uid 50000 (airflow) -> TIDAK bisa menulis, dan dbt gagal dengan
|
|
# [Errno 13] Permission denied: '/opt/airflow/dbt/sicabe/target'
|
|
# Bahkan pesannya pun tidak terlihat karena log-nya juga tidak bisa ditulis
|
|
# (gejalanya: "exit code 2" tanpa keluaran sama sekali). Arahkan ke /tmp.
|
|
DBT_LOG_PATH: ${DBT_LOG_PATH:-/tmp/dbt/logs}
|
|
DBT_TARGET_PATH: ${DBT_TARGET_PATH:-/tmp/dbt/target}
|
|
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:${AIRFLOW_DB_PASSWORD:?err}@${POSTGRES_HOST:-postgres}:${POSTGRES_INTERNAL_PORT:-5432}/airflow
|
|
AIRFLOW__CORE__EXECUTOR: ${AIRFLOW_EXECUTOR:-LocalExecutor}
|
|
# URL browser-facing (link di UI). Default localhost, bukan nama internal
|
|
# airflow-webserver:8080 yang tidak bisa diresolusi browser.
|
|
AIRFLOW__WEBSERVER__BASE_URL: ${AIRFLOW_PUBLIC_URL:-http://localhost:${AIRFLOW_PORT:-12082}}
|
|
# Secret key konsisten untuk JWT internal (task runner auth)
|
|
AIRFLOW__API_AUTH__JWT_SECRET: ${AIRFLOW_SECRET_KEY:?err}
|
|
# SimpleAuthManager users (format: username:password[:team]).
|
|
# ⚠️ Field ketiga adalah TEAM, bukan role — mengisinya memicu crash
|
|
# "user ... is associated to at least one team and multi-team mode is not
|
|
# configured" (dibuktikan di experiments/om20x). Jadi cukup dua field.
|
|
AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: ${AIRFLOW_SIMPLE_AUTH_USERS:-admin:Admin}
|
|
# Password superuser PostgreSQL (untuk DAG akses sicabe_data)
|
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?err}
|
|
# Token OpenMetadata untuk GE Action (validate_to_openmetadata.py) — dipakai
|
|
# task validasi DAG agar hasil Great Expectations TIDAK berhenti di log,
|
|
# tetapi terkirim ke OpenMetadata (Data Quality Test Suite/Case).
|
|
OPENMETADATA_ADMIN_PASSWORD: ${OPENMETADATA_ADMIN_PASSWORD:?err}
|
|
OM_API_URL: http://${OPENMETADATA_HOST:-openmetadata-server}:8585/api
|
|
# dbt → Trino: service account via HTTPS 8443 langsung (self-signed, cert: false)
|
|
DBT_TRINO_HOST: ${DBT_TRINO_HOST:-trino-coordinator}
|
|
DBT_TRINO_USER: ${DBT_TRINO_USER:-svc_dbt}
|
|
DBT_TRINO_PASSWORD: ${TRINO_SVC_DBT_PASSWORD:?err}
|
|
# ─── OpenLineage → OpenMetadata (experiments/lineage-airflow-om) ───
|
|
# Transport dibaca dari KONEKSI Airflow (`config_conn_id`) — token TIDAK
|
|
# ditulis di .env. Koneksi dibuat bootstrap.sh (`openlineage_config` +
|
|
# `openmetadata_lineage_api`). Kosong = provider INERT.
|
|
AIRFLOW__OPENLINEAGE__CONFIG_CONN_ID: ${AIRFLOW_OPENLINEAGE_CONFIG_CONN_ID:-}
|
|
AIRFLOW__OPENLINEAGE__NAMESPACE: ${AIRFLOW_OPENLINEAGE_NAMESPACE:-}
|
|
AIRFLOW__OPENLINEAGE__EXECUTE_IN_THREAD: "true"
|
|
# ─── Metrik → statsd-exporter (M2 observabilitas, Review 19) ───
|
|
AIRFLOW__METRICS__STATSD_ON: "true"
|
|
AIRFLOW__METRICS__STATSD_HOST: ${STATSD_HOST:-statsd-exporter}
|
|
AIRFLOW__METRICS__STATSD_PORT: ${STATSD_PORT:-9125}
|
|
AIRFLOW__METRICS__STATSD_PREFIX: airflow
|
|
# Catatan: Airflow 3.3 memakai SimpleAuthManager (bukan FabAuthManager),
|
|
# jadi _AIRFLOW_WWW_USER_* TIDAK dipakai dan memicu crash di entrypoint.
|
|
# Admin user dibuat oleh airflow-init via 'airflow users create'.
|
|
# Airflow 3.x: 'airflow webserver' diganti 'airflow api-server'
|
|
# Health endpoint: /api/v2/monitor/health (bukan /health)
|
|
# Entrypoint wrapper: install sling CLI dulu, lalu start service
|
|
command: ["bash", "/opt/airflow/scripts/entrypoint.sh", "airflow", "api-server"]
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8080/api/v2/monitor/health"]
|
|
interval: 15s
|
|
timeout: 10s
|
|
retries: 10
|
|
start_period: 30s
|
|
ports:
|
|
- "${AIRFLOW_PORT:-12082}:8080"
|
|
profiles:
|
|
- phase5
|
|
- all
|
|
|
|
airflow-dag-processor:
|
|
image: ${AIRFLOW_IMAGE:-sicabe/airflow:3.3.2-om203-ol}
|
|
container_name: sicabe-airflow-dag-processor
|
|
# Hostname HARUS bisa diresolusi. Secara default Docker memberi hostname = ID
|
|
# container, dan task log server Airflow memakai socket.gethostname() sebagai
|
|
# alamatnya -> muncul "Could not read served logs: HTTPConnectionPool(host=
|
|
# '<id-container>', port=8793) ... Connection refused" (kosmetik, tapi
|
|
# membingungkan). Dengan hostname = nama container, alamat itu resolvable.
|
|
hostname: sicabe-airflow-dag-processor
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
depends_on:
|
|
airflow-init:
|
|
condition: service_completed_successfully
|
|
postgres:
|
|
condition: service_healthy
|
|
volumes:
|
|
- ./config/airflow/airflow.cfg:/opt/airflow/airflow.cfg:ro
|
|
- ./config/airflow/simple_auth_manager_passwords.json:/opt/airflow/simple_auth_manager_passwords.json
|
|
- airflow-dags:/opt/airflow/dags
|
|
- airflow-dags-configs:/opt/airflow/dag_generated_configs
|
|
- airflow-logs:/opt/airflow/logs
|
|
- ./scripts/airflow:/opt/airflow/scripts:ro
|
|
- ./config/airflow/dbt:/opt/airflow/dbt
|
|
- ./config/airflow/great_expectations:/opt/airflow/great_expectations:ro
|
|
- ./config/airflow/sling:/opt/airflow/sling:ro
|
|
# Sertifikat Trino (PEM) untuk klien yang MEMVERIFIKASI TLS — dipakai
|
|
# workflow ingestion OpenMetadata lewat field `verify` (dbt memakai cert:false).
|
|
- ./config/trino/certs:/etc/ssl/trino:ro
|
|
environment:
|
|
# dbt menulis artefak ke <project>/target dan log ke <project>/logs. Direktori
|
|
# dbt di-mount dari repo (pemilik uid host) sementara container berjalan sebagai
|
|
# uid 50000 (airflow) -> TIDAK bisa menulis, dan dbt gagal dengan
|
|
# [Errno 13] Permission denied: '/opt/airflow/dbt/sicabe/target'
|
|
# Bahkan pesannya pun tidak terlihat karena log-nya juga tidak bisa ditulis
|
|
# (gejalanya: "exit code 2" tanpa keluaran sama sekali). Arahkan ke /tmp.
|
|
DBT_LOG_PATH: ${DBT_LOG_PATH:-/tmp/dbt/logs}
|
|
DBT_TARGET_PATH: ${DBT_TARGET_PATH:-/tmp/dbt/target}
|
|
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:${AIRFLOW_DB_PASSWORD:?err}@${POSTGRES_HOST:-postgres}:${POSTGRES_INTERNAL_PORT:-5432}/airflow
|
|
AIRFLOW__CORE__EXECUTOR: ${AIRFLOW_EXECUTOR:-LocalExecutor}
|
|
# Secret key konsisten untuk JWT internal (task runner auth)
|
|
AIRFLOW__API_AUTH__JWT_SECRET: ${AIRFLOW_SECRET_KEY:?err}
|
|
# SimpleAuthManager users (format: username:password[:team]).
|
|
# ⚠️ Field ketiga adalah TEAM, bukan role — mengisinya memicu crash
|
|
# "user ... is associated to at least one team and multi-team mode is not
|
|
# configured" (dibuktikan di experiments/om20x). Jadi cukup dua field.
|
|
AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: ${AIRFLOW_SIMPLE_AUTH_USERS:-admin:Admin}
|
|
# Password superuser PostgreSQL (untuk DAG akses sicabe_data)
|
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?err}
|
|
# Token OpenMetadata untuk GE Action (validate_to_openmetadata.py) — dipakai
|
|
# task validasi DAG agar hasil Great Expectations TIDAK berhenti di log,
|
|
# tetapi terkirim ke OpenMetadata (Data Quality Test Suite/Case).
|
|
OPENMETADATA_ADMIN_PASSWORD: ${OPENMETADATA_ADMIN_PASSWORD:?err}
|
|
OM_API_URL: http://${OPENMETADATA_HOST:-openmetadata-server}:8585/api
|
|
# dbt → Trino: service account via HTTPS 8443 langsung (self-signed, cert: false)
|
|
DBT_TRINO_HOST: ${DBT_TRINO_HOST:-trino-coordinator}
|
|
DBT_TRINO_USER: ${DBT_TRINO_USER:-svc_dbt}
|
|
DBT_TRINO_PASSWORD: ${TRINO_SVC_DBT_PASSWORD:?err}
|
|
# ─── OpenLineage → OpenMetadata (experiments/lineage-airflow-om) ───
|
|
# Transport dibaca dari KONEKSI Airflow (`config_conn_id`) — token TIDAK
|
|
# ditulis di .env. Koneksi dibuat bootstrap.sh (`openlineage_config` +
|
|
# `openmetadata_lineage_api`). Kosong = provider INERT.
|
|
AIRFLOW__OPENLINEAGE__CONFIG_CONN_ID: ${AIRFLOW_OPENLINEAGE_CONFIG_CONN_ID:-}
|
|
AIRFLOW__OPENLINEAGE__NAMESPACE: ${AIRFLOW_OPENLINEAGE_NAMESPACE:-}
|
|
AIRFLOW__OPENLINEAGE__EXECUTE_IN_THREAD: "true"
|
|
# ─── Metrik → statsd-exporter (M2 observabilitas, Review 19) ───
|
|
AIRFLOW__METRICS__STATSD_ON: "true"
|
|
AIRFLOW__METRICS__STATSD_HOST: ${STATSD_HOST:-statsd-exporter}
|
|
AIRFLOW__METRICS__STATSD_PORT: ${STATSD_PORT:-9125}
|
|
AIRFLOW__METRICS__STATSD_PREFIX: airflow
|
|
# Airflow 3.x: DAG parsing dilakukan oleh dag-processor (pisah dari scheduler)
|
|
command: ["bash", "/opt/airflow/scripts/entrypoint.sh", "airflow", "dag-processor"]
|
|
healthcheck:
|
|
# Image Airflow minimal tidak punya pgrep/ps — scan /proc via python
|
|
test: ["CMD-SHELL", "python3 -c \"import os; print('ALIVE' if any('dag-processor' in open(f'/proc/{p}/cmdline').read() for p in os.listdir('/proc') if p.isdigit() and os.path.exists(f'/proc/{p}/cmdline')) else 'DEAD')\" | grep -q ALIVE"]
|
|
interval: 15s
|
|
timeout: 10s
|
|
retries: 10
|
|
start_period: 60s
|
|
profiles:
|
|
- phase5
|
|
- all
|
|
|
|
airflow-scheduler:
|
|
image: ${AIRFLOW_IMAGE:-sicabe/airflow:3.3.2-om203-ol}
|
|
container_name: sicabe-airflow-scheduler
|
|
# Hostname HARUS bisa diresolusi. Secara default Docker memberi hostname = ID
|
|
# container, dan task log server Airflow memakai socket.gethostname() sebagai
|
|
# alamatnya -> muncul "Could not read served logs: HTTPConnectionPool(host=
|
|
# '<id-container>', port=8793) ... Connection refused" (kosmetik, tapi
|
|
# membingungkan). Dengan hostname = nama container, alamat itu resolvable.
|
|
hostname: sicabe-airflow-scheduler
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
depends_on:
|
|
airflow-dag-processor:
|
|
condition: service_started
|
|
airflow-init:
|
|
condition: service_completed_successfully
|
|
postgres:
|
|
condition: service_healthy
|
|
volumes:
|
|
- ./config/airflow/airflow.cfg:/opt/airflow/airflow.cfg:ro
|
|
- ./config/airflow/simple_auth_manager_passwords.json:/opt/airflow/simple_auth_manager_passwords.json
|
|
- airflow-dags:/opt/airflow/dags
|
|
- airflow-dags-configs:/opt/airflow/dag_generated_configs
|
|
- airflow-logs:/opt/airflow/logs
|
|
- ./scripts/airflow:/opt/airflow/scripts:ro
|
|
- ./config/airflow/dbt:/opt/airflow/dbt
|
|
- ./config/airflow/great_expectations:/opt/airflow/great_expectations:ro
|
|
- ./config/airflow/sling:/opt/airflow/sling:ro
|
|
# Sertifikat Trino (PEM) untuk klien yang MEMVERIFIKASI TLS — dipakai
|
|
# workflow ingestion OpenMetadata lewat field `verify` (dbt memakai cert:false).
|
|
- ./config/trino/certs:/etc/ssl/trino:ro
|
|
environment:
|
|
# dbt menulis artefak ke <project>/target dan log ke <project>/logs. Direktori
|
|
# dbt di-mount dari repo (pemilik uid host) sementara container berjalan sebagai
|
|
# uid 50000 (airflow) -> TIDAK bisa menulis, dan dbt gagal dengan
|
|
# [Errno 13] Permission denied: '/opt/airflow/dbt/sicabe/target'
|
|
# Bahkan pesannya pun tidak terlihat karena log-nya juga tidak bisa ditulis
|
|
# (gejalanya: "exit code 2" tanpa keluaran sama sekali). Arahkan ke /tmp.
|
|
DBT_LOG_PATH: ${DBT_LOG_PATH:-/tmp/dbt/logs}
|
|
DBT_TARGET_PATH: ${DBT_TARGET_PATH:-/tmp/dbt/target}
|
|
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:${AIRFLOW_DB_PASSWORD:?err}@${POSTGRES_HOST:-postgres}:${POSTGRES_INTERNAL_PORT:-5432}/airflow
|
|
AIRFLOW__CORE__EXECUTOR: ${AIRFLOW_EXECUTOR:-LocalExecutor}
|
|
# Secret key konsisten untuk JWT internal (task runner auth)
|
|
AIRFLOW__API_AUTH__JWT_SECRET: ${AIRFLOW_SECRET_KEY:?err}
|
|
# SimpleAuthManager users (format: username:password[:team]).
|
|
# ⚠️ Field ketiga adalah TEAM, bukan role — mengisinya memicu crash
|
|
# "user ... is associated to at least one team and multi-team mode is not
|
|
# configured" (dibuktikan di experiments/om20x). Jadi cukup dua field.
|
|
AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: ${AIRFLOW_SIMPLE_AUTH_USERS:-admin:Admin}
|
|
# Password superuser PostgreSQL (untuk DAG akses sicabe_data)
|
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?err}
|
|
# Token OpenMetadata untuk GE Action (validate_to_openmetadata.py) — dipakai
|
|
# task validasi DAG agar hasil Great Expectations TIDAK berhenti di log,
|
|
# tetapi terkirim ke OpenMetadata (Data Quality Test Suite/Case).
|
|
OPENMETADATA_ADMIN_PASSWORD: ${OPENMETADATA_ADMIN_PASSWORD:?err}
|
|
OM_API_URL: http://${OPENMETADATA_HOST:-openmetadata-server}:8585/api
|
|
# dbt → Trino: service account via HTTPS 8443 langsung (self-signed, cert: false)
|
|
DBT_TRINO_HOST: ${DBT_TRINO_HOST:-trino-coordinator}
|
|
DBT_TRINO_USER: ${DBT_TRINO_USER:-svc_dbt}
|
|
DBT_TRINO_PASSWORD: ${TRINO_SVC_DBT_PASSWORD:?err}
|
|
# ─── OpenLineage → OpenMetadata (experiments/lineage-airflow-om) ───
|
|
# Konfigurasi transport dibaca dari KONEKSI Airflow (`config_conn_id`),
|
|
# BUKAN dari env — supaya token lineage TIDAK perlu ditulis di .env.
|
|
# Koneksi dibuat oleh bootstrap.sh:
|
|
# - `openlineage_config` : extra = {"transport": {...}} dengan
|
|
# auth type `airflow_connection_api_key`
|
|
# - `openmetadata_lineage_api` : password = token bot `lineage-bot`
|
|
# ⚠️ Pakai token BOT `lineage-bot` (role LineageBotRole, bisa UNLIMITED).
|
|
# Token admin hanya bertahan 1 JAM → tidak layak untuk konfigurasi.
|
|
# Kosong (default) = provider INERT: tidak mengirim apa pun.
|
|
AIRFLOW__OPENLINEAGE__CONFIG_CONN_ID: ${AIRFLOW_OPENLINEAGE_CONFIG_CONN_ID:-}
|
|
AIRFLOW__OPENLINEAGE__NAMESPACE: ${AIRFLOW_OPENLINEAGE_NAMESPACE:-}
|
|
# WAJIB saat aktif: tanpa ini proses emisi HANG lalu dimatikan setelah
|
|
# `[openlineage] execution_timeout`, sehingga TIDAK ADA event keluar
|
|
# (lihat experiments/lineage-airflow-om/FASE4-HASIL.md).
|
|
AIRFLOW__OPENLINEAGE__EXECUTE_IN_THREAD: "true"
|
|
# ─── Metrik → statsd-exporter (M2 observabilitas, Review 19) ───
|
|
AIRFLOW__METRICS__STATSD_ON: "true"
|
|
AIRFLOW__METRICS__STATSD_HOST: ${STATSD_HOST:-statsd-exporter}
|
|
AIRFLOW__METRICS__STATSD_PORT: ${STATSD_PORT:-9125}
|
|
AIRFLOW__METRICS__STATSD_PREFIX: airflow
|
|
command: ["bash", "/opt/airflow/scripts/entrypoint.sh", "airflow", "scheduler"]
|
|
healthcheck:
|
|
# Cek scheduler masih alive via airflow jobs check (exit 0 = ada job hidup)
|
|
test: ["CMD-SHELL", "airflow jobs check > /dev/null 2>&1 || exit 1"]
|
|
interval: 15s
|
|
timeout: 10s
|
|
retries: 10
|
|
start_period: 60s
|
|
profiles:
|
|
- phase5
|
|
- all
|
|
|
|
|
|
# ============================================================================
|
|
# PHASE 6 — Governance Layer (Lanjutan): OpenMetadata → Grafana
|
|
# Dependency: Semua layer sebelumnya sudah running
|
|
# ============================================================================
|
|
|
|
openmetadata-opensearch:
|
|
# WAJIB OpenSearch 3.x untuk OpenMetadata 2.0.x (minimum 3.0.0).
|
|
# ⚠️ Hindari 3.3.2: bug Lucene 10.3.1 membuat pencarian mengembalikan HTTP 200
|
|
# dengan 0 hasil (shard gagal senyap) dan OM tidak melaporkannya. 3.4.0+ benar.
|
|
# ⚠️ Index 2.19.6 TIDAK bisa dibaca 3.x — index ini derived, OM membangun ulang.
|
|
image: opensearchproject/opensearch:3.8.0
|
|
container_name: sicabe-opensearch
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
environment:
|
|
discovery.type: single-node
|
|
# OpenSearch 3.x memakai env ini; `plugins.security.disabled` (gaya 2.x)
|
|
# tidak lagi dipakai.
|
|
DISABLE_SECURITY_PLUGIN: "true"
|
|
bootstrap.memory_lock: "true"
|
|
OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m"
|
|
# OPENSEARCH_INITIAL_ADMIN_PASSWORD dihapus: tidak dipakai saat security
|
|
# plugin dimatikan, dan menaruh kredensial di compose melanggar aturan
|
|
# "jangan hardcode kredensial".
|
|
ulimits:
|
|
memlock:
|
|
soft: -1
|
|
hard: -1
|
|
nofile:
|
|
soft: 65536
|
|
hard: 65536
|
|
volumes:
|
|
- opensearch-data:/usr/share/opensearch/data
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "curl -fs http://localhost:9200/_cluster/health > /dev/null || exit 1"]
|
|
interval: 15s
|
|
timeout: 10s
|
|
retries: 12
|
|
start_period: 45s
|
|
ports:
|
|
- "${OPENSEARCH_PORT:-12920}:9200"
|
|
profiles:
|
|
- phase6
|
|
- all
|
|
|
|
# ─── One-shot: migrasi schema OpenMetadata (idempotent) ───────────────────
|
|
# OM tidak auto-migrate saat server start → migrasi harus dijalankan lebih
|
|
# dulu, jika tidak server crash-loop (tabel belum ada). OM 2.0.x memakai
|
|
# perintah yang sama seperti 1.13.x.
|
|
openmetadata-migrate:
|
|
image: openmetadata/server:${OPENMETADATA_VERSION:-2.0.3}
|
|
container_name: sicabe-openmetadata-migrate
|
|
restart: "no"
|
|
networks:
|
|
- sicabe-network
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
entrypoint: ["/bin/sh", "-c"]
|
|
command:
|
|
- "cd /opt/openmetadata && ./bootstrap/openmetadata-ops.sh -d migrate --force"
|
|
environment:
|
|
DB_DRIVER_CLASS: org.postgresql.Driver
|
|
DB_HOST: ${POSTGRES_HOST:-postgres}
|
|
DB_PORT: 5432
|
|
DB_DATABASE: openmetadata_db
|
|
DB_USER: openmetadata
|
|
DB_PASSWORD: ${OPENMETADATA_DB_PASSWORD:?err}
|
|
DB_USER_PASSWORD: ${OPENMETADATA_DB_PASSWORD:?err}
|
|
DB_SCHEME: postgresql
|
|
profiles:
|
|
- phase6
|
|
- all
|
|
|
|
openmetadata-server:
|
|
image: openmetadata/server:${OPENMETADATA_VERSION:-2.0.3}
|
|
container_name: sicabe-openmetadata
|
|
labels:
|
|
- "traefik.enable=true"
|
|
- "traefik.http.routers.om.rule=Host(`om.${CLUSTER_DOMAIN:-localhost}`)"
|
|
- "traefik.http.routers.om.entrypoints=websecure"
|
|
- "traefik.http.routers.om.tls=true"
|
|
- "traefik.http.routers.om.tls.certresolver=le"
|
|
- "traefik.http.routers.om.tls.domains[0].main=${CLUSTER_DOMAIN:-localhost}"
|
|
- "traefik.http.routers.om.tls.domains[0].sans=*.${CLUSTER_DOMAIN:-localhost}"
|
|
- "traefik.http.services.om.loadbalancer.server.port=8585"
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
depends_on:
|
|
openmetadata-opensearch:
|
|
condition: service_healthy
|
|
postgres:
|
|
condition: service_healthy
|
|
openmetadata-migrate:
|
|
condition: service_completed_successfully
|
|
volumes:
|
|
- openmetadata-data:/data
|
|
environment:
|
|
DB_DRIVER_CLASS: org.postgresql.Driver
|
|
DB_HOST: ${POSTGRES_HOST:-postgres}
|
|
DB_PORT: 5432
|
|
DB_DATABASE: openmetadata_db
|
|
DB_USER: openmetadata
|
|
DB_PASSWORD: ${OPENMETADATA_DB_PASSWORD:?err}
|
|
# OM 2.0.x membaca DB_PASSWORD; DB_USER_PASSWORD dipertahankan karena
|
|
# sebagian rilis/tooling masih membacanya (migrasi & ops script).
|
|
DB_USER_PASSWORD: ${OPENMETADATA_DB_PASSWORD:?err}
|
|
# DB_SCHEME wajib di-override: default OM adalah mysql.
|
|
DB_SCHEME: postgresql
|
|
# Elasticsearch/OpenSearch connection
|
|
ELASTICSEARCH_HOST: ${OPENSEARCH_HOST:-openmetadata-opensearch}
|
|
ELASTICSEARCH_PORT: 9200
|
|
SEARCH_TYPE: opensearch
|
|
# OM membaca SERVER_PORT (bukan OPENMETADATA_SERVER_PORT); pakai port
|
|
# internal 8585 (bukan port host) agar konsisten dgn mapping 12085:8585.
|
|
SERVER_PORT: 8585
|
|
# Endpoint API OM yang DITULIS OM ke config pipeline yang dibangkitkannya
|
|
# (openmetadata.yaml: metadataApiEndpoint = ${SERVER_HOST_API_URL:-localhost:8585/api}).
|
|
# Default localhost TIDAK bisa dijangkau dari container Airflow -> DAG hasil
|
|
# plugin gagal: 'Transport failure calling http://localhost:8585/api/...'.
|
|
# Karena ingestion/DQ berjalan DI DALAM jaringan compose, pakai nama service.
|
|
SERVER_HOST_API_URL: http://${OPENMETADATA_HOST:-openmetadata-server}:8585/api
|
|
# CATATAN: SERVER_HOST_API_URL sengaja TETAP internal — ia dipakai sisi SERVER
|
|
# (pipelineServiceClientConfiguration.metadataApiEndpoint) oleh Airflow/ingestion
|
|
# untuk memanggil balik OM. Kalau diarahkan ke URL publik, ingestion justru rusak
|
|
# bila nama itu tidak bisa diresolusi container.
|
|
#
|
|
# OPENMETADATA_SERVER_URL = URL yang dipakai MEMBANGUN LINK di email/notifikasi
|
|
# (browser-facing). OM tidak punya env var "UI base URL": UI memakai path relatif,
|
|
# jadi browser memakai origin tempat ia dimuat — tidak perlu diatur.
|
|
OPENMETADATA_SERVER_URL: ${OPENMETADATA_PUBLIC_URL:-http://localhost:${OPENMETADATA_PORT:-12085}}
|
|
# OIDC SSO via Keycloak (bab 4.6.1.5) — set ke 'basic' untuk local login dulu
|
|
AUTHENTICATION_PROVIDER: basic
|
|
# OIDC_CLIENT_ID: openmetadata
|
|
# OIDC_CLIENT_SECRET: "${OPENMETADATA_OIDC_SECRET:-om-secret}"
|
|
# OIDC_DISCOVERY_URI: http://keycloak:8080/realms/${SICABE_REALM:-sicabe}/.well-known/openid-configuration
|
|
# OIDC_SERVER_URL: http://keycloak:8080/realms/${SICABE_REALM:-sicabe}
|
|
# Admin user bootstrap
|
|
OPENMETADATA_ADMIN_PASSWORD: ${OPENMETADATA_ADMIN_PASSWORD:-openmetadata123}
|
|
# ─── Pipeline Service Client (Airflow) ──────────────────────────────────
|
|
# Tanpa ini OM menunjuk http://localhost:8080 (di dalam container = kosong),
|
|
# sehingga tombol "Test Connection" di UI DISABLED dengan tooltip
|
|
# "Platform Service Client Unavailable" — pesan itu label UI, bukan plugin
|
|
# yang kurang, dan bukan spesifik Trino. Endpoint + kredensial Airflow
|
|
# (SimpleAuthManager: admin + AIRFLOW_ADMIN_PASSWORD) membuat tombol aktif
|
|
# sekaligus memungkinkan UI menjalankan ingestion/DQ pipeline lewat Airflow.
|
|
PIPELINE_SERVICE_CLIENT_ENABLED: "true"
|
|
PIPELINE_SERVICE_CLIENT_ENDPOINT: http://${AIRFLOW_WEBSERVER_HOST:-airflow-webserver}:8080
|
|
AIRFLOW_USERNAME: admin
|
|
AIRFLOW_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD:?err}
|
|
healthcheck:
|
|
# Image OM tidak menyediakan curl, pakai wget
|
|
# Healthcheck HARUS pakai port internal container (8585), bukan
|
|
# OPENMETADATA_PORT (12085 = port host) — jika tidak, selalu unhealthy.
|
|
test: ["CMD-SHELL", "wget -q -O - http://localhost:8585/ > /dev/null 2>&1 || exit 1"]
|
|
interval: 20s
|
|
timeout: 15s
|
|
retries: 15
|
|
start_period: 120s
|
|
ports:
|
|
- "${OPENMETADATA_PORT:-12085}:8585"
|
|
profiles:
|
|
- phase6
|
|
- all
|
|
|
|
# ─── statsd-exporter: metrik Airflow (StatsD) → Prometheus ────────────────
|
|
# Airflow 3 mengirim metrik lewat protokol StatsD (UDP). statsd-exporter
|
|
# mengubahnya ke format Prometheus di :9102, yang lalu di-scrape job `airflow`.
|
|
# Ditempatkan di tier `governance` (bersama Prometheus/Grafana) — pada blueprint
|
|
# a3/b5 tier itu se-host dengan `pipeline` (Airflow), sehingga UDP Airflow→statsd
|
|
# tetap lokal. Lihat lib/cluster/inventory.py (TIER_SERVICES).
|
|
#
|
|
# ✅ PENGIRIM: paket Python `statsd` kini BAKED-IN di image Airflow (ditambahkan ke
|
|
# requirements.332.in + image di-rebuild 2026-10-06). Sebelumnya image belum
|
|
# memuatnya sehingga Airflow memakai NoStatsLogger dan tidak ada metrik terkirim.
|
|
statsd-exporter:
|
|
image: prom/statsd-exporter:${STATSD_EXPORTER_VERSION:-v0.28.0}
|
|
container_name: sicabe-statsd-exporter
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
# 9125/udp = listener StatsD (internal, dipakai Airflow); 9102 = metrik Prometheus.
|
|
ports:
|
|
- "${STATSD_EXPORTER_PORT:-12091}:9102"
|
|
profiles:
|
|
- phase6
|
|
- all
|
|
|
|
# ─── Prometheus: sumber metrik untuk datasource Grafana ───────────────────
|
|
# Grafana TIDAK dapat membaca metrik langsung dari MinIO (MinIO = API S3,
|
|
# bukan server Prometheus). Prometheus ini yang meng-scrape metrik MinIO &
|
|
# StarRocks, lalu Grafana membacanya dari sini.
|
|
prometheus:
|
|
image: prom/prometheus:${PROMETHEUS_VERSION:-v3.6.0}
|
|
container_name: sicabe-prometheus
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
extra_hosts:
|
|
# `host.docker.internal` menunjuk Docker host itu sendiri — dipakai target
|
|
# default node.yml. `host-gateway` tersedia sejak Docker 20.10.
|
|
- "host.docker.internal:host-gateway"
|
|
volumes:
|
|
- ./config/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
|
# Daftar target node_exporter (file_sd) — dapat diubah tanpa restart Prometheus.
|
|
- ./config/prometheus/targets:/etc/prometheus/targets:ro
|
|
- prometheus-data:/prometheus
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "wget -q -O - http://localhost:9090/-/healthy >/dev/null 2>&1 || exit 1"]
|
|
interval: 15s
|
|
timeout: 10s
|
|
retries: 10
|
|
start_period: 30s
|
|
ports:
|
|
- "${PROMETHEUS_PORT:-12090}:9090"
|
|
profiles:
|
|
- phase6
|
|
- all
|
|
|
|
grafana:
|
|
image: grafana/grafana:${GRAFANA_VERSION:-13.1}
|
|
container_name: sicabe-grafana
|
|
labels:
|
|
- "traefik.enable=true"
|
|
- "traefik.http.routers.grafana.rule=Host(`grafana.${CLUSTER_DOMAIN:-localhost}`)"
|
|
- "traefik.http.routers.grafana.entrypoints=websecure"
|
|
- "traefik.http.routers.grafana.tls=true"
|
|
- "traefik.http.routers.grafana.tls.certresolver=le"
|
|
- "traefik.http.routers.grafana.tls.domains[0].main=${CLUSTER_DOMAIN:-localhost}"
|
|
- "traefik.http.routers.grafana.tls.domains[0].sans=*.${CLUSTER_DOMAIN:-localhost}"
|
|
- "traefik.http.services.grafana.loadbalancer.server.port=3000"
|
|
restart: unless-stopped
|
|
networks:
|
|
- sicabe-network
|
|
volumes:
|
|
- grafana-data:/var/lib/grafana
|
|
- ./config/grafana/datasources:/etc/grafana/provisioning/datasources:ro
|
|
- ./config/grafana/dashboards:/etc/grafana/provisioning/dashboards:ro
|
|
environment:
|
|
GF_SECURITY_ADMIN_USER: admin
|
|
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-grafana123}
|
|
# Root URL harus pakai host port (bukan port internal 3000) agar
|
|
# redirect_uri OIDC benar saat GRAFANA_PORT != 3000
|
|
GF_SERVER_ROOT_URL: ${GRAFANA_PUBLIC_URL:-http://localhost:${GRAFANA_PORT:-12300}}
|
|
GF_AUTH_GENERIC_OAUTH_ENABLED: "true"
|
|
GF_AUTH_GENERIC_OAUTH_NAME: Keycloak
|
|
GF_AUTH_GENERIC_OAUTH_CLIENT_ID: grafana
|
|
GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: "${GRAFANA_CLIENT_SECRET:?err}"
|
|
GF_AUTH_GENERIC_OAUTH_AUTH_URL: http://localhost:${KEYCLOAK_PORT:-12080}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/auth
|
|
GF_AUTH_GENERIC_OAUTH_TOKEN_URL: "${KEYCLOAK_INTERNAL_URL:-http://${KEYCLOAK_HOST:-keycloak}:${KEYCLOAK_INTERNAL_PORT:-8080}}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/token"
|
|
GF_AUTH_GENERIC_OAUTH_API_URL: "${KEYCLOAK_INTERNAL_URL:-http://${KEYCLOAK_HOST:-keycloak}:${KEYCLOAK_INTERNAL_PORT:-8080}}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/userinfo"
|
|
GF_AUTH_GENERIC_OAUTH_SCOPES: openid profile email
|
|
GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH: contains(roles[*], 'admin') && 'Admin' || 'Viewer'
|
|
GF_AUTH_DISABLE_LOGIN_FORM: "false"
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:3000/api/health"]
|
|
interval: 15s
|
|
timeout: 10s
|
|
retries: 10
|
|
start_period: 30s
|
|
ports:
|
|
- "${GRAFANA_PORT:-12300}:3000"
|
|
profiles:
|
|
- phase6
|
|
- all
|
|
# ── Ingress (T-20, langkah 2) — hanya pada profile `edge` ────────────────────
|
|
# Rute diambil dari LABEL container (provider docker, exposedByDefault=false),
|
|
# jadi daftar rute = label pada layanan; tidak ada konfigurasi rute terpisah.
|
|
# TLS: sertifikat wildcard *.${CLUSTER_DOMAIN} via Let's Encrypt DNS-01.
|
|
# Tanpa kredensial DNS, Traefik TETAP start dan memakai sertifikat self-signed
|
|
# bawaan — jadi routing masih bisa diuji sebelum sertifikat terbit.
|
|
traefik:
|
|
image: traefik:${TRAEFIK_VERSION:-v3.7}
|
|
container_name: sicabe-traefik
|
|
restart: unless-stopped
|
|
profiles:
|
|
- edge
|
|
networks:
|
|
- sicabe-network
|
|
command:
|
|
- "--providers.docker=true"
|
|
- "--providers.docker.exposedbydefault=false"
|
|
- "--providers.file.directory=/etc/traefik/dynamic"
|
|
- "--providers.file.watch=true"
|
|
- "--entrypoints.web.address=:80"
|
|
- "--entrypoints.web.http.redirections.entrypoint.to=websecure"
|
|
- "--entrypoints.web.http.redirections.entrypoint.scheme=https"
|
|
- "--entrypoints.web.http.redirections.entrypoint.permanent=true"
|
|
- "--entrypoints.websecure.address=:443"
|
|
- "--certificatesresolvers.le.acme.email=${TRAEFIK_ACME_EMAIL:-admin@${CLUSTER_DOMAIN:-sicabe.com}}"
|
|
- "--certificatesresolvers.le.acme.storage=/acme/acme.json"
|
|
- "--certificatesresolvers.le.acme.dnschallenge.provider=${TRAEFIK_DNS_PROVIDER:-cloudflare}"
|
|
# Tunggu DNS menyebar sebelum Let's Encrypt memeriksa. Default Traefik 0s — dan
|
|
# untuk DNS-01 dengan nama yang sama (apex + wildcard sama-sama menulis
|
|
# _acme-challenge.<domain>) itu terlalu cepat: LE bisa membaca record lama.
|
|
# Nama opsi yang benar adalah propagation.delayBeforeChecks.
|
|
- "--certificatesresolvers.le.acme.dnschallenge.propagation.delayBeforeChecks=${TRAEFIK_ACME_PROPAGATION_DELAY:-30s}"
|
|
- "--certificatesresolvers.le.acme.caserver=${TRAEFIK_ACME_CASERVER:-https://acme-staging-v02.api.letsencrypt.org/directory}"
|
|
environment:
|
|
CF_DNS_API_TOKEN: ${CF_DNS_API_TOKEN:-}
|
|
ports:
|
|
- "${TRAEFIK_BIND_ADDR:-0.0.0.0}:${TRAEFIK_HTTP_PORT:-80}:80"
|
|
- "${TRAEFIK_BIND_ADDR:-0.0.0.0}:${TRAEFIK_HTTPS_PORT:-443}:443"
|
|
volumes:
|
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
|
# Named volume (bukan bind mount) supaya acme.json dibuat 0600 oleh Traefik:
|
|
# berkas itu WAJIB mode 600, kalau tidak Traefik menolak start.
|
|
- traefik-acme:/acme
|
|
# serversTransports (di atas) tidak bisa lewat label -> berkas dinamis.
|
|
- ./config/traefik/dynamic.yml:/etc/traefik/dynamic/dynamic.yml:ro
|