first commit
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
# =============================================================================
|
||||
# Trino — Access Control via OPA (built-in, deny-by-default)
|
||||
# =============================================================================
|
||||
# access-control.name=opa (built-in Trino). Policy Rego di OPA
|
||||
# (config/opa/policies/trino.rego) menerima input dari Trino dan memutuskan
|
||||
# per-objek/per-kolom secara LOKAL (ACL di config/opa/data.json).
|
||||
#
|
||||
# Grup user berasal dari LDAP group provider (lihat group-provider.properties),
|
||||
# lalu dicocokkan dengan data.roles.<grup> di data.json.
|
||||
#
|
||||
# OPA: http://opa:8181/v1/data/trino/allow
|
||||
# =============================================================================
|
||||
|
||||
access-control.name=opa
|
||||
opa.policy.uri=http://opa:8181/v1/data/trino/allow
|
||||
# Selama debugging biarkan true; matikan saat sudah stabil (log request OPA).
|
||||
opa.log-requests=true
|
||||
@@ -0,0 +1,30 @@
|
||||
# =============================================================================
|
||||
# Trino Iceberg Catalog — Project Nessie (Nessie REST API v2)
|
||||
# =============================================================================
|
||||
# Catalog ini menghubungkan Trino ke tabel Iceberg yang dikelola Nessie
|
||||
# (versioning git-like). Data file + metadata disimpan di MinIO (S3-compatible).
|
||||
#
|
||||
# Catatan: Nessie belum mendukung Iceberg spec v3 → selalu format-version=2
|
||||
# (default Trino = 2, dibiarkan).
|
||||
# =============================================================================
|
||||
|
||||
# --- Connector ---
|
||||
connector.name=iceberg
|
||||
|
||||
# --- Catalog Type: Nessie ---
|
||||
iceberg.catalog.type=nessie
|
||||
iceberg.nessie-catalog.uri=${ENV:NESSIE_URL}/api/v2
|
||||
iceberg.nessie-catalog.ref=main
|
||||
iceberg.nessie-catalog.default-warehouse-dir=s3://${ENV:MINIO_BUCKET_ICEBERG_DATA}/warehouse
|
||||
|
||||
# --- S3 Native Filesystem (MinIO / S3 existing) ---
|
||||
fs.s3.enabled=true
|
||||
s3.endpoint=${ENV:S3_ENDPOINT}
|
||||
s3.path-style-access=true
|
||||
s3.aws-access-key=${ENV:S3_ACCESS_KEY}
|
||||
s3.aws-secret-key=${ENV:S3_SECRET_KEY}
|
||||
s3.region=${ENV:S3_REGION}
|
||||
|
||||
# --- Table Defaults ---
|
||||
iceberg.file-format=PARQUET
|
||||
iceberg.compression-codec=ZSTD
|
||||
@@ -0,0 +1,20 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDVTCCAj2gAwIBAgIUETRg8AYPjCnKkFChXiQiP62JG9YwDQYJKoZIhvcNAQEL
|
||||
BQAwEDEOMAwGA1UEAwwFdHJpbm8wHhcNMjYwOTI4MTIwOTMxWhcNMjgxMjMxMTIw
|
||||
OTMxWjAQMQ4wDAYDVQQDDAV0cmlubzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
|
||||
AQoCggEBAMyEX/NwRqEuT3SweBZeHcUxFKgeFEY89trx9D2nkuP7OMiHRFyeKfuL
|
||||
Kmyyyta9QQDU1EU1cHyhgHwhXLzFxJ6QnlLgSxOP/hTQlF6qL2dvMgKDzBmo51/I
|
||||
e6nwNYlYC9wzOPLNVdqZkqraSk2xmnnMB4fM9L+xisZJOQ7VpprzTNIXigus7q8M
|
||||
CsWPXu+dr9XJSKYeqOvV5jIQGhSE7yyb3zC2nHG/qQ3YiBZc7OMtkUti+UT2/oGK
|
||||
tErYxBj8uCtsqLGCFjpGoioOIFAUmqriBzSrSrOwzkkeF3mI5cok4P4mBmwQcRZk
|
||||
3TBrQW8HrCB8oVq3kNgsuM8dRJZ3NzsCAwEAAaOBpjCBozAdBgNVHQ4EFgQUvcvH
|
||||
CUrnQiy1yYFCsrrcNcYukmswHwYDVR0jBBgwFoAUvcvHCUrnQiy1yYFCsrrcNcYu
|
||||
kmswDwYDVR0TAQH/BAUwAwEB/zBQBgNVHREESTBHggV0cmlub4IJbG9jYWxob3N0
|
||||
ggxzaWNhYmUtdHJpbm+CEXRyaW5vLWNvb3JkaW5hdG9yggx0cmluby13b3JrZXKH
|
||||
BH8AAAEwDQYJKoZIhvcNAQELBQADggEBAMrDCPDqEN/d+LhQ3bNoiVBdjoLemML5
|
||||
G+Torrv7VIcD2VDdUbWQ7WfzXmXS0WTuzVbqD5I6Fus1NK2GFKB0q2kP9whOrmqu
|
||||
lE6Sv2b1g38H3PnjzWQbY34lNTRzJr/PmxvxgTs1GIWFu4fcd7GsNocF3ZciLf/5
|
||||
4Ky51U8JG3ukzPjkFSdAY03aZ2d2Q2VqZkmLJyHcjJDH4+u3zveEbUxSSTgM9Kzx
|
||||
NxaYiqXeUzd65Y+9ALsECuDMQGLkMl9acmgJIO+vH/3sfM9WiDksSN9WJhR3R2UX
|
||||
B5zp85Reh+ehobxQsbIZU+IXR6cEy8SfgQNzxlFVyOQ1/V4MgFckFNw=
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,67 @@
|
||||
# =============================================================================
|
||||
# Trino COORDINATOR — config.properties
|
||||
# -----------------------------------------------------------------------------
|
||||
# Dipisah dari worker sejak awal (tier R11: coord). Coordinator melayani KLIEN
|
||||
# (HTTPS 8443 + OAuth2/Basic) dan mengatur penjadwalan; worker yang mengeksekusi
|
||||
# task (node-scheduler.include-coordinator=false).
|
||||
#
|
||||
# Klien TIDAK melihat worker; worker mendaftar lewat discovery.uri di bawah.
|
||||
# Perubahan dari konfigurasi single-node sebelumnya:
|
||||
# node-scheduler.include-coordinator : true → false
|
||||
# discovery.uri : http://localhost:8080 → http://trino-coordinator:8080
|
||||
# =============================================================================
|
||||
coordinator=true
|
||||
node-scheduler.include-coordinator=false
|
||||
http-server.http.port=8080
|
||||
# HTTPS untuk Web UI OAuth2 — OAuth2 Web UI WAJIB secure request (TLS),
|
||||
# jika tidak Trino mengalihkan /ui/ ke /ui/static/disabled.html.
|
||||
# HTTP (8080) TIDAK lagi melayani tanpa kredensial (allow-insecure-over-http=false);
|
||||
# klien service/pipeline lewat nginx proxy → HTTPS.
|
||||
http-server.https.enabled=true
|
||||
http-server.https.port=8443
|
||||
http-server.https.keystore.path=/etc/trino/keystore.p12
|
||||
http-server.https.keystore.key=trino_ssl_dev
|
||||
# HTTP 8080 tetap dipakai untuk komunikasi INTERNAL coordinator↔worker
|
||||
# (worker mendaftar ke coordinator lewat endpoint ini).
|
||||
discovery.uri=http://trino-coordinator:8080
|
||||
catalog.management=${ENV:CATALOG_MANAGEMENT}
|
||||
|
||||
# Authentication: hybrid — OAuth2 (end user, Keycloak SSO) + password/Basic
|
||||
# (service account). Keduanya berlaku di HTTPS (8443). HTTP polos DIMATIKAN
|
||||
# (allow-insecure-over-http=false) → klien service lewat nginx proxy (HTTP→HTTPS).
|
||||
http-server.authentication.type=oauth2,password
|
||||
http-server.authentication.allow-insecure-over-http=false
|
||||
# Di belakang reverse proxy TLS (Traefik), Trino harus memercayai X-Forwarded-* supaya
|
||||
# skema publik (https) dan alamat klien terbaca benar — termasuk untuk redirect OAuth.
|
||||
# Nilainya dari env: diisi `true` oleh generator saat tls.mode != none, `false` bila tidak.
|
||||
# Trino GAGAL START bila variabel ini tidak ada, jadi compose selalu mendefinisikannya.
|
||||
http-server.process-forwarded=${ENV:TRINO_PROCESS_FORWARDED}
|
||||
# Issuer = URL PUBLIK Keycloak — wajib SAMA PERSIS dengan iss claim JWT.
|
||||
# Diambil dari KEYCLOAK_PUBLIC_URL (bukan ditulis tetap): nilainya berubah antara
|
||||
# single-host (localhost) dan multi-host tanpa TLS (http://<alamat-host-control>:12080).
|
||||
# Kalau di-hardcode, issuer di klaster multi-host tidak cocok dan token ditolak.
|
||||
http-server.authentication.oauth2.issuer=${ENV:KEYCLOAK_PUBLIC_URL}/realms/sicabe
|
||||
# Matikan OIDC discovery — Trino tidak bisa mengakses URL publik dari dalam container.
|
||||
# Endpoint di-set manual: browser pakai URL publik, server-ke-server pakai keycloak:8080.
|
||||
http-server.authentication.oauth2.oidc.discovery=false
|
||||
# Endpoint browser-facing (auth redirect) — harus URL publik juga
|
||||
http-server.authentication.oauth2.auth-url=${ENV:KEYCLOAK_PUBLIC_URL}/realms/sicabe/protocol/openid-connect/auth
|
||||
# Endpoint server-side (token exchange & JWKS) — reachable dari container
|
||||
http-server.authentication.oauth2.token-url=http://keycloak:${ENV:KEYCLOAK_INTERNAL_PORT}/realms/sicabe/protocol/openid-connect/token
|
||||
http-server.authentication.oauth2.jwks-url=http://keycloak:${ENV:KEYCLOAK_INTERNAL_PORT}/realms/sicabe/protocol/openid-connect/certs
|
||||
http-server.authentication.oauth2.client-id=trino
|
||||
http-server.authentication.oauth2.client-secret=${ENV:TRINO_CLIENT_SECRET}
|
||||
# Username Trino = preferred_username (bukan 'sub') agar cocok dengan uid LDAP
|
||||
# pada LDAP group provider.
|
||||
http-server.authentication.oauth2.principal-field=preferred_username
|
||||
# Token Keycloak client 'trino' membawa aud='account'; trust audience ini.
|
||||
http-server.authentication.oauth2.additional-audiences=account
|
||||
|
||||
# Web UI: aktifkan OAuth2 untuk UI. Web UI OAuth2 WAJIB HTTPS (lihat blok https
|
||||
# di atas) — bila request tidak secure, Trino mengalihkan /ui/ ke disabled.html.
|
||||
# allow-insecure-over-http hanya berlaku untuk REST API/JDBC, BUKAN Web UI.
|
||||
web-ui.authentication.type=oauth2
|
||||
|
||||
# Shared secret untuk internal communication (required saat auth enabled)
|
||||
# WAJIB identik dengan worker — tanpa ini worker ditolak coordinator.
|
||||
internal-communication.shared-secret=trino-internal-secret-dev-only
|
||||
@@ -0,0 +1,18 @@
|
||||
# =============================================================================
|
||||
# Trino WORKER — config.properties
|
||||
# -----------------------------------------------------------------------------
|
||||
# Dipisah dari coordinator sejak awal (tier R11: engine). Worker TIDAK melayani
|
||||
# klien: tidak ada HTTPS, tidak ada autentikasi OAuth2/Basic, tidak ada port yang
|
||||
# di-expose ke host. Worker hanya:
|
||||
# 1. mendaftar & menerima task dari coordinator lewat HTTP 8080,
|
||||
# 2. membaca katalog yang sama (etc/catalog/*.properties).
|
||||
#
|
||||
# `internal-communication.shared-secret` WAJIB identik dengan coordinator —
|
||||
# inilah yang mengautentikasi worker (bukan password/OAuth2 klien).
|
||||
# =============================================================================
|
||||
coordinator=false
|
||||
http-server.http.port=8080
|
||||
# Daftar ke coordinator (bukan localhost — worker bukan coordinator-nya sendiri).
|
||||
discovery.uri=http://trino-coordinator:8080
|
||||
|
||||
internal-communication.shared-secret=trino-internal-secret-dev-only
|
||||
@@ -0,0 +1,28 @@
|
||||
# =============================================================================
|
||||
# Trino — LDAP Group Provider
|
||||
# =============================================================================
|
||||
# Grup user di-resolve dari OpenLDAP (source of truth grup). User Trino
|
||||
# = preferred_username dari Keycloak (lihat config.properties principal-field).
|
||||
# Nama grup yang dikembalikan = cn (mis. data-admin, data-engineer, ...),
|
||||
# lalu dicocokkan dengan data.roles.<grup> di config/opa/data.json.
|
||||
# =============================================================================
|
||||
|
||||
group-provider.name=ldap
|
||||
|
||||
# Alamat LDAP dari environment (LDAP_URL) supaya bisa diarahkan ke LDAP EXISTING.
|
||||
# Default compose: ldap://openldap:389 (container lokal).
|
||||
ldap.url=${ENV:LDAP_URL}
|
||||
ldap.allow-insecure=true
|
||||
# DN akun BIND (bukan harus admin): boleh akun read-only dgn hak baca ke ou=users/ou=groups.
|
||||
# Diarahkan dari .env (LDAP_ADMIN_DN) supaya LDAP EXISTING dgn bind DN berbeda bisa dipakai.
|
||||
# Env-nya WAJIB diteruskan ke container (lihat docker-compose.yml trino-coordinator).
|
||||
ldap.admin-user=${ENV:LDAP_ADMIN_DN}
|
||||
ldap.admin-password=${ENV:LDAP_ADMIN_PASSWORD}
|
||||
ldap.user-base-dn=ou=users,dc=sicabe,dc=local
|
||||
ldap.user-search-filter=(uid={0})
|
||||
ldap.group-name-attribute=cn
|
||||
|
||||
# Filtering mode: cari group di ou=groups yg member-nya = DN user
|
||||
ldap.use-group-filter=true
|
||||
ldap.group-base-dn=ou=groups,dc=sicabe,dc=local
|
||||
ldap.group-search-member-attribute=member
|
||||
@@ -0,0 +1,31 @@
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — konfigurasi sidecar `trino-jmx-exporter` (M3 observabilitas)
|
||||
# =============================================================================
|
||||
# Dibaca `jmx_prometheus_standalone.jar` (image bitnami/jmx-exporter):
|
||||
# java -jar jmx_prometheus_standalone.jar 9404 /config/config.yml
|
||||
#
|
||||
# hostPort : target JMX coordinator via RMI — HARUS sama dengan port &
|
||||
# java.rmi.server.hostname di config/trino/jvm-coordinator.config.
|
||||
#
|
||||
# includeObjectNames : PEMBATAS CARDINALITY. Tanpa ini (`rules: ".*"` saja)
|
||||
# jmx-exporter memaparkan SEMUA MBean (~9.700 metrik) — termasuk
|
||||
# io.trino.filesystem.s3, io.airlift.*, org.eclipse.jetty.*, java.lang.*,
|
||||
# trino.server, trino.security, dst yang tidak dipakai.
|
||||
# Di sini dibatasi ke domain EKSEKUSI Trino yang DIPAKAI saja:
|
||||
# trino.execution:* → QueryManager, QueryExecution, TaskManager
|
||||
# trino.execution.executor.* → task executor (timesharing/dedicated)
|
||||
# trino.execution.resourcegroups:* → resource group
|
||||
# SENGAJA DIKECUALIKAN: trino.execution.scheduler[.faulttolerant]*
|
||||
# (≈1.614 metrik statistik penjadwalan FT — stack ini tidak memakai
|
||||
# fault-tolerant execution). Tambahkan kembali bila FT dipakai.
|
||||
# (Konsekuensi: metrik JVM `java_lang_memory_*` juga tidak ada.)
|
||||
---
|
||||
hostPort: trino-coordinator:9080
|
||||
lowercaseOutputName: true
|
||||
lowercaseOutputLabelNames: true
|
||||
includeObjectNames:
|
||||
- "trino.execution:*"
|
||||
- "trino.execution.executor.*:*"
|
||||
- "trino.execution.resourcegroups:*"
|
||||
rules:
|
||||
- pattern: ".*"
|
||||
@@ -0,0 +1,32 @@
|
||||
# =============================================================================
|
||||
# Trino — JVM Tuning
|
||||
# Referensi arsitektur: bab 4.3.1.7
|
||||
# =============================================================================
|
||||
-server
|
||||
# Heap COORDINATOR: ringan (penjadwalan + auth); task dieksekusi worker.
|
||||
-Xmx512m
|
||||
-XX:+UseG1GC
|
||||
-XX:G1HeapRegionSize=32M
|
||||
-XX:+UseGCOverheadLimit
|
||||
-XX:+ExplicitGCInvokesConcurrent
|
||||
-XX:+HeapDumpOnOutOfMemoryError
|
||||
-XX:+ExitOnOutOfMemoryError
|
||||
-XX:HeapDumpPath=/data/trino/heapdump
|
||||
# Diperlukan oleh Trino 450 untuk Java < 22.0.2 (JDK-8329528)
|
||||
-XX:+UnlockDiagnosticVMOptions
|
||||
-XX:G1NumCollectionsKeepPinned=10000000
|
||||
-Djdk.attach.allowAttachSelf=true
|
||||
-Djava.security.egd=file:/dev/urandom
|
||||
# ─── JMX remote (M3 observabilitas, Review 19) ───────────────────────────────
|
||||
# Dibaca sidecar `trino-jmx-exporter` via RMI. Port 9080 hanya dalam jaringan
|
||||
# docker `sicabe-network` (tidak ada `ports:` yang mengeksposnya ke host), karena
|
||||
# itu authenticate=false dapat diterima di sini.
|
||||
# ⚠️ `java.rmi.server.hostname` WAJIB = nama container (bukan default = ID container),
|
||||
# agar stub RMI bisa diresolusi sidecar di jaringan yang sama.
|
||||
-Dcom.sun.management.jmxremote
|
||||
-Dcom.sun.management.jmxremote.port=9080
|
||||
-Dcom.sun.management.jmxremote.rmi.port=9080
|
||||
-Dcom.sun.management.jmxremote.authenticate=false
|
||||
-Dcom.sun.management.jmxremote.ssl=false
|
||||
-Dcom.sun.management.jmxremote.local.only=false
|
||||
-Djava.rmi.server.hostname=trino-coordinator
|
||||
@@ -0,0 +1,19 @@
|
||||
# =============================================================================
|
||||
# Trino — JVM Tuning
|
||||
# Referensi arsitektur: bab 4.3.1.7
|
||||
# =============================================================================
|
||||
-server
|
||||
# Heap WORKER: menampung eksekusi task (scan, join, agregasi).
|
||||
-Xmx1G
|
||||
-XX:+UseG1GC
|
||||
-XX:G1HeapRegionSize=32M
|
||||
-XX:+UseGCOverheadLimit
|
||||
-XX:+ExplicitGCInvokesConcurrent
|
||||
-XX:+HeapDumpOnOutOfMemoryError
|
||||
-XX:+ExitOnOutOfMemoryError
|
||||
-XX:HeapDumpPath=/data/trino/heapdump
|
||||
# Diperlukan oleh Trino 450 untuk Java < 22.0.2 (JDK-8329528)
|
||||
-XX:+UnlockDiagnosticVMOptions
|
||||
-XX:G1NumCollectionsKeepPinned=10000000
|
||||
-Djdk.attach.allowAttachSelf=true
|
||||
-Djava.security.egd=file:/dev/urandom
|
||||
Binary file not shown.
@@ -0,0 +1,6 @@
|
||||
# =============================================================================
|
||||
# Trino — Logging Configuration
|
||||
# =============================================================================
|
||||
io.trino=INFO
|
||||
io.trino.server.security=DEBUG
|
||||
io.trino.plugin.iceberg=INFO
|
||||
@@ -0,0 +1,13 @@
|
||||
# =============================================================================
|
||||
# Trino — Password authenticator (HTTP Basic) untuk SERVICE ACCOUNT
|
||||
# -----------------------------------------------------------------------------
|
||||
# Dipakai bersama OAuth2 (lihat config.properties: type=oauth2,password):
|
||||
# - End user : OAuth2 (Keycloak SSO) → HTTPS Web UI/JDBC
|
||||
# - Service : HTTP Basic (file bcrypt) → lewat nginx proxy (HTTP→HTTPS)
|
||||
#
|
||||
# password.db di-generate oleh bootstrap.sh/install.sh dari .env.
|
||||
# Pra-syarat: `htpasswd` tersedia di host. JANGAN commit (lihat .gitignore).
|
||||
# =============================================================================
|
||||
password-authenticator.name=file
|
||||
file.password-file=/etc/trino/password.db
|
||||
file.refresh-period=5s
|
||||
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"rootGroups": [
|
||||
{
|
||||
"name": "global",
|
||||
"softMemoryLimit": "80%",
|
||||
"hardConcurrencyLimit": 100,
|
||||
"maxQueued": 1000,
|
||||
"schedulingPolicy": "weighted",
|
||||
"jmxExport": true,
|
||||
"subGroups": [
|
||||
{
|
||||
"name": "data_definition",
|
||||
"softMemoryLimit": "10%",
|
||||
"hardConcurrencyLimit": 5,
|
||||
"maxQueued": 100,
|
||||
"schedulingWeight": 1
|
||||
},
|
||||
{
|
||||
"name": "data_analyst",
|
||||
"softMemoryLimit": "30%",
|
||||
"hardConcurrencyLimit": 20,
|
||||
"maxQueued": 200,
|
||||
"schedulingWeight": 2
|
||||
},
|
||||
{
|
||||
"name": "data_engineer",
|
||||
"softMemoryLimit": "40%",
|
||||
"hardConcurrencyLimit": 30,
|
||||
"maxQueued": 300,
|
||||
"schedulingWeight": 3
|
||||
},
|
||||
{
|
||||
"name": "pipeline",
|
||||
"softMemoryLimit": "20%",
|
||||
"hardConcurrencyLimit": 10,
|
||||
"maxQueued": 100,
|
||||
"schedulingWeight": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"selectors": [
|
||||
{
|
||||
"userGroup": "admin",
|
||||
"group": "global.data_engineer"
|
||||
},
|
||||
{
|
||||
"source": ".*pipeline.*",
|
||||
"group": "global.pipeline"
|
||||
},
|
||||
{
|
||||
"queryType": "SELECT",
|
||||
"group": "global.data_analyst"
|
||||
},
|
||||
{
|
||||
"queryType": "DATA_DEFINITION",
|
||||
"group": "global.data_definition"
|
||||
},
|
||||
{
|
||||
"user": "admin",
|
||||
"group": "global.data_engineer"
|
||||
},
|
||||
{
|
||||
"source": ".*",
|
||||
"group": "global.data_analyst"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
# =============================================================================
|
||||
# Trino — Resource Groups & Query Queuing
|
||||
# Referensi arsitektur: bab 4.3.1.5
|
||||
# =============================================================================
|
||||
resource-groups.configuration-manager=file
|
||||
resource-groups.config-file=/etc/trino/resource-groups.json
|
||||
Reference in New Issue
Block a user