56 lines
2.4 KiB
YAML
56 lines
2.4 KiB
YAML
# =============================================================================
|
|
# eBdesk SiCABE — Ingestion agent OpenMetadata: dbt (LINEAGE bronze→silver→gold)
|
|
# =============================================================================
|
|
# Mengapa file ini ada:
|
|
# Ingestion metadata (`ingest-trino.yaml`) hanya mengambil tabel & kolom — TIDAK
|
|
# menghasilkan lineage, karena lineage berasal dari definisi transformasi, bukan
|
|
# dari katalog. Transformasi bronze→silver→gold dijalankan oleh dbt, sehingga
|
|
# lineage-nya diambil dari ARTEFAK dbt:
|
|
# target/manifest.json (grafik model: ref()/source() → relasi antar tabel)
|
|
# target/catalog.json (skema kolom per model → lineage antar KOLOM)
|
|
# target/run_results.json(hasil eksekusi/test)
|
|
#
|
|
# Artefak dibuat di dalam container Airflow (project di /home/airflow/dbt/sicabe):
|
|
# dbt docs generate --profile trino --target prod
|
|
#
|
|
# ⚠️ Catatan: `target/` berada di layer container (bukan bind-mount), jadi artefak
|
|
# HILANG bila container Airflow di-recreate → jalankan `dbt docs generate` lagi
|
|
# sebelum ingestion dbt.
|
|
#
|
|
# Jalankan (butuh OM_JWT_TOKEN):
|
|
# ./scripts/ingest-openmetadata.sh --workflow config/openmetadata/ingest-dbt.yaml
|
|
# =============================================================================
|
|
source:
|
|
type: dbt
|
|
# Service database tempat model dbt ditautkan (harus sama dengan service Trino).
|
|
serviceName: trino_sicabe
|
|
sourceConfig:
|
|
config:
|
|
# Enum-nya huruf besar: DbtConfigType.DBT = 'DBT' (bukan 'dbt').
|
|
type: DBT
|
|
dbtConfigSource:
|
|
dbtConfigType: local
|
|
# Artefak dbt dibaca dari DBT_TARGET_PATH (lihat docker-compose.yml), BUKAN dari
|
|
# <project>/target di dalam bind-mount repo — bind-mount itu tidak bisa ditulis
|
|
# container (uid host vs uid 50000), sehingga artefaknya memang tidak pernah ada
|
|
# di sana dan ingestion lineage selalu menghasilkan 0 relasi.
|
|
dbtManifestFilePath: /tmp/dbt/target/manifest.json
|
|
dbtCatalogFilePath: /tmp/dbt/target/catalog.json
|
|
dbtRunResultsFilePath: /tmp/dbt/target/run_results.json
|
|
dbtUpdateDescriptions: true
|
|
dbtUpdateOwners: true
|
|
includeTags: false
|
|
searchAcrossDatabases: false
|
|
|
|
sink:
|
|
type: metadata-rest
|
|
config: {}
|
|
|
|
workflowConfig:
|
|
loggerLevel: INFO
|
|
openMetadataServerConfig:
|
|
hostPort: http://openmetadata-server:8585/api
|
|
authProvider: openmetadata
|
|
securityConfig:
|
|
jwtToken: ${OM_JWT_TOKEN}
|