first commit

2026-10-09 02:35:14 +07:00
commit 4ca7a9a58e
108 changed files with 23035 additions and 0 deletions
@@ -0,0 +1,55 @@
+# =============================================================================
+# eBdesk SiCABE — Ingestion agent OpenMetadata: dbt (LINEAGE bronze→silver→gold)
+# =============================================================================
+# Mengapa file ini ada:
+#   Ingestion metadata (`ingest-trino.yaml`) hanya mengambil tabel & kolom — TIDAK
+#   menghasilkan lineage, karena lineage berasal dari definisi transformasi, bukan
+#   dari katalog. Transformasi bronze→silver→gold dijalankan oleh dbt, sehingga
+#   lineage-nya diambil dari ARTEFAK dbt:
+#     target/manifest.json   (grafik model: ref()/source() → relasi antar tabel)
+#     target/catalog.json    (skema kolom per model → lineage antar KOLOM)
+#     target/run_results.json(hasil eksekusi/test)
+#
+#   Artefak dibuat di dalam container Airflow (project di /home/airflow/dbt/sicabe):
+#     dbt docs generate --profile trino --target prod
+#
+# ⚠️ Catatan: `target/` berada di layer container (bukan bind-mount), jadi artefak
+#    HILANG bila container Airflow di-recreate → jalankan `dbt docs generate` lagi
+#    sebelum ingestion dbt.
+#
+# Jalankan (butuh OM_JWT_TOKEN):
+#   ./scripts/ingest-openmetadata.sh --workflow config/openmetadata/ingest-dbt.yaml
+# =============================================================================
+source:
+  type: dbt
+  # Service database tempat model dbt ditautkan (harus sama dengan service Trino).
+  serviceName: trino_sicabe
+  sourceConfig:
+    config:
+      # Enum-nya huruf besar: DbtConfigType.DBT = 'DBT' (bukan 'dbt').
+      type: DBT
+      dbtConfigSource:
+        dbtConfigType: local
+        # Artefak dbt dibaca dari DBT_TARGET_PATH (lihat docker-compose.yml), BUKAN dari
+        # <project>/target di dalam bind-mount repo — bind-mount itu tidak bisa ditulis
+        # container (uid host vs uid 50000), sehingga artefaknya memang tidak pernah ada
+        # di sana dan ingestion lineage selalu menghasilkan 0 relasi.
+        dbtManifestFilePath: /tmp/dbt/target/manifest.json
+        dbtCatalogFilePath: /tmp/dbt/target/catalog.json
+        dbtRunResultsFilePath: /tmp/dbt/target/run_results.json
+      dbtUpdateDescriptions: true
+      dbtUpdateOwners: true
+      includeTags: false
+      searchAcrossDatabases: false
+
+sink:
+  type: metadata-rest
+  config: {}
+
+workflowConfig:
+  loggerLevel: INFO
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: ${OM_JWT_TOKEN}