feat(dbt): apply smart contract schema inference and align gold shift-left quality tests

This commit is contained in:
2026-10-21 16:00:40 +07:00
parent 5de6e26181
commit 75f205137d
6 changed files with 91 additions and 60 deletions
+21 -20
View File
@@ -8,7 +8,7 @@ governance:
tier: Tier.Tier1
certification: Certification.Gold
tasks:
- task_id: node-7278
- task_id: node-5630
type: source_db
label: Source Database (Environmental Permits)
enabled: true
@@ -21,7 +21,7 @@ tasks:
target_type: minio_s3
watermark_column: updated_at
ingestion_mode: incremental_watermark
- task_id: node-8718
- task_id: node-6855
type: source_gws
label: Source GWS (Waste Reports)
enabled: true
@@ -35,7 +35,7 @@ tasks:
drive_folder_id: 1OSRugXZvfjoY9M0yySKT
selected_filename: Laporan_Sampah_Kabupaten_2026.xlsx
header_row: 2
- task_id: node-0372
- task_id: node-8156
type: source_api_dlt
label: Source REST API dlt (Sensor SPKU)
enabled: true
@@ -56,14 +56,14 @@ tasks:
landing_prefix: api/spku_hourly/
bronze_table: bronze_spku_telemetry
url: https://api.spku.dlh.id/v1/telemetry
- task_id: node-2418
- task_id: node-9404
type: duckdb_transform
label: DuckDB Bronze Ingestion & Dedup
enabled: true
dependencies:
- node-7278
- node-8718
- node-0372
- node-5630
- node-6855
- node-8156
config:
layer: silver
source_parquet_path: s3://dlh-bronze/staging/environmental_permits/*.parquet
@@ -83,12 +83,12 @@ tasks:
AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC)
= 1
- task_id: node-4451
- task_id: node-0708
type: dbt_transform
label: dbt Silver Cleansed Permits Enriched
enabled: true
dependencies:
- node-2418
- node-9404
config:
dbt_layer: silver
dbt_model: silver_permits_enriched
@@ -118,12 +118,12 @@ tasks:
enforce_contract: true
use_cosmos: true
transformation_engine: cosmos
- task_id: node-6068
- task_id: node-2234
type: ge_quality
label: GE Silver Compliance Quality Gate
enabled: true
dependencies:
- node-4451
- node-0708
config:
ge_scope_level: table
ge_table_name: silver_permits_enriched
@@ -141,12 +141,12 @@ tasks:
error_message: Unexpected rows detected in table
suite: bronze
fail: Stop DAG
- task_id: node-8030
- task_id: node-3694
type: dbt_transform
label: dbt Gold Regulatory Compliance Mart
enabled: true
dependencies:
- node-6068
- node-2234
config:
dbt_layer: gold
dbt_model: gold_compliance_summary_mart
@@ -162,21 +162,22 @@ tasks:
\ CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_permits_enriched')\
\ }}\nGROUP BY permit_type, status"
test_configs:
unique_columns:
- stasiun_id
not_null_columns:
- stasiun_id
- ispu_val
unique_columns: &id003 []
not_null_columns: &id004
- permit_type
- status
accepted_values: {}
enforce_contract: false
unique_columns: *id003
not_null_columns: *id004
use_cosmos: true
transformation_engine: cosmos
- task_id: node-9428
- task_id: node-4977
type: sitomat_callback
label: SITOMAT Governance Callback
enabled: true
dependencies:
- node-8030
- node-3694
config:
layer: gold
artifact_ref: lakehouse.gold.gold_compliance_summary_mart