feat(dbt): apply smart contract schema inference and align gold shift-left quality tests
This commit is contained in:
@@ -8,7 +8,7 @@ governance:
|
||||
tier: Tier.Tier1
|
||||
certification: Certification.Gold
|
||||
tasks:
|
||||
- task_id: node-7278
|
||||
- task_id: node-5630
|
||||
type: source_db
|
||||
label: Source Database (Environmental Permits)
|
||||
enabled: true
|
||||
@@ -21,7 +21,7 @@ tasks:
|
||||
target_type: minio_s3
|
||||
watermark_column: updated_at
|
||||
ingestion_mode: incremental_watermark
|
||||
- task_id: node-8718
|
||||
- task_id: node-6855
|
||||
type: source_gws
|
||||
label: Source GWS (Waste Reports)
|
||||
enabled: true
|
||||
@@ -35,7 +35,7 @@ tasks:
|
||||
drive_folder_id: 1OSRugXZvfjoY9M0yySKT
|
||||
selected_filename: Laporan_Sampah_Kabupaten_2026.xlsx
|
||||
header_row: 2
|
||||
- task_id: node-0372
|
||||
- task_id: node-8156
|
||||
type: source_api_dlt
|
||||
label: Source REST API dlt (Sensor SPKU)
|
||||
enabled: true
|
||||
@@ -56,14 +56,14 @@ tasks:
|
||||
landing_prefix: api/spku_hourly/
|
||||
bronze_table: bronze_spku_telemetry
|
||||
url: https://api.spku.dlh.id/v1/telemetry
|
||||
- task_id: node-2418
|
||||
- task_id: node-9404
|
||||
type: duckdb_transform
|
||||
label: DuckDB Bronze Ingestion & Dedup
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-7278
|
||||
- node-8718
|
||||
- node-0372
|
||||
- node-5630
|
||||
- node-6855
|
||||
- node-8156
|
||||
config:
|
||||
layer: silver
|
||||
source_parquet_path: s3://dlh-bronze/staging/environmental_permits/*.parquet
|
||||
@@ -83,12 +83,12 @@ tasks:
|
||||
AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
|
||||
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC)
|
||||
= 1
|
||||
- task_id: node-4451
|
||||
- task_id: node-0708
|
||||
type: dbt_transform
|
||||
label: dbt Silver Cleansed Permits Enriched
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-2418
|
||||
- node-9404
|
||||
config:
|
||||
dbt_layer: silver
|
||||
dbt_model: silver_permits_enriched
|
||||
@@ -118,12 +118,12 @@ tasks:
|
||||
enforce_contract: true
|
||||
use_cosmos: true
|
||||
transformation_engine: cosmos
|
||||
- task_id: node-6068
|
||||
- task_id: node-2234
|
||||
type: ge_quality
|
||||
label: GE Silver Compliance Quality Gate
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-4451
|
||||
- node-0708
|
||||
config:
|
||||
ge_scope_level: table
|
||||
ge_table_name: silver_permits_enriched
|
||||
@@ -141,12 +141,12 @@ tasks:
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-8030
|
||||
- task_id: node-3694
|
||||
type: dbt_transform
|
||||
label: dbt Gold Regulatory Compliance Mart
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-6068
|
||||
- node-2234
|
||||
config:
|
||||
dbt_layer: gold
|
||||
dbt_model: gold_compliance_summary_mart
|
||||
@@ -162,21 +162,22 @@ tasks:
|
||||
\ CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_permits_enriched')\
|
||||
\ }}\nGROUP BY permit_type, status"
|
||||
test_configs:
|
||||
unique_columns:
|
||||
- stasiun_id
|
||||
not_null_columns:
|
||||
- stasiun_id
|
||||
- ispu_val
|
||||
unique_columns: &id003 []
|
||||
not_null_columns: &id004
|
||||
- permit_type
|
||||
- status
|
||||
accepted_values: {}
|
||||
enforce_contract: false
|
||||
unique_columns: *id003
|
||||
not_null_columns: *id004
|
||||
use_cosmos: true
|
||||
transformation_engine: cosmos
|
||||
- task_id: node-9428
|
||||
- task_id: node-4977
|
||||
type: sitomat_callback
|
||||
label: SITOMAT Governance Callback
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-8030
|
||||
- node-3694
|
||||
config:
|
||||
layer: gold
|
||||
artifact_ref: lakehouse.gold.gold_compliance_summary_mart
|
||||
|
||||
Reference in New Issue
Block a user