merica-dlh/merica-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(dbt): apply smart contract schema inference and align gold shift-left quality tests

Browse Source
This commit is contained in:
syahril
2026-10-21 16:00:40 +07:00
parent 5de6e26181
commit 75f205137d
6 changed files with 91 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
configs/e2e_modernized_ingestion_flow.yaml
+21 -20
View File
@@ -8,7 +8,7 @@ governance:
tier: Tier.Tier1
certification: Certification.Gold
tasks:
- task_id: node-7278
- task_id: node-5630
type: source_db
label: Source Database (Environmental Permits)
enabled: true
@@ -21,7 +21,7 @@ tasks:
target_type: minio_s3
watermark_column: updated_at
ingestion_mode: incremental_watermark
- task_id: node-8718
- task_id: node-6855
type: source_gws
label: Source GWS (Waste Reports)
enabled: true
@@ -35,7 +35,7 @@ tasks:
drive_folder_id: 1OSRugXZvfjoY9M0yySKT
selected_filename: Laporan_Sampah_Kabupaten_2026.xlsx
header_row: 2
- task_id: node-0372
- task_id: node-8156
type: source_api_dlt
label: Source REST API dlt (Sensor SPKU)
enabled: true
@@ -56,14 +56,14 @@ tasks:
landing_prefix: api/spku_hourly/
bronze_table: bronze_spku_telemetry
url: https://api.spku.dlh.id/v1/telemetry
- task_id: node-2418
- task_id: node-9404
type: duckdb_transform
label: DuckDB Bronze Ingestion & Dedup
enabled: true
dependencies:
- node-7278
- node-8718
- node-0372
- node-5630
- node-6855
- node-8156
config:
layer: silver
source_parquet_path: s3://dlh-bronze/staging/environmental_permits/*.parquet
@@ -83,12 +83,12 @@ tasks:
AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC)
= 1
- task_id: node-4451
- task_id: node-0708
type: dbt_transform
label: dbt Silver Cleansed Permits Enriched
enabled: true
dependencies:
- node-2418
- node-9404
config:
dbt_layer: silver
dbt_model: silver_permits_enriched
@@ -118,12 +118,12 @@ tasks:
enforce_contract: true
use_cosmos: true
transformation_engine: cosmos
- task_id: node-6068
- task_id: node-2234
type: ge_quality
label: GE Silver Compliance Quality Gate
enabled: true
dependencies:
- node-4451
- node-0708
config:
ge_scope_level: table
ge_table_name: silver_permits_enriched
@@ -141,12 +141,12 @@ tasks:
error_message: Unexpected rows detected in table
suite: bronze
fail: Stop DAG
- task_id: node-8030
- task_id: node-3694
type: dbt_transform
label: dbt Gold Regulatory Compliance Mart
enabled: true
dependencies:
- node-6068
- node-2234
config:
dbt_layer: gold
dbt_model: gold_compliance_summary_mart
@@ -162,21 +162,22 @@ tasks:
\ CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_permits_enriched')\
\ }}\nGROUP BY permit_type, status"
test_configs:
unique_columns:
- stasiun_id
not_null_columns:
- stasiun_id
- ispu_val
unique_columns: &id003 []
not_null_columns: &id004
- permit_type
- status
accepted_values: {}
enforce_contract: false
unique_columns: *id003
not_null_columns: *id004
use_cosmos: true
transformation_engine: cosmos
- task_id: node-9428
- task_id: node-4977
type: sitomat_callback
label: SITOMAT Governance Callback
enabled: true
dependencies:
- node-8030
- node-3694
config:
layer: gold
artifact_ref: lakehouse.gold.gold_compliance_summary_mart
dbt/models/bronze/bronze_environmental_permits.sql
+17 -1
View File
@@ -1,4 +1,20 @@
-- dbt Bronze View Model: environmental_permits
{{ config(materialized='view') }}
SELECT * FROM {{ source('public', 'environmental_permits') }}
SELECT
permit_id,
permit_number,
applicant_name,
company_name,
business_type,
activity_type,
activity_type AS permit_type,
location,
status,
environmental_score,
issued_date,
expiry_date,
created_at,
updated_at
FROM {{ source('public', 'environmental_permits') }}
dbt/models/gold/schema.yml
+2 -3
View File
@@ -4,10 +4,9 @@ models:
description: dbt Model gold_compliance_summary_mart auto-registered with Shift-Left
quality assertions
columns:
- name: ispu_val
- name: permit_type
tests:
- not_null
- name: stasiun_id
- name: status
tests:
- unique
- not_null
dbt/models/silver/schema.yml
+12
View File
@@ -7,16 +7,28 @@ models:
contract:
enforced: true
columns:
- name: company_name
data_type: character varying(255)
- name: dbt_validated_at
data_type: timestamp with time zone
- name: issued_date
data_type: date
- name: permit_id
tests:
- unique
- not_null
data_type: character varying(50)
- name: permit_number
tests:
- unique
data_type: character varying(50)
- name: permit_type
data_type: character varying(50)
- name: status
tests:
- not_null
data_type: character varying(50)
- name: updated_at
tests:
- not_null
data_type: timestamp with time zone
metadata/e2e_modernized_ingestion_flow.json
+38 -35
View File
@@ -6,7 +6,7 @@
"owner_name": "Administrator Lokal (Break-Glass)",
"nodes": [
{
"id": "node-7278",
"id": "node-5630",
"type": "customNode",
"position": {
"x": 80,
@@ -34,7 +34,7 @@
"dragging": false
},
{
"id": "node-8718",
"id": "node-6855",
"type": "customNode",
"position": {
"x": 270,
@@ -63,7 +63,7 @@
"dragging": false
},
{
"id": "node-0372",
"id": "node-8156",
"type": "customNode",
"position": {
"x": 460,
@@ -99,7 +99,7 @@
"dragging": false
},
{
"id": "node-2418",
"id": "node-9404",
"type": "customNode",
"position": {
"x": 650,
@@ -128,7 +128,7 @@
"dragging": false
},
{
"id": "node-4451",
"id": "node-0708",
"type": "customNode",
"position": {
"x": 840,
@@ -180,7 +180,7 @@
"dragging": false
},
{
"id": "node-6068",
"id": "node-2234",
"type": "customNode",
"position": {
"x": 1030,
@@ -216,7 +216,7 @@
"dragging": false
},
{
"id": "node-8030",
"id": "node-3694",
"type": "customNode",
"position": {
"x": 1220,
@@ -236,16 +236,19 @@
"sql_query_preview": "-- dbt Gold Regulatory Compliance Mart\nSELECT \n permit_type,\n status,\n COUNT(permit_id) AS total_permits,\n COUNT(DISTINCT company_name) AS unique_companies,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_permits_enriched') }}\nGROUP BY permit_type, status",
"query": "-- dbt Gold Regulatory Compliance Mart\nSELECT \n permit_type,\n status,\n COUNT(permit_id) AS total_permits,\n COUNT(DISTINCT company_name) AS unique_companies,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_permits_enriched') }}\nGROUP BY permit_type, status",
"test_configs": {
"unique_columns": [
"stasiun_id"
],
"unique_columns": [],
"not_null_columns": [
"stasiun_id",
"ispu_val"
"permit_type",
"status"
],
"accepted_values": {},
"enforce_contract": false
},
"unique_columns": [],
"not_null_columns": [
"permit_type",
"status"
],
"use_cosmos": true,
"transformation_engine": "cosmos"
}
@@ -256,7 +259,7 @@
"dragging": false
},
{
"id": "node-9428",
"id": "node-4977",
"type": "customNode",
"position": {
"x": 1410,
@@ -286,9 +289,9 @@
],
"edges": [
{
"source": "node-7278",
"source": "node-5630",
"sourceHandle": null,
"target": "node-2418",
"target": "node-9404",
"targetHandle": null,
"type": "removableEdge",
"style": {
@@ -296,12 +299,12 @@
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-7278-node-2418"
"id": "reactflow__edge-node-5630-node-9404"
},
{
"source": "node-8718",
"source": "node-6855",
"sourceHandle": null,
"target": "node-2418",
"target": "node-9404",
"targetHandle": null,
"type": "removableEdge",
"style": {
@@ -309,12 +312,12 @@
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-8718-node-2418"
"id": "reactflow__edge-node-6855-node-9404"
},
{
"source": "node-0372",
"source": "node-8156",
"sourceHandle": null,
"target": "node-2418",
"target": "node-9404",
"targetHandle": null,
"type": "removableEdge",
"style": {
@@ -322,12 +325,12 @@
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-0372-node-2418"
"id": "reactflow__edge-node-8156-node-9404"
},
{
"source": "node-2418",
"source": "node-9404",
"sourceHandle": null,
"target": "node-4451",
"target": "node-0708",
"targetHandle": null,
"type": "removableEdge",
"style": {
@@ -335,12 +338,12 @@
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-2418-node-4451"
"id": "reactflow__edge-node-9404-node-0708"
},
{
"source": "node-4451",
"source": "node-0708",
"sourceHandle": null,
"target": "node-6068",
"target": "node-2234",
"targetHandle": null,
"type": "removableEdge",
"style": {
@@ -348,12 +351,12 @@
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-4451-node-6068"
"id": "reactflow__edge-node-0708-node-2234"
},
{
"source": "node-6068",
"source": "node-2234",
"sourceHandle": null,
"target": "node-8030",
"target": "node-3694",
"targetHandle": null,
"type": "removableEdge",
"style": {
@@ -361,12 +364,12 @@
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-6068-node-8030"
"id": "reactflow__edge-node-2234-node-3694"
},
{
"source": "node-8030",
"source": "node-3694",
"sourceHandle": null,
"target": "node-9428",
"target": "node-4977",
"targetHandle": null,
"type": "removableEdge",
"style": {
@@ -374,11 +377,11 @@
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-8030-node-9428"
"id": "reactflow__edge-node-3694-node-4977"
}
],
"notes": [],
"is_draft": false,
"status": "deployed",
"updated_at": "2026-10-21T07:58:01.895850Z"
"updated_at": "2026-10-21T08:59:34.908289Z"
}
metadata/rbac_mappings.json
+1 -1
View File
@@ -22,6 +22,6 @@
"admin_groups": [
"/Platform Administrators"
],
"updated_at": "2026-10-20T04:23:28.672811+00:00",
"updated_at": "2026-10-21T08:56:14.953819+00:00",
"updated_by": "admin"
}