From 75f205137ddce6d08e4218c7b7b0ef557e45d935 Mon Sep 17 00:00:00 2001 From: syahril Date: Wed, 21 Oct 2026 16:00:40 +0700 Subject: [PATCH] feat(dbt): apply smart contract schema inference and align gold shift-left quality tests --- configs/e2e_modernized_ingestion_flow.yaml | 41 ++++++----- .../bronze/bronze_environmental_permits.sql | 18 ++++- dbt/models/gold/schema.yml | 5 +- dbt/models/silver/schema.yml | 12 +++ metadata/e2e_modernized_ingestion_flow.json | 73 ++++++++++--------- metadata/rbac_mappings.json | 2 +- 6 files changed, 91 insertions(+), 60 deletions(-) diff --git a/configs/e2e_modernized_ingestion_flow.yaml b/configs/e2e_modernized_ingestion_flow.yaml index a719305..b9e85ca 100644 --- a/configs/e2e_modernized_ingestion_flow.yaml +++ b/configs/e2e_modernized_ingestion_flow.yaml @@ -8,7 +8,7 @@ governance: tier: Tier.Tier1 certification: Certification.Gold tasks: -- task_id: node-7278 +- task_id: node-5630 type: source_db label: Source Database (Environmental Permits) enabled: true @@ -21,7 +21,7 @@ tasks: target_type: minio_s3 watermark_column: updated_at ingestion_mode: incremental_watermark -- task_id: node-8718 +- task_id: node-6855 type: source_gws label: Source GWS (Waste Reports) enabled: true @@ -35,7 +35,7 @@ tasks: drive_folder_id: 1OSRugXZvfjoY9M0yySKT selected_filename: Laporan_Sampah_Kabupaten_2026.xlsx header_row: 2 -- task_id: node-0372 +- task_id: node-8156 type: source_api_dlt label: Source REST API dlt (Sensor SPKU) enabled: true @@ -56,14 +56,14 @@ tasks: landing_prefix: api/spku_hourly/ bronze_table: bronze_spku_telemetry url: https://api.spku.dlh.id/v1/telemetry -- task_id: node-2418 +- task_id: node-9404 type: duckdb_transform label: DuckDB Bronze Ingestion & Dedup enabled: true dependencies: - - node-7278 - - node-8718 - - node-0372 + - node-5630 + - node-6855 + - node-8156 config: layer: silver source_parquet_path: s3://dlh-bronze/staging/environmental_permits/*.parquet @@ -83,12 +83,12 @@ tasks: AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1 -- task_id: node-4451 +- task_id: node-0708 type: dbt_transform label: dbt Silver Cleansed Permits Enriched enabled: true dependencies: - - node-2418 + - node-9404 config: dbt_layer: silver dbt_model: silver_permits_enriched @@ -118,12 +118,12 @@ tasks: enforce_contract: true use_cosmos: true transformation_engine: cosmos -- task_id: node-6068 +- task_id: node-2234 type: ge_quality label: GE Silver Compliance Quality Gate enabled: true dependencies: - - node-4451 + - node-0708 config: ge_scope_level: table ge_table_name: silver_permits_enriched @@ -141,12 +141,12 @@ tasks: error_message: Unexpected rows detected in table suite: bronze fail: Stop DAG -- task_id: node-8030 +- task_id: node-3694 type: dbt_transform label: dbt Gold Regulatory Compliance Mart enabled: true dependencies: - - node-6068 + - node-2234 config: dbt_layer: gold dbt_model: gold_compliance_summary_mart @@ -162,21 +162,22 @@ tasks: \ CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_permits_enriched')\ \ }}\nGROUP BY permit_type, status" test_configs: - unique_columns: - - stasiun_id - not_null_columns: - - stasiun_id - - ispu_val + unique_columns: &id003 [] + not_null_columns: &id004 + - permit_type + - status accepted_values: {} enforce_contract: false + unique_columns: *id003 + not_null_columns: *id004 use_cosmos: true transformation_engine: cosmos -- task_id: node-9428 +- task_id: node-4977 type: sitomat_callback label: SITOMAT Governance Callback enabled: true dependencies: - - node-8030 + - node-3694 config: layer: gold artifact_ref: lakehouse.gold.gold_compliance_summary_mart diff --git a/dbt/models/bronze/bronze_environmental_permits.sql b/dbt/models/bronze/bronze_environmental_permits.sql index 995424c..dbb4111 100644 --- a/dbt/models/bronze/bronze_environmental_permits.sql +++ b/dbt/models/bronze/bronze_environmental_permits.sql @@ -1,4 +1,20 @@ -- dbt Bronze View Model: environmental_permits {{ config(materialized='view') }} -SELECT * FROM {{ source('public', 'environmental_permits') }} +SELECT + permit_id, + permit_number, + applicant_name, + company_name, + business_type, + activity_type, + activity_type AS permit_type, + location, + status, + environmental_score, + issued_date, + expiry_date, + created_at, + updated_at +FROM {{ source('public', 'environmental_permits') }} + diff --git a/dbt/models/gold/schema.yml b/dbt/models/gold/schema.yml index 3fc2d9e..7ff5053 100644 --- a/dbt/models/gold/schema.yml +++ b/dbt/models/gold/schema.yml @@ -4,10 +4,9 @@ models: description: dbt Model gold_compliance_summary_mart auto-registered with Shift-Left quality assertions columns: - - name: ispu_val + - name: permit_type tests: - not_null - - name: stasiun_id + - name: status tests: - - unique - not_null diff --git a/dbt/models/silver/schema.yml b/dbt/models/silver/schema.yml index d5f1489..70ac457 100644 --- a/dbt/models/silver/schema.yml +++ b/dbt/models/silver/schema.yml @@ -7,16 +7,28 @@ models: contract: enforced: true columns: + - name: company_name + data_type: character varying(255) + - name: dbt_validated_at + data_type: timestamp with time zone + - name: issued_date + data_type: date - name: permit_id tests: - unique - not_null + data_type: character varying(50) - name: permit_number tests: - unique + data_type: character varying(50) + - name: permit_type + data_type: character varying(50) - name: status tests: - not_null + data_type: character varying(50) - name: updated_at tests: - not_null + data_type: timestamp with time zone diff --git a/metadata/e2e_modernized_ingestion_flow.json b/metadata/e2e_modernized_ingestion_flow.json index d062c9d..30bd1e4 100644 --- a/metadata/e2e_modernized_ingestion_flow.json +++ b/metadata/e2e_modernized_ingestion_flow.json @@ -6,7 +6,7 @@ "owner_name": "Administrator Lokal (Break-Glass)", "nodes": [ { - "id": "node-7278", + "id": "node-5630", "type": "customNode", "position": { "x": 80, @@ -34,7 +34,7 @@ "dragging": false }, { - "id": "node-8718", + "id": "node-6855", "type": "customNode", "position": { "x": 270, @@ -63,7 +63,7 @@ "dragging": false }, { - "id": "node-0372", + "id": "node-8156", "type": "customNode", "position": { "x": 460, @@ -99,7 +99,7 @@ "dragging": false }, { - "id": "node-2418", + "id": "node-9404", "type": "customNode", "position": { "x": 650, @@ -128,7 +128,7 @@ "dragging": false }, { - "id": "node-4451", + "id": "node-0708", "type": "customNode", "position": { "x": 840, @@ -180,7 +180,7 @@ "dragging": false }, { - "id": "node-6068", + "id": "node-2234", "type": "customNode", "position": { "x": 1030, @@ -216,7 +216,7 @@ "dragging": false }, { - "id": "node-8030", + "id": "node-3694", "type": "customNode", "position": { "x": 1220, @@ -236,16 +236,19 @@ "sql_query_preview": "-- dbt Gold Regulatory Compliance Mart\nSELECT \n permit_type,\n status,\n COUNT(permit_id) AS total_permits,\n COUNT(DISTINCT company_name) AS unique_companies,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_permits_enriched') }}\nGROUP BY permit_type, status", "query": "-- dbt Gold Regulatory Compliance Mart\nSELECT \n permit_type,\n status,\n COUNT(permit_id) AS total_permits,\n COUNT(DISTINCT company_name) AS unique_companies,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_permits_enriched') }}\nGROUP BY permit_type, status", "test_configs": { - "unique_columns": [ - "stasiun_id" - ], + "unique_columns": [], "not_null_columns": [ - "stasiun_id", - "ispu_val" + "permit_type", + "status" ], "accepted_values": {}, "enforce_contract": false }, + "unique_columns": [], + "not_null_columns": [ + "permit_type", + "status" + ], "use_cosmos": true, "transformation_engine": "cosmos" } @@ -256,7 +259,7 @@ "dragging": false }, { - "id": "node-9428", + "id": "node-4977", "type": "customNode", "position": { "x": 1410, @@ -286,9 +289,9 @@ ], "edges": [ { - "source": "node-7278", + "source": "node-5630", "sourceHandle": null, - "target": "node-2418", + "target": "node-9404", "targetHandle": null, "type": "removableEdge", "style": { @@ -296,12 +299,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-7278-node-2418" + "id": "reactflow__edge-node-5630-node-9404" }, { - "source": "node-8718", + "source": "node-6855", "sourceHandle": null, - "target": "node-2418", + "target": "node-9404", "targetHandle": null, "type": "removableEdge", "style": { @@ -309,12 +312,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-8718-node-2418" + "id": "reactflow__edge-node-6855-node-9404" }, { - "source": "node-0372", + "source": "node-8156", "sourceHandle": null, - "target": "node-2418", + "target": "node-9404", "targetHandle": null, "type": "removableEdge", "style": { @@ -322,12 +325,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-0372-node-2418" + "id": "reactflow__edge-node-8156-node-9404" }, { - "source": "node-2418", + "source": "node-9404", "sourceHandle": null, - "target": "node-4451", + "target": "node-0708", "targetHandle": null, "type": "removableEdge", "style": { @@ -335,12 +338,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-2418-node-4451" + "id": "reactflow__edge-node-9404-node-0708" }, { - "source": "node-4451", + "source": "node-0708", "sourceHandle": null, - "target": "node-6068", + "target": "node-2234", "targetHandle": null, "type": "removableEdge", "style": { @@ -348,12 +351,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-4451-node-6068" + "id": "reactflow__edge-node-0708-node-2234" }, { - "source": "node-6068", + "source": "node-2234", "sourceHandle": null, - "target": "node-8030", + "target": "node-3694", "targetHandle": null, "type": "removableEdge", "style": { @@ -361,12 +364,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-6068-node-8030" + "id": "reactflow__edge-node-2234-node-3694" }, { - "source": "node-8030", + "source": "node-3694", "sourceHandle": null, - "target": "node-9428", + "target": "node-4977", "targetHandle": null, "type": "removableEdge", "style": { @@ -374,11 +377,11 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-8030-node-9428" + "id": "reactflow__edge-node-3694-node-4977" } ], "notes": [], "is_draft": false, "status": "deployed", - "updated_at": "2026-10-21T07:58:01.895850Z" + "updated_at": "2026-10-21T08:59:34.908289Z" } \ No newline at end of file diff --git a/metadata/rbac_mappings.json b/metadata/rbac_mappings.json index 236f272..58a7c77 100644 --- a/metadata/rbac_mappings.json +++ b/metadata/rbac_mappings.json @@ -22,6 +22,6 @@ "admin_groups": [ "/Platform Administrators" ], - "updated_at": "2026-10-20T04:23:28.672811+00:00", + "updated_at": "2026-10-21T08:56:14.953819+00:00", "updated_by": "admin" } \ No newline at end of file