feat(quality): emit Great Expectations test case results directly to OpenMetadata table test suites
This commit is contained in:
@@ -0,0 +1,204 @@
|
||||
pipeline_id: e2e_medallion_permits_flow
|
||||
description: Dynamic Data Lakehouse Pipeline via DLH Studio
|
||||
schedule: 0 */1 * * *
|
||||
mode: flexible
|
||||
tasks:
|
||||
- task_id: node-9377
|
||||
type: source_db
|
||||
label: Source Database
|
||||
enabled: true
|
||||
dependencies: []
|
||||
config:
|
||||
conn: Postgres
|
||||
conn_id: conn_postgres_source_db
|
||||
table: permits
|
||||
table_name: public.stasiun_spku
|
||||
target_type: minio_s3
|
||||
- task_id: node-0405
|
||||
type: ge_quality
|
||||
label: GE Quality Check
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-9377
|
||||
config:
|
||||
ge_scope_level: all
|
||||
ge_table_name: staging_environmental_permits
|
||||
test_table_row_count: true
|
||||
test_table_schema_match: true
|
||||
test_table_table_not_empty: true
|
||||
table_min_rows: 1
|
||||
table_max_rows: 1000000
|
||||
ge_col_table: stasiun_spku
|
||||
ge_target_column: stasiun_id
|
||||
ge_column_test_type: not_null
|
||||
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
|
||||
'IN_REVIEW', 'REJECTED')
|
||||
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
|
||||
'REJECTED')
|
||||
expected: 0
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-1406
|
||||
type: duckdb_transform
|
||||
label: DuckDB Transform
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-0405
|
||||
config:
|
||||
layer: silver
|
||||
source_parquet_path: s3://dlh-bronze/db_cdc/environmental_permits/
|
||||
source_path: s3://dlh-bronze/db_cdc/environmental_permits/
|
||||
target_table: dlh_lakehouse.silver.environmental_permits
|
||||
table_name: dlh_lakehouse.silver.environmental_permits
|
||||
primary_keys: id, updated_at
|
||||
query: SELECT * FROM source_tbl
|
||||
sql_query: SELECT * FROM source_parquet
|
||||
transformation_sql: SELECT permit_id, permit_number, company_name, business_type,
|
||||
location, status, environmental_score, created_at, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
|
||||
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC)
|
||||
= 1
|
||||
- task_id: node-2426
|
||||
type: ge_quality
|
||||
label: GE Quality Check
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-1406
|
||||
config:
|
||||
ge_scope_level: all
|
||||
ge_table_name: bronze_environmental_permits
|
||||
test_table_row_count: true
|
||||
test_table_schema_match: true
|
||||
test_table_table_not_empty: true
|
||||
table_min_rows: 1
|
||||
table_max_rows: 1000000
|
||||
ge_col_table: stasiun_spku
|
||||
ge_target_column: stasiun_id
|
||||
ge_column_test_type: not_null
|
||||
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
|
||||
'IN_REVIEW', 'REJECTED')
|
||||
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
|
||||
'REJECTED')
|
||||
expected: 0
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-3473
|
||||
type: dbt_transform
|
||||
label: dbt Transform
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-2426
|
||||
config:
|
||||
dbt_layer: silver
|
||||
dbt_model: stg_spku_realtime
|
||||
sel: silver
|
||||
tgt: postgres
|
||||
trino_catalog: iceberg
|
||||
sql_query_preview: "-- dbt Silver Cleansed Model with Master Table Enrichment\n\
|
||||
SELECT \n p.permit_id,\n p.permit_number,\n p.company_name,\n p.business_type,\n\
|
||||
\ m.category_name,\n m.risk_level,\n m.compliance_weight,\n p.environmental_score,\n\
|
||||
\ ROUND(p.environmental_score * m.compliance_weight, 2) AS weighted_compliance_score,\n\
|
||||
\ p.status,\n p.location,\n p.updated_at\nFROM {{ ref('bronze_environmental_permits')\
|
||||
\ }} p\nLEFT JOIN {{ ref('ref_business_categories') }} m \n ON p.business_type\
|
||||
\ = m.business_type"
|
||||
query: "-- dbt Silver Cleansed Model with Master Table Enrichment\nSELECT \n \
|
||||
\ p.permit_id,\n p.permit_number,\n p.company_name,\n p.business_type,\n\
|
||||
\ m.category_name,\n m.risk_level,\n m.compliance_weight,\n p.environmental_score,\n\
|
||||
\ ROUND(p.environmental_score * m.compliance_weight, 2) AS weighted_compliance_score,\n\
|
||||
\ p.status,\n p.location,\n p.updated_at\nFROM {{ ref('bronze_environmental_permits')\
|
||||
\ }} p\nLEFT JOIN {{ ref('ref_business_categories') }} m \n ON p.business_type\
|
||||
\ = m.business_type"
|
||||
- task_id: node-4489
|
||||
type: ge_quality
|
||||
label: GE Quality Check
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-3473
|
||||
config:
|
||||
ge_scope_level: all
|
||||
ge_table_name: silver_environmental_compliance
|
||||
test_table_row_count: true
|
||||
test_table_schema_match: true
|
||||
test_table_table_not_empty: true
|
||||
table_min_rows: 1
|
||||
table_max_rows: 1000000
|
||||
ge_col_table: stasiun_spku
|
||||
ge_target_column: stasiun_id
|
||||
ge_column_test_type: not_null
|
||||
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
|
||||
'IN_REVIEW', 'REJECTED')
|
||||
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
|
||||
'REJECTED')
|
||||
expected: 0
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-5491
|
||||
type: dbt_transform
|
||||
label: dbt Transform
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-4489
|
||||
config:
|
||||
dbt_layer: silver
|
||||
dbt_model: stg_spku_realtime
|
||||
sel: silver
|
||||
tgt: postgres
|
||||
trino_catalog: iceberg
|
||||
sql_query_preview: "-- dbt Gold Aggregated Risk Mart\nSELECT \n risk_level,\n\
|
||||
\ business_type,\n category_name,\n COUNT(permit_id) AS total_permits,\n\
|
||||
\ COUNT(CASE WHEN status = 'ACTIVE' THEN 1 END) AS active_permits,\n COUNT(CASE\
|
||||
\ WHEN status = 'SUSPENDED' THEN 1 END) AS suspended_permits,\n ROUND(AVG(environmental_score),\
|
||||
\ 2) AS avg_environmental_score,\n ROUND(AVG(weighted_compliance_score), 2)\
|
||||
\ AS avg_weighted_score,\n CURRENT_TIMESTAMP AS calculated_at\nFROM {{ ref('silver_environmental_compliance')\
|
||||
\ }}\nGROUP BY risk_level, business_type, category_name\nORDER BY avg_weighted_score\
|
||||
\ DESC"
|
||||
query: "-- dbt Gold Aggregated Risk Mart\nSELECT \n risk_level,\n business_type,\n\
|
||||
\ category_name,\n COUNT(permit_id) AS total_permits,\n COUNT(CASE WHEN status\
|
||||
\ = 'ACTIVE' THEN 1 END) AS active_permits,\n COUNT(CASE WHEN status = 'SUSPENDED'\
|
||||
\ THEN 1 END) AS suspended_permits,\n ROUND(AVG(environmental_score), 2) AS\
|
||||
\ avg_environmental_score,\n ROUND(AVG(weighted_compliance_score), 2) AS avg_weighted_score,\n\
|
||||
\ CURRENT_TIMESTAMP AS calculated_at\nFROM {{ ref('silver_environmental_compliance')\
|
||||
\ }}\nGROUP BY risk_level, business_type, category_name\nORDER BY avg_weighted_score\
|
||||
\ DESC"
|
||||
- task_id: node-6528
|
||||
type: ge_quality
|
||||
label: GE Quality Check
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-5491
|
||||
config:
|
||||
ge_scope_level: all
|
||||
ge_table_name: gold_environmental_risk_summary
|
||||
test_table_row_count: true
|
||||
test_table_schema_match: true
|
||||
test_table_table_not_empty: true
|
||||
table_min_rows: 1
|
||||
table_max_rows: 1000000
|
||||
ge_col_table: stasiun_spku
|
||||
ge_target_column: stasiun_id
|
||||
ge_column_test_type: not_null
|
||||
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
|
||||
'IN_REVIEW', 'REJECTED')
|
||||
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
|
||||
'REJECTED')
|
||||
expected: 0
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-7512
|
||||
type: sitomat_callback
|
||||
label: Sitomat Callback
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-6528
|
||||
config:
|
||||
layer: bronze
|
||||
artifact_ref: lakehouse.bronze.dataset_cdc
|
||||
checksum: sha256-verified-pipeline-pass
|
||||
sitomat_base_url: http://localhost:3000
|
||||
openmetadata_fqn: dlh_airflow_service.pipeline.task
|
||||
ge_status: PASSED
|
||||
url: /sitomat/callback
|
||||
stage: bronze
|
||||
Reference in New Issue
Block a user