feat: initial commit for merica-pipeline delivery
This commit is contained in:
MERICA Admin
@@ -0,0 +1,230 @@
|
||||
{
|
||||
"id": "example_permits_medallion_dbt_postgres",
|
||||
"description": "Example Use Case: Environmental Permits Ingestion (Postgres DB) -> Bronze CDC Landing -> GE Quality Bronze -> dbt-postgres Silver -> GE Quality Silver -> dbt-postgres Gold Daily Mart",
|
||||
"schedule": "0 8 * * *",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "node-bronze-src",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 50,
|
||||
"y": 200
|
||||
},
|
||||
"data": {
|
||||
"label": "Bronze DB Source (CDC Landing)",
|
||||
"type": "source_db",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"source_type": "database",
|
||||
"connection_id": "conn_postgres_source_db",
|
||||
"table_name": "public.environmental_permits",
|
||||
"target_table": "bronze_lakehouse.environmental_permits_cdc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "node-ge-bronze",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 300,
|
||||
"y": 200
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Quality Check (Bronze Suite)",
|
||||
"type": "ge_quality",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"expectation_suite": "permits_bronze_suite",
|
||||
"table_name": "bronze_lakehouse.environmental_permits_cdc",
|
||||
"rules": [
|
||||
{
|
||||
"type": "table_row_count_to_be_between",
|
||||
"min_value": 1
|
||||
},
|
||||
{
|
||||
"type": "column_values_to_not_be_null",
|
||||
"column": "permit_id"
|
||||
}
|
||||
],
|
||||
"queries": [
|
||||
{
|
||||
"name": "check_bronze_null_permit_id",
|
||||
"sql": "SELECT count(*) FROM bronze_lakehouse.environmental_permits_cdc WHERE permit_id IS NULL",
|
||||
"expected": 0,
|
||||
"error_message": "Null permit_id detected in Bronze"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "node-sitomat-bronze",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 550,
|
||||
"y": 200
|
||||
},
|
||||
"data": {
|
||||
"label": "Sitomat Bronze Callback",
|
||||
"type": "sitomat_callback",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"layer": "bronze",
|
||||
"artifact_ref": "dlh_lakehouse.bronze_lakehouse.environmental_permits_cdc",
|
||||
"checksum": "auto-sha256"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "node-silver-dbt",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 800,
|
||||
"y": 200
|
||||
},
|
||||
"data": {
|
||||
"label": "dbt Silver Cleanse (dbt-postgres)",
|
||||
"type": "dbt_transform",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"dbt_layer": "silver",
|
||||
"dbt_model": "silver_environmental_permits_clean",
|
||||
"target_table": "dlh_lakehouse.silver.silver_environmental_permits_clean"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "node-ge-silver",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1050,
|
||||
"y": 200
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Quality Check (Silver Suite)",
|
||||
"type": "ge_quality",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"expectation_suite": "permits_silver_suite",
|
||||
"table_name": "silver.silver_environmental_permits_clean",
|
||||
"rules": [
|
||||
{
|
||||
"type": "column_values_to_not_be_null",
|
||||
"column": "permit_id"
|
||||
},
|
||||
{
|
||||
"type": "column_values_to_be_in_set",
|
||||
"column": "permit_status",
|
||||
"allowed_set": [
|
||||
"APPROVED",
|
||||
"IN_REVIEW",
|
||||
"REJECTED"
|
||||
]
|
||||
}
|
||||
],
|
||||
"queries": [
|
||||
{
|
||||
"name": "check_invalid_status_count",
|
||||
"sql": "SELECT count(*) FROM silver.silver_environmental_permits_clean WHERE permit_status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"expected": 0,
|
||||
"error_message": "Invalid status detected in Silver"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "node-sitomat-silver",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1300,
|
||||
"y": 200
|
||||
},
|
||||
"data": {
|
||||
"label": "Sitomat Silver Callback",
|
||||
"type": "sitomat_callback",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"layer": "silver",
|
||||
"artifact_ref": "dlh_lakehouse.silver.silver_environmental_permits_clean",
|
||||
"checksum": "auto-sha256"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "node-gold-dbt",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1550,
|
||||
"y": 200
|
||||
},
|
||||
"data": {
|
||||
"label": "dbt Gold Mart (dbt-postgres)",
|
||||
"type": "dbt_transform",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"dbt_layer": "gold",
|
||||
"dbt_model": "gold_permits_summary_daily",
|
||||
"target_table": "dlh_lakehouse.gold.gold_permits_summary_daily"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "node-sitomat-gold",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1800,
|
||||
"y": 200
|
||||
},
|
||||
"data": {
|
||||
"label": "Sitomat Gold Callback (Data Product)",
|
||||
"type": "sitomat_callback",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"layer": "gold",
|
||||
"artifact_ref": "dlh_lakehouse.gold.gold_permits_summary_daily",
|
||||
"checksum": "auto-sha256"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"id": "e1-src-ge_bronze",
|
||||
"source": "node-bronze-src",
|
||||
"target": "node-ge-bronze"
|
||||
},
|
||||
{
|
||||
"id": "e2-ge_bronze-sitomat_bronze",
|
||||
"source": "node-ge-bronze",
|
||||
"target": "node-sitomat-bronze"
|
||||
},
|
||||
{
|
||||
"id": "e3-sitomat_bronze-silver",
|
||||
"source": "node-sitomat-bronze",
|
||||
"target": "node-silver-dbt"
|
||||
},
|
||||
{
|
||||
"id": "e4-silver-ge_silver",
|
||||
"source": "node-silver-dbt",
|
||||
"target": "node-ge-silver"
|
||||
},
|
||||
{
|
||||
"id": "e5-ge_silver-sitomat_silver",
|
||||
"source": "node-ge-silver",
|
||||
"target": "node-sitomat-silver"
|
||||
},
|
||||
{
|
||||
"id": "e6-sitomat_silver-gold",
|
||||
"source": "node-sitomat-silver",
|
||||
"target": "node-gold-dbt"
|
||||
},
|
||||
{
|
||||
"id": "e7-gold-sitomat_gold",
|
||||
"source": "node-gold-dbt",
|
||||
"target": "node-sitomat-gold"
|
||||
}
|
||||
],
|
||||
"is_draft": false,
|
||||
"status": "deployed"
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
{
|
||||
"id": "ingestion_api_spku_air_quality",
|
||||
"description": "REST API Ingestion for SPKU Realtime Air Quality Data -> MinIO S3 Bronze -> dbt Silver -> GE Quality Suite",
|
||||
"schedule": "0 4 * * *",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "node-bronze-src",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 80,
|
||||
"y": 160
|
||||
},
|
||||
"data": {
|
||||
"label": "Bronze API Source",
|
||||
"type": "source_api",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"source_type": "api",
|
||||
"connection_id": "conn_spku_api_service",
|
||||
"api_endpoint": "https://api.spku.dlh.id/v1/stasiun/spku/data-realtime",
|
||||
"http_method": "GET",
|
||||
"target": {
|
||||
"bucket": "dlh-bronze",
|
||||
"format": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"width": 220,
|
||||
"height": 92
|
||||
},
|
||||
{
|
||||
"id": "node-bronze-storage",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 340,
|
||||
"y": 160
|
||||
},
|
||||
"data": {
|
||||
"label": "MinIO Bronze Storage",
|
||||
"type": "minio_storage",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"bucket": "dlh-bronze",
|
||||
"format": "json"
|
||||
}
|
||||
},
|
||||
"width": 220,
|
||||
"height": 92
|
||||
},
|
||||
{
|
||||
"id": "node-silver-dbt",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 600,
|
||||
"y": 160
|
||||
},
|
||||
"data": {
|
||||
"label": "dbt Silver Model (stg_spku_realtime)",
|
||||
"type": "dbt_transform",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"dbt_layer": "silver",
|
||||
"dbt_model": "stg_spku_realtime"
|
||||
}
|
||||
},
|
||||
"width": 280,
|
||||
"height": 92
|
||||
},
|
||||
{
|
||||
"id": "node-silver-ge",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 860,
|
||||
"y": 160
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Quality (spku_api_suite)",
|
||||
"type": "ge_quality",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"expectation_suite": "spku_api_suite"
|
||||
}
|
||||
},
|
||||
"width": 236,
|
||||
"height": 92
|
||||
},
|
||||
{
|
||||
"id": "node-gold-dbt",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1120,
|
||||
"y": 160
|
||||
},
|
||||
"data": {
|
||||
"label": "dbt Gold Mart (fct_spku_daily_mart)",
|
||||
"type": "dbt_transform",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"dbt_layer": "gold",
|
||||
"dbt_model": "fct_spku_daily_mart"
|
||||
}
|
||||
},
|
||||
"width": 280,
|
||||
"height": 92
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"id": "e1-2",
|
||||
"source": "node-bronze-src",
|
||||
"target": "node-bronze-storage",
|
||||
"type": "removableEdge",
|
||||
"data": {}
|
||||
},
|
||||
{
|
||||
"id": "e2-3",
|
||||
"source": "node-bronze-storage",
|
||||
"target": "node-silver-dbt",
|
||||
"type": "removableEdge",
|
||||
"data": {}
|
||||
},
|
||||
{
|
||||
"id": "e3-4",
|
||||
"source": "node-silver-dbt",
|
||||
"target": "node-silver-ge",
|
||||
"type": "removableEdge",
|
||||
"data": {}
|
||||
},
|
||||
{
|
||||
"id": "e4-5",
|
||||
"source": "node-silver-ge",
|
||||
"target": "node-gold-dbt",
|
||||
"type": "removableEdge",
|
||||
"data": {}
|
||||
}
|
||||
],
|
||||
"is_draft": false,
|
||||
"status": "deployed"
|
||||
}
|
||||
Reference in New Issue
Block a user