first commit

2026-10-09 02:35:14 +07:00
commit 4ca7a9a58e
108 changed files with 23035 additions and 0 deletions
@@ -0,0 +1,16 @@
+# =============================================================================
+# Grafana — Provisioning Dashboards
+# Referensi arsitektur: bab 4.6.3.3 (Setup dashboard bawaan)
+# =============================================================================
+apiVersion: 1
+
+providers:
+  - name: "SiCABE Data Quality"
+    orgId: 1
+    folder: "SiCABE"
+    type: file
+    disableDeletion: false
+    editable: true
+    updateIntervalSeconds: 60
+    options:
+      path: /etc/grafana/provisioning/dashboards/sicabe
@@ -0,0 +1,260 @@
+{
+  "title": "SiCABE — Host Docker (node_exporter)",
+  "uid": "sicabe-hosts",
+  "version": 1,
+  "timezone": "Asia/Jakarta",
+  "schemaVersion": 36,
+  "refresh": "30s",
+  "time": { "from": "now-6h", "to": "now" },
+  "tags": ["sicabe", "host", "node_exporter"],
+  "panels": [
+    {
+      "id": 1,
+      "title": "Host Up",
+      "type": "stat",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 4, "x": 0, "y": 0 },
+      "description": "Status node_exporter per host (:9100, pra-syarat host di luar installer). 1 = up, 0 = down. Job: node.",
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "value_and_name",
+        "orientation": "vertical"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": { "text": "DOWN", "color": "red", "index": 1 },
+                "1": { "text": "UP", "color": "green", "index": 0 }
+              }
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red", "value": null },
+              { "color": "green", "value": 1 }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "targets": [
+        { "refId": "A", "expr": "up{job=\"node\"}", "legendFormat": "{{instance}}" }
+      ]
+    },
+    {
+      "id": 2,
+      "title": "CPU Usage %",
+      "type": "timeseries",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 10, "x": 4, "y": 0 },
+      "description": "Pemakaian CPU per host = 100 × (1 - idle), rata-rata jendela 5m.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "100 * (1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])))",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "id": 3,
+      "title": "Load Average (1/5/15m)",
+      "type": "timeseries",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 10, "x": 14, "y": 0 },
+      "description": "Load average 1/5/15 menit per host. Bandingkan dengan jumlah CPU; load > jumlah CPU = jenuh.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": { "drawStyle": "line", "fillOpacity": 0, "lineWidth": 1, "showPoints": "never" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        { "refId": "A", "expr": "node_load1", "legendFormat": "{{instance}} load1" },
+        { "refId": "B", "expr": "node_load5", "legendFormat": "{{instance}} load5" },
+        { "refId": "C", "expr": "node_load15", "legendFormat": "{{instance}} load15" }
+      ]
+    },
+    {
+      "id": 4,
+      "title": "Memory Used %",
+      "type": "timeseries",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 8 },
+      "description": "Pemakaian memori per host = 100 × (1 - MemAvailable/MemTotal). Ambang: 80% warning, 90% critical.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "never" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "orange", "value": 80 },
+              { "color": "red", "value": 90 }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "id": 5,
+      "title": "Disk I/O",
+      "type": "timeseries",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 8, "x": 8, "y": 8 },
+      "description": "Laju baca/tulis disk per host (rata-rata 5m, semua device digabung).",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "Bps",
+          "custom": { "drawStyle": "line", "fillOpacity": 5, "lineWidth": 1, "showPoints": "never" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum by (instance) (rate(node_disk_read_bytes_total[5m]))",
+          "legendFormat": "{{instance}} read"
+        },
+        {
+          "refId": "B",
+          "expr": "sum by (instance) (rate(node_disk_written_bytes_total[5m]))",
+          "legendFormat": "{{instance}} write"
+        }
+      ]
+    },
+    {
+      "id": 6,
+      "title": "Network Traffic",
+      "type": "timeseries",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 8 },
+      "description": "Laju RX/TX per host, antarmuka fisik saja (lo/veth/docker/bridge dikecualikan).",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "Bps",
+          "custom": { "drawStyle": "line", "fillOpacity": 5, "lineWidth": 1, "showPoints": "never" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|br-.*|virbr.*\"}[5m]))",
+          "legendFormat": "{{instance}} RX"
+        },
+        {
+          "refId": "B",
+          "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|br-.*|virbr.*\"}[5m]))",
+          "legendFormat": "{{instance}} TX"
+        }
+      ]
+    },
+    {
+      "id": 7,
+      "title": "Disk Usage % (per host & mount)",
+      "type": "table",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 16, "x": 0, "y": 16 },
+      "description": "Persentase pemakaian tiap mount. Filesystem pseudo (tmpfs/overlay/squashfs/ramfs) dikecualikan. Ambang: 80% warning, 90% critical.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "decimals": 1,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "orange", "value": 80 },
+              { "color": "red", "value": 90 }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "Value" },
+            "properties": [
+              { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "gradient" } }
+            ]
+          }
+        ]
+      },
+      "options": { "showHeader": true },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "100 * (1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|ramfs\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|ramfs\"})",
+          "format": "table",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 8,
+      "title": "Uptime",
+      "type": "stat",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 16 },
+      "description": "Lama host hidup sejak boot = time() - node_boot_time_seconds.",
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "colorMode": "none",
+        "graphMode": "none",
+        "textMode": "value_and_name",
+        "orientation": "vertical"
+      },
+      "fieldConfig": { "defaults": { "unit": "dtdurations" }, "overrides": [] },
+      "targets": [
+        { "refId": "A", "expr": "time() - node_boot_time_seconds", "legendFormat": "{{instance}}" }
+      ]
+    }
+  ]
+}
@@ -0,0 +1,159 @@
+{
+  "title": "SiCABE — Overview Lapisan & Tata Kelola Data",
+  "version": 6,
+  "timezone": "Asia/Jakarta",
+  "schemaVersion": 36,
+  "refresh": "30s",
+  "time": { "from": "now-6h", "to": "now" },
+  "panels": [
+    {
+      "id": 1,
+      "title": "DAG Failure Rate (Ingestion Layer)",
+      "type": "stat",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 0 },
+      "description": "KPI #7 — rasio DAG run gagal (jendela 1 jam). Ambang: 5% warning, 15% critical. Sumber: airflow_dagrun_duration_{failed,success}_count (Airflow StatsD → statsd-exporter).",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "orange", "value": 5 },
+              { "color": "red", "value": 15 }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "targets": [
+        { "refId": "A", "expr": "100 * (sum(rate(airflow_dagrun_duration_failed_count[1h])) or vector(0)) / clamp_min((sum(rate(airflow_dagrun_duration_failed_count[1h])) or vector(0)) + (sum(rate(airflow_dagrun_duration_success_count[1h])) or vector(0)), 0.0001)" }
+      ]
+    },
+    {
+      "id": 2,
+      "title": "Disk Usage (MinIO — Storage Layer)",
+      "type": "gauge",
+      "datasource": "MinIO Metrics",
+      "gridPos": { "h": 8, "w": 8, "x": 8, "y": 0 },
+      "description": "KPI #1 — kapasitas terpakai (%) = 1 - usable_free/usable_total. Ambang: 80% warning, 90% critical.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "orange", "value": 80 },
+              { "color": "red", "value": 90 }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "targets": [
+        { "refId": "A", "expr": "(1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) * 100" }
+      ]
+    },
+    {
+      "id": 3,
+      "title": "Serving Query Latency P99 (StarRocks)",
+      "type": "gauge",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 0 },
+      "description": "KPI #6 — latensi query P99 StarRocks FE. Ambang: 500 ms warning, 2000 ms critical.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "orange", "value": 500 },
+              { "color": "red", "value": 2000 }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "targets": [
+        { "refId": "A", "expr": "starrocks_fe_query_latency_ms{quantile=\"0.99\"}" }
+      ]
+    },
+    {
+      "id": 4,
+      "title": "Trino — Saturasi Query (coordinator)",
+      "type": "timeseries",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 8 },
+      "description": "KPI #4 (SATURASI) — antrean & kegagalan query Trino coordinator. ⚠️ P99 LATENCY tidak di sini: tidak tersedia via JMX/StatsD (metrik turunan, di luar lingkup M1–M3). Sumber: JMX → trino-jmx-exporter.",
+      "targets": [
+        { "refId": "A", "legendFormat": "running query", "expr": "trino_execution_querymanager_runningqueries" },
+        { "refId": "B", "legendFormat": "queued query", "expr": "trino_execution_querymanager_queuedqueries" },
+        { "refId": "C", "legendFormat": "failed /s", "expr": "sum(rate(trino_execution_querymanager_failedqueries_totalcount[5m]))" }
+      ]
+    },
+    {
+      "id": 5,
+      "title": "S3 API Latency (MinIO) — % panggilan lambat",
+      "type": "timeseries",
+      "datasource": "MinIO Metrics",
+      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 8 },
+      "description": "KPI #2 — rasio panggilan S3 yang lebih lambat dari ambang (jendela 5m). Warning 50 ms; critical ~250 ms (bucket terdekat — MinIO tidak menyediakan bucket 200 ms). Basis metrik minio_s3_requests_ttfb_seconds_distribution bertipe gauge (bukan histogram), sehingga dipakai rasio barbucket, bukan histogram_quantile.",
+      "fieldConfig": {
+        "defaults": { "unit": "percent", "min": 0 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "legendFormat": "% > 50 ms (warning)",
+          "expr": "100 * (1 - sum(rate(minio_s3_requests_ttfb_seconds_distribution{le=\"0.050\"}[5m])) / sum(rate(minio_s3_requests_ttfb_seconds_distribution{le=\"+Inf\"}[5m])))"
+        },
+        {
+          "refId": "B",
+          "legendFormat": "% > 250 ms (critical)",
+          "expr": "100 * (1 - sum(rate(minio_s3_requests_ttfb_seconds_distribution{le=\"0.250\"}[5m])) / sum(rate(minio_s3_requests_ttfb_seconds_distribution{le=\"+Inf\"}[5m])))"
+        }
+      ]
+    },
+    {
+      "id": 6,
+      "title": "Airflow — Durasi DAG & Task",
+      "type": "timeseries",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 },
+      "description": "KPI #5 (perkiraan) — durasi pipeline & task. Airflow pada StatsD ini TIDAK memecah durasi per-task (hanya summary global `airflow_task_duration`), jadi durasi dbt (silver_to_gold/bronze_to_silver) belum bisa diisolasi. Perbaikan menyusul: statsd mapping config (lihat Review 19). DAG P90 = durasi `sicabe_batch_daily`.",
+      "targets": [
+        {
+          "refId": "A",
+          "legendFormat": "rata-rata task (dtk)",
+          "expr": "sum(rate(airflow_task_duration_sum[1h])) / clamp_min(sum(rate(airflow_task_duration_count[1h])), 0.0001)"
+        },
+        {
+          "refId": "B",
+          "legendFormat": "DAG sicabe_batch_daily P90 (dtk)",
+          "expr": "airflow_dagrun_duration_success_sicabe_batch_daily{quantile=\"0.9\"}"
+        }
+      ]
+    },
+    {
+      "id": 7,
+      "title": "Trino — Query Throughput (coordinator)",
+      "type": "timeseries",
+      "datasource": "Prometheus",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 },
+      "description": "Laju siklus hidup query Trino coordinator (per detik, jendela 5m). Sumber: JMX → trino-jmx-exporter (domain trino.execution).",
+      "targets": [
+        { "refId": "A", "legendFormat": "submitted /s", "expr": "sum(rate(trino_execution_querymanager_submittedqueries_totalcount[5m]))" },
+        { "refId": "B", "legendFormat": "completed /s", "expr": "sum(rate(trino_execution_querymanager_completedqueries_totalcount[5m]))" },
+        { "refId": "C", "legendFormat": "failed /s", "expr": "sum(rate(trino_execution_querymanager_failedqueries_totalcount[5m]))" },
+        { "refId": "D", "legendFormat": "canceled /s", "expr": "sum(rate(trino_execution_querymanager_canceledqueries_totalcount[5m]))" }
+      ]
+    }
+  ]
+}
@@ -0,0 +1,42 @@
+# =============================================================================
+# eBdesk SiCABE — Grafana datasource (hasil provisioning)
+# =============================================================================
+# Datasource bertipe `prometheus` WAJIB menunjuk server Prometheus
+# (`http://prometheus:9090`). Menunjuk `http://minio:9000` akan selalu gagal
+# ("error querying the Prometheus API") karena MinIO adalah API S3, bukan
+# server Prometheus — metrik MinIO diambil Prometheus lewat scrape
+# (lihat config/prometheus/prometheus.yml), bukan dibaca langsung oleh Grafana.
+#
+# Datasource OpenMetadata DIHAPUS pada review Fase 6: plugin
+# `grafana-opendatasource-http` TIDAK ADA di katalog Grafana (404) sehingga
+# selalu "Plugin not registered". Tidak ada plugin resmi Grafana → OpenMetadata;
+# arah yang didukung OpenMetadata adalah sebaliknya (OM meng-ingest Grafana).
+# Bila data OM ingin ditampilkan di Grafana, pakai datasource HTTP generik
+# (mis. Infinity) + token API OpenMetadata.
+# =============================================================================
+apiVersion: 1
+
+# Grafana provisioning hanya MENAMBAH/MENGUBAH — datasource yang dihapus dari
+# berkas ini TIDAK ikut terhapus. Karena itu penghapusan ditulis eksplisit di
+# bawah (idempoten, aman dijalankan berulang).
+deleteDatasources:
+  - name: OpenMetadata
+    orgId: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
+
+  # Metrik MinIO tetap memakai instance Prometheus yang sama (job_name: minio),
+  # dinamai terpisah karena dipakai panel "Storage Usage" di dashboard.
+  - name: MinIO Metrics
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    editable: false
+    jsonData:
+      tlsSkipVerify: true