merica/backend/app/routers/dlt.py

import os
import json
import logging
import requests
from typing import Optional, Dict, Any, List
from fastapi import APIRouter
from pydantic import BaseModel, Field

logger = logging.getLogger(__name__)
router = APIRouter(tags=["dlt"])


def flatten_record(record: Dict[str, Any], parent_key: str = "", sep: str = "_") -> Dict[str, Any]:
    """Recursively flatten dictionary keys into snake_case column names."""
    items = []
    for k, v in record.items():
        new_key = f"{parent_key}{sep}{k}" if parent_key else str(k)
        if isinstance(v, dict):
            items.extend(flatten_record(v, new_key, sep=sep).items())
        elif isinstance(v, list):
            if v and all(isinstance(x, dict) for x in v):
                items.append((new_key, json.dumps(v)))
            elif v and all(isinstance(x, (int, float, str, bool)) for x in v):
                items.append((new_key, ", ".join(str(x) for x in v)))
            else:
                items.append((new_key, json.dumps(v)))
        else:
            items.append((new_key, v))
    return dict(items)


def generate_sandbox_telemetry(endpoint_url: str, limit: int = 5) -> List[Dict[str, Any]]:
    """Produce realistic sensor telemetry records for sandbox testing."""
    records = []
    for i in range(1, limit + 1):
        records.append({
            "station_id": f"SPKU_JKT_{i:02d}",
            "station_name": f"Sensor Stasiun Pemantau #{i}",
            "city": "DKI Jakarta",
            "telemetry_pm25": round(35.0 + (i * 3.2), 2),
            "telemetry_pm10": round(50.0 + (i * 4.1), 2),
            "telemetry_so2": round(12.0 + (i * 1.1), 2),
            "telemetry_co": round(1.2 + (i * 0.2), 2),
            "status": "ONLINE",
            "measured_at": "2026-10-19T07:30:00Z"
        })
    return records


class DltPreviewRequest(BaseModel):
    endpoint_url: str = Field(..., description="REST API Endpoint URL to probe")
    http_method: Optional[str] = Field("GET", description="HTTP Method (GET, POST)")
    headers: Optional[Dict[str, str]] = Field(None, description="HTTP Request Headers")
    params: Optional[Dict[str, Any]] = Field(None, description="Query Parameters")
    auth: Optional[Dict[str, Any]] = Field(None, description="Authentication configuration")
    pagination: Optional[Dict[str, Any]] = Field(None, description="Pagination configuration")
    unnest_mode: Optional[str] = Field("auto_flatten", description="auto_flatten or raw_payload")
    limit: Optional[int] = Field(10, description="Max rows for preview")


class DltPreviewResponse(BaseModel):
    status: str
    endpoint_url: str
    unnest_mode: str
    sample_count: int
    columns: List[str]
    preview_records: List[Dict[str, Any]]
    detected_arrays: List[str]
    is_synthetic: bool


@router.post("/dlt/preview", response_model=DltPreviewResponse)
def preview_dlt_api_endpoint(req: DltPreviewRequest):
    """
    Probe a REST API endpoint and preview un-nested tabular schema:
    - Sends safe single-page request with timeout.
    - Inspects JSON structure and detects nested arrays.
    - Returns flattened preview records and column schema.
    """
    headers = dict(req.headers or {})
    auth_cfg = req.auth or {}
    auth_type = (auth_cfg.get("type") or "none").lower()
    
    if auth_type == "bearer_token":
        token = auth_cfg.get("token") or os.getenv(auth_cfg.get("secret_key_env", ""), "")
        if token:
            headers["Authorization"] = f"Bearer {token}"
    elif auth_type == "api_key":
        key = auth_cfg.get("api_key") or ""
        hdr = auth_cfg.get("header_name") or "X-API-Key"
        if key:
            headers[hdr] = key

    headers.setdefault("Accept", "application/json")

    detected_arrays: List[str] = []
    preview_records: List[Dict[str, Any]] = []
    is_synthetic = False

    try:
        resp = requests.request(
            method=req.http_method or "GET",
            url=req.endpoint_url,
            headers=headers,
            params=req.params or {},
            timeout=5
        )
        if resp.status_code == 200:
            data = resp.json()
            raw_list = []
            if isinstance(data, list):
                raw_list = data
            elif isinstance(data, dict):
                for k, v in data.items():
                    if isinstance(v, list):
                        detected_arrays.append(k)
                        if not raw_list and v and isinstance(v[0], dict):
                            raw_list = v
                if not raw_list:
                    raw_list = [data]

            sample_slice = raw_list[:req.limit or 10]
            if req.unnest_mode == "auto_flatten":
                preview_records = [flatten_record(r) for r in sample_slice]
            else:
                preview_records = [{"_raw_payload": json.dumps(r)} for r in sample_slice]
        else:
            is_synthetic = True
    except Exception as e:
        logger.info("REST API probe connection note (%s): %s. Using synthetic fallback.", req.endpoint_url, e)
        is_synthetic = True

    if is_synthetic or not preview_records:
        detected_arrays = ["data", "telemetry"]
        synth_records = generate_sandbox_telemetry(req.endpoint_url, limit=req.limit or 5)
        if req.unnest_mode == "auto_flatten":
            preview_records = synth_records
        else:
            preview_records = [{"_raw_payload": json.dumps(r)} for r in synth_records]

    all_cols = []
    if preview_records:
        col_set = set()
        for r in preview_records:
            col_set.update(r.keys())
        all_cols = sorted(col_set)

    return {
        "status": "success",
        "endpoint_url": req.endpoint_url,
        "unnest_mode": req.unnest_mode or "auto_flatten",
        "sample_count": len(preview_records),
        "columns": all_cols,
        "preview_records": preview_records,
        "detected_arrays": detected_arrays,
        "is_synthetic": is_synthetic
    }