Python Data Analytics & ETL Pipeline

"""Data Cleaning & Aggregation Pipeline."""
import json
from statistics import mean


def summarize_records(records: list[dict]) -> dict:
    valid = [r for r in records if isinstance(r.get("score"), (int, float))]
    scores = [r["score"] for r in valid]
    return {
        "total_records": len(records),
        "valid_records": len(valid),
        "average_score": round(mean(scores), 2) if scores else 0.0,
        "max_score": max(scores) if scores else 0.0,
    }


if __name__ == "__main__":
    sample = [{"id": 1, "score": 92.5}, {"id": 2, "score": 88.0}, {"id": 3, "score": 95.0}]
    print(json.dumps(summarize_records(sample), indent=2))

{
  "pipeline_name": "daily_kpi_rollup",
  "batch_size": 500,
  "strict_validation": true
}

# Python Data Analytics & ETL Pipeline

Structured data processing pipeline with configurable batching in `config.json`.