"""
Raw data validation gate.

Run this immediately after downloading any new raw CSV, before it ever
touches the preprocessing pipeline. Production data pipelines fail
silently and expensively when a schema assumption breaks three steps
downstream — this catches it at step zero with a clear, actionable error.

Usage:
    python -m ml_pipeline.data_collection.validate_raw_data \
        --file data/raw/imd/rainfall.csv --schema climate
"""

import argparse
import sys
from pathlib import Path

import pandas as pd

REQUIRED_COLUMNS = {
    "climate": {"district", "year", "month", "rainfall_mm"},
    "crop_yield": {"district", "year", "crop", "yield_ton_ha", "area_ha"},
    "market": {"district", "commodity", "date", "modal_price"},
}


def validate(file_path: Path, schema: str) -> list[str]:
    issues: list[str] = []

    if not file_path.exists():
        return [f"File does not exist: {file_path}"]

    try:
        df = pd.read_csv(file_path, nrows=5000)  # sample for speed on huge files
    except Exception as exc:
        return [f"File could not be parsed as CSV: {exc}"]

    if df.empty:
        issues.append("File loaded but contains zero rows")
        return issues

    required = REQUIRED_COLUMNS.get(schema)
    if required is None:
        issues.append(f"Unknown schema '{schema}'. Known schemas: {list(REQUIRED_COLUMNS.keys())}")
        return issues

    actual_columns = {c.strip().lower() for c in df.columns}
    missing = required - actual_columns
    if missing:
        issues.append(
            f"Missing required columns: {sorted(missing)}. "
            f"Found columns: {sorted(actual_columns)}. "
            f"Check if column names need renaming before this file is used."
        )

    null_fraction = df.isnull().mean()
    high_null_cols = null_fraction[null_fraction > 0.5].index.tolist()
    if high_null_cols:
        issues.append(
            f"Columns with >50% missing values: {high_null_cols}. "
            f"Verify this is expected before proceeding — high nulls often "
            f"indicate a parsing or join-key mismatch upstream."
        )

    duplicate_fraction = df.duplicated().mean()
    if duplicate_fraction > 0.1:
        issues.append(
            f"{duplicate_fraction:.0%} of sampled rows are exact duplicates. "
            f"Check the source export for accidental row repetition."
        )

    return issues


def main() -> None:
    parser = argparse.ArgumentParser(description="Validate a raw data file before preprocessing")
    parser.add_argument("--file", required=True, help="Path to the raw CSV file")
    parser.add_argument("--schema", required=True, choices=list(REQUIRED_COLUMNS.keys()))
    args = parser.parse_args()

    issues = validate(Path(args.file), args.schema)

    if issues:
        print(f"VALIDATION FAILED for {args.file}:")
        for issue in issues:
            print(f"  - {issue}")
        sys.exit(1)
    else:
        print(f"VALIDATION PASSED for {args.file}")


if __name__ == "__main__":
    main()