"""
Agmarknet market price collector via data.gov.in's public API.

This is the cleanest of the three data sources — a real documented REST
API, no scraping. Register for a free API key at https://data.gov.in
and set DATA_GOV_IN_API_KEY in your environment before running this.

Usage:
    python -m ml_pipeline.data_collection.fetch_agmarknet \
        --api-key YOUR_KEY --state "Tamil Nadu" --output data/raw/agmarknet/prices.csv
"""

import argparse
import time
from pathlib import Path

import pandas as pd
import requests

RESOURCE_ID = "9ef84268-d588-465a-a308-a864a43d0070"  # current daily prices resource
BASE_URL = f"https://api.data.gov.in/resource/{RESOURCE_ID}"
PAGE_SIZE = 200  # data.gov.in is frequently slow; smaller pages reduce the
                  # chance of a single request timing out before it returns
MAX_RETRIES = 5
RETRY_BACKOFF_SECONDS = 8
REQUEST_TIMEOUT_SECONDS = 60  # this API is known to be slow, especially
                                # during India business hours; 30s was too
                                # tight and caused spurious timeouts even
                                # when the API was working normally


def fetch_page(api_key: str, offset: int, state_filter: str | None) -> list[dict]:
    params = {
        "api-key": api_key,
        "format": "json",
        "limit": PAGE_SIZE,
        "offset": offset,
    }
    if state_filter:
        params["filters[state]"] = state_filter

    for attempt in range(1, MAX_RETRIES + 1):
        try:
            resp = requests.get(BASE_URL, params=params, timeout=REQUEST_TIMEOUT_SECONDS)
            resp.raise_for_status()
            data = resp.json()
            return data.get("records", [])
        except requests.RequestException as exc:
            if attempt == MAX_RETRIES:
                raise RuntimeError(
                    f"Failed to fetch data.gov.in page at offset={offset} "
                    f"after {MAX_RETRIES} attempts: {exc}"
                ) from exc
            wait = RETRY_BACKOFF_SECONDS * attempt
            print(f"Request failed (attempt {attempt}/{MAX_RETRIES}), retrying in {wait}s: {exc}")
            time.sleep(wait)
    return []


def fetch_all_records(api_key: str, state_filter: str | None) -> pd.DataFrame:
    all_records: list[dict] = []
    offset = 0

    while True:
        records = fetch_page(api_key, offset, state_filter)
        if not records:
            break
        all_records.extend(records)
        print(f"Fetched {len(all_records)} records so far...")
        offset += PAGE_SIZE
        time.sleep(0.5)  # be a polite API citizen, do not hammer a free government API

    if not all_records:
        raise ValueError(
            "No records returned. Check your API key, state filter, and that "
            "the resource id is still current — data.gov.in occasionally "
            "rotates resource ids for this dataset."
        )

    return pd.DataFrame(all_records)


def main() -> None:
    parser = argparse.ArgumentParser(description="Fetch Agmarknet prices from data.gov.in")
    parser.add_argument("--api-key", required=True, help="data.gov.in API key")
    parser.add_argument("--state", default=None, help="Filter to one state, e.g. 'Tamil Nadu'")
    parser.add_argument("--output", required=True, help="Output CSV path")
    args = parser.parse_args()

    print(f"Fetching Agmarknet records (state filter: {args.state or 'none'})...")
    df = fetch_all_records(args.api_key, args.state)

    output_path = Path(args.output)
    output_path.parent.mkdir(parents=True, exist_ok=True)
    df.to_csv(output_path, index=False)

    print(f"Saved {len(df)} records to {output_path}")
    print(f"Columns: {list(df.columns)}")


if __name__ == "__main__":
    main()