
    9YGj`)                        d Z ddlZddlZddlZddlZddlmZ ddlZddl	Z
ddlmZ  ej                  ej                  d        ej                  e      Zddddddddddddd	Zd
Zde
j(                  de
j(                  fdZdede
j(                  fdZde
j.                  defdZde
j(                  de
j(                  de
j(                  fdZde
j(                  de
j(                  fdZde
j.                  defdZdedefdZde
j.                  defdZde
j(                  de
j(                  de
j(                  de
j(                  de
j(                  f
dZ ddZ!edk(  r e!        yy) u  
Feature engineering pipeline.

Produces data/processed/final_feature_matrix.csv — the single training
table every model in ml_pipeline/training/ reads from. Also produces
data/processed/climate_clean.csv (already created by clean_climate_data.py,
read directly by the live API's FeatureBuilder for historical averages).

This script must be re-run any time the raw source data is refreshed
(new IMD/ICRISAT/Agmarknet export). Models trained on a stale feature
matrix will silently drift from what the live system computes at
inference time — keep this pipeline run as part of your retraining
checklist, not a one-time setup step.

Usage:
    python -m ml_pipeline.feature_engineering.build_feature_matrix         --climate data/processed/climate_clean.csv         --crop data/processed/crop_clean.csv         --market data/processed/market_clean.csv         --crop-lookup data/lookup/crops.json         --output data/processed/final_feature_matrix.csv
    N)Path)statsz%%(asctime)s %(levelname)s %(message)s)levelformat      )         	   
         r   r            i  climatereturnc                    | j                  g d      j                  d      } | | d   t        k     j                  ddg      j	                  dd	      j                         }t        j                  | |ddgd
      } | d   j                         }| d   j                  |      | d<   | d   j                  | d   j                               | d<   | d   j                  dd      | d<   | d   | d   z
  | d   z  | d<   | j                  d      d   j                  d      | d<   | j                  d      d   j                  d       | d<   | d   j                  | d         | d<   | d   j                  t              | d<   d| j                  vrt         j#                  d       d| d<   d| j                  vrt         j#                  d       d| d<   | S )NdistrictyearmonthT)dropr   r   r   )rainfall_mmmean)r   std)hist_mean_rainfallhist_std_rainfallleftonhowr   r   r   r   r   rainfall_anomalyrainfall_lag1c                 D    | j                  dd      j                         S )Nr   r   min_periods)rollingr   xs    aC:\Crop_Prediction\Backend\crop-ai-system\ml_pipeline\feature_engineering\build_feature_matrix.py<lambda>z&add_climate_features.<locals>.<lambda>H   s    QYYqaY8==?     rainfall_ma3season_indextemp_cuL   No temp_c column in climate data — using a flat 30.0C default for all rowsg      >@humidityuM   No humidity column in climate data — using a flat 70.0 default for all rowsg     Q@)sort_valuesreset_indexHISTORICAL_CUTOFF_YEARgroupbyaggpdmerger   fillnar   replaceshift	transformmap
SEASON_MAPcolumnsloggerwarning)r   histfallback_means      r,   add_climate_featuresrE   0   s   !!"?@LLRVLWG76?&<<=EE	W	c20 
  km 	 hhw*g)>FKGM*//1M$+,@$A$H$H$WG !#*+>#?#F#Fw}G]GaGaGc#dG #*+>#?#G#G1#MG  
	'*>"?	?7K^C__   'z:=IOOPQRGO
#M2	?	@ N  '7>>w}?UVGO%g.22:>GN
 w&ef (fg"
Nr.   lookup_pathc                    t        | d      5 }t        j                  |      }d d d        g }j                         D ]+  \  }}d|i}|j	                  |       |j                  |       - t        j                  |      S # 1 sw Y   ^xY w)Nz	utf-8-sig)encodingcrop)openjsonloaditemsupdateappendr8   	DataFrame)rF   f
crops_dictrowsrI   propsrows          r,   load_crop_propertiesrV   [   s    
 
kK	0 "AYYq\
"D!'') etn

5C <<" "s   BB
seriesc                     | j                         j                  }t        |      dk  ryt        j                  t        |            }t        j                  ||      \  }}}}}t        |      S )Nr           )dropnavalueslennparanger   
linregressfloat)rW   r[   r+   slope_s        r,   _compute_sloperc   j   sZ    ]]_##F
6{Q
		#f+A((F3E1aA<r.   rI   
crop_propsc                    t        j                  | |dd      } | | d   j                            d   j                         }t	        |      dkD  r"t
        j                  dt        |       d       | j                  g d      } | j                  d	dg      d
   j                  d       j                  ddgd      }|| d<   | d   j                  d      | d<   | S )NrI   r    r!   water_req_mmr   z@Crops present in yield data but missing from crops.json lookup: zu. Add them to data/lookup/crops.json or these rows will have null agronomic features and get dropped before training.)r   rI   r   r   yield_ton_hac                 R    | j                  dd      j                  t        d      S )Nr   r   r'   Frawr)   applyrc   ss    r,   r-   z#add_crop_features.<locals>.<lambda>   #    1!4::>u:U r.   r   Tr   r   yield_trend_sloperY   )r8   r9   isnulluniquer\   rA   rB   listr3   r6   rl   r4   r:   )rI   rd   missing_propstrend_slopess       r,   add_crop_featuresrw   s   s    88D*V<Dn-4467?FFHM
=ANM"# $VW	
 89Dj&)*>:	U	V	Aq6	- 
 !-D	 $%8 9 @ @ ED	Kr.   marketc                    | j                  g d      } | j                  ddg      d   j                  d      | d<   | j                  ddg      d   j                  d       j	                  ddgd	
      }|j                  d      | d<   | S )Nr   rI   r   r   r   rI   price_per_quintalr   
price_lag1c                 R    | j                  dd      j                  t        d      S )Nr	   r   r'   Fri   rk   rm   s    r,   r-   z%add_market_features.<locals>.<lambda>   ro   r.   r   Trp   rY   demand_trend)r3   r6   r<   rl   r4   r:   )rx   r~   s     r,   add_market_featuresr      s     EFF!>>:v*>?@STZZ[\]F< 	
F+,-@A	U	V	Aq6	- 
 *005F>Mr.   rU   c                     | d   | d   cxk  xr | d   k  nc }| d   | d   dz  k\  }t        | d         dkD  }| j                  d	d
      d
kD  }| d   | d   dz  dz  k  }|r|ry|r|r|s|ry|r|s|ry|s| d   s|syy)N
min_temp_cr1   
max_temp_cr   rf   r   r$   g      ?rg   r   皙?not_suitablesuitableriskydrought_tolerant)absget)rU   temp_okrain_okanomaly_bad	has_yieldextreme_droughts         r,   label_feasibilityr      s    ,3x=EC4EEG- S%82%=>Gc,-.4K*Q.I-(C,?",D+LLO o 7;9 {y 3129r.   ra   c                 :    | dkD  ry| dk  ryt        |       dkD  ryy)Nr   upwardgdownwardg?volatilestable)r   )ra   s    r,   label_trendr      s(    s{		Ud	r.   c                 R    | d   dk(  r| j                  dd      dkD  ry| d   dk(  ryy)	Nfeasibility_labelr   r~   r   highr   lowmedium)r   )rU   s    r,   label_suitabilityr      s9    
:-#''.!2Lq2P	 	!^	3r.   c                    t        |       } t        ||      }t        |      }t        j                  || g dd      }t
        j                  dt        |       d       t        |      dk(  rt        d      t        j                  ||g dd	      }t
        j                  d
t        |       d|d   j                         j                          d       |j                  d      d   j                  d       |d<   |d   j                  d      |d<   |d   j                  d      |d<   t        |      }g d}|j                  |      }t
        j                  d|t        |      z
   d       |j                  t         d      |d<   |d   j                  t"              |d<   |j                  t$        d      |d<   |S )Nr   innerr!   zAfter climate+crop merge: z rowsr   zZero rows after merging climate and crop data. Likely cause: district name spelling mismatch, or year/month ranges that don't overlap between the two sources. Inspect both inputs.rz   r    zAfter market merge: z rows (r{   z missing market price)rI   c                 @    | j                  | j                               S )N)r:   medianr*   s    r,   r-   zbuild.<locals>.<lambda>   s    !((188:& r.   price_volatilityr   r~   rY   )r1   r   r   r   rf   )subsetzDropped z) rows missing required label-input fieldsr   )axisr   rq   trend_labelsuitability_label)rE   rw   r   r8   r9   rA   infor\   
ValueErrorrr   sumr6   r=   r:   rZ   rl   r   r   r   )r   rI   rx   rd   mergedbefore_droprequired_for_labelss          r,   buildr      s   "7+GT:.D (FXXdG(E7SF
KK,S[M?@
6{aJ
 	
 XXff)NTZ[F
KK&s6{m76BU;V;];];_;c;c;e:ff|}~"(.."89L"M"W"W&#F "((:!;!B!B3!GF#N3::3?F>f+K_]]"5]6F
KK(;V455^_`"(,,/@q,"IF"#67==kJF="(,,/@q,"IFMr.   c                     t        j                  d      } | j                  dd       | j                  dd       | j                  dd       | j                  dd       | j                  d	d       | j                         }t        j                  d
       t        j                  |j                        }t        j                  |j                        }t        j                  |j                        }t        t        |j                              }t        j                  d       t        ||||      }t        |      dk  r7t        j!                  dt        |       d       t#        j$                  d       t        |j&                        }|j(                  j+                  dd       |j-                  |d       t        j                  dt        |       d|        t        j                  d|d   j/                                 t        j                  d|d   j/                                 t        j                  d|d   j/                                 y )Nz1Build the final feature matrix for model training)descriptionz	--climateT)requiredz--cropz--marketz--crop-lookupz--outputzLoading inputs...zBuilding feature matrix...d   zOnly u    rows in the final feature matrix. This is too small to train reliable models — review the merge keys and source data coverage before proceeding to training.r   )parentsexist_okF)indexzSaved z	 rows to z Feasibility label distribution:
r   zTrend label distribution:
r   z Suitability label distribution:
r   )argparseArgumentParseradd_argument
parse_argsrA   r   r8   read_csvr   rI   rx   rV   r   crop_lookupr   r\   errorsysexitoutputparentmkdirto_csvvalue_counts)parserargs
climate_dfcrop_df	market_dfrd   final_dfoutput_paths           r,   mainr      s   $$1deF
d3
40

T2
$7

T2D
KK#$T\\*Jkk$))$GDKK(I%d4+;+;&<=J
KK,-Z)Z@H
8}sCM? #B C	

 	t{{#KTD9OOKuO-
KK&Xy>?
KK3H=P4Q4^4^4`3abc
KK-h}.E.R.R.T-UVW
KK3H=P4Q4^4^4`3abcr.   __main__)r   N)"__doc__r   rK   loggingr   pathlibr   numpyr]   pandasr8   scipyr   basicConfigINFO	getLogger__name__rA   r?   r5   rP   rE   rV   Seriesr`   rc   rw   r   strr   r   r   r   r    r.   r,   <module>r      s  .    
       ',,/V W			8	$ 	Q1	qaAQ1
  
(",, (2<< (Vd r|| 299  BLL bll r|| .
 
 
299  2u  299  !2<< !r|| !R\\ !WYWcWc !hjhtht !H!dH zF r.   