+
    .Gj')                        R t ^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RIt^ RI	t
^ RIHt ]P                  ! ]P                  RR7       ]P                  ! ]4      t^^ ^^ ^^ ^	^ ^
^^^^^^^^^^^^^^^/tRtR R ltR	 R
 ltR R ltR R ltR R ltR R ltR R ltR R ltR R ltR R lt]R8X  d
   ]! 4        R# R# )u  
Feature engineering pipeline.

Produces data/processed/final_feature_matrix.csv — the single training
table every model in ml_pipeline/training/ reads from. Also produces
data/processed/climate_clean.csv (already created by clean_climate_data.py,
read directly by the live API's FeatureBuilder for historical averages).

This script must be re-run any time the raw source data is refreshed
(new IMD/ICRISAT/Agmarknet export). Models trained on a stale feature
matrix will silently drift from what the live system computes at
inference time — keep this pipeline run as part of your retraining
checklist, not a one-time setup step.

Usage:
    python -m ml_pipeline.feature_engineering.build_feature_matrix         --climate data/processed/climate_clean.csv         --crop data/processed/crop_clean.csv         --market data/processed/market_clean.csv         --crop-lookup data/lookup/crops.json         --output data/processed/final_feature_matrix.csv
N)Path)statsz%%(asctime)s %(levelname)s %(message)s)levelformati  c                X    V ^8  d   QhR\         P                  R\         P                  /# )   climatereturnpd	DataFrame)r   s   "aC:\Crop_Prediction\backend\crop-ai-system\ml_pipeline\feature_engineering\build_feature_matrix.py__annotate__r   0   s"     ( (",, (2<< (    c                 2   V P                  . RO4      P                  RR7      p W R,          \        8*  ,          P                  R R.4      P	                  RRR7      P                  4       p\
        P                  ! WR R.RR7      p V R,          P                  4       pV R	,          P                  V4      V R	&   V R
,          P                  V R,          P                  4       4      V R
&   V R
,          P                  ^ ^4      V R
&   V R,          V R	,          ,
          V R
,          ,          V R&   V P                  R 4      R,          P                  ^4      V R&   V P                  R 4      R,          P                  R 4      V R&   V R,          P                  V R,          4      V R&   V R,          P                  \        4      V R&   RV P                  9  d   \         P#                  R4       RV R&   RV P                  9  d   \         P#                  R4       RV R&   V # )districtyearmonthT)droprainfall_mm)hist_mean_rainfallhist_std_rainfallleftonhowr   r   rainfall_anomalyrainfall_lag1c                 D    V P                  ^^R7      P                  4       # )   min_periods)rollingmeanxs   &r   <lambda>&add_climate_features.<locals>.<lambda>H   s    QYYqaY8==?r   rainfall_ma3season_indextemp_cuL   No temp_c column in climate data — using a flat 30.0C default for all rowsg      >@humidityuM   No humidity column in climate data — using a flat 70.0 default for all rowsg     Q@r   r   r   )r   r#   )r   std)sort_valuesreset_indexHISTORICAL_CUTOFF_YEARgroupbyaggr   merger#   fillnar-   replaceshift	transformmap
SEASON_MAPcolumnsloggerwarning)r   histfallback_means   &  r   add_climate_featuresr?   0   s   !!"?@LLRVLWG6?&<<=EE	W	c20 
  km 	 hhw*g)>FKGM*//1M$+,@$A$H$H$WG !#*+>#?#F#Fw}G]GaGaGc#dG #*+>#?#G#G1#MG  
	'*>"?	?7K^C__   'z:=IOOPQRGO
#M2	?	@ N  '7>>w}?UVGO%g.22:>GN
 w&ef (fg"
Nr   c                D    V ^8  d   QhR\         R\        P                  /# )r   lookup_pathr	   )r   r   r   )r   s   "r   r   r   [   s      d r|| r   c                 >   \        V R R7      ;_uu_ 4       p\        P                  ! V4      pRRR4       . pXP                  4        F+  w  rERV/pVP	                  V4       VP                  V4       K-  	  \        P                  ! V4      #   + '       g   i     Lg; i)z	utf-8-sig)encodingNcrop)openjsonloaditemsupdateappendr   r   )rA   f
crops_dictrowsrD   propsrows   &      r   load_crop_propertiesrP   [   sz    
 
kK	0	0AYYq\
 
1D!'')tn

5C * << 
1	0s   BB	c                D    V ^8  d   QhR\         P                  R\        /# )r   seriesr	   )r   Seriesfloat)r   s   "r   r   r   j   s      299  r   c                     V P                  4       P                  p\        V4      ^8  d   R# \        P                  ! \        V4      4      p\
        P                  ! W!4      w  p   p\        V4      # )r           )dropnavalueslennparanger   
linregressrT   )rR   rX   r%   slope_s   &    r   _compute_sloper_   j   sX    ]]_##F
6{Q
		#f+A((3E1aA<r   c                x    V ^8  d   QhR\         P                  R\         P                  R\         P                  /# )r   rD   
crop_propsr	   r
   )r   s   "r   r   r   s   s-      BLL bll r|| r   c                    \         P                  ! WR RR7      p W R,          P                  4       ,          R ,          P                  4       p\	        V4      ^ 8  d#   \
        P                  R\        V4       R24       V P                  . RO4      p V P                  RR .4      R,          P                  R 4      P                  ^ ^.R	R
7      pW0R&   V R,          P                  R4      V R&   V # )rD   r   r   water_req_mmz@Crops present in yield data but missing from crops.json lookup: zu. Add them to data/lookup/crops.json or these rows will have null agronomic features and get dropped before training.r   yield_ton_hac                 R    V P                  ^^R7      P                  \        RR7      # )   r    Frawr"   applyr_   ss   &r   r&   #add_crop_features.<locals>.<lambda>   !    1!4::>u:Ur   Tr   r   yield_trend_sloperV   )r   rD   r   )r   r3   isnulluniquerY   r;   r<   listr.   r1   rj   r/   r4   )rD   ra   missing_propstrend_slopess   &&  r   add_crop_featuresrv   s   s    88DV<Dn-4467?FFHM
=ANM"# $VW	
 89Dj&)*>:	U	V	Aq6	- 
 !-	 $%8 9 @ @ ED	Kr   c                X    V ^8  d   QhR\         P                  R\         P                  /# )r   marketr	   r
   )r   s   "r   r   r      s"     
 
 
 
r   c                 &   V P                  . R	O4      p V P                  R R.4      R,          P                  ^4      V R&   V P                  R R.4      R,          P                  R 4      P	                  ^ ^.RR7      pVP                  R4      V R&   V # )
r   rD   price_per_quintal
price_lag1c                 R    V P                  ^^R7      P                  \        RR7      # )   r    Frg   ri   rk   s   &r   r&   %add_market_features.<locals>.<lambda>   rn   r   Tro   rV   demand_trendr   rD   r   r   )r.   r1   r6   rj   r/   r4   )rx   r   s   & r   add_market_featuresr      s     EFF!>>:v*>?@STZZ[\]F< 	
F+,-@A	U	V	Aq6	- 
 *005F>Mr   c                D    V ^8  d   QhR\         P                  R\        /# r   rO   r	   r   rS   str)r   s   "r   r   r      s      299  r   c                    V R ,          V R,          u;8*  ;'       d    V R,          8*  Mu pV R,          V R,          ^,          8  p\        V R,          4      R8  pV P                  R^ 4      ^ 8  pV'       d   V'       d   V'       g   R# V'       d$   V R,          V R,          ^,          R	,          8  d   R
# V'       d   V'       g   V'       d   R# V'       g   V R,          '       g   V'       g   R
# R# )
min_temp_cr*   
max_temp_cr   rc   r   g      ?rd   suitable皙?not_suitableriskydrought_tolerant)absget)rO   temp_okrain_okanomaly_bad	has_yields   &    r   label_feasibilityr      s    ,3x=EEC4EEG- S%82%=>Gc,-.4K*Q.I 7;M*c..AB.F$-NN	;7S!344Yr   c                0    V ^8  d   QhR\         R\        /# )r   r]   r	   )rT   r   )r   s   "r   r   r      s      u  r   c                 N    V R 8  d   R# V R8  d   R# \        V 4      R8  d   R# R# )r   upwarddownwardg?volatilestableg)r   )r]   s   &r   label_trendr      s(    s{		Ud	r   c                D    V ^8  d   QhR\         P                  R\        /# r   r   )r   s   "r   r   r      s      299  r   c                 t    V R ,          R8X  d   V P                  R^ 4      ^ 8  d   R# V R ,          R8X  d   R# R# )feasibility_labelr   r   highr   lowmedium)r   )rO   s   &r   label_suitabilityr      s9    
:-#''.!2Lq2P	 	!^	3r   c          
          V ^8  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  /# )r   r   rD   rx   ra   r	   r
   )r   s   "r   r   r      sI     ! !2<< !r|| !R\\ !WYWcWc !hjhtht !r   c                    \        V 4      p \        W4      p\        V4      p\        P                  ! W. RORR7      p\
        P                  R\        V4       R24       \        V4      ^ 8X  d   \        R4      h\        P                  ! WB. RORR7      p\
        P                  R\        V4       R	VR
,          P                  4       P                  4        R24       VP                  R4      R
,          P                  R 4      VR
&   VR,          P                  R4      VR&   VR,          P                  R4      VR&   \        V4      p. ROpVP                  VR7      p\
        P                  RV\        V4      ,
           R24       VP                  \         ^R7      VR&   VR,          P                  \"        4      VR&   VP                  \$        ^R7      VR&   V# )r   innerr   zAfter climate+crop merge: z rowszZero rows after merging climate and crop data. Likely cause: district name spelling mismatch, or year/month ranges that don't overlap between the two sources. Inspect both inputs.rD   r   zAfter market merge: z rows (rz   z missing market price)c                 @    V P                  V P                  4       4      # )N)r4   medianr$   s   &r   r&   build.<locals>.<lambda>   s    !((188:&r   price_volatilityr   r   rV   )subsetzDropped z) rows missing required label-input fields)axisr   rp   trend_labelsuitability_labelr,   r   )r*   r   r   r   rc   )r?   rv   r   r   r3   r;   inforY   
ValueErrorrq   sumr1   r7   r4   rW   rj   r   r   r   )r   rD   rx   ra   mergedbefore_droprequired_for_labelss   &&&&   r   buildr      s   "7+GT.D (FXXd(E7SF
KK,S[M?@
6{aJ
 	
 XXf)NTZ[F
KK&s6{m76BU;V;];];_;c;c;e:ff|}~"(.."89L"M"W"W&#F "((:!;!B!B3!GF#N3::3?F>f+K_]]"5]6F
KK(;V455^_`"(,,/@q,"IF"#67==kJF="(,,/@q,"IFMr   c                    V ^8  d   QhRR/# )r   r	   N )r   s   "r   r   r      s     !d !dd !dr   c                     \         P                  ! R R7      p V P                  RRR7       V P                  RRR7       V P                  RRR7       V P                  RRR7       V P                  RRR7       V P                  4       p\        P                  R	4       \        P                  ! VP                  4      p\        P                  ! VP                  4      p\        P                  ! VP                  4      p\        \        VP                  4      4      p\        P                  R
4       \        W#WE4      p\        V4      ^d8  d9   \        P!                  R\        V4       R24       \"        P$                  ! ^4       \        VP&                  4      pVP(                  P+                  RRR7       VP-                  VRR7       \        P                  R\        V4       RV 24       \        P                  RVR,          P/                  4        24       \        P                  RVR,          P/                  4        24       \        P                  RVR,          P/                  4        24       R# )z1Build the final feature matrix for model training)descriptionz	--climateT)requiredz--cropz--marketz--crop-lookupz--outputzLoading inputs...zBuilding feature matrix...zOnly u    rows in the final feature matrix. This is too small to train reliable models — review the merge keys and source data coverage before proceeding to training.)parentsexist_okF)indexzSaved z	 rows to z Feasibility label distribution:
r   zTrend label distribution:
r   z Suitability label distribution:
r   N)argparseArgumentParseradd_argument
parse_argsr;   r   r   read_csvr   rD   rx   rP   r   crop_lookupr   rY   errorsysexitoutputparentmkdirto_csvvalue_counts)parserargs
climate_dfcrop_df	market_dfra   final_dfoutput_paths           r   mainr      s   $$1deF
d3
40

T2
$7

T2D
KK#$T\\*Jkk$))$GDKK(I%d4+;+;&<=J
KK,-Z)@H
8}sCM? #B C	

 	t{{#KTD9OOKuO-
KK&Xy>?
KK3H=P4Q4^4^4`3abc
KK-h}.E.R.R.T-UVW
KK3H=P4Q4^4^4`3abcr   __main__)__doc__r   rF   loggingr   pathlibr   numpyrZ   pandasr   scipyr   basicConfigINFO	getLogger__name__r;   r9   r0   r?   rP   r_   rv   r   r   r   r   r   r   r   r   r   <module>r      s   .    
       ',,/V W			8	$ q!Q1a2q"aAq!Q1a
  
(V.
(!H!dH zF r   