{
 "cells": [
  {
   "cell_type": "code",
   "id": "8adcbe0819b88578",
   "metadata": {},
   "source": [
    "from opt_einsum.paths import branch_1\n",
    "!apt-get update\n",
    "!apt-get install graphviz -y\n",
    "\n",
    "!pip install tensorflow\n",
    "!pip install numpy\n",
    "!pip install pandas\n",
    "\n",
    "!pip install keras\n",
    "!pip install scikit-learn\n",
    "!pip install matplotlib\n",
    "!pip install joblib\n",
    "!pip install pyarrow\n",
    "!pip install fastparquet\n",
    "!pip install scipy\n",
    "!pip install seaborn\n",
    "!pip install tqdm\n",
    "!pip install pydot\n",
    "!pip install tensorflow-io\n",
    "!pip install tensorflow-addons\n"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "7a813e3cbca057b7",
   "metadata": {},
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Dense, LSTM, MultiHeadAttention, Dropout, BatchNormalization, LayerNormalization, Input, Activation, Lambda, Bidirectional, Add, MaxPooling1D\n",
    "from tensorflow.keras import regularizers\n",
    "from tensorflow.keras.models import Model\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\n",
    "from tensorflow.keras.optimizers import AdamW\n",
    "import json\n",
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.keras.utils import plot_model\n",
    "import tensorflow_addons as tfa\n",
    "\n",
    "folder_name = datetime.now().strftime(\"%Y-%m-%d_%H-%M\")\n",
    "\n",
    "random_state_value = None"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "b3f525e19f78a1da",
   "metadata": {},
   "source": [
    "def get_season(date):\n",
    "    month = date.month\n",
    "    day = date.day\n",
    "    if (month == 12 and day >= 21) or (month <= 3 and day < 20):\n",
    "        return 'Winter'\n",
    "    elif (month == 3 and day >= 20) or (month <= 6 and day < 21):\n",
    "        return 'Spring'\n",
    "    elif (month == 6 and day >= 21) or (month <= 9 and day < 23):\n",
    "        return 'Summer'\n",
    "    elif (month == 9 and day >= 23) or (month <= 12 and day < 21):\n",
    "        return 'Autumn'\n",
    "    else:\n",
    "        return 'Unknown'\n",
    "\n",
    "\n",
    "def get_time_period(hour):\n",
    "    if 5 <= hour < 12:\n",
    "        return 'Morning'\n",
    "    elif 12 <= hour < 17:\n",
    "        return 'Afternoon'\n",
    "    elif 17 <= hour < 21:\n",
    "        return 'Evening'\n",
    "    else:\n",
    "        return 'Night'\n",
    "\n",
    "\n",
    "def add_time_features(df):\n",
    "    df['datetime'] = pd.to_datetime(df['datetime'])\n",
    "    df['timestamp'] = df['datetime'].astype(np.int64) // 10 ** 9\n",
    "    df['year'] = df['datetime'].dt.year\n",
    "    df['month'] = df['datetime'].dt.month\n",
    "    df['day'] = df['datetime'].dt.day\n",
    "    df['hour'] = df['datetime'].dt.hour\n",
    "    df['minute'] = df['datetime'].dt.minute\n",
    "    df['hour_sin'] = np.sin(df['hour'] * (2 * np.pi / 24))\n",
    "    df['hour_cos'] = np.cos(df['hour'] * (2 * np.pi / 24))\n",
    "    df['day_of_week'] = df['datetime'].dt.dayofweek\n",
    "    df['day_of_year'] = df['datetime'].dt.dayofyear\n",
    "    df['week_of_year'] = df['datetime'].dt.isocalendar().week.astype(int)\n",
    "    df['quarter'] = df['datetime'].dt.quarter\n",
    "    df['is_month_end'] = df['datetime'].dt.is_month_end.astype(int)\n",
    "    df['is_quarter_end'] = df['datetime'].dt.is_quarter_end.astype(int)\n",
    "    df['is_year_end'] = df['datetime'].dt.is_year_end.astype(int)\n",
    "    df['month_sin'] = np.sin(df['month'] * (2 * np.pi / 12))\n",
    "    df['month_cos'] = np.cos(df['month'] * (2 * np.pi / 12))\n",
    "    df['day_of_year_sin'] = np.sin(df['day_of_year'] * (2 * np.pi / 365.25))\n",
    "    df['day_of_year_cos'] = np.cos(df['day_of_year'] * (2 * np.pi / 365.25))\n",
    "    df['season'] = df['datetime'].apply(get_season)\n",
    "    df['time_period'] = df['hour'].apply(get_time_period)\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_solar_features(df):\n",
    "    # Features based only on radiation and other available variables\n",
    "    df['solar_elevation'] = np.sin(df['day_of_year'] * (2 * np.pi / 365.25)) * np.sin(df['hour'] * (2 * np.pi / 24))\n",
    "\n",
    "    # Radiation-specific features\n",
    "    df['radiation_clearsky'] = df['solarradiation'] * (100 - df['cloudcover']) / 100\n",
    "\n",
    "    # Temperature impact on theoretical efficiency\n",
    "    df['temp_efficiency_factor'] = 1 - 0.004 * (df['temp'] - 25)  # Typical temperature coefficient\n",
    "\n",
    "    # Combined features\n",
    "    df['cloud_impact'] = df['cloudcover'] * df['solarradiation']\n",
    "    df['visibility_radiation'] = df['visibility'] * df['solarradiation']\n",
    "    df['clear_sky_index'] = (100 - df['cloudcover']) / 100\n",
    "    df['temp_effect'] = df['temp'] - df['tempmin']\n",
    "\n",
    "    return df\n",
    "\n",
    "def add_solar_specific_features(df):\n",
    "    # Solar position and theoretical calculations\n",
    "    df['day_length'] = 12 + 3 * np.sin(2 * np.pi * (df['day_of_year'] - 81) / 365.25)\n",
    "    df['solar_noon_distance'] = np.abs(12 - df['hour'])\n",
    "    df['solar_potential'] = df['clear_sky_index'] * np.cos(df['solar_noon_distance'] * np.pi / 12)\n",
    "\n",
    "    # Rolling features for radiation\n",
    "    windows = [3, 6, 12]\n",
    "    for w in windows:\n",
    "        df[f'radiation_rolling_{w}h'] = df['solarradiation'].rolling(window=w).mean()\n",
    "\n",
    "    # Theoretical radiation features\n",
    "    df['theoretical_radiation'] = df['solarradiation'] / (df['clear_sky_index'] + 1e-6)\n",
    "\n",
    "    return df\n",
    "\n",
    "def add_radiation_energy_features(df):\n",
    "    # Features based only on radiation\n",
    "    windows = [3, 6, 12]\n",
    "    for w in windows:\n",
    "        # Radiation features\n",
    "        df[f'radiation_rolling_mean_{w}h'] = df['solarradiation'].rolling(window=w).mean()\n",
    "        df[f'radiation_rolling_std_{w}h'] = df['solarradiation'].rolling(window=w).std()\n",
    "\n",
    "    # Daily aggregations for radiation\n",
    "    df['radiation_daily_sum'] = df.groupby(df.index.date)['solarradiation'].transform('sum')\n",
    "    df['radiation_daily_max'] = df.groupby(df.index.date)['solarradiation'].transform('max')\n",
    "\n",
    "    # Lag features for radiation\n",
    "    lags = [1, 2, 3, 6]\n",
    "    for lag in lags:\n",
    "        df[f'radiation_lag_{lag}h'] = df['solarradiation'].shift(lag)\n",
    "\n",
    "    return df\n",
    "\n",
    "def add_advanced_features(df):\n",
    "    df = add_time_features(df)\n",
    "    df = add_solar_features(df)\n",
    "    df = add_solar_specific_features(df)\n",
    "    df = add_radiation_energy_features(df)\n",
    "\n",
    "    df = pd.get_dummies(df, columns=['season', 'time_period'])\n",
    "\n",
    "    df['temp_humidity_interaction'] = df['temp'] * df['humidity'] / 100\n",
    "    df['radiation_temp_interaction'] = df['solarradiation'] * df['temp_efficiency_factor']\n",
    "    df['radiation_cloud_interaction'] = df['solarradiation'] * (1 - df['cloudcover']/100)\n",
    "\n",
    "    # Theoretical maximum based on clear sky conditions\n",
    "    df['theoretical_max_radiation'] = df['solarradiation'] / (df['clear_sky_index'] + 1e-6)\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def prepare_advanced_data(df):\n",
    "    # Apply feature engineering functions\n",
    "    df = add_advanced_features(df)\n",
    "\n",
    "    target_variables = ['solarradiation', 'solarenergy', 'uvindex']\n",
    "\n",
    "    selected_features = [\n",
    "        # Weather features\n",
    "        'temp', 'humidity', 'cloudcover', 'visibility',\n",
    "        'temp_effect',\n",
    "\n",
    "        # Solar radiation features\n",
    "        'solarradiation',\n",
    "        'radiation_clearsky',\n",
    "        'radiation_rolling_mean_3h',\n",
    "        'radiation_rolling_mean_6h',\n",
    "        'radiation_daily_sum',\n",
    "        'radiation_daily_max',\n",
    "        'radiation_lag_1h',\n",
    "        'radiation_lag_3h',\n",
    "\n",
    "        # Temperature efficiency\n",
    "        'temp_efficiency_factor',\n",
    "\n",
    "        # Time features\n",
    "        'hour_sin', 'hour_cos',\n",
    "        'month_sin', 'month_cos',\n",
    "        'day_of_year_sin', 'day_of_year_cos',\n",
    "\n",
    "        # Solar position and potential\n",
    "        'solar_elevation',\n",
    "        'solar_potential',\n",
    "        'day_length',\n",
    "        'solar_noon_distance',\n",
    "\n",
    "        # Clear sky and theoretical features\n",
    "        'clear_sky_index',\n",
    "        'theoretical_max_radiation',\n",
    "\n",
    "        # Interaction features\n",
    "        'radiation_temp_interaction',\n",
    "        'radiation_cloud_interaction',\n",
    "        'temp_humidity_interaction',\n",
    "        'visibility_radiation'\n",
    "    ]\n",
    "\n",
    "    # Add one-hot columns\n",
    "    categorical_columns = [col for col in df.columns if col.startswith(('season_', 'time_period_'))]\n",
    "    final_features = selected_features + categorical_columns\n",
    "\n",
    "    # Dataset preparation\n",
    "    df = df.sort_values('datetime')\n",
    "    df.set_index('datetime', inplace=True)\n",
    "\n",
    "    # Handle missing values\n",
    "    for column in final_features + target_variables:\n",
    "        df[column] = df[column].interpolate(method='time')\n",
    "    df.fillna(0, inplace=True)\n",
    "\n",
    "    # Temporal split\n",
    "    data_after_2010 = df[df['year'] >= 2010].copy()\n",
    "    data_before_2010 = df[df['year'] < 2010].copy()\n",
    "\n",
    "    X = data_after_2010[final_features]\n",
    "    y = data_after_2010['solarenergy']\n",
    "    X_to_predict = data_before_2010[final_features]\n",
    "\n",
    "    # Train-test split\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=random_state_value)\n",
    "\n",
    "    # Scaling\n",
    "    scaler = RobustScaler()\n",
    "    X_train_scaled = scaler.fit_transform(X_train)\n",
    "    X_test_scaled = scaler.transform(X_test)\n",
    "    X_to_predict_scaled = scaler.transform(X_to_predict)\n",
    "\n",
    "    return X_train_scaled, X_test_scaled, y_train, y_test, scaler, final_features, X_to_predict_scaled\n",
    "\n",
    "\n",
    "def create_sequence_data(X, sequence_length=24):\n",
    "    \"\"\"\n",
    "    Converts data into sequences for LSTM input\n",
    "    sequence_length represents how many previous hours to consider\n",
    "    \"\"\"\n",
    "    sequences = []\n",
    "    for i in range(len(X) - sequence_length + 1):\n",
    "        sequences.append(X[i:i + sequence_length])\n",
    "    return np.array(sequences)\n",
    "\n",
    "\n",
    "def prepare_hybrid_data(df):\n",
    "    X_train_scaled, X_test_scaled, y_train, y_test, scaler, features, X_to_predict_scaled = prepare_advanced_data(df)\n",
    "\n",
    "    # Convert data into sequences\n",
    "    sequence_length = 24  # 24 hours of historical data\n",
    "\n",
    "    X_train_seq = create_sequence_data(X_train_scaled, sequence_length)\n",
    "    X_test_seq = create_sequence_data(X_test_scaled, sequence_length)\n",
    "\n",
    "    # Adjust y by removing the first (sequence_length-1) elements\n",
    "    y_train = y_train[sequence_length - 1:]\n",
    "    y_test = y_test[sequence_length - 1:]\n",
    "\n",
    "    X_to_predict_seq = create_sequence_data(X_to_predict_scaled, sequence_length)\n",
    "\n",
    "    return X_train_seq, X_test_seq, y_train, y_test, scaler, features, X_to_predict_seq"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "9dff3259-b376-4cfc-89d8-ab2ea18aaa5e",
   "metadata": {},
   "source": [
    "def create_residual_lstm_layer(x, units, dropout_rate, l2_reg=0.01, return_sequences=True, survival_probability=0.8):\n",
    "    \"\"\"\n",
    "    Creates a residual LSTM layer with bidirectional processing, normalization, and dropout\n",
    "    Parameters:\n",
    "        x: input tensor\n",
    "        units: number of LSTM units\n",
    "        dropout_rate: dropout probability\n",
    "        l2_reg: L2 regularization factor\n",
    "        return_sequences: whether to return sequences or just the final output\n",
    "        survival_probability: probability of layer survival for stochastic depth\n",
    "    \"\"\"\n",
    "    residual = x\n",
    "    x = Bidirectional(LSTM(units, return_sequences=return_sequences, kernel_regularizer=regularizers.l2(l2_reg)))(x)\n",
    "    x = LayerNormalization()(x)\n",
    "    x = Dropout(dropout_rate)(x)\n",
    "\n",
    "    if return_sequences:\n",
    "        # Adjust residual dimensions if needed\n",
    "        if int(residual.shape[-1]) != 2 * units:\n",
    "            residual = Dense(2 * units, activation='linear')(residual)\n",
    "        x = tfa.layers.StochasticDepth(survival_probability)([x, residual])\n",
    "    return x\n",
    "\n",
    "def attention_block(x, units, num_heads=8, survival_probability=0.8):\n",
    "    \"\"\"\n",
    "    Creates an attention block with multi-head attention and layer normalization\n",
    "    Parameters:\n",
    "        x: input tensor\n",
    "        units: dimensionality of the attention layer\n",
    "        num_heads: number of attention heads\n",
    "        survival_probability: probability of layer survival for stochastic depth\n",
    "    \"\"\"\n",
    "    attention = MultiHeadAttention(num_heads=num_heads, key_dim=units)(x, x)\n",
    "    x = tfa.layers.StochasticDepth(survival_probability)([x, attention])\n",
    "    x = LayerNormalization()(x)\n",
    "    return x\n",
    "\n",
    "def create_solarradiation_model(input_shape, folder_name, l2_lambda=0.005):\n",
    "    \"\"\"\n",
    "    Creates a deep learning model for solar radiation prediction\n",
    "    Parameters:\n",
    "        input_shape: shape of input data\n",
    "        folder_name: directory to save model architecture visualization\n",
    "        l2_lambda: L2 regularization factor\n",
    "    \"\"\"\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    # Define progressive hyperparameters for model depth\n",
    "    survival_probs = [0.9, 0.8, 0.7]  # Decreasing survival probabilities\n",
    "    attention_survival_probs = [0.85, 0.75, 0.65]  # Decreasing attention survival probabilities\n",
    "    lstm_units = [256, 128, 64]  # Decreasing LSTM units\n",
    "    dropout_rates = [0.4, 0.3, 0.2]  # Decreasing dropout rates\n",
    "    attention_heads = [32, 24, 16]  # Decreasing attention heads\n",
    "\n",
    "    # Build LSTM layers with attention blocks\n",
    "    x = inputs\n",
    "    for i in range(3):\n",
    "        # Add residual LSTM layer\n",
    "        x = create_residual_lstm_layer(\n",
    "            x,\n",
    "            units=lstm_units[i],\n",
    "            dropout_rate=dropout_rates[i],\n",
    "            l2_reg=l2_lambda,\n",
    "            return_sequences=True,\n",
    "            survival_probability=survival_probs[i]\n",
    "        )\n",
    "        # Add attention block\n",
    "        x = attention_block(\n",
    "            x,\n",
    "            units=lstm_units[i],\n",
    "            num_heads=attention_heads[i],\n",
    "            survival_probability=attention_survival_probs[i]\n",
    "        )\n",
    "        if i < 2:  # No pooling after last LSTM layer\n",
    "            x = MaxPooling1D()(x)\n",
    "\n",
    "    # Final LSTM layer for sequence aggregation\n",
    "    x = create_residual_lstm_layer(\n",
    "        x,\n",
    "        units=32,\n",
    "        dropout_rate=0.1,\n",
    "        l2_reg=l2_lambda,\n",
    "        return_sequences=False,\n",
    "        survival_probability=0.6\n",
    "    )\n",
    "\n",
    "    # Dense layers for final prediction\n",
    "    dense_units = [64, 32]\n",
    "    dense_dropout = [0.2, 0.1]\n",
    "\n",
    "    for units, dropout in zip(dense_units, dense_dropout):\n",
    "        x = Dense(units, kernel_regularizer=regularizers.l2(l2_lambda))(x)\n",
    "        x = BatchNormalization()(x)\n",
    "        x = Activation('swish')(x)\n",
    "        x = Dropout(dropout)(x)\n",
    "\n",
    "    # Output layer with value clipping\n",
    "    outputs = Dense(1)(x)\n",
    "    outputs = Lambda(lambda x: tf.clip_by_value(x, 0, 1500))(outputs)\n",
    "\n",
    "    model = Model(inputs=inputs, outputs=outputs, name=\"SolarRadiationModel\")\n",
    "\n",
    "    # Configure optimizer with weight decay\n",
    "    optimizer = AdamW(\n",
    "        learning_rate=0.0003,\n",
    "        beta_1=0.9,\n",
    "        beta_2=0.999,\n",
    "        epsilon=1e-07,\n",
    "        weight_decay=0.001\n",
    "    )\n",
    "\n",
    "    # Custom evaluation metrics\n",
    "    def rmse(y_true, y_pred):\n",
    "        \"\"\"Root Mean Square Error\"\"\"\n",
    "        return tf.sqrt(tf.reduce_mean(tf.square(y_true - y_pred)))\n",
    "\n",
    "    def mape(y_true, y_pred):\n",
    "        \"\"\"Mean Absolute Percentage Error\"\"\"\n",
    "        epsilon = 1e-7\n",
    "        return tf.reduce_mean(tf.abs((y_true - y_pred) / (y_true + epsilon))) * 100\n",
    "\n",
    "    # Combined loss function\n",
    "    def hybrid_loss(y_true, y_pred):\n",
    "        \"\"\"Weighted combination of MSE and MAE\"\"\"\n",
    "        mse = tf.reduce_mean(tf.square(y_true - y_pred))\n",
    "        mae = tf.reduce_mean(tf.abs(y_true - y_pred))\n",
    "        return 0.7 * mse + 0.3 * mae\n",
    "\n",
    "    # Compile model with custom loss and metrics\n",
    "    model.compile(\n",
    "        optimizer=optimizer,\n",
    "        loss=hybrid_loss,\n",
    "        metrics=[\n",
    "            'mae',\n",
    "            rmse,\n",
    "            mape\n",
    "        ]\n",
    "    )\n",
    "    model.summary()\n",
    "\n",
    "    # Save model architecture visualization\n",
    "    plot_model(model,\n",
    "               to_file=f'{folder_name}_model_architecture.png',\n",
    "               show_shapes=True,\n",
    "               show_layer_names=True,\n",
    "               dpi=150,\n",
    "               show_layer_activations=True)\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "def evaluate_solarenergy_predictions(y_true, y_pred, hour=None, folder_name=None):\n",
    "    \"\"\"\n",
    "    Comprehensive evaluation of solar energy predictions with detailed analysis and visualizations\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    y_true : array-like\n",
    "        Actual solar energy values (kWh)\n",
    "    y_pred : array-like\n",
    "        Predicted solar energy values (kWh)\n",
    "    hour : array-like, optional\n",
    "        Array of hours corresponding to predictions, for temporal analysis\n",
    "    folder_name : str, optional\n",
    "        Folder to save analysis plots\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    dict\n",
    "        Dictionary containing all calculated metrics\n",
    "    \"\"\"\n",
    "    import os\n",
    "    from datetime import datetime\n",
    "    import numpy as np\n",
    "    import pandas as pd\n",
    "    import matplotlib.pyplot as plt\n",
    "    import seaborn as sns\n",
    "    from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score, confusion_matrix\n",
    "\n",
    "    # Data conversion and preparation\n",
    "    y_true = np.array(y_true).ravel()\n",
    "    y_pred = np.array(y_pred).ravel()\n",
    "    errors = y_pred - y_true\n",
    "\n",
    "    # Basic metrics\n",
    "    mae_raw = mean_absolute_error(y_true, y_pred)\n",
    "    rmse_raw = np.sqrt(mean_squared_error(y_true, y_pred))\n",
    "    r2_raw = r2_score(y_true, y_pred)\n",
    "    mape = np.mean(np.abs((y_true - y_pred) / (y_true + 1e-7))) * 100\n",
    "\n",
    "    # Accuracy for error margins\n",
    "    within_5_percent = np.mean(np.abs((y_pred - y_true) / (y_true + 1e-7)) <= 0.05)\n",
    "    within_10_percent = np.mean(np.abs((y_pred - y_true) / (y_true + 1e-7)) <= 0.10)\n",
    "    within_20_percent = np.mean(np.abs((y_pred - y_true) / (y_true + 1e-7)) <= 0.20)\n",
    "\n",
    "    # Define solar energy levels\n",
    "    def get_energy_level(value):\n",
    "        if value <= 0.5:\n",
    "            return 'Very Low'\n",
    "        elif value <= 2.0:\n",
    "            return 'Low'\n",
    "        elif value <= 4.0:\n",
    "            return 'Moderate'\n",
    "        elif value <= 6.0:\n",
    "            return 'High'\n",
    "        elif value <= 8.0:\n",
    "            return 'Very High'\n",
    "        else:\n",
    "            return 'Extreme'\n",
    "\n",
    "    # Calculate levels\n",
    "    y_true_levels = [get_energy_level(v) for v in y_true]\n",
    "    y_pred_levels = [get_energy_level(v) for v in y_pred]\n",
    "    level_accuracy = np.mean([t == p for t, p in zip(y_true_levels, y_pred_levels)])\n",
    "\n",
    "    # Print main metrics\n",
    "    print(\"\\nSolar Energy Prediction Metrics:\")\n",
    "    print(\"\\nAbsolute Metrics:\")\n",
    "    print(f\"MAE: {mae_raw:.4f} kWh\")\n",
    "    print(f\"RMSE: {rmse_raw:.4f} kWh\")\n",
    "    print(f\"R² Score: {r2_raw:.3f}\")\n",
    "    print(f\"MAPE: {mape:.2f}%\")\n",
    "\n",
    "    print(\"\\nPercentage Accuracy:\")\n",
    "    print(f\"Within ±5%: {within_5_percent*100:.1f}%\")\n",
    "    print(f\"Within ±10%: {within_10_percent*100:.1f}%\")\n",
    "    print(f\"Within ±20%: {within_20_percent*100:.1f}%\")\n",
    "\n",
    "    print(\"\\nLevel Accuracy:\")\n",
    "    print(f\"Level Accuracy: {level_accuracy*100:.1f}%\")\n",
    "\n",
    "    # Confusion matrix for levels\n",
    "    cm = confusion_matrix(y_true_levels, y_pred_levels)\n",
    "    print(\"\\nConfusion Matrix for Levels:\")\n",
    "    cm_df = pd.DataFrame(\n",
    "        cm,\n",
    "        columns=['Very Low', 'Low', 'Moderate', 'High', 'Very High', 'Extreme'],\n",
    "        index=['Very Low', 'Low', 'Moderate', 'High', 'Very High', 'Extreme']\n",
    "    )\n",
    "    print(cm_df)\n",
    "\n",
    "    # Analysis by time periods\n",
    "    if hour is not None:\n",
    "        day_periods = {\n",
    "            'Morning (5-11)': (5, 11),\n",
    "            'Noon (11-13)': (11, 13),\n",
    "            'Afternoon (13-17)': (13, 17),\n",
    "            'Evening (17-21)': (17, 21),\n",
    "            'Night (21-5)': (21, 5)\n",
    "        }\n",
    "\n",
    "        print(\"\\nAnalysis by Time Period:\")\n",
    "        for period, (start, end) in day_periods.items():\n",
    "            if start < end:\n",
    "                mask = (hour >= start) & (hour < end)\n",
    "            else:\n",
    "                mask = (hour >= start) | (hour < end)\n",
    "\n",
    "            if np.any(mask):\n",
    "                period_mae = mean_absolute_error(y_true[mask], y_pred[mask])\n",
    "                period_mape = np.mean(np.abs((y_true[mask] - y_pred[mask]) / (y_true[mask] + 1e-7))) * 100\n",
    "                period_mean = np.mean(y_true[mask])\n",
    "                print(f\"\\n{period}:\")\n",
    "                print(f\"MAE: {period_mae:.4f} kWh\")\n",
    "                print(f\"MAPE: {period_mape:.2f}%\")\n",
    "                print(f\"Mean Energy: {period_mean:.4f} kWh\")\n",
    "\n",
    "    # Visualizations\n",
    "    if folder_name is not None:\n",
    "        try:\n",
    "            os.makedirs(folder_name, exist_ok=True)\n",
    "            timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
    "\n",
    "            # Figure 1: Main analysis\n",
    "            plt.figure(figsize=(20, 15))\n",
    "\n",
    "            # Plot 1: Scatter plot\n",
    "            plt.subplot(3, 2, 1)\n",
    "            plt.scatter(y_true, y_pred, alpha=0.5)\n",
    "            plt.plot([y_true.min(), y_true.max()], [y_true.min(), y_true.max()], 'r--', lw=2)\n",
    "            plt.xlabel('Actual Energy (kWh)')\n",
    "            plt.ylabel('Predicted Energy (kWh)')\n",
    "            plt.title('Actual vs Predicted Values')\n",
    "            plt.grid(True)\n",
    "\n",
    "            # Plot 2: Absolute errors distribution\n",
    "            plt.subplot(3, 2, 2)\n",
    "            plt.hist(errors, bins=50, alpha=0.7)\n",
    "            plt.xlabel('Prediction Error (kWh)')\n",
    "            plt.ylabel('Frequency')\n",
    "            plt.title('Error Distribution')\n",
    "            plt.grid(True)\n",
    "\n",
    "            # Plot 3: Percentage errors distribution\n",
    "            plt.subplot(3, 2, 3)\n",
    "            percentage_errors = ((y_pred - y_true) / (y_true + 1e-7)) * 100\n",
    "            plt.hist(np.clip(percentage_errors, -100, 100), bins=50, alpha=0.7)\n",
    "            plt.xlabel('Percentage Error (%)')\n",
    "            plt.ylabel('Frequency')\n",
    "            plt.title('Percentage Error Distribution')\n",
    "            plt.grid(True)\n",
    "\n",
    "            # Plot 4: Errors vs Actual Values\n",
    "            plt.subplot(3, 2, 4)\n",
    "            plt.scatter(y_true, errors, alpha=0.5)\n",
    "            plt.axhline(y=0, color='r', linestyle='--')\n",
    "            plt.xlabel('Actual Energy (kWh)')\n",
    "            plt.ylabel('Error (kWh)')\n",
    "            plt.title('Errors vs Actual Values')\n",
    "            plt.grid(True)\n",
    "\n",
    "            # Plot 5: Box plot errors by level\n",
    "            plt.subplot(3, 2, 5)\n",
    "            sns.boxplot(x=[get_energy_level(v) for v in y_true], y=errors)\n",
    "            plt.xticks(rotation=45)\n",
    "            plt.xlabel('Energy Level')\n",
    "            plt.ylabel('Error (kWh)')\n",
    "            plt.title('Error Distribution by Level')\n",
    "\n",
    "            # Plot 6: Confusion matrix\n",
    "            plt.subplot(3, 2, 6)\n",
    "            sns.heatmap(cm_df, annot=True, fmt='d', cmap='Blues')\n",
    "            plt.title('Confusion Matrix')\n",
    "            plt.xticks(rotation=45)\n",
    "            plt.yticks(rotation=45)\n",
    "\n",
    "            plt.tight_layout()\n",
    "            filename = os.path.join(folder_name, f'solar_energy_analysis_{timestamp}.png')\n",
    "            plt.savefig(filename, dpi=300, bbox_inches='tight')\n",
    "            print(f\"\\nPlot saved as: {filename}\")\n",
    "            plt.close()\n",
    "\n",
    "            # Additional plot for temporal analysis if hour is available\n",
    "            if hour is not None:\n",
    "                plt.figure(figsize=(15, 8))\n",
    "                plt.scatter(hour, errors, alpha=0.5)\n",
    "                plt.axhline(y=0, color='r', linestyle='--')\n",
    "                plt.xlabel('Hour of Day')\n",
    "                plt.ylabel('Error (kWh)')\n",
    "                plt.title('Error Distribution by Hour of Day')\n",
    "                plt.grid(True)\n",
    "\n",
    "                filename = os.path.join(folder_name, f'hourly_error_analysis_{timestamp}.png')\n",
    "                plt.savefig(filename, dpi=300, bbox_inches='tight')\n",
    "                plt.close()\n",
    "\n",
    "        except Exception as e:\n",
    "            print(f\"\\nError saving plots: {str(e)}\")\n",
    "\n",
    "    # Additional metrics\n",
    "    print(\"\\nError Statistics:\")\n",
    "    print(f\"Mean errors: {np.mean(errors):.4f} kWh\")\n",
    "    print(f\"Standard deviation of errors: {np.std(errors):.4f} kWh\")\n",
    "    print(f\"Median error: {np.median(errors):.4f} kWh\")\n",
    "    print(f\"95th percentile absolute error: {np.percentile(np.abs(errors), 95):.4f} kWh\")\n",
    "\n",
    "    print(\"\\nProduction Statistics:\")\n",
    "    print(f\"Mean actual energy: {np.mean(y_true):.4f} kWh\")\n",
    "    print(f\"Mean predicted energy: {np.mean(y_pred):.4f} kWh\")\n",
    "    print(f\"Maximum actual energy: {np.max(y_true):.4f} kWh\")\n",
    "    print(f\"Maximum predicted energy: {np.max(y_pred):.4f} kWh\")\n",
    "\n",
    "    # Return metrics in structured format\n",
    "    metrics = {\n",
    "        'absolute': {\n",
    "            'mae': mae_raw,\n",
    "            'rmse': rmse_raw,\n",
    "            'r2': r2_raw,\n",
    "            'mape': mape\n",
    "        },\n",
    "        'percentage_accuracy': {\n",
    "            'within_5_percent': within_5_percent,\n",
    "            'within_10_percent': within_10_percent,\n",
    "            'within_20_percent': within_20_percent\n",
    "        },\n",
    "        'categorical': {\n",
    "            'level_accuracy': level_accuracy\n",
    "        },\n",
    "        'error_stats': {\n",
    "            'mean': float(np.mean(errors)),\n",
    "            'std': float(np.std(errors)),\n",
    "            'median': float(np.median(errors)),\n",
    "            'p95_abs': float(np.percentile(np.abs(errors), 95))\n",
    "        },\n",
    "        'production_stats': {\n",
    "            'mean_true': float(np.mean(y_true)),\n",
    "            'mean_pred': float(np.mean(y_pred)),\n",
    "            'max_true': float(np.max(y_true)),\n",
    "            'max_pred': float(np.max(y_pred))\n",
    "        }\n",
    "    }\n",
    "\n",
    "    return metrics\n",
    "\n",
    "\n",
    "def plot_training_history(history, folder_name=None):\n",
    "    \"\"\"\n",
    "    Display and save loss and metrics plots during training\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    history : tensorflow.keras.callbacks.History\n",
    "        History object returned by model training\n",
    "    folder_name : str\n",
    "        Folder to save the plot\n",
    "    \"\"\"\n",
    "    import os\n",
    "\n",
    "    try:\n",
    "        # Create figure\n",
    "        plt.figure(figsize=(12, 4))\n",
    "\n",
    "        # Loss Plot\n",
    "        plt.subplot(1, 2, 1)\n",
    "        plt.plot(history.history['loss'], label='Training Loss')\n",
    "        plt.plot(history.history['val_loss'], label='Validation Loss')\n",
    "        plt.title('Model Loss')\n",
    "        plt.xlabel('Epoch')\n",
    "        plt.ylabel('Loss')\n",
    "        plt.legend()\n",
    "        plt.grid(True)\n",
    "\n",
    "        # MAE Plot\n",
    "        plt.subplot(1, 2, 2)\n",
    "        plt.plot(history.history['mae'], label='Training MAE')\n",
    "        plt.plot(history.history['val_mae'], label='Validation MAE')\n",
    "        plt.title('Model MAE')\n",
    "        plt.xlabel('Epoch')\n",
    "        plt.ylabel('MAE')\n",
    "        plt.legend()\n",
    "        plt.grid(True)\n",
    "\n",
    "        plt.tight_layout()\n",
    "\n",
    "        if folder_name is not None:\n",
    "            os.makedirs(folder_name, exist_ok=True)\n",
    "            # Generate filename with timestamp\n",
    "            filename = os.path.join(folder_name, 'training_history.png')\n",
    "\n",
    "            # Save figure\n",
    "            plt.savefig(filename, dpi=300, bbox_inches='tight')\n",
    "            print(f\"\\nTraining history plot saved as: {filename}\")\n",
    "\n",
    "        # Save numerical data in CSV format\n",
    "        history_df = pd.DataFrame({\n",
    "            'epoch': range(1, len(history.history['loss']) + 1),\n",
    "            'training_loss': history.history['loss'],\n",
    "            'validation_loss': history.history['val_loss'],\n",
    "            'training_mae': history.history['mae'],\n",
    "            'validation_mae': history.history['val_mae']\n",
    "        })\n",
    "\n",
    "        if folder_name is not None:\n",
    "            csv_filename = os.path.join(folder_name, 'training_history.csv')\n",
    "            history_df.to_csv(csv_filename, index=False)\n",
    "            print(f\"Training history data saved as: {csv_filename}\")\n",
    "\n",
    "        # Calculate and save final statistics\n",
    "        final_stats = {\n",
    "            'final_training_loss': history.history['loss'][-1],\n",
    "            'final_validation_loss': history.history['val_loss'][-1],\n",
    "            'final_training_mae': history.history['mae'][-1],\n",
    "            'final_validation_mae': history.history['val_mae'][-1],\n",
    "            'best_validation_loss': min(history.history['val_loss']),\n",
    "            'best_validation_mae': min(history.history['val_mae']),\n",
    "            'epochs': len(history.history['loss']),\n",
    "        }\n",
    "\n",
    "        if folder_name is not None:\n",
    "            # Save statistics in JSON format\n",
    "            stats_filename = os.path.join(folder_name, 'training_stats.json')\n",
    "            with open(stats_filename, 'w') as f:\n",
    "                json.dump(final_stats, f, indent=4)\n",
    "            print(f\"Final statistics saved as: {stats_filename}\")\n",
    "\n",
    "        # Print main statistics\n",
    "        print(\"\\nFinal Training Statistics:\")\n",
    "        print(f\"Final Loss (train/val): {final_stats['final_training_loss']:.4f}/{final_stats['final_validation_loss']:.4f}\")\n",
    "        print(f\"Final MAE (train/val): {final_stats['final_training_mae']:.4f}/{final_stats['final_validation_mae']:.4f}\")\n",
    "        print(f\"Best validation loss: {final_stats['best_validation_loss']:.4f}\")\n",
    "        print(f\"Best validation MAE: {final_stats['best_validation_mae']:.4f}\")\n",
    "\n",
    "        plt.show()\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"\\nError creating or saving plots: {str(e)}\")\n",
    "\n",
    "\n",
    "def train_hybrid_model(model, X_train, y_train, X_test, y_test, epochs=100, batch_size=32, folder_name='solarradiation_index'):\n",
    "    \"\"\"\n",
    "    Advanced training function for the hybrid solar energy index model with detailed monitoring\n",
    "    and training management.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    model : keras.Model\n",
    "        The compiled hybrid model\n",
    "    X_train : numpy.ndarray\n",
    "        Training data\n",
    "    y_train : numpy.ndarray\n",
    "        Training targets\n",
    "    X_test : numpy.ndarray\n",
    "        Validation data\n",
    "    y_test : numpy.ndarray\n",
    "        Validation targets\n",
    "    epochs : int, optional\n",
    "        Maximum number of training epochs\n",
    "    batch_size : int, optional\n",
    "        Batch size\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    history : keras.callbacks.History\n",
    "        Training history with all metrics\n",
    "    \"\"\"\n",
    "\n",
    "    # Advanced training callbacks\n",
    "    callbacks = [\n",
    "        # Early Stopping\n",
    "        EarlyStopping(\n",
    "            monitor='val_loss',\n",
    "            patience=15,\n",
    "            restore_best_weights=True,\n",
    "            mode='min',\n",
    "            verbose=1,\n",
    "            min_delta=1e-4\n",
    "        ),\n",
    "        # ReduceLROnPlateau for MAE\n",
    "        ReduceLROnPlateau(\n",
    "            monitor='mae',\n",
    "            factor=0.2,\n",
    "            patience=5,\n",
    "            verbose=1,\n",
    "            mode='min',\n",
    "            min_delta=1e-4,\n",
    "            cooldown=3,\n",
    "            min_lr=1e-7\n",
    "        ),\n",
    "        # ReduceLROnPlateau for loss\n",
    "        ReduceLROnPlateau(\n",
    "            monitor='val_loss',\n",
    "            factor=0.2,\n",
    "            patience=3,\n",
    "            verbose=1,\n",
    "            mode='min',\n",
    "            min_delta=1e-4,\n",
    "            cooldown=2,\n",
    "            min_lr=1e-7\n",
    "        ),\n",
    "        # Model Checkpoint\n",
    "        tf.keras.callbacks.ModelCheckpoint(\n",
    "            filepath=f'{folder_name}_best_model.h5',\n",
    "            monitor='val_loss',\n",
    "            save_best_only=True,\n",
    "            mode='min',\n",
    "            save_weights_only=False\n",
    "        ),\n",
    "        # TensorBoard\n",
    "        tf.keras.callbacks.TensorBoard(\n",
    "            log_dir=f'./logs_{folder_name}',\n",
    "            histogram_freq=1,\n",
    "            write_graph=True,\n",
    "            update_freq='epoch',\n",
    "            profile_batch=0\n",
    "        ),\n",
    "        # Lambda Callback for solar radiation monitoring\n",
    "        tf.keras.callbacks.LambdaCallback(\n",
    "            on_epoch_end=lambda epoch, logs: (\n",
    "                lambda y_pred: print(\n",
    "                    f\"\\nEpoch {epoch + 1}:\"\n",
    "                    f\"\\nPredictions out of range (0-1500 W/m²): \"\n",
    "                    f\"{np.sum((y_pred < 0) | (y_pred > 1500))}\"\n",
    "                    f\"\\nMAPE: {np.mean(np.abs((y_test - y_pred) / (y_test + 1e-7))) * 100:.2f}%\"\n",
    "                    f\"\\nPredictions within ±10%: \"\n",
    "                    f\"{np.mean(np.abs((y_pred - y_test) / (y_test + 1e-7)) <= 0.10) * 100:.2f}%\"\n",
    "                )\n",
    "            )(model.predict(X_test)) if epoch % 20 == 0 else None\n",
    "        )\n",
    "    ]\n",
    "\n",
    "    try:\n",
    "        history = model.fit(\n",
    "            X_train, y_train,\n",
    "            validation_data=(X_test, y_test),\n",
    "            epochs=epochs,\n",
    "            batch_size=batch_size,\n",
    "            callbacks=callbacks,\n",
    "            verbose=1,\n",
    "            shuffle=False,\n",
    "            validation_freq=1,\n",
    "        )\n",
    "\n",
    "        # Post-training analysis\n",
    "        print(\"\\nTraining completed successfully!\")\n",
    "\n",
    "        # Final evaluation on test set\n",
    "        test_loss, test_mae, test_mse = model.evaluate(X_test, y_test, verbose=0)\n",
    "        print(f\"\\nFinal metrics on test set:\")\n",
    "        print(f\"Loss: {test_loss:.4f}\")\n",
    "        print(f\"MAE: {test_mae:.4f}\")\n",
    "        print(f\"MSE: {test_mse:.4f}\")\n",
    "\n",
    "        # Prediction analysis\n",
    "        predictions = model.predict(X_test)\n",
    "        out_of_range = np.sum((predictions < 0) | (predictions > 11))\n",
    "        print(f\"\\nPredictions out of range: {out_of_range} ({out_of_range / len(predictions) * 100:.2f}%)\")\n",
    "\n",
    "        plot_training_history(history, folder_name=folder_name)\n",
    "\n",
    "        return history\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"\\nError during training: {str(e)}\")\n",
    "        raise\n",
    "\n",
    "    finally:\n",
    "        # Memory cleanup\n",
    "        tf.keras.backend.clear_session()\n",
    "\n",
    "\n",
    "def integrate_predictions(df, predictions, sequence_length=24):\n",
    "    \"\"\"\n",
    "    Integrates solar energy index predictions into the original dataset for data before 2010.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    df : pandas.DataFrame\n",
    "        Original dataset\n",
    "    predictions : numpy.ndarray\n",
    "        Array of solar energy index predictions\n",
    "    sequence_length : int\n",
    "        Sequence length used for predictions\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    pandas.DataFrame\n",
    "        Updated dataset with solar energy index predictions\n",
    "    \"\"\"\n",
    "    # Convert datetime to datetime format if not already\n",
    "    df['datetime'] = pd.to_datetime(df['datetime'])\n",
    "\n",
    "    # Identify rows before 2010\n",
    "    mask_pre_2010 = df['datetime'].dt.year < 2010\n",
    "\n",
    "    # Create temporary DataFrame with predictions\n",
    "    dates_pre_2010 = df[mask_pre_2010]['datetime'].iloc[sequence_length - 1:]\n",
    "    predictions_df = pd.DataFrame({\n",
    "        'datetime': dates_pre_2010,\n",
    "        'solarenergy_predicted': predictions.flatten()\n",
    "    })\n",
    "\n",
    "    # Merge with original dataset\n",
    "    df = df.merge(predictions_df, on='datetime', how='left')\n",
    "\n",
    "    # Update solarenergy column where missing\n",
    "    df['solarenergy'] = df['solarenergy'].fillna(df['solarenergy_predicted'])\n",
    "\n",
    "    # Remove temporary column\n",
    "    df = df.drop('solarenergy_predicted', axis=1)\n",
    "\n",
    "    print(f\"Added {len(predictions)} predictions to dataset\")\n",
    "    print(f\"Rows with solar energy index after integration: {df['solarenergy'].notna().sum()}\")\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def train_solarenergy_bounded_model(df):\n",
    "    \"\"\"\n",
    "    Training of model with specific constraints for solar energy index\n",
    "    \"\"\"\n",
    "    print(\"Initializing solar energy index model training...\")\n",
    "\n",
    "    try:\n",
    "        # Data preparation\n",
    "        print(\"\\n1. Preparing data...\")\n",
    "        X_train_seq, X_test_seq, y_train, y_test, scaler, features, X_to_predict_seq = prepare_hybrid_data(df)\n",
    "\n",
    "        print(f\"Training data shape: {X_train_seq.shape}\")\n",
    "        print(f\"Test data shape: {X_test_seq.shape}\")\n",
    "\n",
    "        # Data quality verification\n",
    "        if np.isnan(X_train_seq).any() or np.isnan(y_train).any():\n",
    "            raise ValueError(\"Found NaN values in training data\")\n",
    "\n",
    "        # Model creation\n",
    "        print(\"\\n2. Creating model...\")\n",
    "        input_shape = (X_train_seq.shape[1], X_train_seq.shape[2])\n",
    "        model = create_solarradiation_model(input_shape, folder_name)\n",
    "\n",
    "        print(\"\\n4. Starting training...\")\n",
    "        history = train_hybrid_model(\n",
    "            model=model,\n",
    "            X_train=X_train_seq,\n",
    "            y_train=y_train,\n",
    "            X_test=X_test_seq,\n",
    "            y_test=y_test,\n",
    "            epochs=100,\n",
    "            batch_size=128,\n",
    "            folder_name=folder_name\n",
    "        )\n",
    "\n",
    "        print(\"\\n5. Generating predictions...\")\n",
    "        predictions = model.predict(X_test_seq)\n",
    "        predictions = np.clip(predictions, 0, 11)\n",
    "\n",
    "        print(\"\\n6. Evaluating model...\")\n",
    "        metrics = evaluate_solarenergy_predictions(y_test, predictions, folder_name=folder_name)\n",
    "\n",
    "        # Create results dictionary\n",
    "        training_results = {\n",
    "            'model_params': {\n",
    "                'input_shape': input_shape,\n",
    "                'n_features': len(features),\n",
    "                'sequence_length': X_train_seq.shape[1]\n",
    "            },\n",
    "            'training_params': {\n",
    "                'batch_size': 32,\n",
    "                'total_epochs': len(history.history['loss']),\n",
    "                'best_epoch': np.argmin(history.history['val_loss']) + 1,\n",
    "            },\n",
    "            'performance_metrics': {\n",
    "                'final_loss': float(history.history['val_loss'][-1]),\n",
    "                'final_mae': float(history.history['val_mae'][-1]),\n",
    "                'best_val_loss': float(min(history.history['val_loss'])),\n",
    "                'out_of_range_predictions': int(np.sum((predictions < 0) | (predictions > 11)))\n",
    "            }\n",
    "        }\n",
    "\n",
    "        print(\"\\n7. Predicting missing data results...\")\n",
    "        to_predict_predictions = model.predict(X_to_predict_seq)\n",
    "        to_predict_predictions = np.clip(to_predict_predictions, 0, 11)\n",
    "\n",
    "        print(\"\\n8. Integrating predictions into original dataset...\")\n",
    "        df_updated = integrate_predictions(df.copy(), to_predict_predictions)\n",
    "\n",
    "        df_updated.to_parquet('../../sources/weather_data_complete.parquet')\n",
    "\n",
    "        # Add prediction statistics to training_results\n",
    "        training_results['prediction_stats'] = {\n",
    "            'n_predictions_added': len(to_predict_predictions),\n",
    "            'mean_predicted_solarenergy': float(to_predict_predictions.mean()),\n",
    "            'min_predicted_solarenergy': float(to_predict_predictions.min()),\n",
    "            'max_predicted_solarenergy': float(to_predict_predictions.max()),\n",
    "        }\n",
    "\n",
    "        print(\"\\nTraining completed successfully!\")\n",
    "\n",
    "        return model, scaler, features, history, predictions, y_test, metrics, training_results\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"\\nError during training: {str(e)}\")\n",
    "        raise\n",
    "\n",
    "    finally:\n",
    "        # Memory cleanup\n",
    "        tf.keras.backend.clear_session()"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {},
   "source": [
    "df = pd.read_parquet('../../sources/weather_data_solarradiation.parquet')\n",
    "\n",
    "model, scaler, features, history, predictions, y_test, metrics, training_results = train_solarenergy_bounded_model(df)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "08fd4208-0afb-4bf1-bdef-b10b4065fe55",
   "metadata": {},
   "source": [
    "def plot_error_analysis(y_true, y_pred, folder_name=None):\n",
    "    \"\"\"\n",
    "    Function to visualize prediction error analysis\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    y_true : array-like\n",
    "        Actual values\n",
    "    y_pred : array-like\n",
    "        Predicted values\n",
    "    folder_name : str, optional\n",
    "        Folder to save plots. If None, plots won't be saved.\n",
    "    \"\"\"\n",
    "    import os\n",
    "    from datetime import datetime\n",
    "\n",
    "    # Convert to 1D numpy arrays if necessary\n",
    "    if isinstance(y_true, pd.Series):\n",
    "        y_true = y_true.values\n",
    "    if isinstance(y_pred, pd.Series):\n",
    "        y_pred = y_pred.values\n",
    "\n",
    "    y_true = y_true.ravel()\n",
    "    y_pred = y_pred.ravel()\n",
    "\n",
    "    # Calculate errors\n",
    "    errors = y_pred - y_true\n",
    "\n",
    "    # Create main figure\n",
    "    fig = plt.figure(figsize=(15, 5))\n",
    "\n",
    "    # Plot 1: Error Distribution\n",
    "    plt.subplot(1, 3, 1)\n",
    "    plt.hist(errors, bins=50, alpha=0.7)\n",
    "    plt.title('Prediction Error Distribution')\n",
    "    plt.xlabel('Error')\n",
    "    plt.ylabel('Frequency')\n",
    "\n",
    "    # Plot 2: Actual vs Predicted\n",
    "    plt.subplot(1, 3, 2)\n",
    "    plt.scatter(y_true, y_pred, alpha=0.5)\n",
    "    plt.plot([y_true.min(), y_true.max()], [y_true.min(), y_true.max()], 'r--', lw=2)\n",
    "    plt.title('Actual vs Predicted Values')\n",
    "    plt.xlabel('Actual Values')\n",
    "    plt.ylabel('Predicted Values')\n",
    "\n",
    "    # Plot 3: Errors vs Actual Values\n",
    "    plt.subplot(1, 3, 3)\n",
    "    plt.scatter(y_true, errors, alpha=0.5)\n",
    "    plt.axhline(y=0, color='r', linestyle='--')\n",
    "    plt.title('Errors vs Actual Values')\n",
    "    plt.xlabel('Actual Values')\n",
    "    plt.ylabel('Error')\n",
    "\n",
    "    plt.tight_layout()\n",
    "\n",
    "    # Save plot if folder is specified\n",
    "    if folder_name is not None:\n",
    "        try:\n",
    "            # Create folder if it doesn't exist\n",
    "            os.makedirs(folder_name, exist_ok=True)\n",
    "\n",
    "            # Generate filename with timestamp\n",
    "            timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
    "            filename = os.path.join(folder_name, f'error_analysis_{timestamp}.png')\n",
    "\n",
    "            # Save figure\n",
    "            plt.savefig(filename, dpi=300, bbox_inches='tight')\n",
    "            print(f\"\\nPlot saved as: {filename}\")\n",
    "        except Exception as e:\n",
    "            print(f\"\\nError saving plot: {str(e)}\")\n",
    "\n",
    "    plt.show()\n",
    "\n",
    "    # Print error statistics\n",
    "    print(\"\\nError Statistics:\")\n",
    "    print(f\"MAE: {np.mean(np.abs(errors)):.4f}\")\n",
    "    print(f\"MSE: {np.mean(errors ** 2):.4f}\")\n",
    "    print(f\"RMSE: {np.sqrt(np.mean(errors ** 2)):.4f}\")\n",
    "    print(f\"Mean errors: {np.mean(errors):.4f}\")\n",
    "    print(f\"Std errors: {np.std(errors):.4f}\")\n",
    "\n",
    "    # Calculate percentage of errors within thresholds\n",
    "    thresholds = [0.5, 1.0, 1.5, 2.0]\n",
    "    for threshold in thresholds:\n",
    "        within_threshold = np.mean(np.abs(errors) <= threshold) * 100\n",
    "        print(f\"Predictions within ±{threshold}: {within_threshold:.1f}%\")\n",
    "\n",
    "\n",
    "plot_error_analysis(y_test, predictions, folder_name=folder_name)"
   ],
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
