{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8adcbe0819b88578",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nfrom opt_einsum.paths import branch_1\\n!apt-get update\\n!apt-get install graphviz -y\\n\\n!pip install tensorflow\\n!pip install numpy\\n!pip install pandas\\n\\n!pip install keras\\n!pip install scikit-learn\\n!pip install matplotlib\\n!pip install joblib\\n!pip install pyarrow\\n!pip install fastparquet\\n!pip install scipy\\n!pip install seaborn\\n!pip install tqdm\\n!pip install pydot\\n!pip install tensorflow-io\\n'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "from opt_einsum.paths import branch_1\n",
    "!apt-get update\n",
    "!apt-get install graphviz -y\n",
    "\n",
    "!pip install tensorflow\n",
    "!pip install numpy\n",
    "!pip install pandas\n",
    "\n",
    "!pip install keras\n",
    "!pip install scikit-learn\n",
    "!pip install matplotlib\n",
    "!pip install joblib\n",
    "!pip install pyarrow\n",
    "!pip install fastparquet\n",
    "!pip install scipy\n",
    "!pip install seaborn\n",
    "!pip install tqdm\n",
    "!pip install pydot\n",
    "!pip install tensorflow-io\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7a813e3cbca057b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-10 22:44:08.491015: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-11-10 22:44:08.491086: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-11-10 22:44:08.491139: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-11-10 22:44:08.502469: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Dense, LSTM, Conv1D, MultiHeadAttention, Dropout, BatchNormalization, \\\n",
    "    LayerNormalization, GlobalAveragePooling1D, GlobalMaxPooling1D, Concatenate, Input, Reshape, Activation, Lambda, \\\n",
    "    Bidirectional, Add, Multiply, MaxPooling1D\n",
    "from tensorflow.keras import regularizers\n",
    "from tensorflow.keras.models import Model\n",
    "import tensorflow.keras.backend as K\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\n",
    "from tensorflow.keras.optimizers import Adam, AdamW\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "import joblib\n",
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from tensorflow.keras.utils import plot_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b3f525e19f78a1da",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_season(date):\n",
    "    month = date.month\n",
    "    day = date.day\n",
    "    if (month == 12 and day >= 21) or (month <= 3 and day < 20):\n",
    "        return 'Winter'\n",
    "    elif (month == 3 and day >= 20) or (month <= 6 and day < 21):\n",
    "        return 'Spring'\n",
    "    elif (month == 6 and day >= 21) or (month <= 9 and day < 23):\n",
    "        return 'Summer'\n",
    "    elif (month == 9 and day >= 23) or (month <= 12 and day < 21):\n",
    "        return 'Autumn'\n",
    "    else:\n",
    "        return 'Unknown'\n",
    "\n",
    "\n",
    "def get_time_period(hour):\n",
    "    if 5 <= hour < 12:\n",
    "        return 'Morning'\n",
    "    elif 12 <= hour < 17:\n",
    "        return 'Afternoon'\n",
    "    elif 17 <= hour < 21:\n",
    "        return 'Evening'\n",
    "    else:\n",
    "        return 'Night'\n",
    "\n",
    "\n",
    "def add_time_features(df):\n",
    "    df['datetime'] = pd.to_datetime(df['datetime'])\n",
    "    df['timestamp'] = df['datetime'].astype(np.int64) // 10 ** 9\n",
    "    df['year'] = df['datetime'].dt.year\n",
    "    df['month'] = df['datetime'].dt.month\n",
    "    df['day'] = df['datetime'].dt.day\n",
    "    df['hour'] = df['datetime'].dt.hour\n",
    "    df['minute'] = df['datetime'].dt.minute\n",
    "    df['hour_sin'] = np.sin(df['hour'] * (2 * np.pi / 24))\n",
    "    df['hour_cos'] = np.cos(df['hour'] * (2 * np.pi / 24))\n",
    "    df['day_of_week'] = df['datetime'].dt.dayofweek\n",
    "    df['day_of_year'] = df['datetime'].dt.dayofyear\n",
    "    df['week_of_year'] = df['datetime'].dt.isocalendar().week.astype(int)\n",
    "    df['quarter'] = df['datetime'].dt.quarter\n",
    "    df['is_month_end'] = df['datetime'].dt.is_month_end.astype(int)\n",
    "    df['is_quarter_end'] = df['datetime'].dt.is_quarter_end.astype(int)\n",
    "    df['is_year_end'] = df['datetime'].dt.is_year_end.astype(int)\n",
    "    df['month_sin'] = np.sin(df['month'] * (2 * np.pi / 12))\n",
    "    df['month_cos'] = np.cos(df['month'] * (2 * np.pi / 12))\n",
    "    df['day_of_year_sin'] = np.sin(df['day_of_year'] * (2 * np.pi / 365.25))\n",
    "    df['day_of_year_cos'] = np.cos(df['day_of_year'] * (2 * np.pi / 365.25))\n",
    "    df['season'] = df['datetime'].apply(get_season)\n",
    "    df['time_period'] = df['hour'].apply(get_time_period)\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_solar_features(df):\n",
    "    # Calcolo dell'angolo solare\n",
    "    df['solar_angle'] = np.sin(df['day_of_year'] * (2 * np.pi / 365.25)) * np.sin(df['hour'] * (2 * np.pi / 24))\n",
    "\n",
    "    # Interazioni tra features rilevanti\n",
    "    df['cloud_temp_interaction'] = df['cloudcover'] * df['temp']\n",
    "    df['visibility_cloud_interaction'] = df['visibility'] * (100 - df['cloudcover'])\n",
    "\n",
    "    # Feature derivate\n",
    "    df['clear_sky_index'] = (100 - df['cloudcover']) / 100\n",
    "    df['temp_gradient'] = df['temp'] - df['tempmin']\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_solar_specific_features(df):\n",
    "    # Angolo solare e durata del giorno\n",
    "    df['day_length'] = 12 + 3 * np.sin(2 * np.pi * (df['day_of_year'] - 81) / 365.25)\n",
    "    df['solar_noon'] = 12 - df['hour']\n",
    "    df['solar_elevation'] = np.sin(2 * np.pi * df['day_of_year'] / 365.25) * np.cos(2 * np.pi * df['solar_noon'] / 24)\n",
    "\n",
    "    # Interazioni\n",
    "    df['cloud_elevation'] = df['cloudcover'] * df['solar_elevation']\n",
    "    df['visibility_elevation'] = df['visibility'] * df['solar_elevation']\n",
    "\n",
    "    # Rolling features con finestre più ampie\n",
    "    df['cloud_rolling_12h'] = df['cloudcover'].rolling(window=12).mean()\n",
    "    df['temp_rolling_12h'] = df['temp'].rolling(window=12).mean()\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "\n",
    "def add_advanced_features(df):\n",
    "    # Features esistenti\n",
    "    df = add_time_features(df)\n",
    "    df = add_solar_features(df)\n",
    "    df = add_solar_specific_features(df)\n",
    "\n",
    "    if not isinstance(df.index, pd.DatetimeIndex):\n",
    "        df.index = pd.to_datetime(df.index)\n",
    "\n",
    "    # One-hot encoding per le feature categoriche\n",
    "    df = pd.get_dummies(df, columns=['season', 'time_period'])\n",
    "\n",
    "    # Aggiungi interazioni tra variabili meteorologiche\n",
    "    df['temp_humidity'] = df['temp'] * df['humidity']\n",
    "    df['temp_cloudcover'] = df['temp'] * df['cloudcover']\n",
    "    df['visibility_cloudcover'] = df['visibility'] * df['cloudcover']\n",
    "\n",
    "    # Features derivate per la radiazione solare\n",
    "    df['clear_sky_factor'] = (100 - df['cloudcover']) / 100\n",
    "    df['day_length'] = np.sin(df['day_of_year_sin']) * 12 + 12  # approssimazione della durata del giorno\n",
    "\n",
    "    # Lag features\n",
    "    df['temp_1h_lag'] = df['temp'].shift(1)\n",
    "    df['cloudcover_1h_lag'] = df['cloudcover'].shift(1)\n",
    "    df['humidity_1h_lag'] = df['humidity'].shift(1)\n",
    "\n",
    "    # Rolling means\n",
    "    df['temp_rolling_mean_6h'] = df['temp'].rolling(window=6).mean()\n",
    "    df['cloudcover_rolling_mean_6h'] = df['cloudcover'].rolling(window=6).mean()\n",
    "\n",
    "    df['temp_humidity_interaction'] = df['temp'] * df['humidity'] / 100\n",
    "    \n",
    "    # Indicatore di condizioni estreme\n",
    "    df['extreme_conditions'] = ((df['temp'] > df['temp'].quantile(0.75)) & \n",
    "                              (df['humidity'] < df['humidity'].quantile(0.25))).astype(int)\n",
    "    \n",
    "    # Feature composite per la trasparenza atmosferica\n",
    "    df['atmospheric_transparency'] = (100 - df['cloudcover']) * (df['visibility'] / 10)\n",
    "    \n",
    "    # Indicatori temporali più granulari per mezze stagioni\n",
    "    df['is_transition_season'] = ((df['season_Spring'] | df['season_Autumn'])).astype(int)\n",
    "    \n",
    "    # Interazione tra angolo solare e copertura nuvolosa normalizzata\n",
    "    df['solar_cloud_effect'] = df['solar_elevation'] * (100 - df['cloudcover']) / 100\n",
    "    \n",
    "    # Indicatore di stabilità atmosferica\n",
    "    df['pressure_stability'] = df.groupby(df.index.date if isinstance(df.index, pd.DatetimeIndex) \n",
    "                                        else df.index.to_series().dt.date)['pressure'].transform(\n",
    "        lambda x: x.std()\n",
    "    ).fillna(0)\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def prepare_advanced_data(df):\n",
    "    # Applicazione delle funzioni di feature engineering\n",
    "    df = add_advanced_features(df)\n",
    "\n",
    "    target_variables = ['solarradiation', 'solarenergy', 'uvindex']\n",
    "    # Selezione delle feature più rilevanti per UV index\n",
    "    selected_features = [\n",
    "        # Features meteorologiche base\n",
    "        'temp', 'humidity', 'cloudcover', 'visibility', 'pressure',\n",
    "\n",
    "        # Features temporali cicliche\n",
    "        'hour_sin', 'hour_cos', 'month_sin', 'month_cos',\n",
    "        'day_of_year_sin', 'day_of_year_cos',\n",
    "\n",
    "        # Features solari\n",
    "        'solar_angle', 'solar_elevation', 'day_length',\n",
    "        'clear_sky_index', 'solar_noon',\n",
    "\n",
    "        # Interazioni\n",
    "        'cloud_temp_interaction', 'visibility_cloud_interaction',\n",
    "        'cloud_elevation', 'visibility_elevation',\n",
    "\n",
    "        # Rolling features\n",
    "        'cloud_rolling_12h', 'temp_rolling_12h',\n",
    "        'temp_rolling_mean_6h', 'cloudcover_rolling_mean_6h',\n",
    "\n",
    "        # Features categoriche (da encodare)\n",
    "        'season', 'time_period'\n",
    "    ]\n",
    "\n",
    "    # Aggiorna la lista delle feature con le colonne one-hot\n",
    "    categorical_columns = [col for col in df.columns if col.startswith(('season_', 'time_period_'))]\n",
    "    final_features = [f for f in selected_features if f not in ['season', 'time_period']] + categorical_columns\n",
    "\n",
    "    df = df.sort_values('datetime')\n",
    "    df.set_index('datetime', inplace=True)\n",
    "    # Rimozione delle righe con valori NaN (create dai rolling features)\n",
    "    #df = df.dropna()\n",
    "    columns_to_interpolate = final_features + target_variables\n",
    "    for column in columns_to_interpolate:\n",
    "        df[column] = df[column].interpolate(method='time')\n",
    "\n",
    "    # Rimuovi eventuali valori mancanti residui\n",
    "    #df.dropna(subset=features + selected_features, inplace=True)\n",
    "    df.fillna(0, inplace=True)\n",
    "\n",
    "    data_after_2010 = df[df['year'] >= 2010].copy()\n",
    "    data_before_2010 = df[df['year'] < 2010].copy()\n",
    "\n",
    "    print(\"\\nNumero di record dopo 2010:\", len(data_after_2010))\n",
    "    print(\"Numero di record prima 2010:\", len(data_before_2010))\n",
    "\n",
    "    X = data_after_2010[final_features]\n",
    "\n",
    "    #print(X.head())\n",
    "    #print(X.columns)\n",
    "    y = data_after_2010['uvindex']\n",
    "\n",
    "    X_to_predict = data_before_2010[final_features]\n",
    "\n",
    "    # Split dei dati\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=42)\n",
    "\n",
    "    # Scaling delle feature\n",
    "    scaler = StandardScaler()\n",
    "    X_train_scaled = scaler.fit_transform(X_train)\n",
    "    X_test_scaled = scaler.transform(X_test)\n",
    "\n",
    "    X_to_predict_scaled = scaler.transform(X_to_predict)\n",
    "\n",
    "    return X_train_scaled, X_test_scaled, y_train, y_test, scaler, final_features, X_to_predict_scaled\n",
    "\n",
    "\n",
    "def create_sequence_data(X, sequence_length=24):\n",
    "    \"\"\"\n",
    "    Converte i dati in sequenze per l'input LSTM\n",
    "    sequence_length rappresenta quante ore precedenti considerare\n",
    "    \"\"\"\n",
    "    sequences = []\n",
    "    for i in range(len(X) - sequence_length + 1):\n",
    "        sequences.append(X[i:i + sequence_length])\n",
    "    return np.array(sequences)\n",
    "\n",
    "\n",
    "def prepare_hybrid_data(df):\n",
    "    # Utilizziamo la preparazione dati esistente\n",
    "    X_train_scaled, X_test_scaled, y_train, y_test, scaler, features, X_to_predict_scaled = prepare_advanced_data(df)\n",
    "\n",
    "    # Convertiamo i dati in sequenze\n",
    "    sequence_length = 24  # 24 ore di dati storici\n",
    "\n",
    "    X_train_seq = create_sequence_data(X_train_scaled, sequence_length)\n",
    "    X_test_seq = create_sequence_data(X_test_scaled, sequence_length)\n",
    "\n",
    "    # Adattiamo le y rimuovendo i primi (sequence_length-1) elementi\n",
    "    y_train = y_train[sequence_length - 1:]\n",
    "    y_test = y_test[sequence_length - 1:]\n",
    "\n",
    "    X_to_predict_seq = create_sequence_data(X_to_predict_scaled, sequence_length)\n",
    "\n",
    "    return X_train_seq, X_test_seq, y_train, y_test, scaler, features, X_to_predict_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9dff3259-b376-4cfc-89d8-ab2ea18aaa5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Funzioni per costruire il modello LSTM avanzato\n",
    "def create_residual_lstm_layer(x, units, dropout_rate, l2_reg=0.01, return_sequences=True):\n",
    "    residual = x\n",
    "    x = Bidirectional(LSTM(units, return_sequences=return_sequences, kernel_regularizer=regularizers.l2(l2_reg)))(x)\n",
    "    x = LayerNormalization()(x)\n",
    "    x = Dropout(dropout_rate)(x)\n",
    "    # Adjust residual dimension and handle return_sequences\n",
    "    if return_sequences:\n",
    "        if int(residual.shape[-1]) != 2 * units:\n",
    "            residual = Dense(2 * units, activation='linear')(residual)\n",
    "        x = Add()([x, residual])\n",
    "    return x\n",
    "\n",
    "\n",
    "def attention_block(x, units, num_heads=8):\n",
    "    attention = MultiHeadAttention(num_heads=num_heads, key_dim=units)(x, x)\n",
    "    x = Add()([x, attention])\n",
    "    x = LayerNormalization()(x)\n",
    "    return x\n",
    "\n",
    "\n",
    "def create_optimized_model(input_shape,folder_name, l2_lambda=0.005):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    # Primi due layer LSTM con sequenze\n",
    "    x = create_residual_lstm_layer(inputs, 128, 0.4, l2_lambda, return_sequences=True)\n",
    "    x = create_residual_lstm_layer(x, 64, 0.3, l2_lambda, return_sequences=True)\n",
    "    x = create_residual_lstm_layer(x, 32, 0.2, l2_lambda, return_sequences=True)\n",
    "\n",
    "    # Attention e MaxPooling mentre abbiamo ancora la sequenza\n",
    "    x = attention_block(x, 32, num_heads=16)\n",
    "    x = MaxPooling1D()(x)\n",
    "\n",
    "    # Ultimo layer LSTM senza sequenze\n",
    "    x = create_residual_lstm_layer(x, 32, 0.1, l2_lambda, return_sequences=False)\n",
    "\n",
    "    # Dense layers\n",
    "    x = Dense(32, kernel_regularizer=regularizers.l2(l2_lambda))(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('swish')(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "\n",
    "    x = Dense(16, kernel_regularizer=regularizers.l2(l2_lambda))(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('swish')(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    \n",
    "    outputs = Dense(1)(x)\n",
    "    outputs = Lambda(lambda x: tf.clip_by_value(x, 0, 11))(outputs)\n",
    "    \n",
    "    model = Model(inputs=inputs, outputs=outputs, name=\"UvModel\")\n",
    "    \n",
    "    # 4. Optimizer con parametri conservativi\n",
    "    optimizer = Adam(\n",
    "        learning_rate=0.0005,  # Learning rate più basso\n",
    "        beta_1=0.9,\n",
    "        beta_2=0.999,\n",
    "        epsilon=1e-07\n",
    "    )\n",
    "    \n",
    "    model.compile(\n",
    "        optimizer=optimizer,\n",
    "        loss='huber',\n",
    "        metrics=['mae', 'mse']\n",
    "    )\n",
    "    model.summary()\n",
    "\n",
    "    plot_model(model, \n",
    "          to_file=f'{folder_name}_model_architecture.png',\n",
    "          show_shapes=True,  # Mostra le dimensioni dei tensori\n",
    "          show_layer_names=True,  # Mostra i nomi dei layer\n",
    "          dpi=96,  # Risoluzione dell'immagine\n",
    "          show_layer_activations=True) \n",
    "\n",
    "    return model\n",
    "\n",
    "def evaluate_uv_predictions(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    Valutazione specifica per UV index con metriche categoriche\n",
    "    \"\"\"\n",
    "    # Converti in numpy array se necessario\n",
    "    y_true = np.array(y_true)\n",
    "    y_pred = np.array(y_pred)\n",
    "\n",
    "    # Arrotonda le predizioni al più vicino intero\n",
    "    y_pred_rounded = np.round(y_pred)\n",
    "\n",
    "    # Clip dei valori tra 0 e 11\n",
    "    y_pred_clipped = np.clip(y_pred_rounded, 0, 11)\n",
    "\n",
    "    # Calcolo metriche\n",
    "    mae = mean_absolute_error(y_true, y_pred_clipped)\n",
    "    rmse = np.sqrt(mean_squared_error(y_true, y_pred_clipped))\n",
    "    r2 = r2_score(y_true, y_pred_clipped)\n",
    "\n",
    "    # Calcolo accuratezza per diversi margini di errore\n",
    "    exact_accuracy = np.mean(y_pred_clipped == y_true.ravel())\n",
    "    one_off_accuracy = np.mean(np.abs(y_pred_clipped - y_true.ravel()) <= 1)\n",
    "    two_off_accuracy = np.mean(np.abs(y_pred_clipped - y_true.ravel()) <= 2)\n",
    "\n",
    "    print(\"\\nUV Index Prediction Metrics:\")\n",
    "    print(f\"MAE: {mae:.3f}\")\n",
    "    print(f\"RMSE: {rmse:.3f}\")\n",
    "    print(f\"R² Score: {r2:.3f}\")\n",
    "    print(f\"Exact Match Accuracy: {exact_accuracy:.3f}\")\n",
    "    print(f\"±1 Accuracy: {one_off_accuracy:.3f}\")\n",
    "    print(f\"±2 Accuracy: {two_off_accuracy:.3f}\")\n",
    "\n",
    "    # Confusion Matrix per livelli di UV\n",
    "    def get_uv_level(value):\n",
    "        if value <= 2:\n",
    "            return 'Low'\n",
    "        elif value <= 5:\n",
    "            return 'Moderate'\n",
    "        elif value <= 7:\n",
    "            return 'High'\n",
    "        elif value <= 10:\n",
    "            return 'Very High'\n",
    "        else:\n",
    "            return 'Extreme'\n",
    "\n",
    "    y_true_levels = [get_uv_level(v) for v in y_true.ravel()]\n",
    "    y_pred_levels = [get_uv_level(v) for v in y_pred_clipped]\n",
    "\n",
    "    print(\"\\nUV Level Confusion Matrix:\")\n",
    "    print(pd.crosstab(\n",
    "        pd.Series(y_true_levels, name='Actual'),\n",
    "        pd.Series(y_pred_levels, name='Predicted')\n",
    "    ))\n",
    "\n",
    "    return mae, rmse, r2, exact_accuracy, one_off_accuracy\n",
    "\n",
    "\n",
    "def plot_uv_predictions(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    Visualizzazione delle predizioni specifica per UV index\n",
    "    \"\"\"\n",
    "    # Converti in numpy array se necessario\n",
    "    y_true = np.array(y_true).ravel()\n",
    "    y_pred = np.array(y_pred).ravel()\n",
    "\n",
    "    plt.figure(figsize=(15, 5))\n",
    "\n",
    "    # Plot 1: Actual vs Predicted\n",
    "    plt.subplot(1, 2, 1)\n",
    "    plt.scatter(y_true, y_pred, alpha=0.5)\n",
    "    plt.plot([0, 11], [0, 11], 'r--', lw=2)\n",
    "    plt.xlabel('Actual UV Index')\n",
    "    plt.ylabel('Predicted UV Index')\n",
    "    plt.title('Actual vs Predicted UV Index')\n",
    "    plt.grid(True)\n",
    "\n",
    "    # Plot 2: Distribution of Errors\n",
    "    plt.subplot(1, 2, 2)\n",
    "    errors = y_pred - y_true\n",
    "    plt.hist(errors, bins=20, alpha=0.7)\n",
    "    plt.xlabel('Prediction Error')\n",
    "    plt.ylabel('Frequency')\n",
    "    plt.title('Distribution of Prediction Errors')\n",
    "    plt.grid(True)\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "def train_hybrid_model(model, X_train, y_train, X_test, y_test, epochs=100, batch_size=32, folder_name='uv_index'):\n",
    "    \"\"\"\n",
    "    Funzione di training avanzata per il modello ibrido UV index con monitoraggio dettagliato\n",
    "    e gestione del training.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    model : keras.Model\n",
    "        Il modello ibrido compilato\n",
    "    X_train : numpy.ndarray\n",
    "        Dati di training\n",
    "    y_train : numpy.ndarray\n",
    "        Target di training\n",
    "    X_test : numpy.ndarray\n",
    "        Dati di validation\n",
    "    y_test : numpy.ndarray\n",
    "        Target di validation\n",
    "    epochs : int, optional\n",
    "        Numero massimo di epoche di training\n",
    "    batch_size : int, optional\n",
    "        Dimensione del batch\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    history : keras.callbacks.History\n",
    "        Storia del training con tutte le metriche\n",
    "    \"\"\"\n",
    "\n",
    "    # Callbacks avanzati per il training\n",
    "    callbacks = [\n",
    "        # Early Stopping avanzato\n",
    "        EarlyStopping(\n",
    "        monitor='mae',\n",
    "        patience=15,              # Aumentato per dare più tempo dopo le riduzioni del LR\n",
    "        restore_best_weights=True,\n",
    "        mode='min',\n",
    "        verbose=1,\n",
    "        min_delta=1e-6           # Reso più sensibile\n",
    "        ),\n",
    "        # Learning Rate Schedule molto più aggressivo\n",
    "        ReduceLROnPlateau(\n",
    "            monitor='mae',\n",
    "            factor=0.05,             # Riduzione molto più aggressiva (95% di riduzione)\n",
    "            patience=3,              # Ridotto per reagire più velocemente\n",
    "            verbose=1,\n",
    "            mode='min',\n",
    "            min_delta=1e-6,\n",
    "            cooldown=2,              # Ridotto per permettere riduzioni più frequenti\n",
    "            min_lr=1e-7              # LR minimo più basso\n",
    "        ),\n",
    "        # Aggiungiamo un secondo scheduler per riduzioni ancora più graduali\n",
    "        ReduceLROnPlateau(\n",
    "            monitor='val_loss',\n",
    "            factor=0.2,              # Riduzione più moderata come backup\n",
    "            patience=2,              # Ancora più reattivo\n",
    "            verbose=1,\n",
    "            mode='min',\n",
    "            min_delta=1e-6,\n",
    "            cooldown=1,\n",
    "            min_lr=1e-7\n",
    "        ),\n",
    "        # Model Checkpoint per salvare i migliori modelli\n",
    "        tf.keras.callbacks.ModelCheckpoint(\n",
    "            filepath=f'{folder_name}_best_uv_model.h5',\n",
    "            monitor='mae',\n",
    "            save_best_only=True,\n",
    "            mode='min'\n",
    "        ),\n",
    "        # TensorBoard callback per il monitoraggio\n",
    "        tf.keras.callbacks.TensorBoard(\n",
    "            log_dir=f'./logs_{folder_name}',\n",
    "            histogram_freq=1,\n",
    "            write_graph=True,\n",
    "            update_freq='epoch'\n",
    "        ),\n",
    "\n",
    "        # Custom Callback per monitorare le predizioni fuori range\n",
    "        tf.keras.callbacks.LambdaCallback(\n",
    "            on_epoch_end=lambda epoch, logs: print(\n",
    "                f\"\\nEpoch {epoch + 1}: Predizioni fuori range: \"\n",
    "                f\"{np.sum((model.predict(X_test) < 0) | (model.predict(X_test) > 11))}\"\n",
    "            ) if epoch % 20 == 0 else None\n",
    "        )\n",
    "    ]\n",
    "\n",
    "    # Calcolo dei class weights se non forniti\n",
    "    '''\n",
    "    if class_weights is None:\n",
    "        # Discretizziamo i valori UV per il calcolo dei pesi\n",
    "        y_discrete = np.round(y_train).astype(int)\n",
    "        class_weights = compute_class_weight(\n",
    "            'balanced',\n",
    "            classes=np.unique(y_discrete),\n",
    "            y=y_discrete\n",
    "        )\n",
    "        class_weights = dict(enumerate(class_weights))\n",
    "    '''\n",
    "    # Training con gestione degli errori e logging\n",
    "    try:\n",
    "        history = model.fit(\n",
    "            X_train, y_train,\n",
    "            validation_data=(X_test, y_test),\n",
    "            epochs=epochs,\n",
    "            batch_size=batch_size,\n",
    "            callbacks=callbacks,\n",
    "            #class_weight=class_weights,\n",
    "            verbose=1,\n",
    "            shuffle=False,        # Abilitato shuffle\n",
    "            validation_freq=1,\n",
    "        )\n",
    "\n",
    "        # Analisi post-training\n",
    "        print(\"\\nTraining completato con successo!\")\n",
    "\n",
    "        # Valutazione finale sul test set\n",
    "        test_loss, test_mae, test_mse = model.evaluate(X_test, y_test, verbose=0)\n",
    "        print(f\"\\nMetriche finali sul test set:\")\n",
    "        print(f\"Loss: {test_loss:.4f}\")\n",
    "        print(f\"MAE: {test_mae:.4f}\")\n",
    "        print(f\"MSE: {test_mse:.4f}\")\n",
    "\n",
    "        # Analisi delle predizioni\n",
    "        predictions = model.predict(X_test)\n",
    "        out_of_range = np.sum((predictions < 0) | (predictions > 11))\n",
    "        print(f\"\\nPredizioni fuori range: {out_of_range} ({out_of_range / len(predictions) * 100:.2f}%)\")\n",
    "\n",
    "        # Plot della loss durante il training\n",
    "        plt.figure(figsize=(12, 4))\n",
    "\n",
    "        plt.subplot(1, 2, 1)\n",
    "        plt.plot(history.history['loss'], label='Training Loss')\n",
    "        plt.plot(history.history['val_loss'], label='Validation Loss')\n",
    "        plt.title('Model Loss')\n",
    "        plt.xlabel('Epoch')\n",
    "        plt.ylabel('Loss')\n",
    "        plt.legend()\n",
    "\n",
    "        plt.subplot(1, 2, 2)\n",
    "        plt.plot(history.history['mae'], label='Training MAE')\n",
    "        plt.plot(history.history['val_mae'], label='Validation MAE')\n",
    "        plt.title('Model MAE')\n",
    "        plt.xlabel('Epoch')\n",
    "        plt.ylabel('MAE')\n",
    "        plt.legend()\n",
    "\n",
    "        plt.tight_layout()\n",
    "        plt.show()\n",
    "\n",
    "        # Salvataggio dei risultati del training\n",
    "        training_results = {\n",
    "            'final_loss': float(test_loss),\n",
    "            'final_mae': float(test_mae),\n",
    "            'final_mse': float(test_mse),\n",
    "            'out_of_range_predictions': int(out_of_range),\n",
    "            'training_time': int(len(history.history['loss'])),\n",
    "            'best_epoch': int(np.argmin(history.history['val_loss'])) + 1\n",
    "        }\n",
    "\n",
    "        # Salvataggio su file\n",
    "        with open('training_results.json', 'w') as f:\n",
    "            json.dump(training_results, f, indent=4)\n",
    "\n",
    "        return history\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"\\nErrore durante il training: {str(e)}\")\n",
    "        raise\n",
    "\n",
    "    finally:\n",
    "        # Pulizia della memoria\n",
    "        tf.keras.backend.clear_session()\n",
    "\n",
    "\n",
    "def calculate_class_weights(y_train, n_classes=12):\n",
    "    \"\"\"\n",
    "    Calcola i pesi delle classi per bilanciare il dataset UV index.\n",
    "    \n",
    "    Parameters:\n",
    "    -----------\n",
    "    y_train : numpy.ndarray\n",
    "        Array dei valori UV di training\n",
    "    n_classes : int, optional\n",
    "        Numero di classi possibili (0-11 per UV index, quindi 12 classi)\n",
    "        \n",
    "    Returns:\n",
    "    --------\n",
    "    dict:\n",
    "        Dizionario con i pesi per ogni classe\n",
    "    \"\"\"\n",
    "    # Arrotonda i valori UV al più vicino intero e converti in intero\n",
    "    y_discrete = np.clip(np.round(y_train), 0, 11).astype(int)\n",
    "\n",
    "    # Calcola la frequenza di ogni classe\n",
    "    unique, counts = np.unique(y_discrete, return_counts=True)\n",
    "    total_samples = len(y_discrete)\n",
    "\n",
    "    # Calcola i pesi inversamente proporzionali alla frequenza\n",
    "    weights = {}\n",
    "    for i in range(n_classes):\n",
    "        if i in unique:\n",
    "            # Se la classe è presente, calcola il peso\n",
    "            weight = total_samples / (len(unique) * counts[unique == i][0])\n",
    "        else:\n",
    "            # Se la classe non è presente, assegna un peso neutro\n",
    "            weight = 1.0\n",
    "        weights[i] = weight\n",
    "\n",
    "    return weights\n",
    "\n",
    "\n",
    "def integrate_predictions(df, predictions, sequence_length=24):\n",
    "    \"\"\"\n",
    "    Integra le predizioni dell'UV index nel dataset originale per i dati precedenti al 2010.\n",
    "    \n",
    "    Parameters:\n",
    "    -----------\n",
    "    df : pandas.DataFrame\n",
    "        Dataset originale\n",
    "    predictions : numpy.ndarray\n",
    "        Array delle predizioni UV index\n",
    "    sequence_length : int\n",
    "        Lunghezza della sequenza usata per le predizioni\n",
    "        \n",
    "    Returns:\n",
    "    --------\n",
    "    pandas.DataFrame\n",
    "        Dataset aggiornato con le predizioni UV index\n",
    "    \"\"\"\n",
    "    # Converti datetime in formato datetime se non lo è già\n",
    "    df['datetime'] = pd.to_datetime(df['datetime'])\n",
    "\n",
    "    # Identifica le righe precedenti al 2010\n",
    "    mask_pre_2010 = df['datetime'].dt.year < 2010\n",
    "\n",
    "    # Crea un DataFrame temporaneo con le predizioni\n",
    "    dates_pre_2010 = df[mask_pre_2010]['datetime'].iloc[sequence_length - 1:]\n",
    "    predictions_df = pd.DataFrame({\n",
    "        'datetime': dates_pre_2010,\n",
    "        'uvindex_predicted': predictions.flatten()\n",
    "    })\n",
    "\n",
    "    # Merge con il dataset originale\n",
    "    df = df.merge(predictions_df, on='datetime', how='left')\n",
    "\n",
    "    # Aggiorna la colonna uvindex dove manca\n",
    "    df['uvindex'] = df['uvindex'].fillna(df['uvindex_predicted'])\n",
    "\n",
    "    # Rimuovi la colonna temporanea\n",
    "    df = df.drop('uvindex_predicted', axis=1)\n",
    "\n",
    "    print(f\"Aggiunte {len(predictions)} predizioni al dataset\")\n",
    "    print(f\"Righe con UV index dopo l'integrazione: {df['uvindex'].notna().sum()}\")\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def train_uvindex_bounded_model(df):\n",
    "    \"\"\"\n",
    "    Training del modello con vincoli specifici per UV index\n",
    "    \"\"\"\n",
    "    print(\"Inizializzazione del training del modello UV index...\")\n",
    "\n",
    "    try:\n",
    "        folder_name = datetime.now().strftime(\"%Y-%m-%d_%H-%M\")\n",
    "        # Preparazione dei dati\n",
    "        print(\"\\n1. Preparazione dei dati...\")\n",
    "        X_train_seq, X_test_seq, y_train, y_test, scaler, features, X_to_predict_seq = prepare_hybrid_data(df)\n",
    "\n",
    "        print(f\"Shape dei dati di training: {X_train_seq.shape}\")\n",
    "        print(f\"Shape dei dati di test: {X_test_seq.shape}\")\n",
    "\n",
    "        # Verifica della qualità dei dati\n",
    "        if np.isnan(X_train_seq).any() or np.isnan(y_train).any():\n",
    "            raise ValueError(\"Trovati valori NaN nei dati di training\")\n",
    "\n",
    "        # Creazione del modello\n",
    "        print(\"\\n2. Creazione del modello...\")\n",
    "        input_shape = (X_train_seq.shape[1], X_train_seq.shape[2])\n",
    "        model = create_optimized_model(input_shape, folder_name)\n",
    "\n",
    "        print(\"\\n4. Avvio del training...\")\n",
    "        history = train_hybrid_model(\n",
    "            model=model,\n",
    "            X_train=X_train_seq,\n",
    "            y_train=y_train,\n",
    "            X_test=X_test_seq,\n",
    "            y_test=y_test,\n",
    "            #class_weights=class_weights,  # Ora passiamo direttamente il dizionario\n",
    "            epochs=100,\n",
    "            batch_size=128,\n",
    "            folder_name=folder_name\n",
    "        )\n",
    "\n",
    "        print(\"\\n5. Generazione delle predizioni...\")\n",
    "        predictions = model.predict(X_test_seq)\n",
    "        predictions = np.clip(predictions, 0, 11)\n",
    "\n",
    "        print(\"\\n6. Valutazione del modello...\")\n",
    "        metrics = evaluate_uv_predictions(y_test, predictions)\n",
    "\n",
    "        print(\"\\n7. Visualizzazione risultati...\")\n",
    "        plot_uv_predictions(y_test, predictions)\n",
    "\n",
    "        # Creazione del dizionario dei risultati\n",
    "        training_results = {\n",
    "            'model_params': {\n",
    "                'input_shape': input_shape,\n",
    "                'n_features': len(features),\n",
    "                'sequence_length': X_train_seq.shape[1]\n",
    "            },\n",
    "            'training_params': {\n",
    "                'batch_size': 32,\n",
    "                'total_epochs': len(history.history['loss']),\n",
    "                'best_epoch': np.argmin(history.history['val_loss']) + 1,\n",
    "                #'class_weights': {str(k): float(v) for k, v in class_weights.items()}\n",
    "            },\n",
    "            'performance_metrics': {\n",
    "                'final_loss': float(history.history['val_loss'][-1]),\n",
    "                'final_mae': float(history.history['val_mae'][-1]),\n",
    "                'best_val_loss': float(min(history.history['val_loss'])),\n",
    "                'out_of_range_predictions': int(np.sum((predictions < 0) | (predictions > 11)))\n",
    "            }\n",
    "        }\n",
    "\n",
    "        print(\"\\n8. Predizione dei dati mancanti risultati...\")\n",
    "        to_predict_predictions = model.predict(X_to_predict_seq)\n",
    "        to_predict_predictions = np.clip(to_predict_predictions, 0, 11)\n",
    "\n",
    "        print(\"\\n9. Integrazione delle predizioni nel dataset originale...\")\n",
    "        df_updated = integrate_predictions(df.copy(), to_predict_predictions)\n",
    "\n",
    "        # Aggiungi statistiche sulle predizioni al training_results\n",
    "        training_results['prediction_stats'] = {\n",
    "            'n_predictions_added': len(to_predict_predictions),\n",
    "            'mean_predicted_uv': float(to_predict_predictions.mean()),\n",
    "            'min_predicted_uv': float(to_predict_predictions.min()),\n",
    "            'max_predicted_uv': float(to_predict_predictions.max()),\n",
    "        }\n",
    "\n",
    "        print(\"\\nTraining completato con successo!\")\n",
    "\n",
    "        return model, scaler, features, history, predictions, y_test, metrics, training_results\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"\\nErrore durante il training: {str(e)}\")\n",
    "        raise\n",
    "\n",
    "    finally:\n",
    "        # Pulizia della memoria\n",
    "        tf.keras.backend.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "initial_id",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inizializzazione del training del modello UV index...\n",
      "\n",
      "1. Preparazione dei dati...\n",
      "\n",
      "Numero di record dopo 2010: 129777\n",
      "Numero di record prima 2010: 227902\n",
      "Shape dei dati di training: (64865, 24, 33)\n",
      "Shape dei dati di test: (64866, 24, 33)\n",
      "\n",
      "2. Creazione del modello...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-10 22:48:30.467084: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1886] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 22456 MB memory:  -> device: 0, name: NVIDIA GeForce RTX 3090, pci bus id: 0000:e1:00.0, compute capability: 8.6\n",
      "2024-11-10 22:48:30.995563: I tensorflow/tsl/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"UvModel\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                Output Shape                 Param #   Connected to                  \n",
      "==================================================================================================\n",
      " input_1 (InputLayer)        [(None, 24, 33)]             0         []                            \n",
      "                                                                                                  \n",
      " bidirectional (Bidirection  (None, 24, 256)              165888    ['input_1[0][0]']             \n",
      " al)                                                                                              \n",
      "                                                                                                  \n",
      " layer_normalization (Layer  (None, 24, 256)              512       ['bidirectional[0][0]']       \n",
      " Normalization)                                                                                   \n",
      "                                                                                                  \n",
      " dropout (Dropout)           (None, 24, 256)              0         ['layer_normalization[0][0]'] \n",
      "                                                                                                  \n",
      " dense (Dense)               (None, 24, 256)              8704      ['input_1[0][0]']             \n",
      "                                                                                                  \n",
      " add (Add)                   (None, 24, 256)              0         ['dropout[0][0]',             \n",
      "                                                                     'dense[0][0]']               \n",
      "                                                                                                  \n",
      " bidirectional_1 (Bidirecti  (None, 24, 128)              164352    ['add[0][0]']                 \n",
      " onal)                                                                                            \n",
      "                                                                                                  \n",
      " layer_normalization_1 (Lay  (None, 24, 128)              256       ['bidirectional_1[0][0]']     \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dropout_1 (Dropout)         (None, 24, 128)              0         ['layer_normalization_1[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_1 (Dense)             (None, 24, 128)              32896     ['add[0][0]']                 \n",
      "                                                                                                  \n",
      " add_1 (Add)                 (None, 24, 128)              0         ['dropout_1[0][0]',           \n",
      "                                                                     'dense_1[0][0]']             \n",
      "                                                                                                  \n",
      " bidirectional_2 (Bidirecti  (None, 24, 64)               41216     ['add_1[0][0]']               \n",
      " onal)                                                                                            \n",
      "                                                                                                  \n",
      " layer_normalization_2 (Lay  (None, 24, 64)               128       ['bidirectional_2[0][0]']     \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dropout_2 (Dropout)         (None, 24, 64)               0         ['layer_normalization_2[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_2 (Dense)             (None, 24, 64)               8256      ['add_1[0][0]']               \n",
      "                                                                                                  \n",
      " add_2 (Add)                 (None, 24, 64)               0         ['dropout_2[0][0]',           \n",
      "                                                                     'dense_2[0][0]']             \n",
      "                                                                                                  \n",
      " multi_head_attention (Mult  (None, 24, 64)               132672    ['add_2[0][0]',               \n",
      " iHeadAttention)                                                     'add_2[0][0]']               \n",
      "                                                                                                  \n",
      " add_3 (Add)                 (None, 24, 64)               0         ['add_2[0][0]',               \n",
      "                                                                     'multi_head_attention[0][0]']\n",
      "                                                                                                  \n",
      " layer_normalization_3 (Lay  (None, 24, 64)               128       ['add_3[0][0]']               \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " max_pooling1d (MaxPooling1  (None, 12, 64)               0         ['layer_normalization_3[0][0]'\n",
      " D)                                                                 ]                             \n",
      "                                                                                                  \n",
      " bidirectional_3 (Bidirecti  (None, 64)                   24832     ['max_pooling1d[0][0]']       \n",
      " onal)                                                                                            \n",
      "                                                                                                  \n",
      " layer_normalization_4 (Lay  (None, 64)                   128       ['bidirectional_3[0][0]']     \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dropout_3 (Dropout)         (None, 64)                   0         ['layer_normalization_4[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_3 (Dense)             (None, 32)                   2080      ['dropout_3[0][0]']           \n",
      "                                                                                                  \n",
      " batch_normalization (Batch  (None, 32)                   128       ['dense_3[0][0]']             \n",
      " Normalization)                                                                                   \n",
      "                                                                                                  \n",
      " activation (Activation)     (None, 32)                   0         ['batch_normalization[0][0]'] \n",
      "                                                                                                  \n",
      " dropout_4 (Dropout)         (None, 32)                   0         ['activation[0][0]']          \n",
      "                                                                                                  \n",
      " dense_4 (Dense)             (None, 16)                   528       ['dropout_4[0][0]']           \n",
      "                                                                                                  \n",
      " batch_normalization_1 (Bat  (None, 16)                   64        ['dense_4[0][0]']             \n",
      " chNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " activation_1 (Activation)   (None, 16)                   0         ['batch_normalization_1[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dropout_5 (Dropout)         (None, 16)                   0         ['activation_1[0][0]']        \n",
      "                                                                                                  \n",
      " dense_5 (Dense)             (None, 1)                    17        ['dropout_5[0][0]']           \n",
      "                                                                                                  \n",
      " lambda (Lambda)             (None, 1)                    0         ['dense_5[0][0]']             \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 582785 (2.22 MB)\n",
      "Trainable params: 582689 (2.22 MB)\n",
      "Non-trainable params: 96 (384.00 Byte)\n",
      "__________________________________________________________________________________________________\n",
      "\n",
      "4. Avvio del training...\n",
      "Epoch 1/100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-10 22:48:44.916189: I tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:442] Loaded cuDNN version 8905\n",
      "2024-11-10 22:48:47.364274: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x7fb501567b00 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2024-11-10 22:48:47.364334: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): NVIDIA GeForce RTX 3090, Compute Capability 8.6\n",
      "2024-11-10 22:48:47.375943: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:269] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n",
      "2024-11-10 22:48:47.576847: I ./tensorflow/compiler/jit/device_compiler.h:186] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "507/507 [==============================] - ETA: 0s - loss: 3.0211 - mae: 0.8732 - mse: 2.6901"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.11/dist-packages/keras/src/engine/training.py:3079: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n",
      "  saving_api.save_model(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2028/2028 [==============================] - 25s 11ms/step\n",
      "2028/2028 [==============================] - 22s 11ms/step\n",
      "\n",
      "Epoch 1: Predizioni fuori range: 0\n",
      "507/507 [==============================] - 95s 149ms/step - loss: 3.0211 - mae: 0.8732 - mse: 2.6901 - val_loss: 1.1114 - val_mae: 0.5132 - val_mse: 0.8839 - lr: 5.0000e-04\n",
      "Epoch 2/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.8217 - mae: 0.5744 - mse: 1.1119 - val_loss: 0.5794 - val_mae: 0.5285 - val_mse: 0.8990 - lr: 5.0000e-04\n",
      "Epoch 3/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.5241 - mae: 0.5406 - mse: 0.9968 - val_loss: 0.4411 - val_mae: 0.4995 - val_mse: 0.8515 - lr: 5.0000e-04\n",
      "Epoch 4/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.4490 - mae: 0.5363 - mse: 0.9738 - val_loss: 0.3752 - val_mae: 0.4606 - val_mse: 0.7784 - lr: 5.0000e-04\n",
      "Epoch 5/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.4127 - mae: 0.5218 - mse: 0.9290 - val_loss: 0.3470 - val_mae: 0.4698 - val_mse: 0.7271 - lr: 5.0000e-04\n",
      "Epoch 6/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3900 - mae: 0.5160 - mse: 0.9202 - val_loss: 0.3096 - val_mae: 0.4316 - val_mse: 0.6812 - lr: 5.0000e-04\n",
      "Epoch 7/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3934 - mae: 0.5173 - mse: 0.9176 - val_loss: 0.3931 - val_mae: 0.5097 - val_mse: 0.9083 - lr: 5.0000e-04\n",
      "Epoch 8/100\n",
      "507/507 [==============================] - ETA: 0s - loss: 0.3765 - mae: 0.5054 - mse: 0.8823\n",
      "Epoch 8: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "507/507 [==============================] - 22s 44ms/step - loss: 0.3765 - mae: 0.5054 - mse: 0.8823 - val_loss: 0.3501 - val_mae: 0.4814 - val_mse: 0.7773 - lr: 5.0000e-04\n",
      "Epoch 9/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3486 - mae: 0.4816 - mse: 0.8109 - val_loss: 0.2983 - val_mae: 0.4224 - val_mse: 0.6545 - lr: 1.0000e-04\n",
      "Epoch 10/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.3425 - mae: 0.4789 - mse: 0.8051 - val_loss: 0.2930 - val_mae: 0.4210 - val_mse: 0.6593 - lr: 1.0000e-04\n",
      "Epoch 11/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.3360 - mae: 0.4739 - mse: 0.7934 - val_loss: 0.2883 - val_mae: 0.4160 - val_mse: 0.6435 - lr: 1.0000e-04\n",
      "Epoch 12/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.3341 - mae: 0.4755 - mse: 0.7938 - val_loss: 0.2981 - val_mae: 0.4305 - val_mse: 0.6719 - lr: 1.0000e-04\n",
      "Epoch 13/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3315 - mae: 0.4741 - mse: 0.7918 - val_loss: 0.2859 - val_mae: 0.4170 - val_mse: 0.6472 - lr: 1.0000e-04\n",
      "Epoch 14/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.3280 - mae: 0.4706 - mse: 0.7851 - val_loss: 0.2841 - val_mae: 0.4196 - val_mse: 0.6415 - lr: 1.0000e-04\n",
      "Epoch 15/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3253 - mae: 0.4691 - mse: 0.7793 - val_loss: 0.2800 - val_mae: 0.4129 - val_mse: 0.6375 - lr: 1.0000e-04\n",
      "Epoch 16/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3238 - mae: 0.4697 - mse: 0.7778 - val_loss: 0.2866 - val_mae: 0.4247 - val_mse: 0.6558 - lr: 1.0000e-04\n",
      "Epoch 17/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.3220 - mae: 0.4683 - mse: 0.7760 - val_loss: 0.2784 - val_mae: 0.4179 - val_mse: 0.6353 - lr: 1.0000e-04\n",
      "Epoch 18/100\n",
      "507/507 [==============================] - 23s 44ms/step - loss: 0.3181 - mae: 0.4659 - mse: 0.7723 - val_loss: 0.2793 - val_mae: 0.4205 - val_mse: 0.6472 - lr: 1.0000e-04\n",
      "Epoch 19/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.3162 - mae: 0.4650 - mse: 0.7704\n",
      "Epoch 19: ReduceLROnPlateau reducing learning rate to 2.0000000949949027e-05.\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3161 - mae: 0.4648 - mse: 0.7699 - val_loss: 0.2797 - val_mae: 0.4190 - val_mse: 0.6547 - lr: 1.0000e-04\n",
      "Epoch 20/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3123 - mae: 0.4613 - mse: 0.7628 - val_loss: 0.2668 - val_mae: 0.4065 - val_mse: 0.6173 - lr: 2.0000e-05\n",
      "Epoch 21/100\n",
      "2028/2028 [==============================] - 23s 11ms/step\n",
      "2028/2028 [==============================] - 22s 11ms/step\n",
      "\n",
      "Epoch 21: Predizioni fuori range: 0\n",
      "507/507 [==============================] - 70s 139ms/step - loss: 0.3098 - mae: 0.4588 - mse: 0.7522 - val_loss: 0.2658 - val_mae: 0.4054 - val_mse: 0.6173 - lr: 2.0000e-05\n",
      "Epoch 22/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3093 - mae: 0.4590 - mse: 0.7529 - val_loss: 0.2652 - val_mae: 0.4046 - val_mse: 0.6218 - lr: 2.0000e-05\n",
      "Epoch 23/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3086 - mae: 0.4580 - mse: 0.7535 - val_loss: 0.2653 - val_mae: 0.4047 - val_mse: 0.6177 - lr: 2.0000e-05\n",
      "Epoch 24/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.3077 - mae: 0.4577 - mse: 0.7500 - val_loss: 0.2638 - val_mae: 0.4048 - val_mse: 0.6161 - lr: 2.0000e-05\n",
      "Epoch 25/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3056 - mae: 0.4556 - mse: 0.7444 - val_loss: 0.2637 - val_mae: 0.4049 - val_mse: 0.6151 - lr: 2.0000e-05\n",
      "Epoch 26/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.3043 - mae: 0.4543 - mse: 0.7443 - val_loss: 0.2630 - val_mae: 0.4033 - val_mse: 0.6194 - lr: 2.0000e-05\n",
      "Epoch 27/100\n",
      "507/507 [==============================] - 22s 44ms/step - loss: 0.3055 - mae: 0.4561 - mse: 0.7485 - val_loss: 0.2623 - val_mae: 0.4022 - val_mse: 0.6164 - lr: 2.0000e-05\n",
      "Epoch 28/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3042 - mae: 0.4555 - mse: 0.7458 - val_loss: 0.2616 - val_mae: 0.4005 - val_mse: 0.6202 - lr: 2.0000e-05\n",
      "Epoch 29/100\n",
      "507/507 [==============================] - ETA: 0s - loss: 0.3047 - mae: 0.4561 - mse: 0.7492\n",
      "Epoch 29: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.3047 - mae: 0.4561 - mse: 0.7492 - val_loss: 0.2613 - val_mae: 0.4038 - val_mse: 0.6127 - lr: 1.0000e-06\n",
      "Epoch 30/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.3041 - mae: 0.4560 - mse: 0.7475 - val_loss: 0.2610 - val_mae: 0.4028 - val_mse: 0.6124 - lr: 1.0000e-06\n",
      "Epoch 31/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3016 - mae: 0.4529 - mse: 0.7384 - val_loss: 0.2611 - val_mae: 0.4027 - val_mse: 0.6122 - lr: 1.0000e-06\n",
      "Epoch 32/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.3021 - mae: 0.4538 - mse: 0.7404\n",
      "Epoch 32: ReduceLROnPlateau reducing learning rate to 2.000000222324161e-07.\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3020 - mae: 0.4537 - mse: 0.7400 - val_loss: 0.2611 - val_mae: 0.4027 - val_mse: 0.6123 - lr: 1.0000e-06\n",
      "Epoch 33/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.3004 - mae: 0.4514 - mse: 0.7353 - val_loss: 0.2610 - val_mae: 0.4024 - val_mse: 0.6128 - lr: 2.0000e-07\n",
      "Epoch 34/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.3014 - mae: 0.4520 - mse: 0.7420 - val_loss: 0.2609 - val_mae: 0.4023 - val_mse: 0.6129 - lr: 2.0000e-07\n",
      "Epoch 35/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.3020 - mae: 0.4532 - mse: 0.7402 - val_loss: 0.2609 - val_mae: 0.4022 - val_mse: 0.6132 - lr: 2.0000e-07\n",
      "Epoch 36/100\n",
      "507/507 [==============================] - ETA: 0s - loss: 0.3030 - mae: 0.4545 - mse: 0.7438\n",
      "Epoch 36: ReduceLROnPlateau reducing learning rate to 1e-07.\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3030 - mae: 0.4545 - mse: 0.7438 - val_loss: 0.2609 - val_mae: 0.4023 - val_mse: 0.6127 - lr: 1.0000e-07\n",
      "Epoch 37/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3020 - mae: 0.4538 - mse: 0.7388 - val_loss: 0.2609 - val_mae: 0.4022 - val_mse: 0.6129 - lr: 1.0000e-07\n",
      "Epoch 38/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3036 - mae: 0.4552 - mse: 0.7454 - val_loss: 0.2609 - val_mae: 0.4022 - val_mse: 0.6127 - lr: 1.0000e-07\n",
      "Epoch 39/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.3005 - mae: 0.4514 - mse: 0.7366 - val_loss: 0.2608 - val_mae: 0.4021 - val_mse: 0.6128 - lr: 1.0000e-07\n",
      "Epoch 40/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3021 - mae: 0.4536 - mse: 0.7401 - val_loss: 0.2609 - val_mae: 0.4021 - val_mse: 0.6128 - lr: 1.0000e-07\n",
      "Epoch 41/100\n",
      "2028/2028 [==============================] - 22s 11ms/step\n",
      "2028/2028 [==============================] - 23s 11ms/step\n",
      "\n",
      "Epoch 41: Predizioni fuori range: 0\n",
      "507/507 [==============================] - 69s 137ms/step - loss: 0.3031 - mae: 0.4542 - mse: 0.7422 - val_loss: 0.2608 - val_mae: 0.4021 - val_mse: 0.6129 - lr: 1.0000e-07\n",
      "Epoch 42/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3007 - mae: 0.4519 - mse: 0.7347 - val_loss: 0.2609 - val_mae: 0.4022 - val_mse: 0.6128 - lr: 1.0000e-07\n",
      "Epoch 43/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3020 - mae: 0.4528 - mse: 0.7403 - val_loss: 0.2608 - val_mae: 0.4021 - val_mse: 0.6129 - lr: 1.0000e-07\n",
      "Epoch 44/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.3017 - mae: 0.4527 - mse: 0.7425 - val_loss: 0.2609 - val_mae: 0.4020 - val_mse: 0.6128 - lr: 1.0000e-07\n",
      "Epoch 45/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3022 - mae: 0.4539 - mse: 0.7417 - val_loss: 0.2608 - val_mae: 0.4019 - val_mse: 0.6131 - lr: 1.0000e-07\n",
      "Epoch 46/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3028 - mae: 0.4541 - mse: 0.7424 - val_loss: 0.2608 - val_mae: 0.4019 - val_mse: 0.6131 - lr: 1.0000e-07\n",
      "Epoch 47/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3028 - mae: 0.4541 - mse: 0.7436 - val_loss: 0.2608 - val_mae: 0.4020 - val_mse: 0.6129 - lr: 1.0000e-07\n",
      "Epoch 48/100\n",
      "507/507 [==============================] - 22s 44ms/step - loss: 0.3014 - mae: 0.4533 - mse: 0.7365 - val_loss: 0.2608 - val_mae: 0.4019 - val_mse: 0.6128 - lr: 1.0000e-07\n",
      "Epoch 49/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.3010 - mae: 0.4516 - mse: 0.7383 - val_loss: 0.2608 - val_mae: 0.4021 - val_mse: 0.6125 - lr: 1.0000e-07\n",
      "Epoch 50/100\n",
      "507/507 [==============================] - 22s 44ms/step - loss: 0.3035 - mae: 0.4550 - mse: 0.7466 - val_loss: 0.2607 - val_mae: 0.4019 - val_mse: 0.6128 - lr: 1.0000e-07\n",
      "Epoch 51/100\n",
      "507/507 [==============================] - 22s 44ms/step - loss: 0.3031 - mae: 0.4544 - mse: 0.7443 - val_loss: 0.2608 - val_mae: 0.4020 - val_mse: 0.6127 - lr: 1.0000e-07\n",
      "Epoch 52/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3030 - mae: 0.4540 - mse: 0.7462 - val_loss: 0.2609 - val_mae: 0.4021 - val_mse: 0.6124 - lr: 1.0000e-07\n",
      "Epoch 53/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.3019 - mae: 0.4529 - mse: 0.7407 - val_loss: 0.2608 - val_mae: 0.4020 - val_mse: 0.6127 - lr: 1.0000e-07\n",
      "Epoch 54/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.3029 - mae: 0.4538 - mse: 0.7433Restoring model weights from the end of the best epoch: 39.\n",
      "507/507 [==============================] - 22s 44ms/step - loss: 0.3028 - mae: 0.4537 - mse: 0.7430 - val_loss: 0.2608 - val_mae: 0.4020 - val_mse: 0.6126 - lr: 1.0000e-07\n",
      "Epoch 54: early stopping\n",
      "\n",
      "Training completato con successo!\n",
      "\n",
      "Metriche finali sul test set:\n",
      "Loss: 0.2608\n",
      "MAE: 0.4021\n",
      "MSE: 0.6128\n",
      "2028/2028 [==============================] - 22s 11ms/step\n",
      "\n",
      "Predizioni fuori range: 0 (0.00%)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 50512 characters]",
      "text/plain": [
       "<Figure size 1200x400 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "5. Generazione delle predizioni...\n",
      "2028/2028 [==============================] - 23s 11ms/step\n",
      "\n",
      "6. Valutazione del modello...\n",
      "\n",
      "UV Index Prediction Metrics:\n",
      "MAE: 0.384\n",
      "RMSE: 0.809\n",
      "R² Score: 0.911\n",
      "Exact Match Accuracy: 0.331\n",
      "±1 Accuracy: 0.458\n",
      "±2 Accuracy: 0.566\n",
      "\n",
      "UV Level Confusion Matrix:\n",
      "Predicted  High    Low  Moderate  Very High\n",
      "Actual                                     \n",
      "High       3134     11      1252        481\n",
      "Low         129  43324      1888         19\n",
      "Moderate   1208   1663      7320        152\n",
      "Very High  1188      1        78       3018\n",
      "\n",
      "7. Visualizzazione risultati...\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 104208 characters]",
      "text/plain": [
       "<Figure size 1500x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "8. Predizione dei dati mancanti risultati...\n",
      "7122/7122 [==============================] - 78s 11ms/step\n",
      "\n",
      "9. Integrazione delle predizioni nel dataset originale...\n",
      "Aggiunte 227879 predizioni al dataset\n",
      "Righe con UV index dopo l'integrazione: 357615\n",
      "\n",
      "Training completato con successo!\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_parquet('../data/weather_data.parquet')\n",
    "\n",
    "# Esegui il training\n",
    "model, scaler, features, history, predictions, y_test, metrics, training_results = train_uvindex_bounded_model(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "637891db-8d55-4232-a56e-9759dbcc8c2f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Analisi Precisione Predizioni UV Index:\n",
      "Precisione esatta: 61.3%\n",
      "Precisione entro 0.5 punti: 79.4%\n",
      "Precisione entro 1.0 punti: 90.4%\n",
      "Precisione livello di rischio: 88.0%\n",
      "\n",
      "Analisi errori per livello UV:\n",
      "MAE per UV Basso (0-2): 0.138 (n=41346)\n",
      "MAE per UV Moderato (2-5): 0.906 (n=11522)\n",
      "MAE per UV Alto (5-7): 0.877 (n=5468)\n",
      "MAE per UV Molto Alto (7-10): 0.758 (n=6278)\n",
      "MAE per UV Estremo (10-11): 1.528 (n=252)\n",
      "\n",
      "Statistiche degli errori:\n",
      "Media errori: 0.006\n",
      "Deviazione standard errori: 0.783\n",
      "Errore mediano: 0.000\n",
      "95° percentile errore assoluto: 1.723\n"
     ]
    }
   ],
   "source": [
    "def analyze_uv_prediction_quality(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    Analisi dettagliata della qualità delle predizioni UV\n",
    "    \"\"\"\n",
    "    # Converti in numpy array e appiattisci\n",
    "    y_true = np.array(y_true).ravel()\n",
    "    y_pred = np.array(y_pred).ravel()\n",
    "\n",
    "    # Arrotonda le predizioni al più vicino 0.5\n",
    "    y_pred_rounded = np.round(y_pred * 2) / 2\n",
    "\n",
    "    # Calcola diverse metriche di accuratezza usando array numpy\n",
    "    exact_match = np.mean(np.abs(y_pred_rounded - y_true) < 1e-6) * 100  # uso di tolleranza per confronti float\n",
    "    within_half = np.mean(np.abs(y_pred_rounded - y_true) <= 0.5) * 100\n",
    "    within_one = np.mean(np.abs(y_pred_rounded - y_true) <= 1.0) * 100\n",
    "\n",
    "    # Analisi per livello di rischio UV\n",
    "    def get_uv_risk_level(values):\n",
    "        # Vettorizzazione della funzione per array numpy\n",
    "        levels = np.zeros_like(values, dtype=str)\n",
    "        levels[values <= 2] = 'Basso'\n",
    "        levels[(values > 2) & (values <= 5)] = 'Moderato'\n",
    "        levels[(values > 5) & (values <= 7)] = 'Alto'\n",
    "        levels[(values > 7) & (values <= 10)] = 'Molto Alto'\n",
    "        levels[values > 10] = 'Estremo'\n",
    "        return levels\n",
    "\n",
    "    y_true_risk = get_uv_risk_level(y_true)\n",
    "    y_pred_risk = get_uv_risk_level(y_pred_rounded)\n",
    "\n",
    "    risk_accuracy = np.mean(y_true_risk == y_pred_risk) * 100\n",
    "\n",
    "    print(\"Analisi Precisione Predizioni UV Index:\")\n",
    "    print(f\"Precisione esatta: {exact_match:.1f}%\")\n",
    "    print(f\"Precisione entro 0.5 punti: {within_half:.1f}%\")\n",
    "    print(f\"Precisione entro 1.0 punti: {within_one:.1f}%\")\n",
    "    print(f\"Precisione livello di rischio: {risk_accuracy:.1f}%\")\n",
    "\n",
    "    # Distribuzione degli errori per livello UV\n",
    "    uv_ranges = [(0, 2), (2, 5), (5, 7), (7, 10), (10, 11)]\n",
    "    labels = ['Basso', 'Moderato', 'Alto', 'Molto Alto', 'Estremo']\n",
    "\n",
    "    print(\"\\nAnalisi errori per livello UV:\")\n",
    "    for (low, high), label in zip(uv_ranges, labels):\n",
    "        mask = (y_true >= low) & (y_true < high)\n",
    "        if np.sum(mask) > 0:\n",
    "            mae_range = np.mean(np.abs(y_pred[mask] - y_true[mask]))\n",
    "            n_samples = np.sum(mask)\n",
    "            print(f\"MAE per UV {label} ({low}-{high}): {mae_range:.3f} (n={n_samples})\")\n",
    "\n",
    "    # Analisi aggiuntiva della distribuzione degli errori\n",
    "    errors = y_pred - y_true\n",
    "    print(\"\\nStatistiche degli errori:\")\n",
    "    print(f\"Media errori: {np.mean(errors):.3f}\")\n",
    "    print(f\"Deviazione standard errori: {np.std(errors):.3f}\")\n",
    "    print(f\"Errore mediano: {np.median(errors):.3f}\")\n",
    "    print(f\"95° percentile errore assoluto: {np.percentile(np.abs(errors), 95):.3f}\")\n",
    "\n",
    "    return {\n",
    "        'exact_match': exact_match,\n",
    "        'within_half': within_half,\n",
    "        'within_one': within_one,\n",
    "        'risk_accuracy': risk_accuracy,\n",
    "        'error_stats': {\n",
    "            'mean': float(np.mean(errors)),\n",
    "            'std': float(np.std(errors)),\n",
    "            'median': float(np.median(errors)),\n",
    "            'p95_abs': float(np.percentile(np.abs(errors), 95))\n",
    "        }\n",
    "    }\n",
    "\n",
    "\n",
    "# Per utilizzare l'analisi:\n",
    "metrics = analyze_uv_prediction_quality(y_test, predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "08fd4208-0afb-4bf1-bdef-b10b4065fe55",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 132604 characters]",
      "text/plain": [
       "<Figure size 1500x500 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Statistiche degli errori:\n",
      "MAE: 0.4021\n",
      "MSE: 0.6128\n",
      "RMSE: 0.7828\n",
      "Media errori: 0.0062\n",
      "Std errori: 0.7828\n",
      "Predizioni entro ±0.5: 71.1%\n",
      "Predizioni entro ±1.0: 86.1%\n",
      "Predizioni entro ±1.5: 93.2%\n",
      "Predizioni entro ±2.0: 96.5%\n"
     ]
    }
   ],
   "source": [
    "def plot_error_analysis(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    Funzione per visualizzare l'analisi degli errori di predizione\n",
    "    \"\"\"\n",
    "    # Converti in array numpy 1D se necessario\n",
    "    if isinstance(y_true, pd.Series):\n",
    "        y_true = y_true.values\n",
    "    if isinstance(y_pred, pd.Series):\n",
    "        y_pred = y_pred.values\n",
    "    \n",
    "    y_true = y_true.ravel()\n",
    "    y_pred = y_pred.ravel()\n",
    "    \n",
    "    # Calcola gli errori\n",
    "    errors = y_pred - y_true\n",
    "    \n",
    "    plt.figure(figsize=(15, 5))\n",
    "    \n",
    "    # Plot 1: Distribuzione degli errori\n",
    "    plt.subplot(1, 3, 1)\n",
    "    plt.hist(errors, bins=50, alpha=0.7)\n",
    "    plt.title('Distribuzione degli Errori di Predizione')\n",
    "    plt.xlabel('Errore')\n",
    "    plt.ylabel('Frequenza')\n",
    "    \n",
    "    # Plot 2: Actual vs Predicted\n",
    "    plt.subplot(1, 3, 2)\n",
    "    plt.scatter(y_true, y_pred, alpha=0.5)\n",
    "    plt.plot([y_true.min(), y_true.max()], [y_true.min(), y_true.max()], 'r--', lw=2)\n",
    "    plt.title('Valori Reali vs Predetti')\n",
    "    plt.xlabel('Valori Reali')\n",
    "    plt.ylabel('Valori Predetti')\n",
    "    \n",
    "    # Plot 3: Errori vs Valori Reali\n",
    "    plt.subplot(1, 3, 3)\n",
    "    plt.scatter(y_true, errors, alpha=0.5)\n",
    "    plt.axhline(y=0, color='r', linestyle='--')\n",
    "    plt.title('Errori vs Valori Reali')\n",
    "    plt.xlabel('Valori Reali')\n",
    "    plt.ylabel('Errore')\n",
    "    \n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "    \n",
    "    # Stampa statistiche degli errori\n",
    "    print(\"\\nStatistiche degli errori:\")\n",
    "    print(f\"MAE: {np.mean(np.abs(errors)):.4f}\")\n",
    "    print(f\"MSE: {np.mean(errors**2):.4f}\")\n",
    "    print(f\"RMSE: {np.sqrt(np.mean(errors**2)):.4f}\")\n",
    "    print(f\"Media errori: {np.mean(errors):.4f}\")\n",
    "    print(f\"Std errori: {np.std(errors):.4f}\")\n",
    "    \n",
    "    # Calcola percentuali di errori entro certe soglie\n",
    "    thresholds = [0.5, 1.0, 1.5, 2.0]\n",
    "    for threshold in thresholds:\n",
    "        within_threshold = np.mean(np.abs(errors) <= threshold) * 100\n",
    "        print(f\"Predizioni entro ±{threshold}: {within_threshold:.1f}%\")\n",
    "\n",
    "\n",
    "plot_error_analysis(y_test, predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "03bb9564-e518-4662-b3ee-4cfa96cdf696",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 225492 characters]",
      "text/plain": [
       "<Figure size 2000x1000 with 5 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Matrice di Confusione per Livelli di Rischio UV:\n",
      "            Basso  Moderato  Alto  Molto Alto  Estremo\n",
      "Basso        2931        19     0        2128      390\n",
      "Moderato       41     40299     0         995       11\n",
      "Alto           13         0     0           0      239\n",
      "Molto Alto    863      2454     0        8063      142\n",
      "Estremo      1878         1     0         169     4230\n",
      "\n",
      "Analisi Precisione Predizioni UV Index:\n",
      "Precisione esatta (±0.1): 55.1%\n",
      "Precisione entro 0.5 punti: 71.1%\n",
      "Precisione entro 1.0 punti: 86.1%\n",
      "Precisione livello di rischio: 85.6%\n",
      "\n",
      "Analisi errori per livello UV:\n",
      "MAE per UV Basso (0-2): 0.138 (n=41346)\n",
      "MAE per UV Moderato (2-5): 0.906 (n=11522)\n",
      "MAE per UV Alto (5-7): 0.877 (n=5468)\n",
      "MAE per UV Molto Alto (7-10): 0.758 (n=6278)\n",
      "MAE per UV Estremo (10-11): 1.528 (n=252)\n",
      "\n",
      "Statistiche degli errori:\n",
      "Media errori: 0.006\n",
      "Deviazione standard errori: 0.783\n",
      "Errore mediano: 0.000\n",
      "95° percentile errore assoluto: 1.723\n",
      "\n",
      "Distribuzione degli errori:\n",
      "Predizioni entro ±0.5: 71.1%\n",
      "Predizioni entro ±1.0: 86.1%\n",
      "Predizioni entro ±1.5: 93.2%\n",
      "Predizioni entro ±2.0: 96.5%\n"
     ]
    }
   ],
   "source": [
    "def plot_advanced_prediction_analysis(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    Funzione per visualizzare l'analisi degli errori di predizione e la precisione\n",
    "    \"\"\"\n",
    "    # Converti in array numpy 1D se necessario\n",
    "    if isinstance(y_true, pd.Series):\n",
    "        y_true = y_true.values\n",
    "    if isinstance(y_pred, pd.Series):\n",
    "        y_pred = y_pred.values\n",
    "\n",
    "\n",
    "    y_true = y_true.ravel()\n",
    "    y_pred = y_pred.ravel()\n",
    "    \n",
    "    # Calcola gli errori\n",
    "    errors = y_pred - y_true\n",
    "    \n",
    "    # Calcola accuracy per diversi livelli di tolleranza\n",
    "    exact_accuracy = np.mean(np.abs(errors) < 0.1) * 100  # Precisione esatta (±0.1)\n",
    "    accuracy_05 = np.mean(np.abs(errors) <= 0.5) * 100  # Precisione entro ±0.5\n",
    "    accuracy_10 = np.mean(np.abs(errors) <= 1.0) * 100  # Precisione entro ±1.0\n",
    "    \n",
    "    \n",
    "    # Calcola accuracy per livelli di rischio UV\n",
    "    def get_risk_level(uv):\n",
    "        if uv < 2:\n",
    "            return 'Basso'\n",
    "        elif uv < 5:\n",
    "            return 'Moderato'\n",
    "        elif uv < 7:\n",
    "            return 'Alto'\n",
    "        elif uv < 10:\n",
    "            return 'Molto Alto'\n",
    "        else:\n",
    "            return 'Estremo'\n",
    "    \n",
    "    \n",
    "    y_true_risk = [get_risk_level(x) for x in y_true]\n",
    "    y_pred_risk = [get_risk_level(x) for x in y_pred]\n",
    "    risk_accuracy = np.mean(np.array(y_true_risk) == np.array(y_pred_risk)) * 100\n",
    "    \n",
    "    plt.figure(figsize=(20, 10))\n",
    "    \n",
    "    # Plot 1: Distribuzione degli errori\n",
    "    plt.subplot(2, 2, 1)\n",
    "    plt.hist(errors, bins=50, alpha=0.7)\n",
    "    plt.title('Distribuzione degli Errori di Predizione')\n",
    "    plt.xlabel('Errore')\n",
    "    plt.ylabel('Frequenza')\n",
    "    \n",
    "    # Plot 2: Actual vs Predicted\n",
    "    plt.subplot(2, 2, 2)\n",
    "    plt.scatter(y_true, y_pred, alpha=0.5)\n",
    "    plt.plot([y_true.min(), y_true.max()], [y_true.min(), y_true.max()], 'r--', lw=2)\n",
    "    plt.title('Valori Reali vs Predetti')\n",
    "    plt.xlabel('Valori Reali')\n",
    "    plt.ylabel('Valori Predetti')\n",
    "    \n",
    "    # Plot 3: Errori vs Valori Reali\n",
    "    plt.subplot(2, 2, 3)\n",
    "    plt.scatter(y_true, errors, alpha=0.5)\n",
    "    plt.axhline(y=0, color='r', linestyle='--')\n",
    "    plt.title('Errori vs Valori Reali')\n",
    "    plt.xlabel('Valori Reali')\n",
    "    plt.ylabel('Errore')\n",
    "    \n",
    "    # Plot 4: Precisione per intervallo di UV\n",
    "    plt.subplot(2, 2, 4)\n",
    "    \n",
    "    # Definisci gli intervalli UV\n",
    "    uv_ranges = [(0, 2), (2, 5), (5, 7), (7, 10), (10, 11)]\n",
    "    range_labels = ['Basso\\n(0-2)', 'Moderato\\n(2-5)', 'Alto\\n(5-7)', 'Molto Alto\\n(7-10)', 'Estremo\\n(10-11)']\n",
    "    \n",
    "    accuracies = []\n",
    "    counts = []\n",
    "    mae_per_range = []\n",
    "    \n",
    "    for (low, high) in uv_ranges:\n",
    "        mask = (y_true >= low) & (y_true < high)\n",
    "        if mask.any():\n",
    "            # Calcola MAE per questo range\n",
    "            mae = np.mean(np.abs(y_pred[mask] - y_true[mask]))\n",
    "            mae_per_range.append(mae)\n",
    "            # Conta quanti valori in questo range\n",
    "            count = np.sum(mask)\n",
    "            counts.append(count)\n",
    "            # Calcola precisione entro 0.5 punti UV\n",
    "            accuracy = np.mean(np.abs(y_pred[mask] - y_true[mask]) <= 0.5) * 100\n",
    "            accuracies.append(accuracy)\n",
    "    \n",
    "    # Crea il grafico a barre con doppio asse y\n",
    "    fig = plt.gca()\n",
    "    \n",
    "    # Barre per accuratezza\n",
    "    bars = plt.bar(range_labels, accuracies, alpha=0.6, color='skyblue')\n",
    "    plt.ylabel('Precisione (%)')\n",
    "    plt.title('Precisione e MAE per Range UV')\n",
    "    \n",
    "    # Aggiungi etichette sopra le barre\n",
    "    for bar in bars:\n",
    "        height = bar.get_height()\n",
    "        plt.text(bar.get_x() + bar.get_width() / 2., height,\n",
    "                 f'{height:.1f}%\\n(n={counts[bars.index(bar)]})',\n",
    "                 ha='center', va='bottom')\n",
    "    \n",
    "    # Secondo asse y per MAE\n",
    "    ax2 = fig.twinx()\n",
    "    line = ax2.plot(range_labels, mae_per_range, 'r-', marker='o', label='MAE')\n",
    "    ax2.set_ylabel('MAE', color='red')\n",
    "    \n",
    "    # Aggiungi valori MAE\n",
    "    for i, mae in enumerate(mae_per_range):\n",
    "        ax2.text(i, mae, f'MAE: {mae:.3f}', color='red', ha='center', va='bottom')\n",
    "    \n",
    "    plt.xticks(rotation=45)\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "    \n",
    "    cm = confusion_matrix(y_true_risk, y_pred_risk)\n",
    "    risk_levels = ['Basso', 'Moderato', 'Alto', 'Molto Alto', 'Estremo']\n",
    "    cm_df = pd.DataFrame(cm,\n",
    "                         columns=risk_levels,\n",
    "                         index=risk_levels)\n",
    "    \n",
    "    print(\"\\nMatrice di Confusione per Livelli di Rischio UV:\")\n",
    "    print(cm_df)\n",
    "    \n",
    "    print(\"\\nAnalisi Precisione Predizioni UV Index:\")\n",
    "    print(f\"Precisione esatta (±0.1): {exact_accuracy:.1f}%\")\n",
    "    print(f\"Precisione entro 0.5 punti: {accuracy_05:.1f}%\")\n",
    "    print(f\"Precisione entro 1.0 punti: {accuracy_10:.1f}%\")\n",
    "    print(f\"Precisione livello di rischio: {risk_accuracy:.1f}%\")\n",
    "    \n",
    "    print(\"\\nAnalisi errori per livello UV:\")\n",
    "    uv_ranges = [(0, 2, 'Basso'), (2, 5, 'Moderato'), (5, 7, 'Alto'),\n",
    "                 (7, 10, 'Molto Alto'), (10, 11, 'Estremo')]\n",
    "    \n",
    "    for low, high, label in uv_ranges:\n",
    "        mask = (y_true >= low) & (y_true < high)\n",
    "        if mask.any():\n",
    "            mae = np.mean(np.abs(errors[mask]))\n",
    "            n_samples = np.sum(mask)\n",
    "            print(f\"MAE per UV {label} ({low}-{high}): {mae:.3f} (n={n_samples})\")\n",
    "    \n",
    "    print(\"\\nStatistiche degli errori:\")\n",
    "    print(f\"Media errori: {np.mean(errors):.3f}\")\n",
    "    print(f\"Deviazione standard errori: {np.std(errors):.3f}\")\n",
    "    print(f\"Errore mediano: {np.median(errors):.3f}\")\n",
    "    print(f\"95° percentile errore assoluto: {np.percentile(np.abs(errors), 95):.3f}\")\n",
    "    \n",
    "    # Calcola percentuali di errori entro certe soglie\n",
    "    thresholds = [0.5, 1.0, 1.5, 2.0]\n",
    "    print(\"\\nDistribuzione degli errori:\")\n",
    "    for threshold in thresholds:\n",
    "        within_threshold = np.mean(np.abs(errors) <= threshold) * 100\n",
    "        print(f\"Predizioni entro ±{threshold}: {within_threshold:.1f}%\")\n",
    "\n",
    "# Usa la funzione\n",
    "plot_advanced_prediction_analysis(y_test, predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe898941-2338-4157-b624-680bc2c517d8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
