{
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.10.14",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kaggle": {
   "accelerator": "none",
   "dataSources": [
    {
     "sourceId": 9725208,
     "sourceType": "datasetVersion",
     "datasetId": 5950719
    }
   ],
   "dockerImageVersionId": 30787,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook",
   "isGpuEnabled": false
  }
 },
 "nbformat_minor": 4,
 "nbformat": 4,
 "cells": [
  {
   "cell_type": "markdown",
   "source": "# Analisi e Previsione della Produzione di Olio d'Oliva\n\nQuesto notebook esplora la relazione tra i dati meteorologici e la produzione annuale di olio d'oliva, con l'obiettivo di creare un modello predittivo.",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "print(f\"Keras version: {tf.keras.__version__}\")\n",
    "print(f\"TensorFlow version: {tf.__version__}\")\n",
    "\n",
    "# GPU configuration\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "if gpus:\n",
    "    try:\n",
    "        for gpu in gpus:\n",
    "            tf.config.experimental.set_memory_growth(gpu, True)\n",
    "        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
    "        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n",
    "    except RuntimeError as e:\n",
    "        print(e)"
   ],
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-25T21:05:00.337046Z",
     "start_time": "2024-10-25T21:04:03.960543Z"
    },
    "execution": {
     "iopub.status.busy": "2024-10-26T15:56:55.835718Z",
     "iopub.execute_input": "2024-10-26T15:56:55.836122Z",
     "iopub.status.idle": "2024-10-26T15:56:55.913711Z",
     "shell.execute_reply.started": "2024-10-26T15:56:55.836084Z",
     "shell.execute_reply": "2024-10-26T15:56:55.912641Z"
    },
    "trusted": true
   },
   "execution_count": 1,
   "outputs": [
    {
     "name": "stdout",
     "text": "Keras version: 3.3.3\nTensorFlow version: 2.16.1\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "# Test semplice per verificare che la GPU funzioni\n",
    "def test_gpu():\n",
    "    print(\"TensorFlow version:\", tf.__version__)\n",
    "    print(\"\\nDispositivi disponibili:\")\n",
    "    print(tf.config.list_physical_devices())\n",
    "\n",
    "    # Creiamo e moltiplichiamo due tensori sulla GPU\n",
    "    with tf.device('/GPU:0'):\n",
    "        a = tf.random.normal([10000, 10000])\n",
    "        b = tf.random.normal([10000, 10000])\n",
    "        c = tf.matmul(a, b)\n",
    "\n",
    "    print(\"\\nShape del risultato:\", c.shape)\n",
    "    print(\"Device del tensore:\", c.device)\n",
    "    return \"Test completato con successo!\"\n",
    "\n",
    "\n",
    "test_gpu()"
   ],
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-25T21:05:14.642072Z",
     "start_time": "2024-10-25T21:05:11.794331Z"
    },
    "execution": {
     "iopub.status.busy": "2024-10-26T15:16:06.209684Z",
     "iopub.execute_input": "2024-10-26T15:16:06.210435Z",
     "iopub.status.idle": "2024-10-26T15:16:06.278328Z",
     "shell.execute_reply.started": "2024-10-26T15:16:06.210394Z",
     "shell.execute_reply": "2024-10-26T15:16:06.277466Z"
    },
    "trusted": true
   },
   "execution_count": 3,
   "outputs": [
    {
     "name": "stdout",
     "text": "TensorFlow version: 2.16.1\n\nDispositivi disponibili:\n[PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU'), PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n\nShape del risultato: (10000, 10000)\nDevice del tensore: /job:localhost/replica:0/task:0/device:GPU:0\n",
     "output_type": "stream"
    },
    {
     "execution_count": 3,
     "output_type": "execute_result",
     "data": {
      "text/plain": "'Test completato con successo!'"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "source": "!pip install numpy\n!pip install pandas\n\n!pip install keras\n!pip install scikit-learn\n!pip install matplotlib\n!pip install joblib\n!pip install pyarrow\n!pip install fastparquet\n!pip install scipy\n!pip install seaborn\n!pip install pysolar",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-25T21:05:34.003058Z",
     "start_time": "2024-10-25T21:05:20.138514Z"
    },
    "execution": {
     "iopub.status.busy": "2024-10-26T07:18:31.623426Z",
     "iopub.execute_input": "2024-10-26T07:18:31.624250Z",
     "iopub.status.idle": "2024-10-26T07:20:41.178583Z",
     "shell.execute_reply.started": "2024-10-26T07:18:31.624208Z",
     "shell.execute_reply": "2024-10-26T07:20:41.177026Z"
    },
    "trusted": true
   },
   "execution_count": 4,
   "outputs": [
    {
     "name": "stderr",
     "text": "/opt/conda/lib/python3.10/pty.py:89: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n  pid, fd = os.forkpty()\n",
     "output_type": "stream"
    },
    {
     "name": "stdout",
     "text": "Requirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (1.26.4)\nRequirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (2.2.2)\nRequirement already satisfied: numpy>=1.22.4 in /opt/conda/lib/python3.10/site-packages (from pandas) (1.26.4)\nRequirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas) (2.9.0.post0)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas) (2024.1)\nRequirement already satisfied: tzdata>=2022.7 in /opt/conda/lib/python3.10/site-packages (from pandas) (2024.1)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\nRequirement already satisfied: keras in /opt/conda/lib/python3.10/site-packages (3.3.3)\nRequirement already satisfied: absl-py in /opt/conda/lib/python3.10/site-packages (from keras) (1.4.0)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from keras) (1.26.4)\nRequirement already satisfied: rich in /opt/conda/lib/python3.10/site-packages (from keras) (13.7.1)\nRequirement already satisfied: namex in /opt/conda/lib/python3.10/site-packages (from keras) (0.0.8)\nRequirement already satisfied: h5py in /opt/conda/lib/python3.10/site-packages (from keras) (3.11.0)\nRequirement already satisfied: optree in /opt/conda/lib/python3.10/site-packages (from keras) (0.11.0)\nRequirement already satisfied: ml-dtypes in /opt/conda/lib/python3.10/site-packages (from keras) (0.3.2)\nRequirement already satisfied: typing-extensions>=4.0.0 in /opt/conda/lib/python3.10/site-packages (from optree->keras) (4.12.2)\nRequirement already satisfied: markdown-it-py>=2.2.0 in /opt/conda/lib/python3.10/site-packages (from rich->keras) (3.0.0)\nRequirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/conda/lib/python3.10/site-packages (from rich->keras) (2.18.0)\nRequirement already satisfied: mdurl~=0.1 in /opt/conda/lib/python3.10/site-packages (from markdown-it-py>=2.2.0->rich->keras) (0.1.2)\nRequirement already satisfied: scikit-learn in /opt/conda/lib/python3.10/site-packages (1.2.2)\nRequirement already satisfied: numpy>=1.17.3 in /opt/conda/lib/python3.10/site-packages (from scikit-learn) (1.26.4)\nRequirement already satisfied: scipy>=1.3.2 in /opt/conda/lib/python3.10/site-packages (from scikit-learn) (1.14.1)\nRequirement already satisfied: joblib>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from scikit-learn) (1.4.2)\nRequirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from scikit-learn) (3.5.0)\nRequirement already satisfied: matplotlib in /opt/conda/lib/python3.10/site-packages (3.7.5)\nRequirement already satisfied: contourpy>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib) (1.2.1)\nRequirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.10/site-packages (from matplotlib) (0.12.1)\nRequirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.10/site-packages (from matplotlib) (4.53.0)\nRequirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib) (1.4.5)\nRequirement already satisfied: numpy<2,>=1.20 in /opt/conda/lib/python3.10/site-packages (from matplotlib) (1.26.4)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from matplotlib) (21.3)\nRequirement already satisfied: pillow>=6.2.0 in /opt/conda/lib/python3.10/site-packages (from matplotlib) (10.3.0)\nRequirement already satisfied: pyparsing>=2.3.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib) (3.1.2)\nRequirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.10/site-packages (from matplotlib) (2.9.0.post0)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\nRequirement already satisfied: joblib in /opt/conda/lib/python3.10/site-packages (1.4.2)\nRequirement already satisfied: pyarrow in /opt/conda/lib/python3.10/site-packages (16.1.0)\nRequirement already satisfied: numpy>=1.16.6 in /opt/conda/lib/python3.10/site-packages (from pyarrow) (1.26.4)\nCollecting fastparquet\n  Downloading fastparquet-2024.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.1 kB)\nRequirement already satisfied: pandas>=1.5.0 in /opt/conda/lib/python3.10/site-packages (from fastparquet) (2.2.2)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from fastparquet) (1.26.4)\nCollecting cramjam>=2.3 (from fastparquet)\n  Downloading cramjam-2.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.9 kB)\nRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from fastparquet) (2024.6.1)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from fastparquet) (21.3)\nRequirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas>=1.5.0->fastparquet) (2.9.0.post0)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas>=1.5.0->fastparquet) (2024.1)\nRequirement already satisfied: tzdata>=2022.7 in /opt/conda/lib/python3.10/site-packages (from pandas>=1.5.0->fastparquet) (2024.1)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->fastparquet) (3.1.2)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=1.5.0->fastparquet) (1.16.0)\nDownloading fastparquet-2024.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n\u001B[2K   \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m1.7/1.7 MB\u001B[0m \u001B[31m28.6 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0ma \u001B[36m0:00:01\u001B[0m\n\u001B[?25hDownloading cramjam-2.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.4 MB)\n\u001B[2K   \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m2.4/2.4 MB\u001B[0m \u001B[31m66.7 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m:00:01\u001B[0m\n\u001B[?25hInstalling collected packages: cramjam, fastparquet\nSuccessfully installed cramjam-2.9.0 fastparquet-2024.5.0\nRequirement already satisfied: scipy in /opt/conda/lib/python3.10/site-packages (1.14.1)\nRequirement already satisfied: numpy<2.3,>=1.23.5 in /opt/conda/lib/python3.10/site-packages (from scipy) (1.26.4)\nRequirement already satisfied: seaborn in /opt/conda/lib/python3.10/site-packages (0.12.2)\nRequirement already satisfied: numpy!=1.24.0,>=1.17 in /opt/conda/lib/python3.10/site-packages (from seaborn) (1.26.4)\nRequirement already satisfied: pandas>=0.25 in /opt/conda/lib/python3.10/site-packages (from seaborn) (2.2.2)\nRequirement already satisfied: matplotlib!=3.6.1,>=3.1 in /opt/conda/lib/python3.10/site-packages (from seaborn) (3.7.5)\nRequirement already satisfied: contourpy>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (1.2.1)\nRequirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.10/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (0.12.1)\nRequirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.10/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (4.53.0)\nRequirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (1.4.5)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (21.3)\nRequirement already satisfied: pillow>=6.2.0 in /opt/conda/lib/python3.10/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (10.3.0)\nRequirement already satisfied: pyparsing>=2.3.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (3.1.2)\nRequirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.10/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (2.9.0.post0)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas>=0.25->seaborn) (2024.1)\nRequirement already satisfied: tzdata>=2022.7 in /opt/conda/lib/python3.10/site-packages (from pandas>=0.25->seaborn) (2024.1)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib!=3.6.1,>=3.1->seaborn) (1.16.0)\nCollecting pysolar\n  Downloading pysolar-0.11-py3-none-any.whl.metadata (331 bytes)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from pysolar) (1.26.4)\nDownloading pysolar-0.11-py3-none-any.whl (47 kB)\n\u001B[2K   \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m47.1/47.1 kB\u001B[0m \u001B[31m2.4 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n\u001B[?25hInstalling collected packages: pysolar\nSuccessfully installed pysolar-0.11\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": "import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import MinMaxScaler, StandardScaler\n\nfrom tensorflow.keras.layers import Input, Dense, Dropout, Bidirectional, LSTM, LayerNormalization, Add, GlobalAveragePooling1D, Activation, BatchNormalization, MultiHeadAttention, MaxPooling1D\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.regularizers import l2\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfrom datetime import datetime\nimport os\nimport json\nimport joblib\nimport re\nimport pyarrow as pa\nimport pyarrow.parquet as pq\n\nrandom_state_value = 42",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-25T21:05:34.177059Z",
     "start_time": "2024-10-25T21:05:34.012517Z"
    },
    "execution": {
     "iopub.status.busy": "2024-10-26T15:56:55.915237Z",
     "iopub.execute_input": "2024-10-26T15:56:55.915609Z",
     "iopub.status.idle": "2024-10-26T15:56:55.935398Z",
     "shell.execute_reply.started": "2024-10-26T15:56:55.915572Z",
     "shell.execute_reply": "2024-10-26T15:56:55.934148Z"
    },
    "trusted": true
   },
   "execution_count": 2,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "## 1. Caricamento e preparazione dei Dati Meteo",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": "# Function to convert csv to parquet\ndef csv_to_parquet(csv_file, parquet_file, chunksize=100000):\n    writer = None\n\n    for chunk in pd.read_csv(csv_file, chunksize=chunksize):\n        if writer is None:\n\n            table = pa.Table.from_pandas(chunk)\n            writer = pq.ParquetWriter(parquet_file, table.schema)\n        else:\n            table = pa.Table.from_pandas(chunk)\n\n        writer.write_table(table)\n\n    if writer:\n        writer.close()\n\n    print(f\"File conversion completed : {csv_file} -> {parquet_file}\")\n\n\ndef read_json_files(folder_path):\n    all_data = []\n\n    file_list = sorted(os.listdir(folder_path))\n\n    for filename in file_list:\n        if filename.endswith('.json'):\n            file_path = os.path.join(folder_path, filename)\n            try:\n                with open(file_path, 'r') as file:\n                    data = json.load(file)\n                    all_data.extend(data['days'])\n            except Exception as e:\n                print(f\"Error processing file '{filename}': {str(e)}\")\n\n    return all_data\n\n\ndef create_weather_dataset(data):\n    dataset = []\n    seen_datetimes = set()\n\n    for day in data:\n        date = day['datetime']\n        for hour in day['hours']:\n            datetime_str = f\"{date} {hour['datetime']}\"\n\n            # Verifico se questo datetime è già stato visto\n            if datetime_str in seen_datetimes:\n                continue\n\n            seen_datetimes.add(datetime_str)\n\n            if isinstance(hour['preciptype'], list):\n                preciptype = \"__\".join(hour['preciptype'])\n            else:\n                preciptype = hour['preciptype'] if hour['preciptype'] else \"\"\n\n            conditions = hour['conditions'].replace(', ', '__').replace(' ', '_').lower()\n\n            row = {\n                'datetime': datetime_str,\n                'temp': hour['temp'],\n                'feelslike': hour['feelslike'],\n                'humidity': hour['humidity'],\n                'dew': hour['dew'],\n                'precip': hour['precip'],\n                'snow': hour['snow'],\n                'preciptype': preciptype.lower(),\n                'windspeed': hour['windspeed'],\n                'winddir': hour['winddir'],\n                'pressure': hour['pressure'],\n                'cloudcover': hour['cloudcover'],\n                'visibility': hour['visibility'],\n                'solarradiation': hour['solarradiation'],\n                'solarenergy': hour['solarenergy'],\n                'uvindex': hour['uvindex'],\n                'conditions': conditions,\n                'tempmax': day['tempmax'],\n                'tempmin': day['tempmin'],\n                'precipprob': day['precipprob'],\n                'precipcover': day['precipcover']\n            }\n            dataset.append(row)\n\n    dataset.sort(key=lambda x: datetime.strptime(x['datetime'], \"%Y-%m-%d %H:%M:%S\"))\n\n    return pd.DataFrame(dataset)\n\n\nfolder_path = './data/weather'\nraw_data = read_json_files(folder_path)\nweather_data = create_weather_dataset(raw_data)\nweather_data['datetime'] = pd.to_datetime(weather_data['datetime'], errors='coerce')\nweather_data['date'] = weather_data['datetime'].dt.date\nweather_data = weather_data.dropna(subset=['datetime'])\nweather_data['datetime'] = pd.to_datetime(weather_data['datetime'])\nweather_data['year'] = weather_data['datetime'].dt.year\nweather_data['month'] = weather_data['datetime'].dt.month\nweather_data['day'] = weather_data['datetime'].dt.day\nweather_data.head()\n\nweather_data.to_parquet('./data/weather_data.parquet')",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-25T21:05:45.017577Z",
     "start_time": "2024-10-25T21:05:34.194467Z"
    }
   },
   "execution_count": 5,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "# Crea le sequenze per LSTM\n",
    "def create_sequences(timesteps, X, y=None):\n",
    "    \"\"\"\n",
    "    Crea sequenze temporali dai dati.\n",
    "    \n",
    "    Parameters:\n",
    "    -----------\n",
    "    X : array-like\n",
    "        Dati di input\n",
    "    timesteps : int\n",
    "        Numero di timestep per ogni sequenza\n",
    "    y : array-like, optional\n",
    "        Target values. Se None, crea sequenze solo per X\n",
    "        \n",
    "    Returns:\n",
    "    --------\n",
    "    tuple o array\n",
    "        Se y è fornito: (X_sequences, y_sequences)\n",
    "        Se y è None: X_sequences\n",
    "    \"\"\"\n",
    "    Xs = []\n",
    "    for i in range(len(X) - timesteps):\n",
    "        Xs.append(X[i:i + timesteps])\n",
    "\n",
    "    if y is not None:\n",
    "        ys = []\n",
    "        for i in range(len(X) - timesteps):\n",
    "            ys.append(y[i + timesteps])\n",
    "        return np.array(Xs), np.array(ys)\n",
    "\n",
    "    return np.array(Xs)\n",
    "\n",
    "\n",
    "# Funzioni per costruire il modello LSTM avanzato\n",
    "def create_residual_lstm_layer(x, units, dropout_rate, l2_reg=0.01, return_sequences=True):\n",
    "    residual = x\n",
    "    x = Bidirectional(LSTM(units, return_sequences=return_sequences, kernel_regularizer=l2(l2_reg)))(x)\n",
    "    x = LayerNormalization()(x)\n",
    "    x = Dropout(dropout_rate)(x)\n",
    "    # Adjust residual dimension and handle return_sequences\n",
    "    if return_sequences:\n",
    "        if int(residual.shape[-1]) != 2 * units:\n",
    "            residual = Dense(2 * units, activation='linear')(residual)\n",
    "        x = Add()([x, residual])\n",
    "    return x\n",
    "\n",
    "\n",
    "def attention_block(x, units, num_heads=8):\n",
    "    attention = MultiHeadAttention(num_heads=num_heads, key_dim=units)(x, x)\n",
    "    x = Add()([x, attention])\n",
    "    x = LayerNormalization()(x)\n",
    "    return x\n",
    "\n",
    "\n",
    "def build_advanced_model(input_shape, l2_lambda=0.005):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    # Primi due layer LSTM con sequenze\n",
    "    x = create_residual_lstm_layer(inputs, 64, 0.2, l2_lambda, return_sequences=True)\n",
    "    x = create_residual_lstm_layer(x, 32, 0.2, l2_lambda, return_sequences=True)\n",
    "\n",
    "    # Attention e MaxPooling mentre abbiamo ancora la sequenza\n",
    "    x = attention_block(x, 32, num_heads=8)\n",
    "    x = MaxPooling1D()(x)\n",
    "\n",
    "    # Ultimo layer LSTM senza sequenze\n",
    "    x = create_residual_lstm_layer(x, 16, 0.1, l2_lambda, return_sequences=False)\n",
    "\n",
    "    # Dense layers\n",
    "    x = Dense(32, kernel_regularizer=l2(l2_lambda))(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('swish')(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "\n",
    "    x = Dense(16, kernel_regularizer=l2(l2_lambda))(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('swish')(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "\n",
    "    outputs = Dense(1, kernel_regularizer=l2(l2_lambda))(x)\n",
    "\n",
    "    model = Model(inputs=inputs, outputs=outputs)\n",
    "    return model\n",
    "\n",
    "\n",
    "def get_season(date):\n",
    "    month = date.month\n",
    "    day = date.day\n",
    "    if (month == 12 and day >= 21) or (month <= 3 and day < 20):\n",
    "        return 'Winter'\n",
    "    elif (month == 3 and day >= 20) or (month <= 6 and day < 21):\n",
    "        return 'Spring'\n",
    "    elif (month == 6 and day >= 21) or (month <= 9 and day < 23):\n",
    "        return 'Summer'\n",
    "    elif (month == 9 and day >= 23) or (month <= 12 and day < 21):\n",
    "        return 'Autumn'\n",
    "    else:\n",
    "        return 'Unknown'\n",
    "\n",
    "\n",
    "def get_time_period(hour):\n",
    "    if 5 <= hour < 12:\n",
    "        return 'Morning'\n",
    "    elif 12 <= hour < 17:\n",
    "        return 'Afternoon'\n",
    "    elif 17 <= hour < 21:\n",
    "        return 'Evening'\n",
    "    else:\n",
    "        return 'Night'\n",
    "\n",
    "\n",
    "def add_time_features(df):\n",
    "    df['datetime'] = pd.to_datetime(df['datetime'])\n",
    "    df['timestamp'] = df['datetime'].astype(np.int64) // 10 ** 9\n",
    "    df['year'] = df['datetime'].dt.year\n",
    "    df['month'] = df['datetime'].dt.month\n",
    "    df['day'] = df['datetime'].dt.day\n",
    "    df['hour'] = df['datetime'].dt.hour\n",
    "    df['minute'] = df['datetime'].dt.minute\n",
    "    df['hour_sin'] = np.sin(df['hour'] * (2 * np.pi / 24))\n",
    "    df['hour_cos'] = np.cos(df['hour'] * (2 * np.pi / 24))\n",
    "    df['day_of_week'] = df['datetime'].dt.dayofweek\n",
    "    df['day_of_year'] = df['datetime'].dt.dayofyear\n",
    "    df['week_of_year'] = df['datetime'].dt.isocalendar().week.astype(int)\n",
    "    df['quarter'] = df['datetime'].dt.quarter\n",
    "    df['is_month_end'] = df['datetime'].dt.is_month_end.astype(int)\n",
    "    df['is_quarter_end'] = df['datetime'].dt.is_quarter_end.astype(int)\n",
    "    df['is_year_end'] = df['datetime'].dt.is_year_end.astype(int)\n",
    "    df['month_sin'] = np.sin(df['month'] * (2 * np.pi / 12))\n",
    "    df['month_cos'] = np.cos(df['month'] * (2 * np.pi / 12))\n",
    "    df['day_of_year_sin'] = np.sin(df['day_of_year'] * (2 * np.pi / 365.25))\n",
    "    df['day_of_year_cos'] = np.cos(df['day_of_year'] * (2 * np.pi / 365.25))\n",
    "    df['season'] = df['datetime'].apply(get_season)\n",
    "    df['time_period'] = df['hour'].apply(get_time_period)\n",
    "    return df\n",
    "\n",
    "\n",
    "# Carica il dataset\n",
    "weather_data = pd.read_parquet('./data/weather_data.parquet')\n",
    "\n",
    "# Aggiungi le caratteristiche temporali\n",
    "weather_data = add_time_features(weather_data)\n",
    "\n",
    "# Encoding delle variabili categoriali\n",
    "weather_data = pd.get_dummies(weather_data, columns=['season', 'time_period'], drop_first=True)\n",
    "\n",
    "weather_data.to_parquet('/kaggle/working/weather_data_extended.parquet')\n",
    "\n",
    "# Dividi i dati in quelli dopo il 2010 e quelli prima del 2010\n",
    "data_after_2010 = weather_data[weather_data['year'] >= 2010].copy()\n",
    "data_before_2010 = weather_data[weather_data['year'] < 2010].copy()\n",
    "\n",
    "# Aggiorna le target variables se necessario\n",
    "target_variables = ['solarradiation', 'solarenergy', 'uvindex']\n",
    "\n",
    "# Seleziona le features\n",
    "features = [\n",
    "               'temp', 'tempmin', 'tempmax', 'humidity', 'cloudcover', 'windspeed', 'pressure', 'visibility',\n",
    "               'hour_sin', 'hour_cos', 'month_sin', 'month_cos', 'day_of_year_sin', 'day_of_year_cos',\n",
    "           ] + [col for col in weather_data.columns if 'season_' in col or 'time_period_' in col]\n",
    "\n",
    "# Prepara data_after_2010\n",
    "data_after_2010 = data_after_2010.sort_values('datetime')\n",
    "data_after_2010.set_index('datetime', inplace=True)\n",
    "\n",
    "# Interpola eventuali valori mancanti nelle variabili target\n",
    "columns_to_interpolate = target_variables\n",
    "for column in columns_to_interpolate:\n",
    "    data_after_2010[column] = data_after_2010[column].interpolate(method='time')\n",
    "\n",
    "# Rimuovi eventuali valori mancanti residui\n",
    "data_after_2010.dropna(subset=features + target_variables, inplace=True)\n",
    "\n",
    "# Crea X e y\n",
    "X = data_after_2010[features].values\n",
    "y = data_after_2010[target_variables].values\n",
    "\n",
    "# Normalizza le features\n",
    "scaler_X = MinMaxScaler()\n",
    "X_scaled = scaler_X.fit_transform(X)\n",
    "\n",
    "\n",
    "def prepare_multi_target_datasets(X_scaled, y, target_variables):\n",
    "    \"\"\"\n",
    "    Prepara dataset separati per ogni target variable e restituisce anche gli scaler\n",
    "    per un uso successivo in fase di predizione.\n",
    "    \n",
    "    Parameters:\n",
    "    -----------\n",
    "    X_scaled : numpy.ndarray\n",
    "        Features già scalate\n",
    "    y : numpy.ndarray\n",
    "        Target variables (matrice con una colonna per ogni target)\n",
    "    target_variables : list\n",
    "        Lista dei nomi delle variabili target\n",
    "    \n",
    "    Returns:\n",
    "    --------\n",
    "    tuple (dict, dict)\n",
    "        - Primo dict: contiene i dataset per ogni target\n",
    "        - Secondo dict: contiene gli scaler per ogni target\n",
    "    \"\"\"\n",
    "\n",
    "    # Inizializza i dizionari per contenere i dataset e gli scaler\n",
    "    train_datasets = {}\n",
    "    scalers_dict = {}\n",
    "\n",
    "    # Scala e splitta i dati per ogni target\n",
    "    for i, target in enumerate(target_variables):\n",
    "        # Scala il target corrente\n",
    "        scaler = MinMaxScaler()\n",
    "        y_scaled_current = scaler.fit_transform(y[:, i].reshape(-1, 1)).flatten()\n",
    "        scalers_dict[target] = scaler\n",
    "\n",
    "        # Split dei dati per il target corrente\n",
    "        X_train_full, X_test, y_train_full, y_test = train_test_split(\n",
    "            X_scaled,\n",
    "            y_scaled_current,\n",
    "            test_size=0.2,\n",
    "            shuffle=False\n",
    "        )\n",
    "\n",
    "        # Ulteriore split per validation\n",
    "        X_train, X_val, y_train, y_val = train_test_split(\n",
    "            X_train_full,\n",
    "            y_train_full,\n",
    "            test_size=0.2,\n",
    "            shuffle=False\n",
    "        )\n",
    "\n",
    "        # Salva i dataset per questo target\n",
    "        train_datasets[target] = {\n",
    "            'X_train': X_train,\n",
    "            'X_val': X_val,\n",
    "            'X_test': X_test,\n",
    "            'y_train': y_train.reshape(-1, 1),\n",
    "            'y_val': y_val.reshape(-1, 1),\n",
    "            'y_test': y_test.reshape(-1, 1)\n",
    "        }\n",
    "\n",
    "    return train_datasets, scalers_dict\n",
    "\n",
    "\n",
    "datasets, scalers = prepare_multi_target_datasets(X_scaled, y, target_variables)"
   ],
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-26T05:43:32.169183Z",
     "start_time": "2024-10-26T05:43:29.609044Z"
    },
    "execution": {
     "iopub.status.busy": "2024-10-26T12:37:57.305210Z",
     "iopub.execute_input": "2024-10-26T12:37:57.305635Z",
     "iopub.status.idle": "2024-10-26T12:38:00.719541Z",
     "shell.execute_reply.started": "2024-10-26T12:37:57.305597Z",
     "shell.execute_reply": "2024-10-26T12:38:00.718532Z"
    },
    "trusted": true
   },
   "execution_count": 28,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# numero di timesteps (utilizziamo le ultime 24 ore)\n",
    "timesteps = 24\n",
    "\n",
    "# Costruisci il modello per ogni variabile target\n",
    "models = {}\n",
    "histories = {}\n",
    "for i, target in enumerate(target_variables):\n",
    "    target_data = datasets[target]\n",
    "    target_scaler = scalers[target]\n",
    "\n",
    "    X_train = target_data['X_train']\n",
    "    y_train = target_data['y_train']\n",
    "    X_val = target_data['X_val']\n",
    "    y_val = target_data['y_val']\n",
    "    X_test = target_data['X_test']\n",
    "    y_test = target_data['y_test']\n",
    "\n",
    "    num_features = X_train.shape[1]\n",
    "\n",
    "    X_train_seq, y_train_seq = create_sequences(timesteps, X_train, y_train)\n",
    "    X_val_seq, y_val_seq = create_sequences(timesteps, X_val, y_val)\n",
    "    X_test_seq, y_test_seq = create_sequences(timesteps, X_test, y_test)\n",
    "\n",
    "    print(X_train_seq.shape, y_train_seq.shape)\n",
    "    print(X_val_seq.shape, y_val_seq.shape)\n",
    "    print(X_test_seq.shape, y_test_seq.shape)\n",
    "\n",
    "    print(f\"Addestramento del modello per: {target}\")\n",
    "    model = build_advanced_model((timesteps, num_features), l2_lambda=0.001)\n",
    "    optimizer = Adam(learning_rate=0.001, clipnorm=1.0)\n",
    "    model.compile(optimizer=optimizer, loss='mean_squared_error')\n",
    "    early_stopping = EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True)\n",
    "\n",
    "    reduce_lr = ReduceLROnPlateau(\n",
    "        monitor='val_loss',\n",
    "        factor=0.5,\n",
    "        patience=5,\n",
    "        min_lr=1e-6\n",
    "    )\n",
    "    history = model.fit(\n",
    "        X_train_seq, y_train_seq,\n",
    "        validation_data=(X_val_seq, y_val_seq),\n",
    "        epochs=50,\n",
    "        batch_size=180,\n",
    "        callbacks=[early_stopping, reduce_lr],\n",
    "        verbose=1\n",
    "    )\n",
    "    test_loss = model.evaluate(X_test_seq, y_test_seq)\n",
    "    print(f'Test MAE per {target}: {test_loss:.4f}')\n",
    "    models[target] = model\n",
    "    histories[target] = history\n"
   ],
   "metadata": {
    "jupyter": {
     "is_executing": true
    },
    "ExecuteTime": {
     "start_time": "2024-10-26T05:43:33.294101Z"
    },
    "execution": {
     "iopub.status.busy": "2024-10-26T12:38:20.218129Z",
     "iopub.execute_input": "2024-10-26T12:38:20.218982Z",
     "iopub.status.idle": "2024-10-26T13:05:41.767391Z",
     "shell.execute_reply.started": "2024-10-26T12:38:20.218940Z",
     "shell.execute_reply": "2024-10-26T13:05:41.766345Z"
    },
    "trusted": true
   },
   "execution_count": 29,
   "outputs": [
    {
     "name": "stdout",
     "text": "(82972, 24, 21) (82972, 1)\n(20725, 24, 21) (20725, 1)\n(25913, 24, 21) (25913, 1)\nAddestramento del modello per: solarradiation\nEpoch 1/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m24s\u001B[0m 30ms/step - loss: 0.4766 - val_loss: 0.1042 - learning_rate: 0.0010\nEpoch 2/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0848 - val_loss: 0.0360 - learning_rate: 0.0010\nEpoch 3/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0326 - val_loss: 0.0181 - learning_rate: 0.0010\nEpoch 4/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0169 - val_loss: 0.0122 - learning_rate: 0.0010\nEpoch 5/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0108 - val_loss: 0.0095 - learning_rate: 0.0010\nEpoch 6/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0085 - val_loss: 0.0094 - learning_rate: 0.0010\nEpoch 7/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0079 - val_loss: 0.0103 - learning_rate: 0.0010\nEpoch 8/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m14s\u001B[0m 30ms/step - loss: 0.0071 - val_loss: 0.0081 - learning_rate: 0.0010\nEpoch 9/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0071 - val_loss: 0.0094 - learning_rate: 0.0010\nEpoch 10/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0071 - val_loss: 0.0078 - learning_rate: 0.0010\nEpoch 11/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0070 - val_loss: 0.0122 - learning_rate: 0.0010\nEpoch 12/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0068 - val_loss: 0.0081 - learning_rate: 0.0010\nEpoch 13/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0068 - val_loss: 0.0087 - learning_rate: 0.0010\nEpoch 14/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0068 - val_loss: 0.0094 - learning_rate: 0.0010\nEpoch 15/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0068 - val_loss: 0.0067 - learning_rate: 0.0010\nEpoch 16/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0066 - val_loss: 0.0072 - learning_rate: 0.0010\nEpoch 17/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0063 - val_loss: 0.0065 - learning_rate: 0.0010\nEpoch 18/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0066 - val_loss: 0.0063 - learning_rate: 0.0010\nEpoch 19/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0064 - val_loss: 0.0078 - learning_rate: 0.0010\nEpoch 20/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0064 - val_loss: 0.0114 - learning_rate: 0.0010\nEpoch 21/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0061 - val_loss: 0.0064 - learning_rate: 0.0010\nEpoch 22/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0062 - val_loss: 0.0073 - learning_rate: 0.0010\nEpoch 23/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0061 - val_loss: 0.0068 - learning_rate: 0.0010\nEpoch 24/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0058 - val_loss: 0.0066 - learning_rate: 5.0000e-04\nEpoch 25/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0057 - val_loss: 0.0086 - learning_rate: 5.0000e-04\nEpoch 26/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0058 - val_loss: 0.0067 - learning_rate: 5.0000e-04\nEpoch 27/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0058 - val_loss: 0.0059 - learning_rate: 5.0000e-04\nEpoch 28/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0057 - val_loss: 0.0062 - learning_rate: 5.0000e-04\nEpoch 29/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0055 - val_loss: 0.0070 - learning_rate: 5.0000e-04\nEpoch 30/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0054 - val_loss: 0.0052 - learning_rate: 5.0000e-04\nEpoch 31/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0054 - val_loss: 0.0057 - learning_rate: 5.0000e-04\nEpoch 32/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0056 - val_loss: 0.0069 - learning_rate: 5.0000e-04\nEpoch 33/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0054 - val_loss: 0.0062 - learning_rate: 5.0000e-04\nEpoch 34/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0054 - val_loss: 0.0056 - learning_rate: 5.0000e-04\nEpoch 35/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0054 - val_loss: 0.0057 - learning_rate: 5.0000e-04\nEpoch 36/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0052 - val_loss: 0.0059 - learning_rate: 2.5000e-04\nEpoch 37/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0052 - val_loss: 0.0060 - learning_rate: 2.5000e-04\nEpoch 38/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0051 - val_loss: 0.0050 - learning_rate: 2.5000e-04\nEpoch 39/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0051 - val_loss: 0.0052 - learning_rate: 2.5000e-04\nEpoch 40/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0050 - val_loss: 0.0055 - learning_rate: 2.5000e-04\nEpoch 41/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m14s\u001B[0m 29ms/step - loss: 0.0051 - val_loss: 0.0051 - learning_rate: 2.5000e-04\nEpoch 42/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0049 - val_loss: 0.0049 - learning_rate: 2.5000e-04\nEpoch 43/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0050 - val_loss: 0.0055 - learning_rate: 2.5000e-04\nEpoch 44/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0049 - val_loss: 0.0065 - learning_rate: 2.5000e-04\nEpoch 45/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0049 - val_loss: 0.0054 - learning_rate: 2.5000e-04\nEpoch 46/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0049 - val_loss: 0.0058 - learning_rate: 2.5000e-04\nEpoch 47/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0050 - val_loss: 0.0052 - learning_rate: 2.5000e-04\nEpoch 48/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0048 - val_loss: 0.0053 - learning_rate: 1.2500e-04\nEpoch 49/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0048 - val_loss: 0.0051 - learning_rate: 1.2500e-04\nEpoch 50/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m14s\u001B[0m 30ms/step - loss: 0.0048 - val_loss: 0.0057 - learning_rate: 1.2500e-04\n\u001B[1m810/810\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m5s\u001B[0m 6ms/step - loss: 0.0050\nTest MAE per solarradiation: 0.0061\n(82972, 24, 21) (82972, 1)\n(20725, 24, 21) (20725, 1)\n(25913, 24, 21) (25913, 1)\nAddestramento del modello per: solarenergy\nEpoch 1/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m24s\u001B[0m 31ms/step - loss: 0.5212 - val_loss: 0.1362 - learning_rate: 0.0010\nEpoch 2/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m14s\u001B[0m 30ms/step - loss: 0.1183 - val_loss: 0.0555 - learning_rate: 0.0010\nEpoch 3/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0496 - val_loss: 0.0281 - learning_rate: 0.0010\nEpoch 4/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m14s\u001B[0m 30ms/step - loss: 0.0256 - val_loss: 0.0169 - learning_rate: 0.0010\nEpoch 5/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0155 - val_loss: 0.0119 - learning_rate: 0.0010\nEpoch 6/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0110 - val_loss: 0.0093 - learning_rate: 0.0010\nEpoch 7/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0087 - val_loss: 0.0070 - learning_rate: 0.0010\nEpoch 8/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0079 - val_loss: 0.0073 - learning_rate: 0.0010\nEpoch 9/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m14s\u001B[0m 30ms/step - loss: 0.0074 - val_loss: 0.0073 - learning_rate: 0.0010\nEpoch 10/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m14s\u001B[0m 29ms/step - loss: 0.0070 - val_loss: 0.0065 - learning_rate: 0.0010\nEpoch 11/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0069 - val_loss: 0.0091 - learning_rate: 0.0010\nEpoch 12/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0070 - val_loss: 0.0081 - learning_rate: 0.0010\nEpoch 13/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0069 - val_loss: 0.0072 - learning_rate: 0.0010\nEpoch 14/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m14s\u001B[0m 29ms/step - loss: 0.0068 - val_loss: 0.0078 - learning_rate: 0.0010\nEpoch 15/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0071 - val_loss: 0.0091 - learning_rate: 0.0010\nEpoch 16/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0065 - val_loss: 0.0076 - learning_rate: 5.0000e-04\nEpoch 17/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0064 - val_loss: 0.0070 - learning_rate: 5.0000e-04\nEpoch 18/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0063 - val_loss: 0.0065 - learning_rate: 5.0000e-04\nEpoch 19/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0063 - val_loss: 0.0081 - learning_rate: 5.0000e-04\nEpoch 20/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0062 - val_loss: 0.0068 - learning_rate: 5.0000e-04\n\u001B[1m810/810\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m5s\u001B[0m 6ms/step - loss: 0.0060\nTest MAE per solarenergy: 0.0070\n(82972, 24, 21) (82972, 1)\n(20725, 24, 21) (20725, 1)\n(25913, 24, 21) (25913, 1)\nAddestramento del modello per: uvindex\nEpoch 1/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m26s\u001B[0m 32ms/step - loss: 0.4562 - val_loss: 0.1049 - learning_rate: 0.0010\nEpoch 2/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m14s\u001B[0m 30ms/step - loss: 0.0864 - val_loss: 0.0411 - learning_rate: 0.0010\nEpoch 3/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m14s\u001B[0m 29ms/step - loss: 0.0345 - val_loss: 0.0201 - learning_rate: 0.0010\nEpoch 4/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m14s\u001B[0m 30ms/step - loss: 0.0186 - val_loss: 0.0128 - learning_rate: 0.0010\nEpoch 5/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0131 - val_loss: 0.0122 - learning_rate: 0.0010\nEpoch 6/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0110 - val_loss: 0.0132 - learning_rate: 0.0010\nEpoch 7/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0101 - val_loss: 0.0096 - learning_rate: 0.0010\nEpoch 8/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0095 - val_loss: 0.0088 - learning_rate: 0.0010\nEpoch 9/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0090 - val_loss: 0.0088 - learning_rate: 0.0010\nEpoch 10/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0094 - val_loss: 0.0105 - learning_rate: 0.0010\nEpoch 11/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0091 - val_loss: 0.0105 - learning_rate: 0.0010\nEpoch 12/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0090 - val_loss: 0.0095 - learning_rate: 0.0010\nEpoch 13/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0087 - val_loss: 0.0093 - learning_rate: 0.0010\nEpoch 14/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0085 - val_loss: 0.0087 - learning_rate: 5.0000e-04\nEpoch 15/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0081 - val_loss: 0.0095 - learning_rate: 5.0000e-04\nEpoch 16/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m14s\u001B[0m 29ms/step - loss: 0.0081 - val_loss: 0.0088 - learning_rate: 5.0000e-04\nEpoch 17/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0080 - val_loss: 0.0102 - learning_rate: 5.0000e-04\nEpoch 18/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0082 - val_loss: 0.0085 - learning_rate: 5.0000e-04\nEpoch 19/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0082 - val_loss: 0.0098 - learning_rate: 5.0000e-04\nEpoch 20/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0079 - val_loss: 0.0093 - learning_rate: 5.0000e-04\nEpoch 21/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0080 - val_loss: 0.0103 - learning_rate: 5.0000e-04\nEpoch 22/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0080 - val_loss: 0.0098 - learning_rate: 5.0000e-04\nEpoch 23/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0078 - val_loss: 0.0096 - learning_rate: 5.0000e-04\nEpoch 24/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0076 - val_loss: 0.0079 - learning_rate: 2.5000e-04\nEpoch 25/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0075 - val_loss: 0.0088 - learning_rate: 2.5000e-04\nEpoch 26/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0075 - val_loss: 0.0099 - learning_rate: 2.5000e-04\nEpoch 27/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0075 - val_loss: 0.0091 - learning_rate: 2.5000e-04\nEpoch 28/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0073 - val_loss: 0.0088 - learning_rate: 2.5000e-04\nEpoch 29/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0072 - val_loss: 0.0083 - learning_rate: 2.5000e-04\nEpoch 30/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0071 - val_loss: 0.0089 - learning_rate: 1.2500e-04\nEpoch 31/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0072 - val_loss: 0.0078 - learning_rate: 1.2500e-04\nEpoch 32/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0071 - val_loss: 0.0099 - learning_rate: 1.2500e-04\nEpoch 33/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0070 - val_loss: 0.0074 - learning_rate: 1.2500e-04\nEpoch 34/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0068 - val_loss: 0.0094 - learning_rate: 1.2500e-04\nEpoch 35/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0070 - val_loss: 0.0085 - learning_rate: 1.2500e-04\nEpoch 36/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0069 - val_loss: 0.0078 - learning_rate: 1.2500e-04\nEpoch 37/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0070 - val_loss: 0.0073 - learning_rate: 1.2500e-04\nEpoch 38/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0070 - val_loss: 0.0079 - learning_rate: 1.2500e-04\nEpoch 39/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0068 - val_loss: 0.0075 - learning_rate: 6.2500e-05\nEpoch 40/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0066 - val_loss: 0.0077 - learning_rate: 6.2500e-05\nEpoch 41/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0068 - val_loss: 0.0074 - learning_rate: 6.2500e-05\nEpoch 42/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0068 - val_loss: 0.0078 - learning_rate: 6.2500e-05\nEpoch 43/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0068 - val_loss: 0.0077 - learning_rate: 6.2500e-05\nEpoch 44/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0066 - val_loss: 0.0076 - learning_rate: 3.1250e-05\nEpoch 45/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0067 - val_loss: 0.0070 - learning_rate: 3.1250e-05\nEpoch 46/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 29ms/step - loss: 0.0065 - val_loss: 0.0079 - learning_rate: 3.1250e-05\nEpoch 47/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0066 - val_loss: 0.0072 - learning_rate: 3.1250e-05\nEpoch 48/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0068 - val_loss: 0.0079 - learning_rate: 3.1250e-05\nEpoch 49/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0068 - val_loss: 0.0079 - learning_rate: 3.1250e-05\nEpoch 50/50\n\u001B[1m461/461\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m13s\u001B[0m 28ms/step - loss: 0.0067 - val_loss: 0.0075 - learning_rate: 3.1250e-05\n\u001B[1m810/810\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m5s\u001B[0m 6ms/step - loss: 0.0066\nTest MAE per uvindex: 0.0078\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": "import joblib\nimport os\n\n\ndef save_models_and_scalers(models, scaler_X, scalers_y, target_variables, base_path='/kaggle/working/models'):\n    \"\"\"\n    Salva i modelli e gli scaler nella cartella models.\n    \n    Parameters:\n    -----------\n    models : dict\n        Dizionario contenente i modelli per ogni variabile target\n    scaler_X : MinMaxScaler\n        Scaler unico per tutte le feature di input\n    scalers_y : dict\n        Dizionario contenente gli scaler per le variabili target\n    target_variables : list\n        Lista delle variabili target\n    base_path : str\n        Percorso base dove salvare i modelli (default: 'models')\n    \"\"\"\n\n    # Crea la cartella se non esiste\n    os.makedirs(base_path, exist_ok=True)\n\n    # Salva lo scaler X generale\n    scaler_x_path = os.path.join(base_path, 'scaler_x.joblib')\n    joblib.dump(scaler_X, scaler_x_path)\n\n    # Salva i modelli e gli scaler Y per ogni variabile target\n    for target in target_variables:\n        # Crea una sottocartella per ogni target\n        target_path = os.path.join(base_path, target)\n        os.makedirs(target_path, exist_ok=True)\n\n        # Salva il modello\n        model_path = os.path.join(target_path, 'model.joblib')\n        joblib.dump(models[target], model_path)\n\n        # Salva lo scaler Y\n        scaler_y_path = os.path.join(target_path, 'scaler_y.joblib')\n        joblib.dump(scalers_y[target], scaler_y_path)\n\n    # Salva la lista delle variabili target\n    target_vars_path = os.path.join(base_path, 'target_variables.joblib')\n    joblib.dump(target_variables, target_vars_path)\n\n    print(f\"Modelli e scaler salvati in: {base_path}\")\n\n\ndef load_models_and_scalers(base_path='/kaggle/working/models'):\n    \"\"\"\n    Carica i modelli e gli scaler dalla cartella models.\n    \n    Parameters:\n    -----------\n    base_path : str\n        Percorso della cartella contenente i modelli salvati (default: 'models')\n    \n    Returns:\n    --------\n    tuple\n        (models, scaler_X, scalers_y, target_variables)\n    \"\"\"\n\n    # Carica la lista delle variabili target\n    target_vars_path = os.path.join(base_path, 'target_variables.joblib')\n    target_variables = joblib.load(target_vars_path)\n\n    # Carica lo scaler X generale\n    scaler_x_path = os.path.join(base_path, 'scaler_x.joblib')\n    scaler_X = joblib.load(scaler_x_path)\n\n    # Inizializza i dizionari\n    models = {}\n    scalers_y = {}\n\n    # Carica i modelli e gli scaler per ogni variabile target\n    for target in target_variables:\n        target_path = os.path.join(base_path, target)\n\n        # Carica il modello\n        model_path = os.path.join(target_path, 'model.joblib')\n        models[target] = joblib.load(model_path)\n\n        # Carica lo scaler Y\n        scaler_y_path = os.path.join(target_path, 'scaler_y.joblib')\n        scalers_y[target] = joblib.load(scaler_y_path)\n\n    print(f\"Modelli e scaler caricati da: {base_path}\")\n    return models, scaler_X, scalers_y, target_variables\n\n\n",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-10-26T13:05:41.770201Z",
     "iopub.execute_input": "2024-10-26T13:05:41.770625Z",
     "iopub.status.idle": "2024-10-26T13:05:41.785155Z",
     "shell.execute_reply.started": "2024-10-26T13:05:41.770576Z",
     "shell.execute_reply": "2024-10-26T13:05:41.784399Z"
    },
    "trusted": true
   },
   "execution_count": 30,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": "save_models_and_scalers(models, scaler_X, scalers, target_variables)",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-10-26T13:05:41.786135Z",
     "iopub.execute_input": "2024-10-26T13:05:41.786420Z",
     "iopub.status.idle": "2024-10-26T13:05:42.220174Z",
     "shell.execute_reply.started": "2024-10-26T13:05:41.786389Z",
     "shell.execute_reply": "2024-10-26T13:05:42.219274Z"
    },
    "trusted": true
   },
   "execution_count": 31,
   "outputs": [
    {
     "name": "stdout",
     "text": "Modelli e scaler salvati in: /kaggle/working/models\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": "# Previsione delle variabili mancanti per data_before_2010\n# Prepara data_before_2010\ndata_before_2010 = data_before_2010.sort_values('datetime')\ndata_before_2010.set_index('datetime', inplace=True)\n\n# Assicurati che le features non abbiano valori mancanti\ndata_before_2010[features] = data_before_2010[features].ffill()\ndata_before_2010[features] = data_before_2010[features].bfill()",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-24T16:14:44.770508Z",
     "start_time": "2024-10-24T13:29:15.181470Z"
    },
    "execution": {
     "iopub.status.busy": "2024-10-26T13:05:42.221912Z",
     "iopub.execute_input": "2024-10-26T13:05:42.222221Z",
     "iopub.status.idle": "2024-10-26T13:05:42.343085Z",
     "shell.execute_reply.started": "2024-10-26T13:05:42.222188Z",
     "shell.execute_reply": "2024-10-26T13:05:42.342136Z"
    },
    "trusted": true
   },
   "execution_count": 32,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": "models, scaler_X, scalers_y, target_variables = load_models_and_scalers()\n\ntimesteps = 24\n# Crea X per data_before_2010\nX_before = data_before_2010[features].values\nX_before_scaled = scaler_X.transform(X_before)\n\n# Crea le sequenze per LSTM\nX_before_seq = create_sequences(timesteps, X_before_scaled)\n\n# Prevedi le variabili mancanti\nfor i, target in enumerate(target_variables):\n    print(\"Shape di X_before_seq:\", X_before_seq.shape)\n    print(f\"Previsione di {target} per data_before_2010\")\n    y_pred_scaled = models[target].predict(X_before_seq)\n    print(\"Shape delle predizioni:\", y_pred_scaled.shape)\n    # Ricostruisci i valori originali\n    scaler = scalers_y[target]\n    y_pred = scaler.inverse_transform(y_pred_scaled)\n\n    # Allinea le previsioni con le date corrette\n    dates = data_before_2010.index[timesteps:]\n    data_before_2010.loc[dates, target] = y_pred\n\n# Gestisci eventuali valori iniziali mancanti\ndata_before_2010[target_variables] = data_before_2010[target_variables].bfill()\n\n# Combina data_before_2010 e data_after_2010\nweather_data_complete = pd.concat([data_before_2010, data_after_2010], axis=0)\nweather_data_complete = weather_data_complete.sort_index()\n\n# Salva il dataset completo\nweather_data_complete.reset_index(inplace=True)\nweather_data_complete.to_parquet('/kaggle/working/weather_data_complete.parquet', index=False)\n",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-24T18:50:48.087413Z",
     "start_time": "2024-10-24T18:47:52.511763Z"
    },
    "execution": {
     "iopub.status.busy": "2024-10-26T13:05:42.344137Z",
     "iopub.execute_input": "2024-10-26T13:05:42.344431Z",
     "iopub.status.idle": "2024-10-26T13:07:47.910572Z",
     "shell.execute_reply.started": "2024-10-26T13:05:42.344400Z",
     "shell.execute_reply": "2024-10-26T13:07:47.909535Z"
    },
    "trusted": true
   },
   "execution_count": 33,
   "outputs": [
    {
     "name": "stdout",
     "text": "Modelli e scaler caricati da: /kaggle/working/models\nShape di X_before_seq: (227878, 24, 21)\nPrevisione di solarradiation per data_before_2010\n\u001B[1m7122/7122\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m38s\u001B[0m 5ms/step\nShape delle predizioni: (227878, 1)\nShape di X_before_seq: (227878, 24, 21)\nPrevisione di solarenergy per data_before_2010\n\u001B[1m7122/7122\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m37s\u001B[0m 5ms/step\nShape delle predizioni: (227878, 1)\nShape di X_before_seq: (227878, 24, 21)\nPrevisione di uvindex per data_before_2010\n\u001B[1m7122/7122\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m36s\u001B[0m 5ms/step\nShape delle predizioni: (227878, 1)\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "## 2. Esplorazione dei Dati Meteo",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": "weather_data = pd.read_parquet('/kaggle/working/weather_data_complete.parquet')",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-23T06:10:46.688323Z",
     "start_time": "2024-10-23T06:10:46.586185Z"
    },
    "execution": {
     "iopub.status.busy": "2024-10-26T13:08:14.248728Z",
     "iopub.execute_input": "2024-10-26T13:08:14.249138Z",
     "iopub.status.idle": "2024-10-26T13:08:14.382810Z",
     "shell.execute_reply.started": "2024-10-26T13:08:14.249101Z",
     "shell.execute_reply": "2024-10-26T13:08:14.381817Z"
    },
    "trusted": true
   },
   "execution_count": 34,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": "# Visualizzazione delle tendenze temporali\nfig, axes = plt.subplots(5, 1, figsize=(15, 20))\nweather_data.set_index('date')['temp'].plot(ax=axes[0], title='Temperatura Media Giornaliera')\nweather_data.set_index('date')['humidity'].plot(ax=axes[1], title='Umidità Media Giornaliera')\nweather_data.set_index('date')['solarradiation'].plot(ax=axes[2], title='Radiazione Solare Giornaliera')\nweather_data.set_index('date')['solarenergy'].plot(ax=axes[3], title='Radiazione Solare Giornaliera')\nweather_data.set_index('date')['precip'].plot(ax=axes[4], title='Precipitazioni Giornaliere')\nplt.tight_layout()\nplt.show()",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-23T06:10:50.718574Z",
     "start_time": "2024-10-23T06:10:46.901554Z"
    },
    "execution": {
     "iopub.status.busy": "2024-10-26T13:08:22.673121Z",
     "iopub.execute_input": "2024-10-26T13:08:22.673513Z",
     "iopub.status.idle": "2024-10-26T13:08:27.772406Z",
     "shell.execute_reply.started": "2024-10-26T13:08:22.673463Z",
     "shell.execute_reply": "2024-10-26T13:08:27.771460Z"
    },
    "trusted": true
   },
   "execution_count": 35,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 1500x2000 with 5 Axes>",
      "image/png": "[encoded data, 295344 characters]"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "## 3. Simulazione dei Dati di Produzione Annuale",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "\n",
    "# Esempio di utilizzo\n",
    "olive_varieties = pd.read_csv('./data/variety_olive_oil_production.csv')\n",
    "\n",
    "\n",
    "def add_olive_water_consumption_correlation(dataset):\n",
    "    # Dati simulati per il fabbisogno d'acqua e la correlazione con la temperatura\n",
    "    fabbisogno_acqua = {\n",
    "        \"Nocellara dell'Etna\": {\"Primavera\": 1200, \"Estate\": 2000, \"Autunno\": 1000, \"Inverno\": 500, \"Temperatura Ottimale\": 18, \"Resistenza\": \"Media\"},\n",
    "        \"Leccino\": {\"Primavera\": 1000, \"Estate\": 1800, \"Autunno\": 800, \"Inverno\": 400, \"Temperatura Ottimale\": 20, \"Resistenza\": \"Alta\"},\n",
    "        \"Frantoio\": {\"Primavera\": 1100, \"Estate\": 1900, \"Autunno\": 900, \"Inverno\": 450, \"Temperatura Ottimale\": 19, \"Resistenza\": \"Alta\"},\n",
    "        \"Coratina\": {\"Primavera\": 1300, \"Estate\": 2200, \"Autunno\": 1100, \"Inverno\": 550, \"Temperatura Ottimale\": 17, \"Resistenza\": \"Media\"},\n",
    "        \"Moraiolo\": {\"Primavera\": 1150, \"Estate\": 2100, \"Autunno\": 900, \"Inverno\": 480, \"Temperatura Ottimale\": 18, \"Resistenza\": \"Media\"},\n",
    "        \"Pendolino\": {\"Primavera\": 1050, \"Estate\": 1850, \"Autunno\": 850, \"Inverno\": 430, \"Temperatura Ottimale\": 20, \"Resistenza\": \"Alta\"},\n",
    "        \"Taggiasca\": {\"Primavera\": 1000, \"Estate\": 1750, \"Autunno\": 800, \"Inverno\": 400, \"Temperatura Ottimale\": 19, \"Resistenza\": \"Alta\"},\n",
    "        \"Canino\": {\"Primavera\": 1100, \"Estate\": 1900, \"Autunno\": 900, \"Inverno\": 450, \"Temperatura Ottimale\": 18, \"Resistenza\": \"Media\"},\n",
    "        \"Itrana\": {\"Primavera\": 1200, \"Estate\": 2000, \"Autunno\": 1000, \"Inverno\": 500, \"Temperatura Ottimale\": 17, \"Resistenza\": \"Media\"},\n",
    "        \"Ogliarola\": {\"Primavera\": 1150, \"Estate\": 1950, \"Autunno\": 900, \"Inverno\": 480, \"Temperatura Ottimale\": 18, \"Resistenza\": \"Media\"},\n",
    "        \"Biancolilla\": {\"Primavera\": 1050, \"Estate\": 1800, \"Autunno\": 850, \"Inverno\": 430, \"Temperatura Ottimale\": 19, \"Resistenza\": \"Alta\"}\n",
    "    }\n",
    "\n",
    "    # Calcola il fabbisogno idrico annuale per ogni varietà\n",
    "    for varieta in fabbisogno_acqua:\n",
    "        fabbisogno_acqua[varieta][\"Annuale\"] = sum([fabbisogno_acqua[varieta][stagione] for stagione in [\"Primavera\", \"Estate\", \"Autunno\", \"Inverno\"]])\n",
    "\n",
    "    # Aggiungiamo le nuove colonne al dataset\n",
    "    dataset[\"Fabbisogno Acqua Primavera (m³/ettaro)\"] = dataset[\"Varietà di Olive\"].apply(lambda x: fabbisogno_acqua[x][\"Primavera\"])\n",
    "    dataset[\"Fabbisogno Acqua Estate (m³/ettaro)\"] = dataset[\"Varietà di Olive\"].apply(lambda x: fabbisogno_acqua[x][\"Estate\"])\n",
    "    dataset[\"Fabbisogno Acqua Autunno (m³/ettaro)\"] = dataset[\"Varietà di Olive\"].apply(lambda x: fabbisogno_acqua[x][\"Autunno\"])\n",
    "    dataset[\"Fabbisogno Acqua Inverno (m³/ettaro)\"] = dataset[\"Varietà di Olive\"].apply(lambda x: fabbisogno_acqua[x][\"Inverno\"])\n",
    "    dataset[\"Fabbisogno Idrico Annuale (m³/ettaro)\"] = dataset[\"Varietà di Olive\"].apply(lambda x: fabbisogno_acqua[x][\"Annuale\"])\n",
    "    dataset[\"Temperatura Ottimale\"] = dataset[\"Varietà di Olive\"].apply(lambda x: fabbisogno_acqua[x][\"Temperatura Ottimale\"])\n",
    "    dataset[\"Resistenza alla Siccità\"] = dataset[\"Varietà di Olive\"].apply(lambda x: fabbisogno_acqua[x][\"Resistenza\"])\n",
    "\n",
    "    return dataset\n",
    "\n",
    "\n",
    "olive_varieties = add_olive_water_consumption_correlation(olive_varieties)\n",
    "\n",
    "olive_varieties.to_parquet(\"./data/olive_varieties.parquet\")"
   ],
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-23T06:10:51.081621Z",
     "start_time": "2024-10-23T06:10:51.044080Z"
    },
    "execution": {
     "iopub.status.busy": "2024-10-26T13:09:36.079724Z",
     "iopub.execute_input": "2024-10-26T13:09:36.080418Z",
     "iopub.status.idle": "2024-10-26T13:09:36.109877Z",
     "shell.execute_reply.started": "2024-10-26T13:09:36.080378Z",
     "shell.execute_reply": "2024-10-26T13:09:36.109129Z"
    },
    "trusted": true
   },
   "execution_count": 38,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "simulated_data = pd.read_parquet(\"./data/simulated_data.parquet\")\n",
    "\n",
    "\n",
    "def clean_column_names(df):\n",
    "    # Funzione per pulire i nomi delle colonne\n",
    "    new_columns = []\n",
    "    for col in df.columns:\n",
    "        # Usa regex per separare le varietà\n",
    "        varieties = re.findall(r'([a-z]+)_([a-z_]+)', col)\n",
    "        if varieties:\n",
    "            new_columns.append(f\"{varieties[0][0]}_{varieties[0][1]}\")\n",
    "        else:\n",
    "            new_columns.append(col)\n",
    "    return new_columns\n",
    "\n",
    "\n",
    "def prepare_comparison_data(simulated_data, olive_varieties):\n",
    "    # Pulisci i nomi delle colonne\n",
    "    df = simulated_data.copy()\n",
    "\n",
    "    df.columns = clean_column_names(df)\n",
    "    df = encode_techniques(df)\n",
    "\n",
    "    all_varieties = olive_varieties['Varietà di Olive'].unique()\n",
    "    varieties = [clean_column_name(variety) for variety in all_varieties]\n",
    "    comparison_data = []\n",
    "\n",
    "    for variety in varieties:\n",
    "        olive_prod_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_olive_prod')), None)\n",
    "        oil_prod_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_avg_oil_prod')), None)\n",
    "        tech_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_tech')), None)\n",
    "        water_need_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_water_need')), None)\n",
    "\n",
    "        if olive_prod_col and oil_prod_col and tech_col and water_need_col:\n",
    "            variety_data = df[[olive_prod_col, oil_prod_col, tech_col, water_need_col]]\n",
    "            variety_data = variety_data[variety_data[tech_col] != 0]  # Esclude le righe dove la tecnica è 0\n",
    "\n",
    "            if not variety_data.empty:\n",
    "                avg_olive_prod = pd.to_numeric(variety_data[olive_prod_col], errors='coerce').mean()\n",
    "                avg_oil_prod = pd.to_numeric(variety_data[oil_prod_col], errors='coerce').mean()\n",
    "                avg_water_need = pd.to_numeric(variety_data[water_need_col], errors='coerce').mean()\n",
    "                efficiency = avg_oil_prod / avg_olive_prod if avg_olive_prod > 0 else 0\n",
    "                water_efficiency = avg_oil_prod / avg_water_need if avg_water_need > 0 else 0\n",
    "\n",
    "                comparison_data.append({\n",
    "                    'Variety': variety,\n",
    "                    'Avg Olive Production (kg/ha)': avg_olive_prod,\n",
    "                    'Avg Oil Production (L/ha)': avg_oil_prod,\n",
    "                    'Avg Water Need (m³/ha)': avg_water_need,\n",
    "                    'Oil Efficiency (L/kg)': efficiency,\n",
    "                    'Water Efficiency (L oil/m³ water)': water_efficiency\n",
    "                })\n",
    "\n",
    "    return pd.DataFrame(comparison_data)\n",
    "\n",
    "\n",
    "def plot_variety_comparison(comparison_data, metric):\n",
    "    plt.figure(figsize=(12, 6))\n",
    "    bars = plt.bar(comparison_data['Variety'], comparison_data[metric])\n",
    "    plt.title(f'Comparison of {metric} across Olive Varieties')\n",
    "    plt.xlabel('Variety')\n",
    "    plt.ylabel(metric)\n",
    "    plt.xticks(rotation=45, ha='right')\n",
    "\n",
    "    for bar in bars:\n",
    "        height = bar.get_height()\n",
    "        plt.text(bar.get_x() + bar.get_width() / 2., height,\n",
    "                 f'{height:.2f}',\n",
    "                 ha='center', va='bottom')\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "def plot_efficiency_vs_production(comparison_data):\n",
    "    plt.figure(figsize=(10, 6))\n",
    "\n",
    "    plt.scatter(comparison_data['Avg Olive Production (kg/ha)'],\n",
    "                comparison_data['Oil Efficiency (L/kg)'],\n",
    "                s=100)\n",
    "\n",
    "    for i, row in comparison_data.iterrows():\n",
    "        plt.annotate(row['Variety'],\n",
    "                     (row['Avg Olive Production (kg/ha)'], row['Oil Efficiency (L/kg)']),\n",
    "                     xytext=(5, 5), textcoords='offset points')\n",
    "\n",
    "    plt.title('Oil Efficiency vs Olive Production by Variety')\n",
    "    plt.xlabel('Average Olive Production (kg/ha)')\n",
    "    plt.ylabel('Oil Efficiency (L oil / kg olives)')\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "def plot_water_efficiency_vs_production(comparison_data):\n",
    "    plt.figure(figsize=(10, 6))\n",
    "\n",
    "    plt.scatter(comparison_data['Avg Olive Production (kg/ha)'],\n",
    "                comparison_data['Water Efficiency (L oil/m³ water)'],\n",
    "                s=100)\n",
    "\n",
    "    for i, row in comparison_data.iterrows():\n",
    "        plt.annotate(row['Variety'],\n",
    "                     (row['Avg Olive Production (kg/ha)'], row['Water Efficiency (L oil/m³ water)']),\n",
    "                     xytext=(5, 5), textcoords='offset points')\n",
    "\n",
    "    plt.title('Water Efficiency vs Olive Production by Variety')\n",
    "    plt.xlabel('Average Olive Production (kg/ha)')\n",
    "    plt.ylabel('Water Efficiency (L oil / m³ water)')\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "def plot_water_need_vs_oil_production(comparison_data):\n",
    "    plt.figure(figsize=(10, 6))\n",
    "\n",
    "    plt.scatter(comparison_data['Avg Water Need (m³/ha)'],\n",
    "                comparison_data['Avg Oil Production (L/ha)'],\n",
    "                s=100)\n",
    "\n",
    "    for i, row in comparison_data.iterrows():\n",
    "        plt.annotate(row['Variety'],\n",
    "                     (row['Avg Water Need (m³/ha)'], row['Avg Oil Production (L/ha)']),\n",
    "                     xytext=(5, 5), textcoords='offset points')\n",
    "\n",
    "    plt.title('Oil Production vs Water Need by Variety')\n",
    "    plt.xlabel('Average Water Need (m³/ha)')\n",
    "    plt.ylabel('Average Oil Production (L/ha)')\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "def analyze_by_technique(simulated_data, olive_varieties):\n",
    "    # Pulisci i nomi delle colonne\n",
    "    df = simulated_data.copy()\n",
    "\n",
    "    df.columns = clean_column_names(df)\n",
    "    df = encode_techniques(df)\n",
    "    all_varieties = olive_varieties['Varietà di Olive'].unique()\n",
    "    varieties = [clean_column_name(variety) for variety in all_varieties]\n",
    "\n",
    "    technique_data = []\n",
    "\n",
    "    for variety in varieties:\n",
    "        olive_prod_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_olive_prod')), None)\n",
    "        oil_prod_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_avg_oil_prod')), None)\n",
    "        tech_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_tech')), None)\n",
    "        water_need_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_water_need')), None)\n",
    "\n",
    "        if olive_prod_col and oil_prod_col and tech_col and water_need_col:\n",
    "            variety_data = df[[olive_prod_col, oil_prod_col, tech_col, water_need_col]]\n",
    "            variety_data = variety_data[variety_data[tech_col] != 0]\n",
    "\n",
    "            if not variety_data.empty:\n",
    "                for tech in variety_data[tech_col].unique():\n",
    "                    tech_data = variety_data[variety_data[tech_col] == tech]\n",
    "\n",
    "                    avg_olive_prod = pd.to_numeric(tech_data[olive_prod_col], errors='coerce').mean()\n",
    "                    avg_oil_prod = pd.to_numeric(tech_data[oil_prod_col], errors='coerce').mean()\n",
    "                    avg_water_need = pd.to_numeric(tech_data[water_need_col], errors='coerce').mean()\n",
    "\n",
    "                    efficiency = avg_oil_prod / avg_olive_prod if avg_olive_prod > 0 else 0\n",
    "                    water_efficiency = avg_oil_prod / avg_water_need if avg_water_need > 0 else 0\n",
    "\n",
    "                    technique_data.append({\n",
    "                        'Variety': variety,\n",
    "                        'Technique': tech,\n",
    "                        'Technique String': decode_single_technique(tech),\n",
    "                        'Avg Olive Production (kg/ha)': avg_olive_prod,\n",
    "                        'Avg Oil Production (L/ha)': avg_oil_prod,\n",
    "                        'Avg Water Need (m³/ha)': avg_water_need,\n",
    "                        'Oil Efficiency (L/kg)': efficiency,\n",
    "                        'Water Efficiency (L oil/m³ water)': water_efficiency\n",
    "                    })\n",
    "\n",
    "    return pd.DataFrame(technique_data)\n",
    "\n",
    "\n",
    "# Esecuzione dell'analisi\n",
    "comparison_data = prepare_comparison_data(simulated_data, olive_varieties)\n",
    "\n",
    "# Genera i grafici\n",
    "plot_variety_comparison(comparison_data, 'Avg Olive Production (kg/ha)')\n",
    "plot_variety_comparison(comparison_data, 'Avg Oil Production (L/ha)')\n",
    "plot_variety_comparison(comparison_data, 'Avg Water Need (m³/ha)')\n",
    "plot_variety_comparison(comparison_data, 'Oil Efficiency (L/kg)')\n",
    "plot_variety_comparison(comparison_data, 'Water Efficiency (L oil/m³ water)')\n",
    "plot_efficiency_vs_production(comparison_data)\n",
    "plot_water_efficiency_vs_production(comparison_data)\n",
    "plot_water_need_vs_oil_production(comparison_data)\n",
    "\n",
    "# Analisi per tecnica\n",
    "technique_data = analyze_by_technique(simulated_data, olive_varieties)\n",
    "\n",
    "print(technique_data)\n",
    "\n",
    "# Stampa un sommario statistico\n",
    "print(\"Comparison by Variety:\")\n",
    "print(comparison_data.set_index('Variety'))\n",
    "print(\"\\nBest Varieties by Water Efficiency:\")\n",
    "print(comparison_data.sort_values('Water Efficiency (L oil/m³ water)', ascending=False).head())"
   ],
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-23T06:10:54.639402Z",
     "start_time": "2024-10-23T06:10:52.895228Z"
    },
    "execution": {
     "iopub.status.busy": "2024-10-26T15:17:21.295503Z",
     "iopub.execute_input": "2024-10-26T15:17:21.296071Z",
     "iopub.status.idle": "2024-10-26T15:17:24.809199Z",
     "shell.execute_reply.started": "2024-10-26T15:17:21.296019Z",
     "shell.execute_reply": "2024-10-26T15:17:24.808170Z"
    },
    "trusted": true
   },
   "execution_count": 5,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 1200x600 with 1 Axes>",
      "image/png": "[encoded data, 73480 characters]"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 1200x600 with 1 Axes>",
      "image/png": "[encoded data, 66164 characters]"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 1200x600 with 1 Axes>",
      "image/png": "[encoded data, 74072 characters]"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 1200x600 with 1 Axes>",
      "image/png": "[encoded data, 56492 characters]"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 1200x600 with 1 Axes>",
      "image/png": "[encoded data, 63268 characters]"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 1000x600 with 1 Axes>",
      "image/png": "[encoded data, 54112 characters]"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 1000x600 with 1 Axes>",
      "image/png": "[encoded data, 55140 characters]"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 1000x600 with 1 Axes>",
      "image/png": "[encoded data, 51684 characters]"
     },
     "metadata": {}
    },
    {
     "name": "stdout",
     "text": "               Variety  Technique Technique String  \\\n0   nocellara_delletna          1        intensiva   \n1   nocellara_delletna          3     tradizionale   \n2   nocellara_delletna          2   superintensiva   \n3              leccino          3     tradizionale   \n4              leccino          2   superintensiva   \n5              leccino          1        intensiva   \n6             frantoio          1        intensiva   \n7             frantoio          3     tradizionale   \n8             frantoio          2   superintensiva   \n9             coratina          3     tradizionale   \n10            coratina          1        intensiva   \n11            coratina          2   superintensiva   \n12           taggiasca          3     tradizionale   \n13           taggiasca          2   superintensiva   \n14           taggiasca          1        intensiva   \n15           pendolino          2   superintensiva   \n16           pendolino          3     tradizionale   \n17           pendolino          1        intensiva   \n18            moraiolo          2   superintensiva   \n19            moraiolo          1        intensiva   \n20            moraiolo          3     tradizionale   \n\n    Avg Olive Production (kg/ha)  Avg Oil Production (L/ha)  \\\n0                   13430.576751                2930.127881   \n1                    9674.594156                2111.747584   \n2                   18128.939877                3961.033183   \n3                   10863.137762                2135.295311   \n4                   20893.668331                4106.104750   \n5                   16631.230839                3269.836854   \n6                   20533.298664                5041.380818   \n7                   13751.647760                3373.784292   \n8                   24222.148425                5947.098618   \n9                   12761.677272                3272.963510   \n10                  16874.784488                4337.367800   \n11                  19198.548070                4929.046332   \n12                   6788.842503                1369.815268   \n13                  16688.411956                3374.200957   \n14                  10919.652136                2204.218746   \n15                  18649.529533                3357.886798   \n16                  10809.553659                1945.612618   \n17                  13047.902582                2347.841759   \n18                  18121.613448                3957.376155   \n19                  13526.923929                2952.003495   \n20                   8600.211719                1878.644161   \n\n    Avg Water Need (m³/ha)  Oil Efficiency (L/kg)  \\\n0             32641.443613               0.218168   \n1             33394.127828               0.218278   \n2             33749.458187               0.218492   \n3             24750.228273               0.196563   \n4             25458.172070               0.196524   \n5             25353.171069               0.196608   \n6             28798.034223               0.245522   \n7             29007.653412               0.245337   \n8             28447.820160               0.245523   \n9             39699.411494               0.256468   \n10            39355.797117               0.257032   \n11            38234.955778               0.256741   \n12            26170.119339               0.201774   \n13            26761.688671               0.202188   \n14            26200.342381               0.201858   \n15            25462.338590               0.180052   \n16            25787.938681               0.179990   \n17            24859.533372               0.179940   \n18            33127.025856               0.218379   \n19            33672.044343               0.218232   \n20            31962.026367               0.218442   \n\n    Water Efficiency (L oil/m³ water)  \n0                            0.089767  \n1                            0.063237  \n2                            0.117366  \n3                            0.086274  \n4                            0.161288  \n5                            0.128972  \n6                            0.175060  \n7                            0.116307  \n8                            0.209053  \n9                            0.082444  \n10                           0.110209  \n11                           0.128915  \n12                           0.052343  \n13                           0.126083  \n14                           0.084129  \n15                           0.131877  \n16                           0.075447  \n17                           0.094444  \n18                           0.119461  \n19                           0.087669  \n20                           0.058777  \nComparison by Variety:\n                    Avg Olive Production (kg/ha)  Avg Oil Production (L/ha)  \\\nVariety                                                                       \nnocellara_delletna                  13765.063494                3005.420035   \nleccino                             16071.119630                3158.949427   \nfrantoio                            19481.215830                4782.219901   \ncoratina                            16329.215793                4192.916534   \ntaggiasca                           11450.414937                2313.006641   \npendolino                           14306.733353                2575.267304   \nmoraiolo                            13422.146167                2930.615844   \n\n                    Avg Water Need (m³/ha)  Oil Efficiency (L/kg)  \\\nVariety                                                             \nnocellara_delletna            33261.678785               0.218337   \nleccino                       25179.459455               0.196561   \nfrantoio                      28751.682839               0.245479   \ncoratina                      39088.379510               0.256774   \ntaggiasca                     26377.580453               0.202002   \npendolino                     25371.399964               0.180004   \nmoraiolo                      32927.361924               0.218342   \n\n                    Water Efficiency (L oil/m³ water)  \nVariety                                                \nnocellara_delletna                           0.090357  \nleccino                                      0.125457  \nfrantoio                                     0.166328  \ncoratina                                     0.107268  \ntaggiasca                                    0.087688  \npendolino                                    0.101503  \nmoraiolo                                     0.089002  \n\nBest Varieties by Water Efficiency:\n              Variety  Avg Olive Production (kg/ha)  \\\n2            frantoio                  19481.215830   \n1             leccino                  16071.119630   \n3            coratina                  16329.215793   \n5           pendolino                  14306.733353   \n0  nocellara_delletna                  13765.063494   \n\n   Avg Oil Production (L/ha)  Avg Water Need (m³/ha)  Oil Efficiency (L/kg)  \\\n2                4782.219901            28751.682839               0.245479   \n1                3158.949427            25179.459455               0.196561   \n3                4192.916534            39088.379510               0.256774   \n5                2575.267304            25371.399964               0.180004   \n0                3005.420035            33261.678785               0.218337   \n\n   Water Efficiency (L oil/m³ water)  \n2                           0.166328  \n1                           0.125457  \n3                           0.107268  \n5                           0.101503  \n0                           0.090357  \n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "def preprocess_weather_data(weather_df):\n",
    "    # Calcola statistiche mensili per ogni anno\n",
    "    monthly_weather = weather_df.groupby(['year', 'month']).agg({\n",
    "        'temp': ['mean', 'min', 'max'],\n",
    "        'humidity': 'mean',\n",
    "        'precip': 'sum',\n",
    "        'windspeed': 'mean',\n",
    "        'cloudcover': 'mean',\n",
    "        'solarradiation': 'sum',\n",
    "        'solarenergy': 'sum',\n",
    "        'uvindex': 'max'\n",
    "    }).reset_index()\n",
    "\n",
    "    monthly_weather.columns = ['year', 'month'] + [f'{col[0]}_{col[1]}' for col in monthly_weather.columns[2:]]\n",
    "    return monthly_weather\n",
    "\n",
    "\n",
    "def get_growth_phase(month):\n",
    "    if month in [12, 1, 2]:\n",
    "        return 'dormancy'\n",
    "    elif month in [3, 4, 5]:\n",
    "        return 'flowering'\n",
    "    elif month in [6, 7, 8]:\n",
    "        return 'fruit_set'\n",
    "    else:\n",
    "        return 'ripening'\n",
    "\n",
    "\n",
    "def calculate_weather_effect(row, optimal_temp):\n",
    "    # Effetti base\n",
    "    temp_effect = -0.1 * (row['temp_mean'] - optimal_temp) ** 2\n",
    "    rain_effect = -0.05 * (row['precip_sum'] - 600) ** 2 / 10000\n",
    "    sun_effect = 0.1 * row['solarenergy_sum'] / 1000\n",
    "\n",
    "    # Fattori di scala basati sulla fase di crescita\n",
    "    if row['growth_phase'] == 'dormancy':\n",
    "        temp_scale = 0.5\n",
    "        rain_scale = 0.2\n",
    "        sun_scale = 0.1\n",
    "    elif row['growth_phase'] == 'flowering':\n",
    "        temp_scale = 2.0\n",
    "        rain_scale = 1.5\n",
    "        sun_scale = 1.0\n",
    "    elif row['growth_phase'] == 'fruit_set':\n",
    "        temp_scale = 1.5\n",
    "        rain_scale = 1.0\n",
    "        sun_scale = 0.8\n",
    "    else:  # ripening\n",
    "        temp_scale = 1.0\n",
    "        rain_scale = 0.5\n",
    "        sun_scale = 1.2\n",
    "\n",
    "    # Calcolo dell'effetto combinato\n",
    "    combined_effect = (\n",
    "            temp_scale * temp_effect +\n",
    "            rain_scale * rain_effect +\n",
    "            sun_scale * sun_effect\n",
    "    )\n",
    "\n",
    "    # Aggiustamenti specifici per fase\n",
    "    if row['growth_phase'] == 'flowering':\n",
    "        combined_effect -= 0.5 * max(0, row['precip_sum'] - 50)  # Penalità per pioggia eccessiva durante la fioritura\n",
    "    elif row['growth_phase'] == 'fruit_set':\n",
    "        combined_effect += 0.3 * max(0, row['temp_mean'] - (optimal_temp + 5))  # Bonus per temperature più alte durante la formazione dei frutti\n",
    "\n",
    "    return combined_effect\n",
    "\n",
    "\n",
    "def calculate_water_need(weather_data, base_need, optimal_temp):\n",
    "    # Calcola il fabbisogno idrico basato su temperatura e precipitazioni\n",
    "    temp_factor = 1 + 0.05 * (weather_data['temp_mean'] - optimal_temp)  # Aumenta del 5% per ogni grado sopra l'ottimale\n",
    "    rain_factor = 1 - 0.001 * weather_data['precip_sum']  # Diminuisce leggermente con l'aumentare delle precipitazioni\n",
    "    return base_need * temp_factor * rain_factor\n",
    "\n",
    "\n",
    "def clean_column_name(name):\n",
    "    # Rimuove caratteri speciali e spazi, converte in snake_case e abbrevia\n",
    "    name = re.sub(r'[^a-zA-Z0-9\\s]', '', name)  # Rimuove caratteri speciali\n",
    "    name = name.lower().replace(' ', '_')  # Converte in snake_case\n",
    "\n",
    "    # Abbreviazioni comuni\n",
    "    abbreviations = {\n",
    "        'production': 'prod',\n",
    "        'percentage': 'pct',\n",
    "        'hectare': 'ha',\n",
    "        'tonnes': 't',\n",
    "        'litres': 'l',\n",
    "        'minimum': 'min',\n",
    "        'maximum': 'max',\n",
    "        'average': 'avg'\n",
    "    }\n",
    "\n",
    "    for full, abbr in abbreviations.items():\n",
    "        name = name.replace(full, abbr)\n",
    "\n",
    "    return name\n",
    "\n",
    "\n",
    "def create_technique_mapping(olive_varieties, mapping_path='models/technique_mapping.joblib'):\n",
    "    # Estrai tutte le tecniche uniche dal dataset e convertile in lowercase\n",
    "    all_techniques = olive_varieties['Tecnica di Coltivazione'].str.lower().unique()\n",
    "\n",
    "    # Crea il mapping partendo da 1\n",
    "    technique_mapping = {tech: i + 1 for i, tech in enumerate(sorted(all_techniques))}\n",
    "\n",
    "    # Salva il mapping\n",
    "    os.makedirs(os.path.dirname(mapping_path), exist_ok=True)\n",
    "    joblib.dump(technique_mapping, mapping_path)\n",
    "\n",
    "    return technique_mapping\n",
    "\n",
    "\n",
    "def encode_techniques(df, mapping_path='models/technique_mapping.joblib'):\n",
    "    if not os.path.exists(mapping_path):\n",
    "        raise FileNotFoundError(f\"Mapping not found at {mapping_path}. Run create_technique_mapping first.\")\n",
    "\n",
    "    technique_mapping = joblib.load(mapping_path)\n",
    "\n",
    "    # Trova tutte le colonne delle tecniche\n",
    "    tech_columns = [col for col in df.columns if col.endswith('_tech')]\n",
    "\n",
    "    # Applica il mapping a tutte le colonne delle tecniche\n",
    "    for col in tech_columns:\n",
    "        df[col] = df[col].str.lower().map(technique_mapping).fillna(0).astype(int)\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def decode_techniques(df, mapping_path='models/technique_mapping.joblib'):\n",
    "    if not os.path.exists(mapping_path):\n",
    "        raise FileNotFoundError(f\"Mapping not found at {mapping_path}\")\n",
    "\n",
    "    technique_mapping = joblib.load(mapping_path)\n",
    "    reverse_mapping = {v: k for k, v in technique_mapping.items()}\n",
    "    reverse_mapping[0] = ''  # Aggiungi un mapping per 0 a stringa vuota\n",
    "\n",
    "    # Trova tutte le colonne delle tecniche\n",
    "    tech_columns = [col for col in df.columns if col.endswith('_tech')]\n",
    "\n",
    "    # Applica il reverse mapping a tutte le colonne delle tecniche\n",
    "    for col in tech_columns:\n",
    "        df[col] = df[col].map(reverse_mapping)\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def decode_single_technique(technique_value, mapping_path='models/technique_mapping.joblib'):\n",
    "    if not os.path.exists(mapping_path):\n",
    "        raise FileNotFoundError(f\"Mapping not found at {mapping_path}\")\n",
    "\n",
    "    technique_mapping = joblib.load(mapping_path)\n",
    "    reverse_mapping = {v: k for k, v in technique_mapping.items()}\n",
    "    reverse_mapping[0] = ''\n",
    "\n",
    "    return reverse_mapping.get(technique_value, '')\n",
    "\n",
    "\n",
    "def simulate_olive_production(weather_data, olive_varieties, num_simulations=5, random_seed=None):\n",
    "    \"\"\"\n",
    "    Simula la produzione di olive per diverse zone e varietà, considerando variazioni meteo specifiche per zona.\n",
    "    \n",
    "    Args:\n",
    "        weather_data: DataFrame con dati meteorologici storici\n",
    "        olive_varieties: DataFrame con informazioni sulle varietà di olive\n",
    "        num_simulations: Numero di simulazioni/zone da generare\n",
    "        random_seed: Seme per la riproducibilità dei risultati\n",
    "    \n",
    "    Returns:\n",
    "        DataFrame con i risultati delle simulazioni per tutte le zone\n",
    "    \"\"\"\n",
    "    if random_seed is not None:\n",
    "        np.random.seed(random_seed)\n",
    "\n",
    "    create_technique_mapping(olive_varieties)\n",
    "    monthly_weather = preprocess_weather_data(weather_data)\n",
    "    all_results = []\n",
    "\n",
    "    # Preparazione dati varietà\n",
    "    all_varieties = olive_varieties['Varietà di Olive'].unique()\n",
    "    variety_techniques = {\n",
    "        variety: olive_varieties[olive_varieties['Varietà di Olive'] == variety]['Tecnica di Coltivazione'].unique()\n",
    "        for variety in all_varieties\n",
    "    }\n",
    "\n",
    "    # Per ogni simulazione (anno)\n",
    "    for sim in range(num_simulations):\n",
    "        print(f'Simulation {sim}')\n",
    "        # Seleziona anno di base per questa simulazione\n",
    "        selected_year = np.random.choice(monthly_weather['year'].unique())\n",
    "        base_weather = monthly_weather[monthly_weather['year'] == selected_year].copy()\n",
    "        base_weather.loc[:, 'growth_phase'] = base_weather['month'].apply(get_growth_phase)\n",
    "\n",
    "        # Per ogni zona nella simulazione\n",
    "        for zone in range(num_simulations):\n",
    "            # Crea una copia dei dati meteo per questa zona specifica\n",
    "            zone_weather = base_weather.copy()\n",
    "\n",
    "            # Genera variazioni meteorologiche specifiche per questa zona\n",
    "            zone_weather['temp_mean'] *= np.random.uniform(0.95, 1.05, len(zone_weather))\n",
    "            zone_weather['precip_sum'] *= np.random.uniform(0.9, 1.1, len(zone_weather))\n",
    "            zone_weather['solarenergy_sum'] *= np.random.uniform(0.95, 1.05, len(zone_weather))\n",
    "\n",
    "            # Genera caratteristiche specifiche della zona\n",
    "            num_varieties = np.random.randint(1, 4)  # 1-3 varietà per zona\n",
    "            selected_varieties = np.random.choice(all_varieties, size=num_varieties, replace=False)\n",
    "            hectares = np.random.uniform(1, 10)  # Dimensione del terreno\n",
    "            percentages = np.random.dirichlet(np.ones(num_varieties))  # Distribuzione delle varietà\n",
    "\n",
    "            # Inizializzazione contatori annuali\n",
    "            annual_production = 0\n",
    "            annual_min_oil = 0\n",
    "            annual_max_oil = 0\n",
    "            annual_avg_oil = 0\n",
    "            annual_water_need = 0\n",
    "\n",
    "            # Inizializzazione dizionario dati varietà\n",
    "            variety_data = {clean_column_name(variety): {\n",
    "                'tech': '',\n",
    "                'pct': 0,\n",
    "                'prod_t_ha': 0,\n",
    "                'oil_prod_t_ha': 0,\n",
    "                'oil_prod_l_ha': 0,\n",
    "                'min_yield_pct': 0,\n",
    "                'max_yield_pct': 0,\n",
    "                'min_oil_prod_l_ha': 0,\n",
    "                'max_oil_prod_l_ha': 0,\n",
    "                'avg_oil_prod_l_ha': 0,\n",
    "                'l_per_t': 0,\n",
    "                'min_l_per_t': 0,\n",
    "                'max_l_per_t': 0,\n",
    "                'avg_l_per_t': 0,\n",
    "                'olive_prod': 0,\n",
    "                'min_oil_prod': 0,\n",
    "                'max_oil_prod': 0,\n",
    "                'avg_oil_prod': 0,\n",
    "                'water_need': 0\n",
    "            } for variety in all_varieties}\n",
    "\n",
    "            # Simula produzione per ogni varietà selezionata\n",
    "            for i, variety in enumerate(selected_varieties):\n",
    "                # Seleziona tecnica di coltivazione casuale per questa varietà\n",
    "                technique = np.random.choice(variety_techniques[variety])\n",
    "                percentage = percentages[i]\n",
    "\n",
    "                # Ottieni informazioni specifiche della varietà\n",
    "                variety_info = olive_varieties[\n",
    "                    (olive_varieties['Varietà di Olive'] == variety) &\n",
    "                    (olive_varieties['Tecnica di Coltivazione'] == technique)\n",
    "                    ].iloc[0]\n",
    "\n",
    "                # Calcola produzione base con variabilità\n",
    "                base_production = variety_info['Produzione (tonnellate/ettaro)'] * 1000 * percentage * hectares / 12\n",
    "                base_production *= np.random.uniform(0.9, 1.1)  # Aggiungi variabilità alla produzione base\n",
    "\n",
    "                # Calcola effetti meteo sulla produzione\n",
    "                weather_effect = zone_weather.apply(\n",
    "                    lambda row: calculate_weather_effect(row, variety_info['Temperatura Ottimale']),\n",
    "                    axis=1\n",
    "                )\n",
    "                monthly_production = base_production * (1 + weather_effect / 10000)\n",
    "                monthly_production *= np.random.uniform(0.95, 1.05, len(zone_weather))\n",
    "\n",
    "                # Calcola produzione annuale per questa varietà\n",
    "                annual_variety_production = monthly_production.sum()\n",
    "\n",
    "                # Calcola rese di olio con variabilità\n",
    "                min_yield_factor = np.random.uniform(0.95, 1.05)\n",
    "                max_yield_factor = np.random.uniform(0.95, 1.05)\n",
    "                avg_yield_factor = (min_yield_factor + max_yield_factor) / 2\n",
    "\n",
    "                min_oil_production = annual_variety_production * variety_info['Min Litri per Tonnellata'] / 1000 * min_yield_factor\n",
    "                max_oil_production = annual_variety_production * variety_info['Max Litri per Tonnellata'] / 1000 * max_yield_factor\n",
    "                avg_oil_production = annual_variety_production * variety_info['Media Litri per Tonnellata'] / 1000 * avg_yield_factor\n",
    "\n",
    "                # Calcola fabbisogno idrico\n",
    "                base_water_need = (\n",
    "                                          variety_info['Fabbisogno Acqua Primavera (m³/ettaro)'] +\n",
    "                                          variety_info['Fabbisogno Acqua Estate (m³/ettaro)'] +\n",
    "                                          variety_info['Fabbisogno Acqua Autunno (m³/ettaro)'] +\n",
    "                                          variety_info['Fabbisogno Acqua Inverno (m³/ettaro)']\n",
    "                                  ) / 4  # Media stagionale\n",
    "\n",
    "                monthly_water_need = zone_weather.apply(\n",
    "                    lambda row: calculate_water_need(row, base_water_need, variety_info['Temperatura Ottimale']),\n",
    "                    axis=1\n",
    "                )\n",
    "                monthly_water_need *= np.random.uniform(0.95, 1.05, len(monthly_water_need))\n",
    "                annual_variety_water_need = monthly_water_need.sum() * percentage * hectares\n",
    "\n",
    "                # Aggiorna totali annuali\n",
    "                annual_production += annual_variety_production\n",
    "                annual_min_oil += min_oil_production\n",
    "                annual_max_oil += max_oil_production\n",
    "                annual_avg_oil += avg_oil_production\n",
    "                annual_water_need += annual_variety_water_need\n",
    "\n",
    "                # Aggiorna dati varietà\n",
    "                clean_variety = clean_column_name(variety)\n",
    "                variety_data[clean_variety].update({\n",
    "                    'tech': clean_column_name(technique),\n",
    "                    'pct': percentage,\n",
    "                    'prod_t_ha': variety_info['Produzione (tonnellate/ettaro)'] * np.random.uniform(0.95, 1.05),\n",
    "                    'oil_prod_t_ha': variety_info['Produzione Olio (tonnellate/ettaro)'] * np.random.uniform(0.95, 1.05),\n",
    "                    'oil_prod_l_ha': variety_info['Produzione Olio (litri/ettaro)'] * np.random.uniform(0.95, 1.05),\n",
    "                    'min_yield_pct': variety_info['Min % Resa'] * min_yield_factor,\n",
    "                    'max_yield_pct': variety_info['Max % Resa'] * max_yield_factor,\n",
    "                    'min_oil_prod_l_ha': variety_info['Min Produzione Olio (litri/ettaro)'] * min_yield_factor,\n",
    "                    'max_oil_prod_l_ha': variety_info['Max Produzione Olio (litri/ettaro)'] * max_yield_factor,\n",
    "                    'avg_oil_prod_l_ha': variety_info['Media Produzione Olio (litri/ettaro)'] * avg_yield_factor,\n",
    "                    'l_per_t': variety_info['Litri per Tonnellata'] * np.random.uniform(0.98, 1.02),\n",
    "                    'min_l_per_t': variety_info['Min Litri per Tonnellata'] * min_yield_factor,\n",
    "                    'max_l_per_t': variety_info['Max Litri per Tonnellata'] * max_yield_factor,\n",
    "                    'avg_l_per_t': variety_info['Media Litri per Tonnellata'] * avg_yield_factor,\n",
    "                    'olive_prod': annual_variety_production,\n",
    "                    'min_oil_prod': min_oil_production,\n",
    "                    'max_oil_prod': max_oil_production,\n",
    "                    'avg_oil_prod': avg_oil_production,\n",
    "                    'water_need': annual_variety_water_need\n",
    "                })\n",
    "\n",
    "            # Appiattisci i dati delle varietà per il DataFrame finale\n",
    "            flattened_variety_data = {\n",
    "                f'{variety}_{key}': value\n",
    "                for variety, data in variety_data.items()\n",
    "                for key, value in data.items()\n",
    "            }\n",
    "\n",
    "            # Aggiungi il risultato con tutti i dati della zona\n",
    "            all_results.append({\n",
    "                'simulation_id': sim + 1,\n",
    "                'zone_id': zone + 1,\n",
    "                'year': selected_year,\n",
    "                'temp_mean': zone_weather['temp_mean'].mean(),\n",
    "                'precip_sum': zone_weather['precip_sum'].sum(),\n",
    "                'solar_energy_sum': zone_weather['solarenergy_sum'].sum(),\n",
    "                'ha': hectares,\n",
    "                'zone': f\"zone_{zone + 1}\",\n",
    "                'olive_prod': annual_production,\n",
    "                'min_oil_prod': annual_min_oil,\n",
    "                'max_oil_prod': annual_max_oil,\n",
    "                'avg_oil_prod': annual_avg_oil,\n",
    "                'total_water_need': annual_water_need,\n",
    "                **flattened_variety_data\n",
    "            })\n",
    "\n",
    "    # Crea DataFrame finale con tutti i risultati\n",
    "    df_results = pd.DataFrame(all_results)\n",
    "    return df_results\n",
    "\n",
    "\n",
    "olive_varieties = pd.read_parquet(\"./data/olive_varieties.parquet\")\n",
    "\n",
    "weather_data = pd.read_parquet('./data/weather_data_complete.parquet')\n",
    "\n",
    "simulated_data = simulate_olive_production(weather_data, olive_varieties, 1000, random_state_value)\n",
    "\n",
    "simulated_data.to_parquet(\"./data/simulated_data.parquet\")\n",
    "\n",
    "\n",
    "# Funzione per visualizzare il mapping delle tecniche\n",
    "def print_technique_mapping(mapping_path='./data/models/technique_mapping.joblib'):\n",
    "    if not os.path.exists(mapping_path):\n",
    "        print(\"Mapping file not found.\")\n",
    "        return\n",
    "\n",
    "    mapping = joblib.load(mapping_path)\n",
    "    print(\"Technique Mapping:\")\n",
    "    for technique, code in mapping.items():\n",
    "        print(f\"{technique}: {code}\")\n",
    "\n",
    "\n",
    "# Visualizza il mapping delle tecniche\n",
    "print_technique_mapping()"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-10-26T16:00:01.832064Z",
     "iopub.execute_input": "2024-10-26T16:00:01.832611Z"
    },
    "trusted": true,
    "ExecuteTime": {
     "end_time": "2024-10-26T18:10:50.923580Z",
     "start_time": "2024-10-26T16:43:33.447610Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Simulation 0\n",
      "Simulation 1\n",
      "Simulation 2\n",
      "Simulation 3\n",
      "Simulation 4\n",
      "Simulation 5\n",
      "Simulation 6\n",
      "Simulation 7\n",
      "Simulation 8\n",
      "Simulation 9\n",
      "Simulation 10\n",
      "Simulation 11\n",
      "Simulation 12\n",
      "Simulation 13\n",
      "Simulation 14\n",
      "Simulation 15\n",
      "Simulation 16\n",
      "Simulation 17\n",
      "Simulation 18\n",
      "Simulation 19\n",
      "Simulation 20\n",
      "Simulation 21\n",
      "Simulation 22\n",
      "Simulation 23\n",
      "Simulation 24\n",
      "Simulation 25\n",
      "Simulation 26\n",
      "Simulation 27\n",
      "Simulation 28\n",
      "Simulation 29\n",
      "Simulation 30\n",
      "Simulation 31\n",
      "Simulation 32\n",
      "Simulation 33\n",
      "Simulation 34\n",
      "Simulation 35\n",
      "Simulation 36\n",
      "Simulation 37\n",
      "Simulation 38\n",
      "Simulation 39\n",
      "Simulation 40\n",
      "Simulation 41\n",
      "Simulation 42\n",
      "Simulation 43\n",
      "Simulation 44\n",
      "Simulation 45\n",
      "Simulation 46\n",
      "Simulation 47\n",
      "Simulation 48\n",
      "Simulation 49\n",
      "Simulation 50\n",
      "Simulation 51\n",
      "Simulation 52\n",
      "Simulation 53\n",
      "Simulation 54\n",
      "Simulation 55\n",
      "Simulation 56\n",
      "Simulation 57\n",
      "Simulation 58\n",
      "Simulation 59\n",
      "Simulation 60\n",
      "Simulation 61\n",
      "Simulation 62\n",
      "Simulation 63\n",
      "Simulation 64\n",
      "Simulation 65\n",
      "Simulation 66\n",
      "Simulation 67\n",
      "Simulation 68\n",
      "Simulation 69\n",
      "Simulation 70\n",
      "Simulation 71\n",
      "Simulation 72\n",
      "Simulation 73\n",
      "Simulation 74\n",
      "Simulation 75\n",
      "Simulation 76\n",
      "Simulation 77\n",
      "Simulation 78\n",
      "Simulation 79\n",
      "Simulation 80\n",
      "Simulation 81\n",
      "Simulation 82\n",
      "Simulation 83\n",
      "Simulation 84\n",
      "Simulation 85\n",
      "Simulation 86\n",
      "Simulation 87\n",
      "Simulation 88\n",
      "Simulation 89\n",
      "Simulation 90\n",
      "Simulation 91\n",
      "Simulation 92\n",
      "Simulation 93\n",
      "Simulation 94\n",
      "Simulation 95\n",
      "Simulation 96\n",
      "Simulation 97\n",
      "Simulation 98\n",
      "Simulation 99\n",
      "Simulation 100\n",
      "Simulation 101\n",
      "Simulation 102\n",
      "Simulation 103\n",
      "Simulation 104\n",
      "Simulation 105\n",
      "Simulation 106\n",
      "Simulation 107\n",
      "Simulation 108\n",
      "Simulation 109\n",
      "Simulation 110\n",
      "Simulation 111\n",
      "Simulation 112\n",
      "Simulation 113\n",
      "Simulation 114\n",
      "Simulation 115\n",
      "Simulation 116\n",
      "Simulation 117\n",
      "Simulation 118\n",
      "Simulation 119\n",
      "Simulation 120\n",
      "Simulation 121\n",
      "Simulation 122\n",
      "Simulation 123\n",
      "Simulation 124\n",
      "Simulation 125\n",
      "Simulation 126\n",
      "Simulation 127\n",
      "Simulation 128\n",
      "Simulation 129\n",
      "Simulation 130\n",
      "Simulation 131\n",
      "Simulation 132\n",
      "Simulation 133\n",
      "Simulation 134\n",
      "Simulation 135\n",
      "Simulation 136\n",
      "Simulation 137\n",
      "Simulation 138\n",
      "Simulation 139\n",
      "Simulation 140\n",
      "Simulation 141\n",
      "Simulation 142\n",
      "Simulation 143\n",
      "Simulation 144\n",
      "Simulation 145\n",
      "Simulation 146\n",
      "Simulation 147\n",
      "Simulation 148\n",
      "Simulation 149\n",
      "Simulation 150\n",
      "Simulation 151\n",
      "Simulation 152\n",
      "Simulation 153\n",
      "Simulation 154\n",
      "Simulation 155\n",
      "Simulation 156\n",
      "Simulation 157\n",
      "Simulation 158\n",
      "Simulation 159\n",
      "Simulation 160\n",
      "Simulation 161\n",
      "Simulation 162\n",
      "Simulation 163\n",
      "Simulation 164\n",
      "Simulation 165\n",
      "Simulation 166\n",
      "Simulation 167\n",
      "Simulation 168\n",
      "Simulation 169\n",
      "Simulation 170\n",
      "Simulation 171\n",
      "Simulation 172\n",
      "Simulation 173\n",
      "Simulation 174\n",
      "Simulation 175\n",
      "Simulation 176\n",
      "Simulation 177\n",
      "Simulation 178\n",
      "Simulation 179\n",
      "Simulation 180\n",
      "Simulation 181\n",
      "Simulation 182\n",
      "Simulation 183\n",
      "Simulation 184\n",
      "Simulation 185\n",
      "Simulation 186\n",
      "Simulation 187\n",
      "Simulation 188\n",
      "Simulation 189\n",
      "Simulation 190\n",
      "Simulation 191\n",
      "Simulation 192\n",
      "Simulation 193\n",
      "Simulation 194\n",
      "Simulation 195\n",
      "Simulation 196\n",
      "Simulation 197\n",
      "Simulation 198\n",
      "Simulation 199\n",
      "Simulation 200\n",
      "Simulation 201\n",
      "Simulation 202\n",
      "Simulation 203\n",
      "Simulation 204\n",
      "Simulation 205\n",
      "Simulation 206\n",
      "Simulation 207\n",
      "Simulation 208\n",
      "Simulation 209\n",
      "Simulation 210\n",
      "Simulation 211\n",
      "Simulation 212\n",
      "Simulation 213\n",
      "Simulation 214\n",
      "Simulation 215\n",
      "Simulation 216\n",
      "Simulation 217\n",
      "Simulation 218\n",
      "Simulation 219\n",
      "Simulation 220\n",
      "Simulation 221\n",
      "Simulation 222\n",
      "Simulation 223\n",
      "Simulation 224\n",
      "Simulation 225\n",
      "Simulation 226\n",
      "Simulation 227\n",
      "Simulation 228\n",
      "Simulation 229\n",
      "Simulation 230\n",
      "Simulation 231\n",
      "Simulation 232\n",
      "Simulation 233\n",
      "Simulation 234\n",
      "Simulation 235\n",
      "Simulation 236\n",
      "Simulation 237\n",
      "Simulation 238\n",
      "Simulation 239\n",
      "Simulation 240\n",
      "Simulation 241\n",
      "Simulation 242\n",
      "Simulation 243\n",
      "Simulation 244\n",
      "Simulation 245\n",
      "Simulation 246\n",
      "Simulation 247\n",
      "Simulation 248\n",
      "Simulation 249\n",
      "Simulation 250\n",
      "Simulation 251\n",
      "Simulation 252\n",
      "Simulation 253\n",
      "Simulation 254\n",
      "Simulation 255\n",
      "Simulation 256\n",
      "Simulation 257\n",
      "Simulation 258\n",
      "Simulation 259\n",
      "Simulation 260\n",
      "Simulation 261\n",
      "Simulation 262\n",
      "Simulation 263\n",
      "Simulation 264\n",
      "Simulation 265\n",
      "Simulation 266\n",
      "Simulation 267\n",
      "Simulation 268\n",
      "Simulation 269\n",
      "Simulation 270\n",
      "Simulation 271\n",
      "Simulation 272\n",
      "Simulation 273\n",
      "Simulation 274\n",
      "Simulation 275\n",
      "Simulation 276\n",
      "Simulation 277\n",
      "Simulation 278\n",
      "Simulation 279\n",
      "Simulation 280\n",
      "Simulation 281\n",
      "Simulation 282\n",
      "Simulation 283\n",
      "Simulation 284\n",
      "Simulation 285\n",
      "Simulation 286\n",
      "Simulation 287\n",
      "Simulation 288\n",
      "Simulation 289\n",
      "Simulation 290\n",
      "Simulation 291\n",
      "Simulation 292\n",
      "Simulation 293\n",
      "Simulation 294\n",
      "Simulation 295\n",
      "Simulation 296\n",
      "Simulation 297\n",
      "Simulation 298\n",
      "Simulation 299\n",
      "Simulation 300\n",
      "Simulation 301\n",
      "Simulation 302\n",
      "Simulation 303\n",
      "Simulation 304\n",
      "Simulation 305\n",
      "Simulation 306\n",
      "Simulation 307\n",
      "Simulation 308\n",
      "Simulation 309\n",
      "Simulation 310\n",
      "Simulation 311\n",
      "Simulation 312\n",
      "Simulation 313\n",
      "Simulation 314\n",
      "Simulation 315\n",
      "Simulation 316\n",
      "Simulation 317\n",
      "Simulation 318\n",
      "Simulation 319\n",
      "Simulation 320\n",
      "Simulation 321\n",
      "Simulation 322\n",
      "Simulation 323\n",
      "Simulation 324\n",
      "Simulation 325\n",
      "Simulation 326\n",
      "Simulation 327\n",
      "Simulation 328\n",
      "Simulation 329\n",
      "Simulation 330\n",
      "Simulation 331\n",
      "Simulation 332\n",
      "Simulation 333\n",
      "Simulation 334\n",
      "Simulation 335\n",
      "Simulation 336\n",
      "Simulation 337\n",
      "Simulation 338\n",
      "Simulation 339\n",
      "Simulation 340\n",
      "Simulation 341\n",
      "Simulation 342\n",
      "Simulation 343\n",
      "Simulation 344\n",
      "Simulation 345\n",
      "Simulation 346\n",
      "Simulation 347\n",
      "Simulation 348\n",
      "Simulation 349\n",
      "Simulation 350\n",
      "Simulation 351\n",
      "Simulation 352\n",
      "Simulation 353\n",
      "Simulation 354\n",
      "Simulation 355\n",
      "Simulation 356\n",
      "Simulation 357\n",
      "Simulation 358\n",
      "Simulation 359\n",
      "Simulation 360\n",
      "Simulation 361\n",
      "Simulation 362\n",
      "Simulation 363\n",
      "Simulation 364\n",
      "Simulation 365\n",
      "Simulation 366\n",
      "Simulation 367\n",
      "Simulation 368\n",
      "Simulation 369\n",
      "Simulation 370\n",
      "Simulation 371\n",
      "Simulation 372\n",
      "Simulation 373\n",
      "Simulation 374\n",
      "Simulation 375\n",
      "Simulation 376\n",
      "Simulation 377\n",
      "Simulation 378\n",
      "Simulation 379\n",
      "Simulation 380\n",
      "Simulation 381\n",
      "Simulation 382\n",
      "Simulation 383\n",
      "Simulation 384\n",
      "Simulation 385\n",
      "Simulation 386\n",
      "Simulation 387\n",
      "Simulation 388\n",
      "Simulation 389\n",
      "Simulation 390\n",
      "Simulation 391\n",
      "Simulation 392\n",
      "Simulation 393\n",
      "Simulation 394\n",
      "Simulation 395\n",
      "Simulation 396\n",
      "Simulation 397\n",
      "Simulation 398\n",
      "Simulation 399\n",
      "Simulation 400\n",
      "Simulation 401\n",
      "Simulation 402\n",
      "Simulation 403\n",
      "Simulation 404\n",
      "Simulation 405\n",
      "Simulation 406\n",
      "Simulation 407\n",
      "Simulation 408\n",
      "Simulation 409\n",
      "Simulation 410\n",
      "Simulation 411\n",
      "Simulation 412\n",
      "Simulation 413\n",
      "Simulation 414\n",
      "Simulation 415\n",
      "Simulation 416\n",
      "Simulation 417\n",
      "Simulation 418\n",
      "Simulation 419\n",
      "Simulation 420\n",
      "Simulation 421\n",
      "Simulation 422\n",
      "Simulation 423\n",
      "Simulation 424\n",
      "Simulation 425\n",
      "Simulation 426\n",
      "Simulation 427\n",
      "Simulation 428\n",
      "Simulation 429\n",
      "Simulation 430\n",
      "Simulation 431\n",
      "Simulation 432\n",
      "Simulation 433\n",
      "Simulation 434\n",
      "Simulation 435\n",
      "Simulation 436\n",
      "Simulation 437\n",
      "Simulation 438\n",
      "Simulation 439\n",
      "Simulation 440\n",
      "Simulation 441\n",
      "Simulation 442\n",
      "Simulation 443\n",
      "Simulation 444\n",
      "Simulation 445\n",
      "Simulation 446\n",
      "Simulation 447\n",
      "Simulation 448\n",
      "Simulation 449\n",
      "Simulation 450\n",
      "Simulation 451\n",
      "Simulation 452\n",
      "Simulation 453\n",
      "Simulation 454\n",
      "Simulation 455\n",
      "Simulation 456\n",
      "Simulation 457\n",
      "Simulation 458\n",
      "Simulation 459\n",
      "Simulation 460\n",
      "Simulation 461\n",
      "Simulation 462\n",
      "Simulation 463\n",
      "Simulation 464\n",
      "Simulation 465\n",
      "Simulation 466\n",
      "Simulation 467\n",
      "Simulation 468\n",
      "Simulation 469\n",
      "Simulation 470\n",
      "Simulation 471\n",
      "Simulation 472\n",
      "Simulation 473\n",
      "Simulation 474\n",
      "Simulation 475\n",
      "Simulation 476\n",
      "Simulation 477\n",
      "Simulation 478\n",
      "Simulation 479\n",
      "Simulation 480\n",
      "Simulation 481\n",
      "Simulation 482\n",
      "Simulation 483\n",
      "Simulation 484\n",
      "Simulation 485\n",
      "Simulation 486\n",
      "Simulation 487\n",
      "Simulation 488\n",
      "Simulation 489\n",
      "Simulation 490\n",
      "Simulation 491\n",
      "Simulation 492\n",
      "Simulation 493\n",
      "Simulation 494\n",
      "Simulation 495\n",
      "Simulation 496\n",
      "Simulation 497\n",
      "Simulation 498\n",
      "Simulation 499\n",
      "Simulation 500\n",
      "Simulation 501\n",
      "Simulation 502\n",
      "Simulation 503\n",
      "Simulation 504\n",
      "Simulation 505\n",
      "Simulation 506\n",
      "Simulation 507\n",
      "Simulation 508\n",
      "Simulation 509\n",
      "Simulation 510\n",
      "Simulation 511\n",
      "Simulation 512\n",
      "Simulation 513\n",
      "Simulation 514\n",
      "Simulation 515\n",
      "Simulation 516\n",
      "Simulation 517\n",
      "Simulation 518\n",
      "Simulation 519\n",
      "Simulation 520\n",
      "Simulation 521\n",
      "Simulation 522\n",
      "Simulation 523\n",
      "Simulation 524\n",
      "Simulation 525\n",
      "Simulation 526\n",
      "Simulation 527\n",
      "Simulation 528\n",
      "Simulation 529\n",
      "Simulation 530\n",
      "Simulation 531\n",
      "Simulation 532\n",
      "Simulation 533\n",
      "Simulation 534\n",
      "Simulation 535\n",
      "Simulation 536\n",
      "Simulation 537\n",
      "Simulation 538\n",
      "Simulation 539\n",
      "Simulation 540\n",
      "Simulation 541\n",
      "Simulation 542\n",
      "Simulation 543\n",
      "Simulation 544\n",
      "Simulation 545\n",
      "Simulation 546\n",
      "Simulation 547\n",
      "Simulation 548\n",
      "Simulation 549\n",
      "Simulation 550\n",
      "Simulation 551\n",
      "Simulation 552\n",
      "Simulation 553\n",
      "Simulation 554\n",
      "Simulation 555\n",
      "Simulation 556\n",
      "Simulation 557\n",
      "Simulation 558\n",
      "Simulation 559\n",
      "Simulation 560\n",
      "Simulation 561\n",
      "Simulation 562\n",
      "Simulation 563\n",
      "Simulation 564\n",
      "Simulation 565\n",
      "Simulation 566\n",
      "Simulation 567\n",
      "Simulation 568\n",
      "Simulation 569\n",
      "Simulation 570\n",
      "Simulation 571\n",
      "Simulation 572\n",
      "Simulation 573\n",
      "Simulation 574\n",
      "Simulation 575\n",
      "Simulation 576\n",
      "Simulation 577\n",
      "Simulation 578\n",
      "Simulation 579\n",
      "Simulation 580\n",
      "Simulation 581\n",
      "Simulation 582\n",
      "Simulation 583\n",
      "Simulation 584\n",
      "Simulation 585\n",
      "Simulation 586\n",
      "Simulation 587\n",
      "Simulation 588\n",
      "Simulation 589\n",
      "Simulation 590\n",
      "Simulation 591\n",
      "Simulation 592\n",
      "Simulation 593\n",
      "Simulation 594\n",
      "Simulation 595\n",
      "Simulation 596\n",
      "Simulation 597\n",
      "Simulation 598\n",
      "Simulation 599\n",
      "Simulation 600\n",
      "Simulation 601\n",
      "Simulation 602\n",
      "Simulation 603\n",
      "Simulation 604\n",
      "Simulation 605\n",
      "Simulation 606\n",
      "Simulation 607\n",
      "Simulation 608\n",
      "Simulation 609\n",
      "Simulation 610\n",
      "Simulation 611\n",
      "Simulation 612\n",
      "Simulation 613\n",
      "Simulation 614\n",
      "Simulation 615\n",
      "Simulation 616\n",
      "Simulation 617\n",
      "Simulation 618\n",
      "Simulation 619\n",
      "Simulation 620\n",
      "Simulation 621\n",
      "Simulation 622\n",
      "Simulation 623\n",
      "Simulation 624\n",
      "Simulation 625\n",
      "Simulation 626\n",
      "Simulation 627\n",
      "Simulation 628\n",
      "Simulation 629\n",
      "Simulation 630\n",
      "Simulation 631\n",
      "Simulation 632\n",
      "Simulation 633\n",
      "Simulation 634\n",
      "Simulation 635\n",
      "Simulation 636\n",
      "Simulation 637\n",
      "Simulation 638\n",
      "Simulation 639\n",
      "Simulation 640\n",
      "Simulation 641\n",
      "Simulation 642\n",
      "Simulation 643\n",
      "Simulation 644\n",
      "Simulation 645\n",
      "Simulation 646\n",
      "Simulation 647\n",
      "Simulation 648\n",
      "Simulation 649\n",
      "Simulation 650\n",
      "Simulation 651\n",
      "Simulation 652\n",
      "Simulation 653\n",
      "Simulation 654\n",
      "Simulation 655\n",
      "Simulation 656\n",
      "Simulation 657\n",
      "Simulation 658\n",
      "Simulation 659\n",
      "Simulation 660\n",
      "Simulation 661\n",
      "Simulation 662\n",
      "Simulation 663\n",
      "Simulation 664\n",
      "Simulation 665\n",
      "Simulation 666\n",
      "Simulation 667\n",
      "Simulation 668\n",
      "Simulation 669\n",
      "Simulation 670\n",
      "Simulation 671\n",
      "Simulation 672\n",
      "Simulation 673\n",
      "Simulation 674\n",
      "Simulation 675\n",
      "Simulation 676\n",
      "Simulation 677\n",
      "Simulation 678\n",
      "Simulation 679\n",
      "Simulation 680\n",
      "Simulation 681\n",
      "Simulation 682\n",
      "Simulation 683\n",
      "Simulation 684\n",
      "Simulation 685\n",
      "Simulation 686\n",
      "Simulation 687\n",
      "Simulation 688\n",
      "Simulation 689\n",
      "Simulation 690\n",
      "Simulation 691\n",
      "Simulation 692\n",
      "Simulation 693\n",
      "Simulation 694\n",
      "Simulation 695\n",
      "Simulation 696\n",
      "Simulation 697\n",
      "Simulation 698\n",
      "Simulation 699\n",
      "Simulation 700\n",
      "Simulation 701\n",
      "Simulation 702\n",
      "Simulation 703\n",
      "Simulation 704\n",
      "Simulation 705\n",
      "Simulation 706\n",
      "Simulation 707\n",
      "Simulation 708\n",
      "Simulation 709\n",
      "Simulation 710\n",
      "Simulation 711\n",
      "Simulation 712\n",
      "Simulation 713\n",
      "Simulation 714\n",
      "Simulation 715\n",
      "Simulation 716\n",
      "Simulation 717\n",
      "Simulation 718\n",
      "Simulation 719\n",
      "Simulation 720\n",
      "Simulation 721\n",
      "Simulation 722\n",
      "Simulation 723\n",
      "Simulation 724\n",
      "Simulation 725\n",
      "Simulation 726\n",
      "Simulation 727\n",
      "Simulation 728\n",
      "Simulation 729\n",
      "Simulation 730\n",
      "Simulation 731\n",
      "Simulation 732\n",
      "Simulation 733\n",
      "Simulation 734\n",
      "Simulation 735\n",
      "Simulation 736\n",
      "Simulation 737\n",
      "Simulation 738\n",
      "Simulation 739\n",
      "Simulation 740\n",
      "Simulation 741\n",
      "Simulation 742\n",
      "Simulation 743\n",
      "Simulation 744\n",
      "Simulation 745\n",
      "Simulation 746\n",
      "Simulation 747\n",
      "Simulation 748\n",
      "Simulation 749\n",
      "Simulation 750\n",
      "Simulation 751\n",
      "Simulation 752\n",
      "Simulation 753\n",
      "Simulation 754\n",
      "Simulation 755\n",
      "Simulation 756\n",
      "Simulation 757\n",
      "Simulation 758\n",
      "Simulation 759\n",
      "Simulation 760\n",
      "Simulation 761\n",
      "Simulation 762\n",
      "Simulation 763\n",
      "Simulation 764\n",
      "Simulation 765\n",
      "Simulation 766\n",
      "Simulation 767\n",
      "Simulation 768\n",
      "Simulation 769\n",
      "Simulation 770\n",
      "Simulation 771\n",
      "Simulation 772\n",
      "Simulation 773\n",
      "Simulation 774\n",
      "Simulation 775\n",
      "Simulation 776\n",
      "Simulation 777\n",
      "Simulation 778\n",
      "Simulation 779\n",
      "Simulation 780\n",
      "Simulation 781\n",
      "Simulation 782\n",
      "Simulation 783\n",
      "Simulation 784\n",
      "Simulation 785\n",
      "Simulation 786\n",
      "Simulation 787\n",
      "Simulation 788\n",
      "Simulation 789\n",
      "Simulation 790\n",
      "Simulation 791\n",
      "Simulation 792\n",
      "Simulation 793\n",
      "Simulation 794\n",
      "Simulation 795\n",
      "Simulation 796\n",
      "Simulation 797\n",
      "Simulation 798\n",
      "Simulation 799\n",
      "Simulation 800\n",
      "Simulation 801\n",
      "Simulation 802\n",
      "Simulation 803\n",
      "Simulation 804\n",
      "Simulation 805\n",
      "Simulation 806\n",
      "Simulation 807\n",
      "Simulation 808\n",
      "Simulation 809\n",
      "Simulation 810\n",
      "Simulation 811\n",
      "Simulation 812\n",
      "Simulation 813\n",
      "Simulation 814\n",
      "Simulation 815\n",
      "Simulation 816\n",
      "Simulation 817\n",
      "Simulation 818\n",
      "Simulation 819\n",
      "Simulation 820\n",
      "Simulation 821\n",
      "Simulation 822\n",
      "Simulation 823\n",
      "Simulation 824\n",
      "Simulation 825\n",
      "Simulation 826\n",
      "Simulation 827\n",
      "Simulation 828\n",
      "Simulation 829\n",
      "Simulation 830\n",
      "Simulation 831\n",
      "Simulation 832\n",
      "Simulation 833\n",
      "Simulation 834\n",
      "Simulation 835\n",
      "Simulation 836\n",
      "Simulation 837\n",
      "Simulation 838\n",
      "Simulation 839\n",
      "Simulation 840\n",
      "Simulation 841\n",
      "Simulation 842\n",
      "Simulation 843\n",
      "Simulation 844\n",
      "Simulation 845\n",
      "Simulation 846\n",
      "Simulation 847\n",
      "Simulation 848\n",
      "Simulation 849\n",
      "Simulation 850\n",
      "Simulation 851\n",
      "Simulation 852\n",
      "Simulation 853\n",
      "Simulation 854\n",
      "Simulation 855\n",
      "Simulation 856\n",
      "Simulation 857\n",
      "Simulation 858\n",
      "Simulation 859\n",
      "Simulation 860\n",
      "Simulation 861\n",
      "Simulation 862\n",
      "Simulation 863\n",
      "Simulation 864\n",
      "Simulation 865\n",
      "Simulation 866\n",
      "Simulation 867\n",
      "Simulation 868\n",
      "Simulation 869\n",
      "Simulation 870\n",
      "Simulation 871\n",
      "Simulation 872\n",
      "Simulation 873\n",
      "Simulation 874\n",
      "Simulation 875\n",
      "Simulation 876\n",
      "Simulation 877\n",
      "Simulation 878\n",
      "Simulation 879\n",
      "Simulation 880\n",
      "Simulation 881\n",
      "Simulation 882\n",
      "Simulation 883\n",
      "Simulation 884\n",
      "Simulation 885\n",
      "Simulation 886\n",
      "Simulation 887\n",
      "Simulation 888\n",
      "Simulation 889\n",
      "Simulation 890\n",
      "Simulation 891\n",
      "Simulation 892\n",
      "Simulation 893\n",
      "Simulation 894\n",
      "Simulation 895\n",
      "Simulation 896\n",
      "Simulation 897\n",
      "Simulation 898\n",
      "Simulation 899\n",
      "Simulation 900\n",
      "Simulation 901\n",
      "Simulation 902\n",
      "Simulation 903\n",
      "Simulation 904\n",
      "Simulation 905\n",
      "Simulation 906\n",
      "Simulation 907\n",
      "Simulation 908\n",
      "Simulation 909\n",
      "Simulation 910\n",
      "Simulation 911\n",
      "Simulation 912\n",
      "Simulation 913\n",
      "Simulation 914\n",
      "Simulation 915\n",
      "Simulation 916\n",
      "Simulation 917\n",
      "Simulation 918\n",
      "Simulation 919\n",
      "Simulation 920\n",
      "Simulation 921\n",
      "Simulation 922\n",
      "Simulation 923\n",
      "Simulation 924\n",
      "Simulation 925\n",
      "Simulation 926\n",
      "Simulation 927\n",
      "Simulation 928\n",
      "Simulation 929\n",
      "Simulation 930\n",
      "Simulation 931\n",
      "Simulation 932\n",
      "Simulation 933\n",
      "Simulation 934\n",
      "Simulation 935\n",
      "Simulation 936\n",
      "Simulation 937\n",
      "Simulation 938\n",
      "Simulation 939\n",
      "Simulation 940\n",
      "Simulation 941\n",
      "Simulation 942\n",
      "Simulation 943\n",
      "Simulation 944\n",
      "Simulation 945\n",
      "Simulation 946\n",
      "Simulation 947\n",
      "Simulation 948\n",
      "Simulation 949\n",
      "Simulation 950\n",
      "Simulation 951\n",
      "Simulation 952\n",
      "Simulation 953\n",
      "Simulation 954\n",
      "Simulation 955\n",
      "Simulation 956\n",
      "Simulation 957\n",
      "Simulation 958\n",
      "Simulation 959\n",
      "Simulation 960\n",
      "Simulation 961\n",
      "Simulation 962\n",
      "Simulation 963\n",
      "Simulation 964\n",
      "Simulation 965\n",
      "Simulation 966\n",
      "Simulation 967\n",
      "Simulation 968\n",
      "Simulation 969\n",
      "Simulation 970\n",
      "Simulation 971\n",
      "Simulation 972\n",
      "Simulation 973\n",
      "Simulation 974\n",
      "Simulation 975\n",
      "Simulation 976\n",
      "Simulation 977\n",
      "Simulation 978\n",
      "Simulation 979\n",
      "Simulation 980\n",
      "Simulation 981\n",
      "Simulation 982\n",
      "Simulation 983\n",
      "Simulation 984\n",
      "Simulation 985\n",
      "Simulation 986\n",
      "Simulation 987\n",
      "Simulation 988\n",
      "Simulation 989\n",
      "Simulation 990\n",
      "Simulation 991\n",
      "Simulation 992\n",
      "Simulation 993\n",
      "Simulation 994\n",
      "Simulation 995\n",
      "Simulation 996\n",
      "Simulation 997\n",
      "Simulation 998\n",
      "Simulation 999\n",
      "Mapping file not found.\n"
     ]
    }
   ],
   "execution_count": 15
  },
  {
   "cell_type": "markdown",
   "source": "## 4. Analisi della Relazione tra Meteo e Produzione",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": "def get_full_data(simulated_data, olive_varieties):\n    # Assumiamo che simulated_data contenga già tutti i dati necessari\n    # Includiamo solo le colonne rilevanti\n    relevant_columns = ['year', 'temp_mean', 'precip_sum', 'solar_energy_sum', 'ha', 'zone', 'olive_prod']\n\n    # Aggiungiamo le colonne specifiche per varietà\n    all_varieties = olive_varieties['Varietà di Olive'].unique()\n    varieties = [clean_column_name(variety) for variety in all_varieties]\n    for variety in varieties:\n        relevant_columns.extend([f'{variety}_olive_prod', f'{variety}_tech'])\n\n    return simulated_data[relevant_columns].copy()\n\n\ndef analyze_correlations(full_data, variety):\n    # Filtra i dati per la varietà specifica\n    variety_data = full_data[[col for col in full_data.columns if not col.startswith('_') or col.startswith(f'{variety}_')]]\n\n    # Rinomina le colonne per chiarezza\n    variety_data = variety_data.rename(columns={\n        f'{variety}_olive_prod': 'olive_production',\n        f'{variety}_tech': 'technique'\n    })\n\n    # Matrice di correlazione\n    plt.figure(figsize=(12, 10))\n    corr_matrix = variety_data[['temp_mean', 'precip_sum', 'solar_energy_sum', 'olive_production']].corr()\n    sns.heatmap(corr_matrix, annot=True, cmap='coolwarm')\n    plt.title(f'Matrice di Correlazione - {variety}')\n    plt.tight_layout()\n    plt.show()\n\n    # Scatter plots\n    fig, axes = plt.subplots(2, 2, figsize=(20, 20))\n    fig.suptitle(f'Relazione tra Fattori Meteorologici e Produzione di Olive - {variety}', fontsize=16)\n\n    for ax, var in zip(axes.flat, ['temp_mean', 'precip_sum', 'solar_energy_sum', 'ha']):\n        sns.scatterplot(data=variety_data, x=var, y='olive_production', hue='technique', ax=ax)\n        ax.set_title(f'{var.capitalize()} vs Produzione Olive')\n        ax.set_xlabel(var.capitalize())\n        ax.set_ylabel('Produzione Olive (kg/ettaro)')\n\n    plt.tight_layout()\n    plt.show()\n\n\n# Uso delle funzioni\nfull_data = get_full_data(simulated_data, olive_varieties)\n\n# Assumiamo che 'selected_variety' sia definito altrove nel codice\n# Per esempio:\nselected_variety = 'nocellara_delletna'\n\nanalyze_correlations(full_data, selected_variety)",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-23T06:10:55.903873Z",
     "start_time": "2024-10-23T06:10:54.655058Z"
    },
    "execution": {
     "iopub.status.busy": "2024-10-26T15:17:59.167980Z",
     "iopub.execute_input": "2024-10-26T15:17:59.168844Z",
     "iopub.status.idle": "2024-10-26T15:18:02.822974Z",
     "shell.execute_reply.started": "2024-10-26T15:17:59.168801Z",
     "shell.execute_reply": "2024-10-26T15:18:02.821839Z"
    },
    "trusted": true
   },
   "execution_count": 6,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 1200x1000 with 2 Axes>",
      "image/png": "[encoded data, 59020 characters]"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 2000x2000 with 4 Axes>",
      "image/png": "[encoded data, 1945160 characters]"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "## 5. Preparazione del Modello di Machine Learning",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": "def prepare_data(df, olive_varieties_df):\n    # Crea una copia del DataFrame per evitare modifiche all'originale\n    df = df.copy()\n\n    # Ordina per zona e anno\n    df = df.sort_values(['zone', 'year'])\n\n    # Definisci le feature\n    temporal_features = ['temp_mean', 'precip_sum', 'solar_energy_sum']\n    static_features = ['ha']  # Feature statiche base\n    target_features = ['olive_prod', 'min_oil_prod', 'max_oil_prod', 'avg_oil_prod', 'total_water_need']\n\n    # Ottieni le varietà pulite\n    varieties = [clean_column_name(variety) for variety in olive_varieties_df['Varietà di Olive']]\n\n    # Crea la struttura delle feature per ogni varietà\n    variety_features = [\n        'tech', 'pct', 'prod_t_ha', 'oil_prod_t_ha', 'oil_prod_l_ha',\n        'min_yield_pct', 'max_yield_pct', 'min_oil_prod_l_ha', 'max_oil_prod_l_ha',\n        'avg_oil_prod_l_ha', 'l_per_t', 'min_l_per_t', 'max_l_per_t', 'avg_l_per_t'\n    ]\n\n    # Prepara dizionari per le nuove colonne\n    new_columns = {}\n\n    # Prepara le feature per ogni varietà\n    for variety in varieties:\n        # Feature esistenti\n        for feature in variety_features:\n            col_name = f\"{variety}_{feature}\"\n            if col_name in df.columns:\n                if feature != 'tech':  # Non includere la colonna tech direttamente\n                    static_features.append(col_name)\n\n        # Feature binarie per le tecniche di coltivazione\n        for technique in ['tradizionale', 'intensiva', 'superintensiva']:\n            col_name = f\"{variety}_{technique}\"\n            new_columns[col_name] = df[f\"{variety}_tech\"].notna() & (\n                    df[f\"{variety}_tech\"].str.lower() == technique\n            ).fillna(False)\n            static_features.append(col_name)\n\n    # Aggiungi tutte le nuove colonne in una volta sola\n    new_df = pd.concat([df] + [pd.Series(v, name=k) for k, v in new_columns.items()], axis=1)\n\n    #print(\"Temporal features:\", temporal_features)\n    #print(\"Static features:\", static_features)\n\n    # Crea scalers separati per ogni tipo di dato\n    scaler_temporal = StandardScaler()\n    scaler_static = StandardScaler()\n    scaler_y = StandardScaler()\n\n    # Prepara array per i dati\n    zones = new_df['zone'].unique()\n    years = new_df['year'].unique()\n    years.sort()\n\n    # Inizializza arrays con le dimensioni corrette\n    X_temporal = np.zeros((len(zones), len(years), len(temporal_features)))\n    X_static = np.zeros((len(zones), len(static_features)))\n    y = np.zeros((len(zones), len(target_features)))\n\n    # Popola gli arrays\n    for i, zone in enumerate(zones):\n        zone_data = new_df[new_df['zone'] == zone]\n        # Dati statici\n        X_static[i] = zone_data[static_features].iloc[0].values\n\n        # Per i target, prendiamo solo l'ultimo anno disponibile per il training\n        # Questo simula una predizione per il prossimo anno\n        last_year_data = zone_data.iloc[-1]\n        y[i] = last_year_data[target_features].values\n\n    #print(\"\\nShape prima dello split:\")\n    #print(\"X_temporal shape:\", X_temporal.shape)\n    #print(\"X_static shape:\", X_static.shape)\n    #print(\"y shape:\", y.shape)\n\n    # Dividi i dati in train, validation e test\n    # Modifica: ora dividiamo per zone invece che per anni\n    n_zones = len(zones)\n    train_idx = int(n_zones * 0.7)  # 70% per training\n    val_idx = int(n_zones * 0.85)  # 15% per validation\n\n    # Split dei dati\n    X_temporal_train = X_temporal[:train_idx]\n    X_temporal_val = X_temporal[train_idx:val_idx]\n    X_temporal_test = X_temporal[val_idx:]\n\n    X_static_train = X_static[:train_idx]\n    X_static_val = X_static[train_idx:val_idx]\n    X_static_test = X_static[val_idx:]\n\n    y_train = y[:train_idx]\n    y_val = y[train_idx:val_idx]\n    y_test = y[val_idx:]\n\n    # Standardizzazione\n    X_temporal_train = scaler_temporal.fit_transform(X_temporal_train.reshape(-1, len(temporal_features))).reshape(X_temporal_train.shape)\n    X_temporal_val = scaler_temporal.transform(X_temporal_val.reshape(-1, len(temporal_features))).reshape(X_temporal_val.shape)\n    X_temporal_test = scaler_temporal.transform(X_temporal_test.reshape(-1, len(temporal_features))).reshape(X_temporal_test.shape)\n\n    X_static_train = scaler_static.fit_transform(X_static_train)\n    X_static_val = scaler_static.transform(X_static_val)\n    X_static_test = scaler_static.transform(X_static_test)\n\n    y_train = scaler_y.fit_transform(y_train)\n    y_val = scaler_y.transform(y_val)\n    y_test = scaler_y.transform(y_test)\n\n    # Prepara i dizionari di input\n    train_data = {'temporal': X_temporal_train, 'static': X_static_train}\n    val_data = {'temporal': X_temporal_val, 'static': X_static_val}\n    test_data = {'temporal': X_temporal_test, 'static': X_static_test}\n\n    #print(\"\\nShape dopo lo split:\")\n    #print(\"Train temporal shape:\", train_data['temporal'].shape)\n    #print(\"Train static shape:\", train_data['static'].shape)\n    #print(\"Train target shape:\", y_train.shape)\n\n    return (train_data, y_train), (val_data, y_val), (test_data, y_test), (scaler_temporal, scaler_static, scaler_y)",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-24T10:25:45.872651Z",
     "start_time": "2024-10-24T10:25:45.859503Z"
    },
    "execution": {
     "iopub.status.busy": "2024-10-26T15:18:55.586033Z",
     "iopub.execute_input": "2024-10-26T15:18:55.586731Z",
     "iopub.status.idle": "2024-10-26T15:18:55.607531Z",
     "shell.execute_reply.started": "2024-10-26T15:18:55.586692Z",
     "shell.execute_reply": "2024-10-26T15:18:55.606536Z"
    },
    "trusted": true
   },
   "execution_count": 7,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "## Divisione train/validation/test:\n",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "simulated_data = pd.read_parquet(\"./data/simulated_data.parquet\")\n",
    "olive_varieties = pd.read_parquet(\"./data/olive_varieties.parquet\")\n",
    "\n",
    "(train_data, train_targets), (val_data, val_targets), (test_data, test_targets), scalers = prepare_data(simulated_data, olive_varieties)\n",
    "\n",
    "scaler_temporal, scaler_static, scaler_y = scalers\n",
    "\n",
    "print(\"Temporal data shape:\", train_data['temporal'].shape)\n",
    "print(\"Static data shape:\", train_data['static'].shape)\n",
    "print(\"Target shape:\", train_targets.shape)"
   ],
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-24T10:25:49.473595Z",
     "start_time": "2024-10-24T10:25:49.199833Z"
    },
    "execution": {
     "iopub.status.busy": "2024-10-26T15:19:03.795160Z",
     "iopub.execute_input": "2024-10-26T15:19:03.796100Z",
     "iopub.status.idle": "2024-10-26T15:19:04.726941Z",
     "shell.execute_reply.started": "2024-10-26T15:19:03.796040Z",
     "shell.execute_reply": "2024-10-26T15:19:04.725679Z"
    },
    "trusted": true
   },
   "execution_count": 8,
   "outputs": [
    {
     "name": "stdout",
     "text": "Temporal data shape: (70, 38, 3)\nStatic data shape: (70, 337)\nTarget shape: (70, 5)\n",
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "## OliveOilTransformer",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "\n",
    "class PositionalEncoding(tf.keras.layers.Layer):\n",
    "    def __init__(self, position, d_model):\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        self.pos_encoding = self.positional_encoding(position, d_model)\n",
    "\n",
    "    def get_angles(self, position, i, d_model):\n",
    "        angles = 1 / tf.pow(10000, (2 * (i // 2)) / tf.cast(d_model, tf.float32))\n",
    "        return position * angles\n",
    "\n",
    "    def positional_encoding(self, position, d_model):\n",
    "        angle_rads = self.get_angles(\n",
    "            position=tf.range(position, dtype=tf.float32)[:, tf.newaxis],\n",
    "            i=tf.range(d_model, dtype=tf.float32)[tf.newaxis, :],\n",
    "            d_model=d_model)\n",
    "\n",
    "        sines = tf.math.sin(angle_rads[:, 0::2])\n",
    "        cosines = tf.math.cos(angle_rads[:, 1::2])\n",
    "\n",
    "        pos_encoding = tf.concat([sines, cosines], axis=-1)\n",
    "        pos_encoding = pos_encoding[tf.newaxis, ...]\n",
    "        return tf.cast(pos_encoding, tf.float32)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        return inputs + self.pos_encoding[:, :tf.shape(inputs)[1], :]\n",
    "\n",
    "\n",
    "class TransformerBlock(tf.keras.layers.Layer):\n",
    "    def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1):\n",
    "        super(TransformerBlock, self).__init__()\n",
    "        self.att = tf.keras.layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n",
    "        self.ffn = tf.keras.Sequential([\n",
    "            tf.keras.layers.Dense(ff_dim, activation=\"relu\"),\n",
    "            tf.keras.layers.Dense(embed_dim),\n",
    "        ])\n",
    "        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.dropout1 = tf.keras.layers.Dropout(rate)\n",
    "        self.dropout2 = tf.keras.layers.Dropout(rate)\n",
    "\n",
    "    def call(self, inputs, training):\n",
    "        attn_output = self.att(inputs, inputs)\n",
    "        attn_output = self.dropout1(attn_output, training=training)\n",
    "        out1 = self.layernorm1(inputs + attn_output)\n",
    "        ffn_output = self.ffn(out1)\n",
    "        ffn_output = self.dropout2(ffn_output, training=training)\n",
    "        return self.layernorm2(out1 + ffn_output)\n",
    "\n",
    "\n",
    "class OliveOilTransformer(tf.keras.Model):\n",
    "    def __init__(self, temporal_shape, static_shape, num_outputs, d_model=64, num_heads=8, ff_dim=64,\n",
    "                 num_transformer_blocks=4, mlp_units=[128, 64], dropout=0.1, mlp_dropout=0.1):\n",
    "        super(OliveOilTransformer, self).__init__()\n",
    "\n",
    "        # Input layers\n",
    "        self.temporal_input = tf.keras.layers.Input(shape=temporal_shape, name='temporal_input')\n",
    "        self.static_input = tf.keras.layers.Input(shape=static_shape, name='static_input')\n",
    "\n",
    "        # Temporal path\n",
    "        self.feature_projection = tf.keras.layers.Dense(d_model)\n",
    "        self.pos_encoding = PositionalEncoding(position=temporal_shape[0], d_model=d_model)\n",
    "\n",
    "        # Transformer blocks\n",
    "        self.transformer_blocks = []\n",
    "        for _ in range(num_transformer_blocks):\n",
    "            self.transformer_blocks.append(TransformerBlock(d_model, num_heads, ff_dim, dropout))\n",
    "\n",
    "        # Global pooling for temporal features\n",
    "        self.global_average_pooling = tf.keras.layers.GlobalAveragePooling1D()\n",
    "\n",
    "        # Static path\n",
    "        self.static_encoder = tf.keras.Sequential([\n",
    "            tf.keras.layers.Dense(64, activation='relu'),\n",
    "            tf.keras.layers.Dropout(dropout),\n",
    "            tf.keras.layers.Dense(32, activation='relu')\n",
    "        ], name='static_encoder')\n",
    "\n",
    "        # Combination layer\n",
    "        self.combination_layer = tf.keras.layers.Concatenate()\n",
    "\n",
    "        # MLP head\n",
    "        self.mlp_layers = []\n",
    "        for dim in mlp_units:\n",
    "            self.mlp_layers.append(tf.keras.layers.Dense(dim, activation=\"relu\"))\n",
    "            self.mlp_layers.append(tf.keras.layers.Dropout(mlp_dropout))\n",
    "\n",
    "        self.final_layer = tf.keras.layers.Dense(num_outputs)\n",
    "\n",
    "        # Build the model\n",
    "        temporal_encoded = self.encode_temporal(self.temporal_input)\n",
    "        static_encoded = self.static_encoder(self.static_input)\n",
    "        combined = self.combination_layer([temporal_encoded, static_encoded])\n",
    "\n",
    "        # Apply MLP head\n",
    "        x = combined\n",
    "        for layer in self.mlp_layers:\n",
    "            x = layer(x)\n",
    "        outputs = self.final_layer(x)\n",
    "\n",
    "        self._model = tf.keras.Model(\n",
    "            inputs={'temporal': self.temporal_input, 'static': self.static_input},\n",
    "            outputs=outputs\n",
    "        )\n",
    "\n",
    "    def encode_temporal(self, x, training=None):\n",
    "        # Project features to d_model dimension\n",
    "        x = self.feature_projection(x)\n",
    "\n",
    "        # Add positional encoding\n",
    "        x = self.pos_encoding(x)\n",
    "\n",
    "        # Apply transformer blocks\n",
    "        for transformer_block in self.transformer_blocks:\n",
    "            x = transformer_block(x, training=training)\n",
    "\n",
    "        # Global pooling\n",
    "        return self.global_average_pooling(x)\n",
    "\n",
    "    def call(self, inputs, training=None):\n",
    "        temporal_input = inputs['temporal']\n",
    "        static_input = inputs['static']\n",
    "\n",
    "        # Encode temporal and static features\n",
    "        temporal_encoded = self.encode_temporal(temporal_input, training)\n",
    "        static_encoded = self.static_encoder(static_input)\n",
    "\n",
    "        # Combine features\n",
    "        combined = self.combination_layer([temporal_encoded, static_encoded])\n",
    "\n",
    "        # Apply MLP head\n",
    "        x = combined\n",
    "        for layer in self.mlp_layers:\n",
    "            x = layer(x, training=training)\n",
    "\n",
    "        # Final output\n",
    "        return self.final_layer(x)\n",
    "\n",
    "    def model(self):\n",
    "        return self._model\n",
    "\n",
    "\n",
    "# Costruisci il modello\n",
    "model = OliveOilTransformer(\n",
    "    temporal_shape=(train_data['temporal'].shape[1], train_data['temporal'].shape[2]),  # (seq_length, num_features)\n",
    "    static_shape=(train_data['static'].shape[1],),  # (static_features,)\n",
    "    num_outputs=train_targets.shape[1]  # numero di target\n",
    ")\n",
    "\n",
    "# Compila il modello\n",
    "model.compile(\n",
    "    optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),\n",
    "    loss='mse',\n",
    "    metrics=['mae']\n",
    ")\n",
    "\n",
    "# Mostra il summary\n",
    "model.model().summary()"
   ],
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-24T09:32:37.506903Z",
     "start_time": "2024-10-24T09:32:36.905756Z"
    },
    "execution": {
     "iopub.status.busy": "2024-10-26T15:26:03.570530Z",
     "iopub.execute_input": "2024-10-26T15:26:03.570965Z",
     "iopub.status.idle": "2024-10-26T15:26:05.011312Z",
     "shell.execute_reply.started": "2024-10-26T15:26:03.570910Z",
     "shell.execute_reply": "2024-10-26T15:26:05.010277Z"
    },
    "trusted": true
   },
   "execution_count": 10,
   "outputs": [
    {
     "name": "stderr",
     "text": "/opt/conda/lib/python3.10/site-packages/keras/src/models/functional.py:112: UserWarning: When providing `inputs` as a dict, all keys in the dict must match the names of the corresponding tensors. Received key 'temporal' mapping to value <KerasTensor shape=(None, 38, 3), dtype=float32, sparse=None, name=temporal_input> which has name 'temporal_input'. Change the tensor name to 'temporal' (via `Input(..., name='temporal')`)\n  warnings.warn(\n/opt/conda/lib/python3.10/site-packages/keras/src/models/functional.py:112: UserWarning: When providing `inputs` as a dict, all keys in the dict must match the names of the corresponding tensors. Received key 'static' mapping to value <KerasTensor shape=(None, 337), dtype=float32, sparse=None, name=static_input> which has name 'static_input'. Change the tensor name to 'static' (via `Input(..., name='static')`)\n  warnings.warn(\n",
     "output_type": "stream"
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "\u001B[1mModel: \"functional_6\"\u001B[0m\n",
      "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"functional_6\"</span>\n</pre>\n"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓\n┃\u001B[1m \u001B[0m\u001B[1mLayer (type)       \u001B[0m\u001B[1m \u001B[0m┃\u001B[1m \u001B[0m\u001B[1mOutput Shape     \u001B[0m\u001B[1m \u001B[0m┃\u001B[1m \u001B[0m\u001B[1m   Param #\u001B[0m\u001B[1m \u001B[0m┃\u001B[1m \u001B[0m\u001B[1mConnected to     \u001B[0m\u001B[1m \u001B[0m┃\n┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩\n│ temporal_input      │ (\u001B[38;5;45mNone\u001B[0m, \u001B[38;5;34m38\u001B[0m, \u001B[38;5;34m3\u001B[0m)     │          \u001B[38;5;34m0\u001B[0m │ -                 │\n│ (\u001B[38;5;33mInputLayer\u001B[0m)        │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ dense (\u001B[38;5;33mDense\u001B[0m)       │ (\u001B[38;5;45mNone\u001B[0m, \u001B[38;5;34m38\u001B[0m, \u001B[38;5;34m64\u001B[0m)    │        \u001B[38;5;34m256\u001B[0m │ temporal_input[\u001B[38;5;34m0\u001B[0m… │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ positional_encoding │ (\u001B[38;5;45mNone\u001B[0m, \u001B[38;5;34m38\u001B[0m, \u001B[38;5;34m64\u001B[0m)    │          \u001B[38;5;34m0\u001B[0m │ dense[\u001B[38;5;34m0\u001B[0m][\u001B[38;5;34m0\u001B[0m]       │\n│ (\u001B[38;5;33mPositionalEncodin…\u001B[0m │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ transformer_block   │ (\u001B[38;5;45mNone\u001B[0m, \u001B[38;5;34m38\u001B[0m, \u001B[38;5;34m64\u001B[0m)    │    \u001B[38;5;34m141,248\u001B[0m │ positional_encod… │\n│ (\u001B[38;5;33mTransformerBlock\u001B[0m)  │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ transformer_block_1 │ (\u001B[38;5;45mNone\u001B[0m, \u001B[38;5;34m38\u001B[0m, \u001B[38;5;34m64\u001B[0m)    │    \u001B[38;5;34m141,248\u001B[0m │ transformer_bloc… │\n│ (\u001B[38;5;33mTransformerBlock\u001B[0m)  │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ transformer_block_2 │ (\u001B[38;5;45mNone\u001B[0m, \u001B[38;5;34m38\u001B[0m, \u001B[38;5;34m64\u001B[0m)    │    \u001B[38;5;34m141,248\u001B[0m │ transformer_bloc… │\n│ (\u001B[38;5;33mTransformerBlock\u001B[0m)  │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ transformer_block_3 │ (\u001B[38;5;45mNone\u001B[0m, \u001B[38;5;34m38\u001B[0m, \u001B[38;5;34m64\u001B[0m)    │    \u001B[38;5;34m141,248\u001B[0m │ transformer_bloc… │\n│ (\u001B[38;5;33mTransformerBlock\u001B[0m)  │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ static_input        │ (\u001B[38;5;45mNone\u001B[0m, \u001B[38;5;34m337\u001B[0m)       │          \u001B[38;5;34m0\u001B[0m │ -                 │\n│ (\u001B[38;5;33mInputLayer\u001B[0m)        │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ global_average_poo… │ (\u001B[38;5;45mNone\u001B[0m, \u001B[38;5;34m64\u001B[0m)        │          \u001B[38;5;34m0\u001B[0m │ transformer_bloc… │\n│ (\u001B[38;5;33mGlobalAveragePool…\u001B[0m │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ static_encoder      │ (\u001B[38;5;45mNone\u001B[0m, \u001B[38;5;34m32\u001B[0m)        │     \u001B[38;5;34m23,712\u001B[0m │ static_input[\u001B[38;5;34m0\u001B[0m][\u001B[38;5;34m…\u001B[0m │\n│ (\u001B[38;5;33mSequential\u001B[0m)        │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ concatenate         │ (\u001B[38;5;45mNone\u001B[0m, \u001B[38;5;34m96\u001B[0m)        │          \u001B[38;5;34m0\u001B[0m │ global_average_p… │\n│ (\u001B[38;5;33mConcatenate\u001B[0m)       │                   │            │ static_encoder[\u001B[38;5;34m0\u001B[0m… │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ dense_11 (\u001B[38;5;33mDense\u001B[0m)    │ (\u001B[38;5;45mNone\u001B[0m, \u001B[38;5;34m128\u001B[0m)       │     \u001B[38;5;34m12,416\u001B[0m │ concatenate[\u001B[38;5;34m0\u001B[0m][\u001B[38;5;34m0\u001B[0m] │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ dropout_15          │ (\u001B[38;5;45mNone\u001B[0m, \u001B[38;5;34m128\u001B[0m)       │          \u001B[38;5;34m0\u001B[0m │ dense_11[\u001B[38;5;34m0\u001B[0m][\u001B[38;5;34m0\u001B[0m]    │\n│ (\u001B[38;5;33mDropout\u001B[0m)           │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ dense_12 (\u001B[38;5;33mDense\u001B[0m)    │ (\u001B[38;5;45mNone\u001B[0m, \u001B[38;5;34m64\u001B[0m)        │      \u001B[38;5;34m8,256\u001B[0m │ dropout_15[\u001B[38;5;34m0\u001B[0m][\u001B[38;5;34m0\u001B[0m]  │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ dropout_16          │ (\u001B[38;5;45mNone\u001B[0m, \u001B[38;5;34m64\u001B[0m)        │          \u001B[38;5;34m0\u001B[0m │ dense_12[\u001B[38;5;34m0\u001B[0m][\u001B[38;5;34m0\u001B[0m]    │\n│ (\u001B[38;5;33mDropout\u001B[0m)           │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ dense_13 (\u001B[38;5;33mDense\u001B[0m)    │ (\u001B[38;5;45mNone\u001B[0m, \u001B[38;5;34m5\u001B[0m)         │        \u001B[38;5;34m325\u001B[0m │ dropout_16[\u001B[38;5;34m0\u001B[0m][\u001B[38;5;34m0\u001B[0m]  │\n└─────────────────────┴───────────────────┴────────────┴───────────────────┘\n",
      "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓\n┃<span style=\"font-weight: bold\"> Layer (type)        </span>┃<span style=\"font-weight: bold\"> Output Shape      </span>┃<span style=\"font-weight: bold\">    Param # </span>┃<span style=\"font-weight: bold\"> Connected to      </span>┃\n┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩\n│ temporal_input      │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">38</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">3</span>)     │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                 │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)        │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ dense (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">38</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │        <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span> │ temporal_input[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>… │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ positional_encoding │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">38</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ dense[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]       │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">PositionalEncodin…</span> │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ transformer_block   │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">38</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │    <span style=\"color: #00af00; text-decoration-color: #00af00\">141,248</span> │ positional_encod… │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TransformerBlock</span>)  │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ transformer_block_1 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">38</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │    <span style=\"color: #00af00; text-decoration-color: #00af00\">141,248</span> │ transformer_bloc… │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TransformerBlock</span>)  │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ transformer_block_2 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">38</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │    <span style=\"color: #00af00; text-decoration-color: #00af00\">141,248</span> │ transformer_bloc… │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TransformerBlock</span>)  │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ transformer_block_3 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">38</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │    <span style=\"color: #00af00; text-decoration-color: #00af00\">141,248</span> │ transformer_bloc… │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TransformerBlock</span>)  │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ static_input        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">337</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                 │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)        │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ global_average_poo… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)        │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ transformer_bloc… │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GlobalAveragePool…</span> │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ static_encoder      │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)        │     <span style=\"color: #00af00; text-decoration-color: #00af00\">23,712</span> │ static_input[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">…</span> │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Sequential</span>)        │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ concatenate         │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">96</span>)        │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ global_average_p… │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Concatenate</span>)       │                   │            │ static_encoder[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>… │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ dense_11 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)    │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)       │     <span style=\"color: #00af00; text-decoration-color: #00af00\">12,416</span> │ concatenate[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>] │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ dropout_15          │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ dense_11[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]    │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)           │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ dense_12 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)    │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)        │      <span style=\"color: #00af00; text-decoration-color: #00af00\">8,256</span> │ dropout_15[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]  │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ dropout_16          │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)        │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ dense_12[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]    │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)           │                   │            │                   │\n├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n│ dense_13 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)    │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">5</span>)         │        <span style=\"color: #00af00; text-decoration-color: #00af00\">325</span> │ dropout_16[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]  │\n└─────────────────────┴───────────────────┴────────────┴───────────────────┘\n</pre>\n"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "\u001B[1m Total params: \u001B[0m\u001B[38;5;34m609,957\u001B[0m (2.33 MB)\n",
      "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">609,957</span> (2.33 MB)\n</pre>\n"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "\u001B[1m Trainable params: \u001B[0m\u001B[38;5;34m609,957\u001B[0m (2.33 MB)\n",
      "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">609,957</span> (2.33 MB)\n</pre>\n"
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "\u001B[1m Non-trainable params: \u001B[0m\u001B[38;5;34m0\u001B[0m (0.00 B)\n",
      "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n</pre>\n"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": "## Model Training",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "source": [
    "# Importazioni necessarie\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\n",
    "\n",
    "# Addestramento\n",
    "history = model.fit(\n",
    "    x=train_data,  # dizionario con 'temporal' e 'static'\n",
    "    y=train_targets,\n",
    "    validation_data=(val_data, val_targets),  # dati di validation già separati\n",
    "    epochs=100,\n",
    "    batch_size=32,\n",
    "    callbacks=[\n",
    "        EarlyStopping(\n",
    "            monitor='val_loss',\n",
    "            patience=10,\n",
    "            restore_best_weights=True\n",
    "        ),\n",
    "        ReduceLROnPlateau(\n",
    "            monitor='val_loss',\n",
    "            factor=0.5,\n",
    "            patience=5,\n",
    "            min_lr=1e-6\n",
    "        )\n",
    "    ],\n",
    "    verbose=1  # mostra la progress bar\n",
    ")\n",
    "\n",
    "# Per visualizzare le performance durante il training\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "def plot_training_history(history):\n",
    "    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5))\n",
    "\n",
    "    # Plot della loss\n",
    "    ax1.plot(history.history['loss'], label='Training Loss')\n",
    "    ax1.plot(history.history['val_loss'], label='Validation Loss')\n",
    "    ax1.set_title('Model Loss')\n",
    "    ax1.set_xlabel('Epoch')\n",
    "    ax1.set_ylabel('Loss')\n",
    "    ax1.legend()\n",
    "\n",
    "    # Plot della MAE\n",
    "    ax2.plot(history.history['mae'], label='Training MAE')\n",
    "    ax2.plot(history.history['val_mae'], label='Validation MAE')\n",
    "    ax2.set_title('Model MAE')\n",
    "    ax2.set_xlabel('Epoch')\n",
    "    ax2.set_ylabel('MAE')\n",
    "    ax2.legend()\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# Visualizza i risultati del training\n",
    "plot_training_history(history)"
   ],
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-24T09:33:43.625381Z",
     "start_time": "2024-10-24T09:33:34.088970Z"
    },
    "execution": {
     "iopub.status.busy": "2024-10-26T15:30:05.572528Z",
     "iopub.execute_input": "2024-10-26T15:30:05.573537Z",
     "iopub.status.idle": "2024-10-26T15:30:07.147659Z",
     "shell.execute_reply.started": "2024-10-26T15:30:05.573491Z",
     "shell.execute_reply": "2024-10-26T15:30:07.146531Z"
    },
    "trusted": true
   },
   "execution_count": 14,
   "outputs": [
    {
     "name": "stdout",
     "text": "Epoch 1/100\n\u001B[1m3/3\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m0s\u001B[0m 62ms/step - loss: 0.9621 - mae: 0.7784 - val_loss: 0.9333 - val_mae: 0.8344 - learning_rate: 2.5000e-04\nEpoch 2/100\n\u001B[1m3/3\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m0s\u001B[0m 20ms/step - loss: 0.9915 - mae: 0.7776 - val_loss: 0.9528 - val_mae: 0.8392 - learning_rate: 2.5000e-04\nEpoch 3/100\n\u001B[1m3/3\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m0s\u001B[0m 20ms/step - loss: 0.8394 - mae: 0.7329 - val_loss: 0.9899 - val_mae: 0.8458 - learning_rate: 2.5000e-04\nEpoch 4/100\n\u001B[1m3/3\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m0s\u001B[0m 20ms/step - loss: 0.8104 - mae: 0.7179 - val_loss: 1.0247 - val_mae: 0.8531 - learning_rate: 2.5000e-04\nEpoch 5/100\n\u001B[1m3/3\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m0s\u001B[0m 19ms/step - loss: 0.7645 - mae: 0.6949 - val_loss: 1.0296 - val_mae: 0.8565 - learning_rate: 2.5000e-04\nEpoch 6/100\n\u001B[1m3/3\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m0s\u001B[0m 20ms/step - loss: 0.8045 - mae: 0.7195 - val_loss: 1.0227 - val_mae: 0.8574 - learning_rate: 2.5000e-04\nEpoch 7/100\n\u001B[1m3/3\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m0s\u001B[0m 20ms/step - loss: 0.7550 - mae: 0.7084 - val_loss: 1.0306 - val_mae: 0.8601 - learning_rate: 1.2500e-04\nEpoch 8/100\n\u001B[1m3/3\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m0s\u001B[0m 20ms/step - loss: 0.8057 - mae: 0.7346 - val_loss: 1.0302 - val_mae: 0.8591 - learning_rate: 1.2500e-04\nEpoch 9/100\n\u001B[1m3/3\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m0s\u001B[0m 20ms/step - loss: 0.8039 - mae: 0.7125 - val_loss: 1.0187 - val_mae: 0.8555 - learning_rate: 1.2500e-04\nEpoch 10/100\n\u001B[1m3/3\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m0s\u001B[0m 20ms/step - loss: 0.7932 - mae: 0.7158 - val_loss: 0.9960 - val_mae: 0.8492 - learning_rate: 1.2500e-04\nEpoch 11/100\n\u001B[1m3/3\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m0s\u001B[0m 20ms/step - loss: 0.7294 - mae: 0.6898 - val_loss: 0.9789 - val_mae: 0.8448 - learning_rate: 1.2500e-04\n",
     "output_type": "stream"
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 1500x500 with 2 Axes>",
      "image/png": "[encoded data, 84192 characters]"
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "source": "next_year_production = model.predict(next_year_weather.mean().to_frame().T)[0]\nprint(f'Previsione produzione di olive per il prossimo anno: {next_year_production:.2f} kg/ettaro')",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-10-26T15:27:20.258169Z",
     "iopub.status.idle": "2024-10-26T15:27:20.258769Z",
     "shell.execute_reply.started": "2024-10-26T15:27:20.258422Z",
     "shell.execute_reply": "2024-10-26T15:27:20.258449Z"
    },
    "trusted": true
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "## 8. Conclusioni e Prossimi Passi\n\nIn questo notebook, abbiamo:\n1. Caricato e analizzato i dati meteorologici\n2. Simulato la produzione annuale di olive basata sui dati meteo\n3. Esplorato le relazioni tra variabili meteorologiche e produzione di olive\n4. Creato e valutato un modello di machine learning per prevedere la produzione\n5. Utilizzato ARIMA per fare previsioni meteo\n6. Previsto la produzione di olive per il prossimo anno\n\nProssimi passi:\n- Raccogliere dati reali sulla produzione di olive per sostituire i dati simulati\n- Esplorare modelli più avanzati, come le reti neurali o i modelli di ensemble\n- Incorporare altri fattori che potrebbero influenzare la produzione, come le pratiche agricole o l'età degli alberi\n- Sviluppare una dashboard interattiva basata su questo modello",
   "metadata": {}
  }
 ]
}
