{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Get:1 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB]\n",
      "Hit:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease\n",
      "Hit:3 http://archive.ubuntu.com/ubuntu jammy InRelease                         \n",
      "Get:4 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB]\n",
      "Hit:5 http://archive.ubuntu.com/ubuntu jammy-backports InRelease\n",
      "Fetched 257 kB in 1s (278 kB/s)\n",
      "Reading package lists... Done\n",
      "Reading package lists... Done\n",
      "Building dependency tree... Done\n",
      "Reading state information... Done\n",
      "graphviz is already the newest version (2.42.2-6ubuntu0.1).\n",
      "0 upgraded, 0 newly installed, 0 to remove and 120 not upgraded.\n",
      "Requirement already satisfied: tensorflow in /usr/local/lib/python3.11/dist-packages (2.14.0)\n",
      "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.0.0)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.6.3)\n",
      "Requirement already satisfied: flatbuffers>=23.5.26 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (23.5.26)\n",
      "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.5.4)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.2.0)\n",
      "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (3.9.0)\n",
      "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (16.0.6)\n",
      "Requirement already satisfied: ml-dtypes==0.2.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.2.0)\n",
      "Requirement already satisfied: numpy>=1.23.5 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.26.0)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (3.3.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from tensorflow) (23.1)\n",
      "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (4.24.3)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.11/dist-packages (from tensorflow) (68.2.2)\n",
      "Requirement already satisfied: six>=1.12.0 in /usr/lib/python3/dist-packages (from tensorflow) (1.16.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.3.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (4.8.0)\n",
      "Requirement already satisfied: wrapt<1.15,>=1.11.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.14.1)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.37.1)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.58.0)\n",
      "Requirement already satisfied: tensorboard<2.15,>=2.14 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.14.0)\n",
      "Requirement already satisfied: tensorflow-estimator<2.15,>=2.14.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.14.0)\n",
      "Requirement already satisfied: keras<2.15,>=2.14.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.14.0)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.11/dist-packages (from astunparse>=1.6.0->tensorflow) (0.41.2)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (2.23.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (1.0.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (3.4.4)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (2.31.0)\n",
      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (0.7.1)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (2.3.7)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (5.3.1)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (0.3.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (4.9)\n",
      "Requirement already satisfied: urllib3>=2.0.5 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (2.0.5)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard<2.15,>=2.14->tensorflow) (1.3.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.15,>=2.14->tensorflow) (3.2.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.15,>=2.14->tensorflow) (3.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.15,>=2.14->tensorflow) (2023.7.22)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.11/dist-packages (from werkzeug>=1.0.1->tensorboard<2.15,>=2.14->tensorflow) (2.1.3)\n",
      "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.11/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (0.5.0)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/lib/python3/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard<2.15,>=2.14->tensorflow) (3.2.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (1.26.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (2.2.3)\n",
      "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (1.26.0)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: keras in /usr/local/lib/python3.11/dist-packages (2.14.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.11/dist-packages (1.5.2)\n",
      "Requirement already satisfied: numpy>=1.19.5 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.26.0)\n",
      "Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.14.1)\n",
      "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.4.2)\n",
      "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (3.5.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: matplotlib in /usr/local/lib/python3.11/dist-packages (3.8.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (4.42.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.4.5)\n",
      "Requirement already satisfied: numpy<2,>=1.21 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.26.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (23.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (10.0.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (3.2.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (2.8.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.11/dist-packages (1.4.2)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: pyarrow in /usr/local/lib/python3.11/dist-packages (18.0.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: fastparquet in /usr/local/lib/python3.11/dist-packages (2024.5.0)\n",
      "Requirement already satisfied: pandas>=1.5.0 in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2.2.3)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (from fastparquet) (1.26.0)\n",
      "Requirement already satisfied: cramjam>=2.3 in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2.9.0)\n",
      "Requirement already satisfied: fsspec in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2024.10.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from fastparquet) (23.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2024.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas>=1.5.0->fastparquet) (1.16.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.11/dist-packages (1.14.1)\n",
      "Requirement already satisfied: numpy<2.3,>=1.23.5 in /usr/local/lib/python3.11/dist-packages (from scipy) (1.26.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: seaborn in /usr/local/lib/python3.11/dist-packages (0.13.2)\n",
      "Requirement already satisfied: numpy!=1.24.0,>=1.20 in /usr/local/lib/python3.11/dist-packages (from seaborn) (1.26.0)\n",
      "Requirement already satisfied: pandas>=1.2 in /usr/local/lib/python3.11/dist-packages (from seaborn) (2.2.3)\n",
      "Requirement already satisfied: matplotlib!=3.6.1,>=3.4 in /usr/local/lib/python3.11/dist-packages (from seaborn) (3.8.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (4.42.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.4.5)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (23.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (10.0.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (3.2.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.2->seaborn) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.2->seaborn) (2024.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib!=3.6.1,>=3.4->seaborn) (1.16.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (4.67.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: pydot in /usr/local/lib/python3.11/dist-packages (3.0.2)\n",
      "Requirement already satisfied: pyparsing>=3.0.9 in /usr/local/lib/python3.11/dist-packages (from pydot) (3.2.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: tensorflow-io in /usr/local/lib/python3.11/dist-packages (0.37.1)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem==0.37.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow-io) (0.37.1)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.11/dist-packages (0.23.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from tensorflow-addons) (23.1)\n",
      "Requirement already satisfied: typeguard<3.0.0,>=2.7 in /usr/local/lib/python3.11/dist-packages (from tensorflow-addons) (2.13.3)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n"
     ]
    }
   ],
   "source": [
    "!apt-get update\n",
    "!apt-get install graphviz -y\n",
    "\n",
    "!pip install tensorflow\n",
    "!pip install numpy\n",
    "!pip install pandas\n",
    "\n",
    "!pip install keras\n",
    "!pip install scikit-learn\n",
    "!pip install matplotlib\n",
    "!pip install joblib\n",
    "!pip install pyarrow\n",
    "!pip install fastparquet\n",
    "!pip install scipy\n",
    "!pip install seaborn\n",
    "!pip install tqdm\n",
    "!pip install pydot\n",
    "!pip install tensorflow-io\n",
    "!pip install tensorflow-addons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a467d3f0dfd9beab",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-12 13:23:10.750385: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-11-12 13:23:10.750440: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-11-12 13:23:10.750502: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-11-12 13:23:10.761720: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Keras version: 2.14.0\n",
      "TensorFlow version: 2.14.0\n",
      "TensorFlow version: 2.14.0\n",
      "CUDA available: True\n",
      "GPU devices: [PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n",
      "1 Physical GPUs, 1 Logical GPUs\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-12 13:23:13.297044: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1886] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 43404 MB memory:  -> device: 0, name: NVIDIA L40, pci bus id: 0000:01:00.0, compute capability: 8.9\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import keras\n",
    "\n",
    "print(f\"Keras version: {keras.__version__}\")\n",
    "print(f\"TensorFlow version: {tf.__version__}\")\n",
    "print(f\"TensorFlow version: {tf.__version__}\")\n",
    "print(f\"CUDA available: {tf.test.is_built_with_cuda()}\")\n",
    "print(f\"GPU devices: {tf.config.list_physical_devices('GPU')}\")\n",
    "\n",
    "# GPU configuration\n",
    "import tensorflow as tf\n",
    "import os\n",
    "\n",
    "# Limita la crescita della memoria GPU\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "if gpus:\n",
    "    try:\n",
    "        # Imposta la crescita di memoria dinamica\n",
    "        for gpu in gpus:\n",
    "            tf.config.experimental.set_memory_growth(gpu, True)\n",
    "            \n",
    "        # Opzionalmente, limita la memoria GPU massima (uncomment se necessario)\n",
    "        # tf.config.experimental.set_virtual_device_configuration(\n",
    "        #     gpus[0],\n",
    "        #     [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=1024*4)]  # 4GB\n",
    "        # )\n",
    "        \n",
    "        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
    "        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n",
    "    except RuntimeError as e:\n",
    "        print(e)\n",
    "        \n",
    "# Imposta le opzioni di logging\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'  # Riduce i messaggi di log\n",
    "        \n",
    "# Configura la modalità mista di precisione\n",
    "tf.keras.mixed_precision.set_global_policy('float32')\n",
    "\n",
    "# Imposta il seed per la riproducibilità\n",
    "##tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c0155cde4740b0a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.11/dist-packages/tensorflow_addons/utils/tfa_eol_msg.py:23: UserWarning: \n",
      "\n",
      "TensorFlow Addons (TFA) has ended development and introduction of new features.\n",
      "TFA has entered a minimal maintenance and release mode until a planned end of life in May 2024.\n",
      "Please modify downstream libraries to take dependencies from other repositories in our TensorFlow community (e.g. Keras, Keras-CV, and Keras-NLP). \n",
      "\n",
      "For more information see: https://github.com/tensorflow/addons/issues/2807 \n",
      "\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import tensorflow_addons as tfa\n",
    "from datetime import datetime\n",
    "import os\n",
    "import joblib\n",
    "import re\n",
    "from typing import List\n",
    "\n",
    "random_state_value = None\n",
    "execute_name = datetime.now().strftime(\"%Y-%m-%d_%H-%M\")\n",
    "\n",
    "base_project_dir = './'\n",
    "data_dir = '../../sources/'\n",
    "models_project_dir = base_project_dir\n",
    "\n",
    "os.makedirs(base_project_dir, exist_ok=True)\n",
    "os.makedirs(models_project_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1347fb59-50cc-4aa8-b805-ca9403037af5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_column_name(name: str) -> str:\n",
    "    \"\"\"\n",
    "    Rimuove caratteri speciali e spazi, converte in snake_case e abbrevia.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    name : str\n",
    "        Nome della colonna da pulire\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    str\n",
    "        Nome della colonna pulito\n",
    "    \"\"\"\n",
    "    # Rimuove caratteri speciali\n",
    "    name = re.sub(r'[^a-zA-Z0-9\\s]', '', name)\n",
    "    # Converte in snake_case\n",
    "    name = name.lower().replace(' ', '_')\n",
    "\n",
    "    # Abbreviazioni comuni\n",
    "    abbreviations = {\n",
    "        'production': 'prod',\n",
    "        'percentage': 'pct',\n",
    "        'hectare': 'ha',\n",
    "        'tonnes': 't',\n",
    "        'litres': 'l',\n",
    "        'minimum': 'min',\n",
    "        'maximum': 'max',\n",
    "        'average': 'avg'\n",
    "    }\n",
    "\n",
    "    for full, abbr in abbreviations.items():\n",
    "        name = name.replace(full, abbr)\n",
    "\n",
    "    return name\n",
    "\n",
    "\n",
    "def clean_column_names(df: pd.DataFrame) -> List[str]:\n",
    "    \"\"\"\n",
    "    Pulisce tutti i nomi delle colonne in un DataFrame.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    df : pd.DataFrame\n",
    "        DataFrame con le colonne da pulire\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    list\n",
    "        Lista dei nuovi nomi delle colonne puliti\n",
    "    \"\"\"\n",
    "    new_columns = []\n",
    "\n",
    "    for col in df.columns:\n",
    "        # Usa regex per separare le varietà\n",
    "        varieties = re.findall(r'([a-z]+)_([a-z_]+)', col)\n",
    "        if varieties:\n",
    "            new_columns.append(f\"{varieties[0][0]}_{varieties[0][1]}\")\n",
    "        else:\n",
    "            new_columns.append(col)\n",
    "\n",
    "    return new_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4da1f1bb67343e3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_plot(plt, title, output_dir=f'{base_project_dir}/{execute_name}_plots'):\n",
    "    os.makedirs(output_dir, exist_ok=True)\n",
    "    filename = \"\".join(x for x in title if x.isalnum() or x in [' ', '-', '_']).rstrip()\n",
    "    filename = filename.replace(' ', '_').lower()\n",
    "    filepath = os.path.join(output_dir, f\"{filename}.png\")\n",
    "    plt.savefig(filepath, bbox_inches='tight', dpi=300)\n",
    "    print(f\"Plot salvato come: {filepath}\")\n",
    "\n",
    "\n",
    "def encode_techniques(df, mapping_path=f'{data_dir}technique_mapping.joblib'):\n",
    "    if not os.path.exists(mapping_path):\n",
    "        raise FileNotFoundError(f\"Mapping not found at {mapping_path}. Run create_technique_mapping first.\")\n",
    "\n",
    "    technique_mapping = joblib.load(mapping_path)\n",
    "\n",
    "    # Trova tutte le colonne delle tecniche\n",
    "    tech_columns = [col for col in df.columns if col.endswith('_tech')]\n",
    "\n",
    "    # Applica il mapping a tutte le colonne delle tecniche\n",
    "    for col in tech_columns:\n",
    "        df[col] = df[col].str.lower().map(technique_mapping).fillna(0).astype(int)\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def decode_single_technique(technique_value, mapping_path=f'{data_dir}technique_mapping.joblib'):\n",
    "    if not os.path.exists(mapping_path):\n",
    "        raise FileNotFoundError(f\"Mapping not found at {mapping_path}\")\n",
    "\n",
    "    technique_mapping = joblib.load(mapping_path)\n",
    "    reverse_mapping = {v: k for k, v in technique_mapping.items()}\n",
    "    reverse_mapping[0] = ''\n",
    "\n",
    "    return reverse_mapping.get(technique_value, '')\n",
    "\n",
    "\n",
    "def prepare_comparison_data(simulated_data, olive_varieties):\n",
    "    # Pulisci i nomi delle colonne\n",
    "    df = simulated_data.copy()\n",
    "\n",
    "    df.columns = clean_column_names(df)\n",
    "    df = encode_techniques(df)\n",
    "\n",
    "    all_varieties = olive_varieties['Varietà di Olive'].unique()\n",
    "    varieties = [clean_column_name(variety) for variety in all_varieties]\n",
    "    comparison_data = []\n",
    "\n",
    "    for variety in varieties:\n",
    "        olive_prod_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_olive_prod')), None)\n",
    "        oil_prod_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_avg_oil_prod')), None)\n",
    "        tech_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_tech')), None)\n",
    "        water_need_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_water_need')), None)\n",
    "\n",
    "        if olive_prod_col and oil_prod_col and tech_col and water_need_col:\n",
    "            variety_data = df[[olive_prod_col, oil_prod_col, tech_col, water_need_col]]\n",
    "            variety_data = variety_data[variety_data[tech_col] != 0]  # Esclude le righe dove la tecnica è 0\n",
    "\n",
    "            if not variety_data.empty:\n",
    "                avg_olive_prod = pd.to_numeric(variety_data[olive_prod_col], errors='coerce').mean()\n",
    "                avg_oil_prod = pd.to_numeric(variety_data[oil_prod_col], errors='coerce').mean()\n",
    "                avg_water_need = pd.to_numeric(variety_data[water_need_col], errors='coerce').mean()\n",
    "                efficiency = avg_oil_prod / avg_olive_prod if avg_olive_prod > 0 else 0\n",
    "                water_efficiency = avg_oil_prod / avg_water_need if avg_water_need > 0 else 0\n",
    "\n",
    "                comparison_data.append({\n",
    "                    'Variety': variety,\n",
    "                    'Avg Olive Production (kg/ha)': avg_olive_prod,\n",
    "                    'Avg Oil Production (L/ha)': avg_oil_prod,\n",
    "                    'Avg Water Need (m³/ha)': avg_water_need,\n",
    "                    'Oil Efficiency (L/kg)': efficiency,\n",
    "                    'Water Efficiency (L oil/m³ water)': water_efficiency\n",
    "                })\n",
    "\n",
    "    return pd.DataFrame(comparison_data)\n",
    "\n",
    "\n",
    "def plot_variety_comparison(comparison_data, metric):\n",
    "    plt.figure(figsize=(12, 6))\n",
    "    bars = plt.bar(comparison_data['Variety'], comparison_data[metric])\n",
    "    plt.title(f'Comparison of {metric} across Olive Varieties')\n",
    "    plt.xlabel('Variety')\n",
    "    plt.ylabel(metric)\n",
    "    plt.xticks(rotation=45, ha='right')\n",
    "\n",
    "    for bar in bars:\n",
    "        height = bar.get_height()\n",
    "        plt.text(bar.get_x() + bar.get_width() / 2., height,\n",
    "                 f'{height:.2f}',\n",
    "                 ha='center', va='bottom')\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "    save_plot(plt, f'variety_comparison_{metric.lower().replace(\" \", \"_\").replace(\"/\", \"_\").replace(\"(\", \"\").replace(\")\", \"\")}')\n",
    "    plt.close()\n",
    "\n",
    "\n",
    "def plot_efficiency_vs_production(comparison_data):\n",
    "    plt.figure(figsize=(10, 6))\n",
    "\n",
    "    plt.scatter(comparison_data['Avg Olive Production (kg/ha)'],\n",
    "                comparison_data['Oil Efficiency (L/kg)'],\n",
    "                s=100)\n",
    "\n",
    "    for i, row in comparison_data.iterrows():\n",
    "        plt.annotate(row['Variety'],\n",
    "                     (row['Avg Olive Production (kg/ha)'], row['Oil Efficiency (L/kg)']),\n",
    "                     xytext=(5, 5), textcoords='offset points')\n",
    "\n",
    "    plt.title('Oil Efficiency vs Olive Production by Variety')\n",
    "    plt.xlabel('Average Olive Production (kg/ha)')\n",
    "    plt.ylabel('Oil Efficiency (L oil / kg olives)')\n",
    "    plt.tight_layout()\n",
    "    save_plot(plt, 'efficiency_vs_production')\n",
    "    plt.close()\n",
    "\n",
    "\n",
    "def plot_water_efficiency_vs_production(comparison_data):\n",
    "    plt.figure(figsize=(10, 6))\n",
    "\n",
    "    plt.scatter(comparison_data['Avg Olive Production (kg/ha)'],\n",
    "                comparison_data['Water Efficiency (L oil/m³ water)'],\n",
    "                s=100)\n",
    "\n",
    "    for i, row in comparison_data.iterrows():\n",
    "        plt.annotate(row['Variety'],\n",
    "                     (row['Avg Olive Production (kg/ha)'], row['Water Efficiency (L oil/m³ water)']),\n",
    "                     xytext=(5, 5), textcoords='offset points')\n",
    "\n",
    "    plt.title('Water Efficiency vs Olive Production by Variety')\n",
    "    plt.xlabel('Average Olive Production (kg/ha)')\n",
    "    plt.ylabel('Water Efficiency (L oil / m³ water)')\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "    save_plot(plt, 'water_efficiency_vs_production')\n",
    "    plt.close()\n",
    "\n",
    "\n",
    "def plot_water_need_vs_oil_production(comparison_data):\n",
    "    plt.figure(figsize=(10, 6))\n",
    "\n",
    "    plt.scatter(comparison_data['Avg Water Need (m³/ha)'],\n",
    "                comparison_data['Avg Oil Production (L/ha)'],\n",
    "                s=100)\n",
    "\n",
    "    for i, row in comparison_data.iterrows():\n",
    "        plt.annotate(row['Variety'],\n",
    "                     (row['Avg Water Need (m³/ha)'], row['Avg Oil Production (L/ha)']),\n",
    "                     xytext=(5, 5), textcoords='offset points')\n",
    "\n",
    "    plt.title('Oil Production vs Water Need by Variety')\n",
    "    plt.xlabel('Average Water Need (m³/ha)')\n",
    "    plt.ylabel('Average Oil Production (L/ha)')\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "    save_plot(plt, 'water_need_vs_oil_production')\n",
    "    plt.close()\n",
    "\n",
    "\n",
    "def analyze_by_technique(simulated_data, olive_varieties):\n",
    "    # Pulisci i nomi delle colonne\n",
    "    df = simulated_data.copy()\n",
    "\n",
    "    df.columns = clean_column_names(df)\n",
    "    df = encode_techniques(df)\n",
    "    all_varieties = olive_varieties['Varietà di Olive'].unique()\n",
    "    varieties = [clean_column_name(variety) for variety in all_varieties]\n",
    "\n",
    "    technique_data = []\n",
    "\n",
    "    for variety in varieties:\n",
    "        olive_prod_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_olive_prod')), None)\n",
    "        oil_prod_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_avg_oil_prod')), None)\n",
    "        tech_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_tech')), None)\n",
    "        water_need_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_water_need')), None)\n",
    "\n",
    "        if olive_prod_col and oil_prod_col and tech_col and water_need_col:\n",
    "            variety_data = df[[olive_prod_col, oil_prod_col, tech_col, water_need_col]]\n",
    "            variety_data = variety_data[variety_data[tech_col] != 0]\n",
    "\n",
    "            if not variety_data.empty:\n",
    "                for tech in variety_data[tech_col].unique():\n",
    "                    tech_data = variety_data[variety_data[tech_col] == tech]\n",
    "\n",
    "                    avg_olive_prod = pd.to_numeric(tech_data[olive_prod_col], errors='coerce').mean()\n",
    "                    avg_oil_prod = pd.to_numeric(tech_data[oil_prod_col], errors='coerce').mean()\n",
    "                    avg_water_need = pd.to_numeric(tech_data[water_need_col], errors='coerce').mean()\n",
    "\n",
    "                    efficiency = avg_oil_prod / avg_olive_prod if avg_olive_prod > 0 else 0\n",
    "                    water_efficiency = avg_oil_prod / avg_water_need if avg_water_need > 0 else 0\n",
    "\n",
    "                    technique_data.append({\n",
    "                        'Variety': variety,\n",
    "                        'Technique': tech,\n",
    "                        'Technique String': decode_single_technique(tech),\n",
    "                        'Avg Olive Production (kg/ha)': avg_olive_prod,\n",
    "                        'Avg Oil Production (L/ha)': avg_oil_prod,\n",
    "                        'Avg Water Need (m³/ha)': avg_water_need,\n",
    "                        'Oil Efficiency (L/kg)': efficiency,\n",
    "                        'Water Efficiency (L oil/m³ water)': water_efficiency\n",
    "                    })\n",
    "\n",
    "    return pd.DataFrame(technique_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9aa4bf176c4affb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_real_error(model, test_data, test_targets, scaler_y):\n",
    "    # Fare predizioni\n",
    "    predictions = model.predict(test_data)\n",
    "\n",
    "    # Denormalizzare predizioni e target\n",
    "    predictions_real = scaler_y.inverse_transform(predictions)\n",
    "    targets_real = scaler_y.inverse_transform(test_targets)\n",
    "\n",
    "    # Calcolare errore percentuale per ogni target\n",
    "    percentage_errors = []\n",
    "    absolute_errors = []\n",
    "\n",
    "    for i in range(predictions_real.shape[1]):\n",
    "        mae = np.mean(np.abs(predictions_real[:, i] - targets_real[:, i]))\n",
    "        mape = np.mean(np.abs((predictions_real[:, i] - targets_real[:, i]) / targets_real[:, i])) * 100\n",
    "        percentage_errors.append(mape)\n",
    "        absolute_errors.append(mae)\n",
    "\n",
    "    # Stampa risultati per ogni target\n",
    "    target_names = ['olive_prod', 'min_oil_prod', 'max_oil_prod', 'avg_oil_prod', 'total_water_need']\n",
    "\n",
    "    print(\"\\nErrori per target:\")\n",
    "    print(\"-\" * 50)\n",
    "    for i, target in enumerate(target_names):\n",
    "        print(f\"{target}:\")\n",
    "        print(f\"MAE assoluto: {absolute_errors[i]:.2f}\")\n",
    "        print(f\"Errore percentuale medio: {percentage_errors[i]:.2f}%\")\n",
    "        print(f\"Precisione: {100 - percentage_errors[i]:.2f}%\")\n",
    "        print(\"-\" * 50)\n",
    "\n",
    "    return percentage_errors, absolute_errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b3ba2b96ba678389",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'Variety'",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mKeyError\u001B[0m                                  Traceback (most recent call last)",
      "Cell \u001B[0;32mIn[7], line 7\u001B[0m\n\u001B[1;32m      4\u001B[0m comparison_data \u001B[38;5;241m=\u001B[39m prepare_comparison_data(simulated_data, olive_varieties)\n\u001B[1;32m      6\u001B[0m \u001B[38;5;66;03m# Genera i grafici\u001B[39;00m\n\u001B[0;32m----> 7\u001B[0m \u001B[43mplot_variety_comparison\u001B[49m\u001B[43m(\u001B[49m\u001B[43mcomparison_data\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43mAvg Olive Production (kg/ha)\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[1;32m      8\u001B[0m plot_variety_comparison(comparison_data, \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mAvg Oil Production (L/ha)\u001B[39m\u001B[38;5;124m'\u001B[39m)\n\u001B[1;32m      9\u001B[0m plot_variety_comparison(comparison_data, \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mAvg Water Need (m³/ha)\u001B[39m\u001B[38;5;124m'\u001B[39m)\n",
      "Cell \u001B[0;32mIn[5], line 79\u001B[0m, in \u001B[0;36mplot_variety_comparison\u001B[0;34m(comparison_data, metric)\u001B[0m\n\u001B[1;32m     77\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mplot_variety_comparison\u001B[39m(comparison_data, metric):\n\u001B[1;32m     78\u001B[0m     plt\u001B[38;5;241m.\u001B[39mfigure(figsize\u001B[38;5;241m=\u001B[39m(\u001B[38;5;241m12\u001B[39m, \u001B[38;5;241m6\u001B[39m))\n\u001B[0;32m---> 79\u001B[0m     bars \u001B[38;5;241m=\u001B[39m plt\u001B[38;5;241m.\u001B[39mbar(\u001B[43mcomparison_data\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43mVariety\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m]\u001B[49m, comparison_data[metric])\n\u001B[1;32m     80\u001B[0m     plt\u001B[38;5;241m.\u001B[39mtitle(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mComparison of \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mmetric\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m across Olive Varieties\u001B[39m\u001B[38;5;124m'\u001B[39m)\n\u001B[1;32m     81\u001B[0m     plt\u001B[38;5;241m.\u001B[39mxlabel(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mVariety\u001B[39m\u001B[38;5;124m'\u001B[39m)\n",
      "File \u001B[0;32m/usr/local/lib/python3.11/dist-packages/pandas/core/frame.py:4102\u001B[0m, in \u001B[0;36mDataFrame.__getitem__\u001B[0;34m(self, key)\u001B[0m\n\u001B[1;32m   4100\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mcolumns\u001B[38;5;241m.\u001B[39mnlevels \u001B[38;5;241m>\u001B[39m \u001B[38;5;241m1\u001B[39m:\n\u001B[1;32m   4101\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_getitem_multilevel(key)\n\u001B[0;32m-> 4102\u001B[0m indexer \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcolumns\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mget_loc\u001B[49m\u001B[43m(\u001B[49m\u001B[43mkey\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   4103\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m is_integer(indexer):\n\u001B[1;32m   4104\u001B[0m     indexer \u001B[38;5;241m=\u001B[39m [indexer]\n",
      "File \u001B[0;32m/usr/local/lib/python3.11/dist-packages/pandas/core/indexes/range.py:417\u001B[0m, in \u001B[0;36mRangeIndex.get_loc\u001B[0;34m(self, key)\u001B[0m\n\u001B[1;32m    415\u001B[0m         \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mKeyError\u001B[39;00m(key) \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01merr\u001B[39;00m\n\u001B[1;32m    416\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(key, Hashable):\n\u001B[0;32m--> 417\u001B[0m     \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mKeyError\u001B[39;00m(key)\n\u001B[1;32m    418\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_check_indexing_error(key)\n\u001B[1;32m    419\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mKeyError\u001B[39;00m(key)\n",
      "\u001B[0;31mKeyError\u001B[0m: 'Variety'"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1200x600 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "simulated_data = pd.read_parquet(f\"{data_dir}simulated_data.parquet\")\n",
    "olive_varieties = pd.read_parquet(f\"{data_dir}olive_varieties.parquet\")\n",
    "# Esecuzione dell'analisi\n",
    "comparison_data = prepare_comparison_data(simulated_data, olive_varieties)\n",
    "\n",
    "# Genera i grafici\n",
    "plot_variety_comparison(comparison_data, 'Avg Olive Production (kg/ha)')\n",
    "plot_variety_comparison(comparison_data, 'Avg Oil Production (L/ha)')\n",
    "plot_variety_comparison(comparison_data, 'Avg Water Need (m³/ha)')\n",
    "plot_variety_comparison(comparison_data, 'Oil Efficiency (L/kg)')\n",
    "plot_variety_comparison(comparison_data, 'Water Efficiency (L oil/m³ water)')\n",
    "plot_efficiency_vs_production(comparison_data)\n",
    "plot_water_efficiency_vs_production(comparison_data)\n",
    "plot_water_need_vs_oil_production(comparison_data)\n",
    "\n",
    "# Analisi per tecnica\n",
    "technique_data = analyze_by_technique(simulated_data, olive_varieties)\n",
    "\n",
    "print(technique_data)\n",
    "\n",
    "# Stampa un sommario statistico\n",
    "print(\"Comparison by Variety:\")\n",
    "print(comparison_data.set_index('Variety'))\n",
    "print(\"\\nBest Varieties by Water Efficiency:\")\n",
    "print(comparison_data.sort_values('Water Efficiency (L oil/m³ water)', ascending=False).head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbe87b415168368",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_transformer_data(df, olive_varieties_df):\n",
    "    # Crea una copia del DataFrame per evitare modifiche all'originale\n",
    "    df = df.copy()\n",
    "\n",
    "    # Ordina per zona e anno\n",
    "    df = df.sort_values(['zone', 'year'])\n",
    "\n",
    "    # Definisci le feature\n",
    "    temporal_features = ['temp_mean', 'precip_sum', 'solar_energy_sum']\n",
    "    static_features = ['ha']  # Feature statiche base\n",
    "    target_features = ['olive_prod', 'min_oil_prod', 'max_oil_prod', 'avg_oil_prod', 'total_water_need']\n",
    "\n",
    "    # Ottieni le varietà pulite\n",
    "    all_varieties = olive_varieties_df['Varietà di Olive'].unique()\n",
    "    varieties = [clean_column_name(variety) for variety in all_varieties]\n",
    "\n",
    "    # Crea la struttura delle feature per ogni varietà\n",
    "    variety_features = [\n",
    "        'tech', 'pct', 'prod_t_ha', 'oil_prod_t_ha', 'oil_prod_l_ha',\n",
    "        'min_yield_pct', 'max_yield_pct', 'min_oil_prod_l_ha', 'max_oil_prod_l_ha',\n",
    "        'avg_oil_prod_l_ha', 'l_per_t', 'min_l_per_t', 'max_l_per_t', 'avg_l_per_t'\n",
    "    ]\n",
    "\n",
    "    # Prepara dizionari per le nuove colonne\n",
    "    new_columns = {}\n",
    "\n",
    "    # Prepara le feature per ogni varietà\n",
    "    for variety in varieties:\n",
    "        # Feature esistenti\n",
    "        for feature in variety_features:\n",
    "            col_name = f\"{variety}_{feature}\"\n",
    "            if col_name in df.columns:\n",
    "                if feature != 'tech':  # Non includere la colonna tech direttamente\n",
    "                    static_features.append(col_name)\n",
    "\n",
    "        # Feature binarie per le tecniche di coltivazione\n",
    "        for technique in ['tradizionale', 'intensiva', 'superintensiva']:\n",
    "            col_name = f\"{variety}_{technique}\"\n",
    "            new_columns[col_name] = df[f\"{variety}_tech\"].notna() & (\n",
    "                    df[f\"{variety}_tech\"].str.lower() == technique\n",
    "            ).fillna(False)\n",
    "            static_features.append(col_name)\n",
    "\n",
    "    # Aggiungi tutte le nuove colonne in una volta sola\n",
    "    new_df = pd.concat([df] + [pd.Series(v, name=k) for k, v in new_columns.items()], axis=1)\n",
    "\n",
    "    # Ordiniamo per zona e anno per mantenere la continuità temporale\n",
    "    df_sorted = new_df.sort_values(['zone', 'year'])\n",
    "\n",
    "    # Definiamo la dimensione della finestra temporale\n",
    "    window_size = 41\n",
    "\n",
    "    # Liste per raccogliere i dati\n",
    "    temporal_sequences = []\n",
    "    static_features_list = []\n",
    "    targets_list = []\n",
    "\n",
    "    # Iteriamo per ogni zona\n",
    "    for zone in df_sorted['zone'].unique():\n",
    "        zone_data = df_sorted[df_sorted['zone'] == zone].reset_index(drop=True)\n",
    "\n",
    "        if len(zone_data) >= window_size:  # Verifichiamo che ci siano abbastanza dati\n",
    "            # Creiamo sequenze temporali scorrevoli\n",
    "            for i in range(len(zone_data) - window_size + 1):\n",
    "                # Sequenza temporale\n",
    "                temporal_window = zone_data.iloc[i:i + window_size][temporal_features].values\n",
    "                # Verifichiamo che non ci siano valori NaN\n",
    "                if not np.isnan(temporal_window).any():\n",
    "                    temporal_sequences.append(temporal_window)\n",
    "\n",
    "                    # Feature statiche (prendiamo quelle dell'ultimo timestep della finestra)\n",
    "                    static_features_list.append(zone_data.iloc[i + window_size - 1][static_features].values)\n",
    "\n",
    "                    # Target (prendiamo quelli dell'ultimo timestep della finestra)\n",
    "                    targets_list.append(zone_data.iloc[i + window_size - 1][target_features].values)\n",
    "\n",
    "    # Convertiamo in array numpy\n",
    "    X_temporal = np.array(temporal_sequences)\n",
    "    X_static = np.array(static_features_list)\n",
    "    y = np.array(targets_list)\n",
    "\n",
    "    print(f\"Dataset completo - Temporal: {X_temporal.shape}, Static: {X_static.shape}, Target: {y.shape}\")\n",
    "\n",
    "    # Split dei dati (usando indici casuali per una migliore distribuzione)\n",
    "    indices = np.random.permutation(len(X_temporal))\n",
    "\n",
    "    #train_idx = int(len(indices) * 0.7)        # 70% training\n",
    "    #val_idx = int(len(indices) * 0.85)         # 15% validation\n",
    "    # Il resto rimane 15% test\n",
    "\n",
    "    train_idx = int(len(indices) * 0.65)        # 65% training\n",
    "    val_idx = int(len(indices) * 0.85)          # 20% validation\n",
    "    # Il resto rimane 15% test\n",
    "\n",
    "    #train_idx = int(len(indices) * 0.60)       # 60% training\n",
    "    #val_idx = int(len(indices) * 0.85)         # 25% validation\n",
    "    # Il resto rimane 15% test\n",
    "\n",
    "    train_indices = indices[:train_idx]\n",
    "    val_indices = indices[train_idx:val_idx]\n",
    "    test_indices = indices[val_idx:]\n",
    "\n",
    "    # Split dei dati\n",
    "    X_temporal_train = X_temporal[train_indices]\n",
    "    X_temporal_val = X_temporal[val_indices]\n",
    "    X_temporal_test = X_temporal[test_indices]\n",
    "\n",
    "    X_static_train = X_static[train_indices]\n",
    "    X_static_val = X_static[val_indices]\n",
    "    X_static_test = X_static[test_indices]\n",
    "\n",
    "    y_train = y[train_indices]\n",
    "    y_val = y[val_indices]\n",
    "    y_test = y[test_indices]\n",
    "\n",
    "    # Standardizzazione\n",
    "    scaler_temporal = StandardScaler()\n",
    "    scaler_static = StandardScaler()\n",
    "    scaler_y = StandardScaler()\n",
    "\n",
    "    # Standardizzazione dei dati temporali\n",
    "    X_temporal_train = scaler_temporal.fit_transform(X_temporal_train.reshape(-1, len(temporal_features))).reshape(X_temporal_train.shape)\n",
    "    X_temporal_val = scaler_temporal.transform(X_temporal_val.reshape(-1, len(temporal_features))).reshape(X_temporal_val.shape)\n",
    "    X_temporal_test = scaler_temporal.transform(X_temporal_test.reshape(-1, len(temporal_features))).reshape(X_temporal_test.shape)\n",
    "\n",
    "    # Standardizzazione dei dati statici\n",
    "    X_static_train = scaler_static.fit_transform(X_static_train)\n",
    "    X_static_val = scaler_static.transform(X_static_val)\n",
    "    X_static_test = scaler_static.transform(X_static_test)\n",
    "\n",
    "    # Standardizzazione dei target\n",
    "    y_train = scaler_y.fit_transform(y_train)\n",
    "    y_val = scaler_y.transform(y_val)\n",
    "    y_test = scaler_y.transform(y_test)\n",
    "\n",
    "    print(\"\\nShape dopo lo split e standardizzazione:\")\n",
    "    print(f\"Train - Temporal: {X_temporal_train.shape}, Static: {X_static_train.shape}, Target: {y_train.shape}\")\n",
    "    print(f\"Val - Temporal: {X_temporal_val.shape}, Static: {X_static_val.shape}, Target: {y_val.shape}\")\n",
    "    print(f\"Test - Temporal: {X_temporal_test.shape}, Static: {X_static_test.shape}, Target: {y_test.shape}\")\n",
    "\n",
    "    # Prepara i dizionari di input\n",
    "    train_data = {'temporal': X_temporal_train, 'static': X_static_train}\n",
    "    val_data = {'temporal': X_temporal_val, 'static': X_static_val}\n",
    "    test_data = {'temporal': X_temporal_test, 'static': X_static_test}\n",
    "\n",
    "    joblib.dump(scaler_temporal, os.path.join(base_project_dir, f'{execute_name}_scaler_temporal.joblib'))\n",
    "    joblib.dump(scaler_static, os.path.join(base_project_dir, f'{execute_name}_scaler_static.joblib'))\n",
    "    joblib.dump(scaler_y, os.path.join(base_project_dir, f'{execute_name}_scaler_y.joblib'))\n",
    "\n",
    "    return (train_data, y_train), (val_data, y_val), (test_data, y_test), (scaler_temporal, scaler_static, scaler_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c4d5f0f3fafdc2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "simulated_data = pd.read_parquet(f\"{data_dir}simulated_data.parquet\")\n",
    "olive_varieties = pd.read_parquet(f\"{data_dir}olive_varieties.parquet\")\n",
    "\n",
    "(train_data, train_targets), (val_data, val_targets), (test_data, test_targets), scalers = prepare_transformer_data(simulated_data, olive_varieties)\n",
    "\n",
    "scaler_temporal, scaler_static, scaler_y = scalers\n",
    "\n",
    "print(\"Temporal data shape:\", train_data['temporal'].shape)\n",
    "print(\"Static data shape:\", train_data['static'].shape)\n",
    "print(\"Target shape:\", train_targets.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af0c53c96eb322cc",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "604c952c7195f40c",
   "metadata": {},
   "outputs": [],
   "source": [
    "@keras.saving.register_keras_serializable()\n",
    "class DataAugmentation(tf.keras.layers.Layer):\n",
    "    \"\"\"Custom layer per l'augmentation dei dati\"\"\"\n",
    "\n",
    "    def __init__(self, noise_stddev=0.03, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.noise_stddev = noise_stddev\n",
    "\n",
    "    def call(self, inputs, training=None):\n",
    "        if training:\n",
    "            return inputs + tf.random.normal(\n",
    "                shape=tf.shape(inputs),\n",
    "                mean=0.0,\n",
    "                stddev=self.noise_stddev\n",
    "            )\n",
    "        return inputs\n",
    "\n",
    "    def get_config(self):\n",
    "        config = super().get_config()\n",
    "        config.update({\"noise_stddev\": self.noise_stddev})\n",
    "        return config\n",
    "\n",
    "\n",
    "@keras.saving.register_keras_serializable()\n",
    "class PositionalEncoding(tf.keras.layers.Layer):\n",
    "    \"\"\"Custom layer per l'encoding posizionale\"\"\"\n",
    "\n",
    "    def __init__(self, d_model, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.d_model = d_model\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        _, seq_length, _ = input_shape\n",
    "\n",
    "        # Crea la matrice di encoding posizionale\n",
    "        position = tf.range(seq_length, dtype=tf.float32)[:, tf.newaxis]\n",
    "        div_term = tf.exp(\n",
    "            tf.range(0, self.d_model, 2, dtype=tf.float32) *\n",
    "            (-tf.math.log(10000.0) / self.d_model)\n",
    "        )\n",
    "\n",
    "        # Calcola sin e cos\n",
    "        pos_encoding = tf.zeros((1, seq_length, self.d_model))\n",
    "        pos_encoding_even = tf.sin(position * div_term)\n",
    "        pos_encoding_odd = tf.cos(position * div_term)\n",
    "\n",
    "        # Assegna i valori alle posizioni pari e dispari\n",
    "        pos_encoding = tf.concat(\n",
    "            [tf.expand_dims(pos_encoding_even, -1),\n",
    "             tf.expand_dims(pos_encoding_odd, -1)],\n",
    "            axis=-1\n",
    "        )\n",
    "        pos_encoding = tf.reshape(pos_encoding, (1, seq_length, -1))\n",
    "        pos_encoding = pos_encoding[:, :, :self.d_model]\n",
    "\n",
    "        # Salva l'encoding come peso non trainabile\n",
    "        self.pos_encoding = self.add_weight(\n",
    "            shape=(1, seq_length, self.d_model),\n",
    "            initializer=tf.keras.initializers.Constant(pos_encoding),\n",
    "            trainable=False,\n",
    "            name='positional_encoding'\n",
    "        )\n",
    "\n",
    "        super().build(input_shape)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        # Broadcast l'encoding posizionale sul batch\n",
    "        batch_size = tf.shape(inputs)[0]\n",
    "        pos_encoding_tiled = tf.tile(self.pos_encoding, [batch_size, 1, 1])\n",
    "        return inputs + pos_encoding_tiled\n",
    "\n",
    "    def get_config(self):\n",
    "        config = super().get_config()\n",
    "        config.update({\"d_model\": self.d_model})\n",
    "        return config\n",
    "\n",
    "\n",
    "@keras.saving.register_keras_serializable()\n",
    "class WarmUpLearningRateSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
    "    \"\"\"Custom learning rate schedule with linear warmup and exponential decay.\"\"\"\n",
    "\n",
    "    def __init__(self, initial_learning_rate=1e-3, warmup_steps=500, decay_steps=5000):\n",
    "        super().__init__()\n",
    "        self.initial_learning_rate = initial_learning_rate\n",
    "        self.warmup_steps = warmup_steps\n",
    "        self.decay_steps = decay_steps\n",
    "\n",
    "    def __call__(self, step):\n",
    "        warmup_pct = tf.cast(step, tf.float32) / self.warmup_steps\n",
    "        warmup_lr = self.initial_learning_rate * warmup_pct\n",
    "        decay_factor = tf.pow(0.1, tf.cast(step, tf.float32) / self.decay_steps)\n",
    "        decayed_lr = self.initial_learning_rate * decay_factor\n",
    "        return tf.where(step < self.warmup_steps, warmup_lr, decayed_lr)\n",
    "\n",
    "    def get_config(self):\n",
    "        return {\n",
    "            'initial_learning_rate': self.initial_learning_rate,\n",
    "            'warmup_steps': self.warmup_steps,\n",
    "            'decay_steps': self.decay_steps\n",
    "        }\n",
    "\n",
    "\n",
    "def create_olive_oil_transformer(temporal_shape, static_shape, num_outputs,\n",
    "                                 d_model=128, num_heads=8, ff_dim=256,\n",
    "                                 num_transformer_blocks=4, mlp_units=None,\n",
    "                                 dropout=0.2):\n",
    "    \"\"\"\n",
    "    Crea un transformer per la predizione della produzione di olio d'oliva.\n",
    "    \"\"\"\n",
    "    # Input layers\n",
    "    if mlp_units is None:\n",
    "        mlp_units = [256, 128, 64]\n",
    "\n",
    "    temporal_input = tf.keras.layers.Input(shape=temporal_shape, name='temporal')\n",
    "    static_input = tf.keras.layers.Input(shape=static_shape, name='static')\n",
    "\n",
    "    # === TEMPORAL PATH ===\n",
    "    x = tf.keras.layers.LayerNormalization(epsilon=1e-6)(temporal_input)\n",
    "    x = DataAugmentation()(x)\n",
    "\n",
    "    # Temporal projection\n",
    "    x = tf.keras.layers.Dense(\n",
    "        d_model // 2,\n",
    "        activation='swish',\n",
    "        kernel_regularizer=tf.keras.regularizers.l2(1e-5)\n",
    "    )(x)\n",
    "    x = tf.keras.layers.Dropout(dropout)(x)\n",
    "    x = tf.keras.layers.Dense(\n",
    "        d_model,\n",
    "        activation='swish',\n",
    "        kernel_regularizer=tf.keras.regularizers.l2(1e-5)\n",
    "    )(x)\n",
    "\n",
    "    # Positional encoding\n",
    "    x = PositionalEncoding(d_model)(x)\n",
    "\n",
    "    # Transformer blocks\n",
    "    skip_connection = x\n",
    "    for _ in range(num_transformer_blocks):\n",
    "        # Self-attention\n",
    "        attention_output = tf.keras.layers.MultiHeadAttention(\n",
    "            num_heads=num_heads,\n",
    "            key_dim=d_model // num_heads,\n",
    "            value_dim=d_model // num_heads\n",
    "        )(x, x)\n",
    "        attention_output = tf.keras.layers.Dropout(dropout)(attention_output)\n",
    "\n",
    "        # Residual connection con pesi addestrabili\n",
    "        residual_weights = tf.keras.layers.Dense(d_model, activation='sigmoid')(x)\n",
    "        x = tfa.layers.StochasticDepth(survival_probability=0.3)([x, residual_weights * attention_output])\n",
    "        x = tf.keras.layers.LayerNormalization(epsilon=1e-6)(x)\n",
    "\n",
    "        # Feed-forward network\n",
    "        ffn = tf.keras.layers.Dense(ff_dim, activation=\"swish\")(x)\n",
    "        ffn = tf.keras.layers.Dropout(dropout)(ffn)\n",
    "        ffn = tf.keras.layers.Dense(d_model)(ffn)\n",
    "        ffn = tf.keras.layers.Dropout(dropout)(ffn)\n",
    "\n",
    "        # Second residual connection\n",
    "        x = tfa.layers.StochasticDepth()([x, ffn])\n",
    "        x = tf.keras.layers.LayerNormalization(epsilon=1e-6)(x)\n",
    "\n",
    "    # Add final skip connection\n",
    "    x = tfa.layers.StochasticDepth(survival_probability=0.5)([x, skip_connection])\n",
    "\n",
    "    # Temporal pooling\n",
    "    attention_pooled = tf.keras.layers.MultiHeadAttention(\n",
    "        num_heads=num_heads,\n",
    "        key_dim=d_model // 4\n",
    "    )(x, x)\n",
    "    attention_pooled = tf.keras.layers.GlobalAveragePooling1D()(attention_pooled)\n",
    "\n",
    "    # Additional pooling operations\n",
    "    avg_pooled = tf.keras.layers.GlobalAveragePooling1D()(x)\n",
    "    max_pooled = tf.keras.layers.GlobalMaxPooling1D()(x)\n",
    "\n",
    "    # Combine pooling results\n",
    "    temporal_features = tf.keras.layers.Concatenate()(\n",
    "        [attention_pooled, avg_pooled, max_pooled]\n",
    "    )\n",
    "\n",
    "    # === STATIC PATH ===\n",
    "    static_features = tf.keras.layers.LayerNormalization(epsilon=1e-6)(static_input)\n",
    "    for units in [256, 128, 64]:\n",
    "        static_features = tf.keras.layers.Dense(\n",
    "            units,\n",
    "            activation='swish',\n",
    "            kernel_regularizer=tf.keras.regularizers.l2(1e-5)\n",
    "        )(static_features)\n",
    "        static_features = tf.keras.layers.Dropout(dropout)(static_features)\n",
    "\n",
    "    # === FEATURE FUSION ===\n",
    "    combined = tf.keras.layers.Concatenate()([temporal_features, static_features])\n",
    "\n",
    "    # === MLP HEAD ===\n",
    "    x = combined\n",
    "    for units in mlp_units:\n",
    "        x = tf.keras.layers.BatchNormalization()(x)\n",
    "        x = tf.keras.layers.Dense(\n",
    "            units,\n",
    "            activation=\"swish\",\n",
    "            kernel_regularizer=tf.keras.regularizers.l2(1e-5)\n",
    "        )(x)\n",
    "        x = tf.keras.layers.Dropout(dropout)(x)\n",
    "\n",
    "    # Output layer\n",
    "    outputs = tf.keras.layers.Dense(\n",
    "        num_outputs,\n",
    "        activation='linear',\n",
    "        kernel_regularizer=tf.keras.regularizers.l2(1e-5)\n",
    "    )(x)\n",
    "\n",
    "    # Create model\n",
    "    model = tf.keras.Model(\n",
    "        inputs={'temporal': temporal_input, 'static': static_input},\n",
    "        outputs=outputs,\n",
    "        name='OilTransformer'\n",
    "    )\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "def create_transformer_callbacks(target_names, val_data, val_targets):\n",
    "    \"\"\"\n",
    "    Crea i callbacks per il training del modello.\n",
    "    \n",
    "    Parameters:\n",
    "    -----------\n",
    "    target_names : list\n",
    "        Lista dei nomi dei target per il monitoraggio specifico\n",
    "    val_data : dict\n",
    "        Dati di validazione\n",
    "    val_targets : array\n",
    "        Target di validazione\n",
    "    \n",
    "    Returns:\n",
    "    --------\n",
    "    list\n",
    "        Lista dei callbacks configurati\n",
    "    \"\"\"\n",
    "\n",
    "    # Custom Metric per target specifici\n",
    "    class TargetSpecificMetric(tf.keras.callbacks.Callback):\n",
    "        def __init__(self, validation_data, target_names):\n",
    "            super().__init__()\n",
    "            self.validation_data = validation_data\n",
    "            self.target_names = target_names\n",
    "\n",
    "        def on_epoch_end(self, epoch, logs={}):\n",
    "            x_val, y_val = self.validation_data\n",
    "            y_pred = self.model.predict(x_val, verbose=0)\n",
    "\n",
    "            for i, name in enumerate(self.target_names):\n",
    "                mae = np.mean(np.abs(y_val[:, i] - y_pred[:, i]))\n",
    "                logs[f'val_{name}_mae'] = mae\n",
    "\n",
    "\n",
    "    callbacks = [\n",
    "        # Early Stopping\n",
    "        tf.keras.callbacks.EarlyStopping(\n",
    "            monitor='val_loss',\n",
    "            patience=20,\n",
    "            restore_best_weights=True,\n",
    "            min_delta=0.0005,\n",
    "            mode='min'\n",
    "        ),\n",
    "\n",
    "        # Model Checkpoint\n",
    "        tf.keras.callbacks.ModelCheckpoint(\n",
    "            filepath=f'{execute_name}_best_oil_model.h5',\n",
    "            monitor='val_loss',\n",
    "            save_best_only=True,\n",
    "            mode='min',\n",
    "            save_weights_only=True\n",
    "        ),\n",
    "\n",
    "        # Metric per target specifici\n",
    "        TargetSpecificMetric(\n",
    "            validation_data=(val_data, val_targets),\n",
    "            target_names=target_names\n",
    "        ),\n",
    "\n",
    "        # Reduce LR on Plateau\n",
    "        tf.keras.callbacks.ReduceLROnPlateau(\n",
    "            monitor='val_loss',\n",
    "            factor=0.5,\n",
    "            patience=10,\n",
    "            min_lr=1e-6,\n",
    "            verbose=1\n",
    "        ),\n",
    "\n",
    "        # TensorBoard logging\n",
    "        tf.keras.callbacks.TensorBoard(\n",
    "            log_dir=f'./logs_{execute_name}',\n",
    "            histogram_freq=1,\n",
    "            write_graph=True,\n",
    "            update_freq='epoch'\n",
    "        )\n",
    "    ]\n",
    "\n",
    "    return callbacks\n",
    "\n",
    "\n",
    "def compile_model(model, learning_rate=1e-3):\n",
    "    \"\"\"\n",
    "    Compila il modello con le impostazioni standard.\n",
    "    \"\"\"\n",
    "    lr_schedule = WarmUpLearningRateSchedule(\n",
    "        initial_learning_rate=learning_rate,\n",
    "        warmup_steps=500,\n",
    "        decay_steps=5000\n",
    "    )\n",
    "\n",
    "    model.compile(\n",
    "        optimizer=tf.keras.optimizers.AdamW(\n",
    "            learning_rate=lr_schedule,\n",
    "            weight_decay=0.01\n",
    "        ),\n",
    "        loss=tf.keras.losses.Huber(),\n",
    "        metrics=['mae']\n",
    "    )\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "def setup_transformer_training(train_data, train_targets, val_data, val_targets):\n",
    "    \"\"\"\n",
    "    Configura e prepara il transformer con dimensioni dinamiche basate sui dati.\n",
    "    \"\"\"\n",
    "    # Estrai le shape dai dati\n",
    "    temporal_shape = (train_data['temporal'].shape[1], train_data['temporal'].shape[2])\n",
    "    static_shape = (train_data['static'].shape[1],)\n",
    "    num_outputs = train_targets.shape[1]\n",
    "\n",
    "    print(f\"Shape rilevate:\")\n",
    "    print(f\"- Temporal shape: {temporal_shape}\")\n",
    "    print(f\"- Static shape: {static_shape}\")\n",
    "    print(f\"- Numero di output: {num_outputs}\")\n",
    "\n",
    "    # Target names basati sul numero di output\n",
    "    target_names = ['olive_prod', 'min_oil_prod', 'max_oil_prod', 'avg_oil_prod', 'total_water_need']\n",
    "\n",
    "    # Assicurati che il numero di target names corrisponda al numero di output\n",
    "    assert len(target_names) == num_outputs, \\\n",
    "        f\"Il numero di target names ({len(target_names)}) non corrisponde al numero di output ({num_outputs})\"\n",
    "\n",
    "    # Crea il modello con le dimensioni rilevate\n",
    "    model = create_olive_oil_transformer(\n",
    "        temporal_shape=temporal_shape,\n",
    "        static_shape=static_shape,\n",
    "        num_outputs=num_outputs\n",
    "    )\n",
    "\n",
    "    # Compila il modello\n",
    "    model = compile_model(model)\n",
    "\n",
    "    # Crea i callbacks\n",
    "    callbacks = create_transformer_callbacks(target_names, val_data, val_targets)\n",
    "\n",
    "    return model, callbacks, target_names\n",
    "\n",
    "\n",
    "def train_transformer(train_data, train_targets, val_data, val_targets, epochs=150, batch_size=64, save_name='final_model'):\n",
    "    \"\"\"\n",
    "    Funzione principale per l'addestramento del transformer con ottimizzazioni.\n",
    "    \"\"\"\n",
    "    # Conversione dei dati in tf.data.Dataset per una gestione più efficiente della memoria\n",
    "    train_dataset = tf.data.Dataset.from_tensor_slices((train_data, train_targets))\\\n",
    "        .cache()\\\n",
    "        .shuffle(buffer_size=1024)\\\n",
    "        .batch(batch_size)\\\n",
    "        .prefetch(tf.data.AUTOTUNE)\n",
    "\n",
    "    val_dataset = tf.data.Dataset.from_tensor_slices((val_data, val_targets))\\\n",
    "        .cache()\\\n",
    "        .batch(batch_size)\\\n",
    "        .prefetch(tf.data.AUTOTUNE)\n",
    "\n",
    "    # Setup del modello\n",
    "    strategy = tf.distribute.MirroredStrategy() if len(tf.config.list_physical_devices('GPU')) > 1 else tf.distribute.get_strategy()\n",
    "    \n",
    "    with strategy.scope():\n",
    "        model, callbacks, target_names = setup_transformer_training(\n",
    "            train_data, train_targets, val_data, val_targets\n",
    "        )\n",
    "\n",
    "    # Mostra il summary del modello\n",
    "    model.summary()\n",
    "    \n",
    "    try:\n",
    "        keras.utils.plot_model(model, f\"{execute_name}_{save_name}.png\", show_shapes=True)\n",
    "    except Exception as e:\n",
    "        print(f\"Warning: Could not create model plot: {e}\")\n",
    "\n",
    "    # Training con gestione degli errori\n",
    "    try:\n",
    "        history = model.fit(\n",
    "            train_dataset,\n",
    "            validation_data=val_dataset,\n",
    "            epochs=epochs,\n",
    "            callbacks=callbacks,\n",
    "            verbose=1,\n",
    "            workers=4,\n",
    "            use_multiprocessing=True\n",
    "        )\n",
    "    except tf.errors.ResourceExhaustedError:\n",
    "        print(\"Memoria GPU esaurita, riprovo con batch size più piccolo...\")\n",
    "        # Riprova con batch size più piccolo\n",
    "        batch_size = batch_size // 2\n",
    "        train_dataset = train_dataset.unbatch().batch(batch_size)\n",
    "        val_dataset = val_dataset.unbatch().batch(batch_size)\n",
    "        history = model.fit(\n",
    "            train_dataset,\n",
    "            validation_data=val_dataset,\n",
    "            epochs=epochs,\n",
    "            callbacks=callbacks,\n",
    "            verbose=1\n",
    "        )\n",
    "\n",
    "    # Salva il modello finale\n",
    "    try:\n",
    "        save_path = f'{execute_name}_{save_name}.keras'\n",
    "        model.save(save_path, save_format='keras')\n",
    "        \n",
    "        os.makedirs(f'{execute_name}/weights', exist_ok=True)\n",
    "        model.save_weights(f'{execute_name}/weights')\n",
    "        print(f\"\\nModello salvato in: {save_path}\")\n",
    "    except Exception as e:\n",
    "        print(f\"Warning: Could not save model: {e}\")\n",
    "\n",
    "    return model, history"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33ad638941654e12",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35490e902e494c4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "model, history = train_transformer(train_data, train_targets, val_data, val_targets, 150, 256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e2fb5a5341dac92",
   "metadata": {},
   "outputs": [],
   "source": [
    "percentage_errors, absolute_errors = calculate_real_error(model, val_data, val_targets, scaler_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4af58aa9bbc156f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_model_performance(model, data, targets, set_name=\"\"):\n",
    "    \"\"\"\n",
    "    Valuta le performance del modello su un set di dati specifico.\n",
    "    \"\"\"\n",
    "    predictions = model.predict(data, verbose=0)\n",
    "\n",
    "    target_names = ['olive_prod', 'min_oil_prod', 'max_oil_prod', 'avg_oil_prod', 'total_water_need']\n",
    "    metrics = {}\n",
    "\n",
    "    for i, name in enumerate(target_names):\n",
    "        mae = np.mean(np.abs(targets[:, i] - predictions[:, i]))\n",
    "        mse = np.mean(np.square(targets[:, i] - predictions[:, i]))\n",
    "        rmse = np.sqrt(mse)\n",
    "        mape = np.mean(np.abs((targets[:, i] - predictions[:, i]) / (targets[:, i] + 1e-7))) * 100\n",
    "\n",
    "        metrics[f\"{name}_mae\"] = mae\n",
    "        metrics[f\"{name}_rmse\"] = rmse\n",
    "        metrics[f\"{name}_mape\"] = mape\n",
    "\n",
    "    if set_name:\n",
    "        print(f\"\\nPerformance sul set {set_name}:\")\n",
    "        for metric, value in metrics.items():\n",
    "            print(f\"{metric}: {value:.4f}\")\n",
    "\n",
    "    return metrics\n",
    "\n",
    "\n",
    "def retrain_model(base_model, train_data, train_targets,\n",
    "                  val_data, val_targets,\n",
    "                  test_data, test_targets,\n",
    "                  epochs=50, batch_size=128):\n",
    "    \"\"\"\n",
    "    Implementa il retraining del modello con i dati combinati.\n",
    "    \"\"\"\n",
    "    print(\"Valutazione performance iniziali del modello...\")\n",
    "    initial_metrics = {\n",
    "        'train': evaluate_model_performance(base_model, train_data, train_targets, \"training\"),\n",
    "        'val': evaluate_model_performance(base_model, val_data, val_targets, \"validazione\"),\n",
    "        'test': evaluate_model_performance(base_model, test_data, test_targets, \"test\")\n",
    "    }\n",
    "\n",
    "    # Combina i dati per il retraining\n",
    "    combined_data = {\n",
    "        'temporal': np.concatenate([train_data['temporal'], val_data['temporal'], test_data['temporal']]),\n",
    "        'static': np.concatenate([train_data['static'], val_data['static'], test_data['static']])\n",
    "    }\n",
    "    combined_targets = np.concatenate([train_targets, val_targets, test_targets])\n",
    "\n",
    "    # Crea una nuova suddivisione per la validazione\n",
    "    indices = np.arange(len(combined_targets))\n",
    "    np.random.shuffle(indices)\n",
    "\n",
    "    split_idx = int(len(indices) * 0.9)\n",
    "    train_idx, val_idx = indices[:split_idx], indices[split_idx:]\n",
    "\n",
    "    # Prepara i dati per il retraining\n",
    "    retrain_data = {k: v[train_idx] for k, v in combined_data.items()}\n",
    "    retrain_targets = combined_targets[train_idx]\n",
    "    retrain_val_data = {k: v[val_idx] for k, v in combined_data.items()}\n",
    "    retrain_val_targets = combined_targets[val_idx]\n",
    "\n",
    "    # Configura callbacks\n",
    "    callbacks = [\n",
    "        tf.keras.callbacks.EarlyStopping(\n",
    "            monitor='val_loss',\n",
    "            patience=10,\n",
    "            restore_best_weights=True,\n",
    "            min_delta=0.0001\n",
    "        ),\n",
    "        tf.keras.callbacks.ReduceLROnPlateau(\n",
    "            monitor='val_loss',\n",
    "            factor=0.2,\n",
    "            patience=5,\n",
    "            min_lr=1e-6,\n",
    "            verbose=1\n",
    "        ),\n",
    "        tf.keras.callbacks.ModelCheckpoint(\n",
    "            filepath=f'{execute_name}_retrained_best_oil_model.h5',\n",
    "            monitor='val_loss',\n",
    "            save_best_only=True,\n",
    "            mode='min',\n",
    "            save_weights_only=True\n",
    "        )\n",
    "    ]\n",
    "\n",
    "    # Imposta learning rate per il fine-tuning\n",
    "    optimizer = tf.keras.optimizers.AdamW(\n",
    "        learning_rate=tf.keras.optimizers.schedules.ExponentialDecay(\n",
    "            initial_learning_rate=1e-4,\n",
    "            decay_steps=1000,\n",
    "            decay_rate=0.9\n",
    "        ),\n",
    "        weight_decay=0.01\n",
    "    )\n",
    "\n",
    "    # Ricompila il modello con il nuovo optimizer\n",
    "    base_model.compile(\n",
    "        optimizer=optimizer,\n",
    "        loss=tf.keras.losses.Huber(),\n",
    "        metrics=['mae']\n",
    "    )\n",
    "\n",
    "    print(\"\\nAvvio retraining...\")\n",
    "    history = base_model.fit(\n",
    "        retrain_data,\n",
    "        retrain_targets,\n",
    "        validation_data=(retrain_val_data, retrain_val_targets),\n",
    "        epochs=epochs,\n",
    "        batch_size=batch_size,\n",
    "        callbacks=callbacks,\n",
    "        verbose=1\n",
    "    )\n",
    "\n",
    "    print(\"\\nValutazione performance finali...\")\n",
    "    final_metrics = {\n",
    "        'train': evaluate_model_performance(base_model, train_data, train_targets, \"training\"),\n",
    "        'val': evaluate_model_performance(base_model, val_data, val_targets, \"validazione\"),\n",
    "        'test': evaluate_model_performance(base_model, test_data, test_targets, \"test\")\n",
    "    }\n",
    "\n",
    "    # Salva il modello finale\n",
    "    save_path = f'{execute_name}_retrained_model.keras'\n",
    "    os.makedirs(f'{execute_name}_retrained/weights', exist_ok=True)\n",
    "    \n",
    "    base_model.save_weights(f'{execute_name}_retrained/weights')\n",
    "    base_model.save(save_path, save_format='keras')\n",
    "    print(f\"\\nModello riaddestrato salvato in: {save_path}\")\n",
    "\n",
    "    # Report miglioramenti\n",
    "    print(\"\\nMiglioramenti delle performance:\")\n",
    "    for dataset in ['train', 'val', 'test']:\n",
    "        print(f\"\\nSet {dataset}:\")\n",
    "        for metric in initial_metrics[dataset].keys():\n",
    "            initial = initial_metrics[dataset][metric]\n",
    "            final = final_metrics[dataset][metric]\n",
    "            improvement = ((initial - final) / initial) * 100\n",
    "            print(f\"{metric}: {improvement:.2f}% di miglioramento\")\n",
    "\n",
    "    return base_model, history, final_metrics\n",
    "\n",
    "\n",
    "def start_retraining(model_path, train_data, train_targets,\n",
    "                     val_data, val_targets,\n",
    "                     test_data, test_targets,\n",
    "                     epochs=50, batch_size=128):\n",
    "    \"\"\"\n",
    "    Avvia il processo di retraining in modo sicuro.\n",
    "    \"\"\"\n",
    "    try:\n",
    "        print(\"Caricamento del modello...\")\n",
    "        base_model = tf.keras.models.load_model(model_path, compile=False)\n",
    "        print(\"Modello caricato con successo!\")\n",
    "\n",
    "        return retrain_model(\n",
    "            base_model=base_model,\n",
    "            train_data=train_data,\n",
    "            train_targets=train_targets,\n",
    "            val_data=val_data,\n",
    "            val_targets=val_targets,\n",
    "            test_data=test_data,\n",
    "            test_targets=test_targets,\n",
    "            epochs=epochs,\n",
    "            batch_size=batch_size\n",
    "        )\n",
    "    except Exception as e:\n",
    "        print(f\"Errore durante il retraining: {str(e)}\")\n",
    "        raise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "588c7e49371f4a0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path = f'{execute_name}_final_model.keras'\n",
    "\n",
    "retrained_model, retrain_history, final_metrics = start_retraining(\n",
    "    model_path=model_path,\n",
    "    train_data=train_data,\n",
    "    train_targets=train_targets,\n",
    "    val_data=val_data,\n",
    "    val_targets=val_targets,\n",
    "    test_data=test_data,\n",
    "    test_targets=test_targets,\n",
    "    epochs=50,\n",
    "    batch_size=128\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a95606bc-c4bc-418a-acdb-2e24c30dfa81",
   "metadata": {},
   "outputs": [],
   "source": [
    "percentage_errors, absolute_errors = calculate_real_error(retrained_model, val_data, val_targets, scaler_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6f357cb-56a4-4f19-a4e8-77b73a28329d",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ProbabilityFunctions:\n",
    "    \"\"\"\n",
    "    Classe per calcolare e visualizzare PMF e CMF usando TensorFlow.\n",
    "    \"\"\"\n",
    "    \n",
    "    @staticmethod\n",
    "    def calculate_pmf(data, bins=50, range_limits=None):\n",
    "        \"\"\"\n",
    "        Calcola la PMF (Probability Mass Function) usando TensorFlow.\n",
    "        \n",
    "        Args:\n",
    "            data: Tensor dei dati\n",
    "            bins: Numero di bin per l'istogramma\n",
    "            range_limits: Tuple (min, max) per i limiti dell'istogramma\n",
    "            \n",
    "        Returns:\n",
    "            bin_edges: Bordi dei bin\n",
    "            pmf: Valori della PMF normalizzati\n",
    "        \"\"\"\n",
    "        # Converti i dati in tensor se non lo sono già\n",
    "        if not isinstance(data, tf.Tensor):\n",
    "            data = tf.convert_to_tensor(data, dtype=tf.float32)\n",
    "            \n",
    "        # Calcola i limiti se non specificati\n",
    "        if range_limits is None:\n",
    "            range_limits = (tf.reduce_min(data), tf.reduce_max(data))\n",
    "            \n",
    "        # Calcola l'istogramma\n",
    "        counts = tf.histogram_fixed_width(data, range_limits, bins)\n",
    "        \n",
    "        # Normalizza per ottenere la PMF\n",
    "        pmf = tf.cast(counts, tf.float32) / tf.reduce_sum(tf.cast(counts, tf.float32))\n",
    "        \n",
    "        # Calcola i bordi dei bin\n",
    "        bin_width = (range_limits[1] - range_limits[0]) / bins\n",
    "        bin_edges = tf.range(range_limits[0], range_limits[1] + bin_width, bin_width)\n",
    "        \n",
    "        return bin_edges.numpy(), pmf.numpy()\n",
    "    \n",
    "    @staticmethod\n",
    "    def calculate_cmf(pmf):\n",
    "        \"\"\"\n",
    "        Calcola la CMF (Cumulative Mass Function) dalla PMF.\n",
    "        \n",
    "        Args:\n",
    "            pmf: Array della PMF\n",
    "            \n",
    "        Returns:\n",
    "            cmf: Array della CMF\n",
    "        \"\"\"\n",
    "        # Converti in tensor se non lo è già\n",
    "        if not isinstance(pmf, tf.Tensor):\n",
    "            pmf = tf.convert_to_tensor(pmf, dtype=tf.float32)\n",
    "            \n",
    "        # Calcola la CMF usando cumsum\n",
    "        cmf = tf.cumsum(pmf)\n",
    "        \n",
    "        return cmf.numpy()\n",
    "    \n",
    "    @staticmethod\n",
    "    def calculate_statistics(data):\n",
    "        \"\"\"\n",
    "        Calcola statistiche di base usando TensorFlow.\n",
    "        \n",
    "        Args:\n",
    "            data: Tensor dei dati\n",
    "            \n",
    "        Returns:\n",
    "            dict: Dizionario con le statistiche\n",
    "        \"\"\"\n",
    "        if not isinstance(data, tf.Tensor):\n",
    "            data = tf.convert_to_tensor(data, dtype=tf.float32)\n",
    "            \n",
    "        mean = tf.reduce_mean(data)\n",
    "        variance = tf.reduce_variance(data)\n",
    "        std = tf.sqrt(variance)\n",
    "        \n",
    "        return {\n",
    "            'mean': mean.numpy(),\n",
    "            'variance': variance.numpy(),\n",
    "            'std': std.numpy(),\n",
    "            'min': tf.reduce_min(data).numpy(),\n",
    "            'max': tf.reduce_max(data).numpy(),\n",
    "            'median': tf.raw_ops.TopKV2(input=data, k=tf.size(data)//2)[0][-1].numpy()\n",
    "        }\n",
    "    \n",
    "    @staticmethod\n",
    "    def plot_distributions(bin_edges, pmf, cmf, title=\"Probability and Cumulative Distributions\"):\n",
    "        \"\"\"\n",
    "        Visualizza PMF e CMF.\n",
    "        \n",
    "        Args:\n",
    "            bin_edges: Bordi dei bin\n",
    "            pmf: Array della PMF\n",
    "            cmf: Array della CMF\n",
    "            title: Titolo del plot\n",
    "        \"\"\"\n",
    "        import matplotlib.pyplot as plt\n",
    "        \n",
    "        fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10, 8))\n",
    "        \n",
    "        # Plot PMF\n",
    "        ax1.bar(bin_edges[:-1], pmf, width=np.diff(bin_edges), alpha=0.5, \n",
    "                align='edge', label='PMF')\n",
    "        ax1.set_title('Probability Mass Function')\n",
    "        ax1.set_ylabel('Probability')\n",
    "        ax1.grid(True)\n",
    "        ax1.legend()\n",
    "        \n",
    "        # Plot CMF\n",
    "        ax2.plot(bin_edges[:-1], cmf, 'r-', label='CMF')\n",
    "        ax2.set_title('Cumulative Mass Function')\n",
    "        ax2.set_ylabel('Cumulative Probability')\n",
    "        ax2.set_xlabel('Value')\n",
    "        ax2.grid(True)\n",
    "        ax2.legend()\n",
    "        \n",
    "        plt.suptitle(title)\n",
    "        plt.tight_layout()\n",
    "        plt.show()\n",
    "\n",
    "# Esempio di utilizzo\n",
    "def demonstrate_probability_functions():\n",
    "    \"\"\"\n",
    "    Dimostra l'uso delle funzioni di probabilità.\n",
    "    \"\"\"\n",
    "    # Genera dati di esempio\n",
    "    tf.random.set_seed(42)\n",
    "    data = tf.concat([\n",
    "        tf.random.normal([1000], mean=0, stddev=1),\n",
    "        tf.random.normal([500], mean=3, stddev=0.5)\n",
    "    ], axis=0)\n",
    "    \n",
    "    # Inizializza la classe\n",
    "    prob = ProbabilityFunctions()\n",
    "    \n",
    "    # Calcola PMF e CMF\n",
    "    bin_edges, pmf = prob.calculate_pmf(data, bins=50)\n",
    "    cmf = prob.calculate_cmf(pmf)\n",
    "    \n",
    "    # Calcola statistiche\n",
    "    stats = prob.calculate_statistics(data)\n",
    "    \n",
    "    print(\"Statistiche di Base:\")\n",
    "    for key, value in stats.items():\n",
    "        print(f\"{key}: {value:.3f}\")\n",
    "    \n",
    "    # Visualizza le distribuzioni\n",
    "    prob.plot_distributions(bin_edges, pmf, cmf, \"Distribuzioni di Probabilità\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2022b594-6a8b-47ae-b207-112dfd9e3e95",
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyze_model_predictions(model, test_data, test_targets, scaler_y):\n",
    "    \"\"\"\n",
    "    Analizza le distribuzioni di probabilità delle predizioni del modello.\n",
    "    \n",
    "    Args:\n",
    "        model: Modello TensorFlow addestrato\n",
    "        test_data: Dati di test\n",
    "        test_targets: Target di test\n",
    "        scaler_y: Scaler usato per denormalizzare i target\n",
    "    \"\"\"\n",
    "    # Ottieni le predizioni\n",
    "    predictions = model.predict(test_data)\n",
    "    \n",
    "    # Denormalizza predizioni e target\n",
    "    predictions_real = scaler_y.inverse_transform(predictions)\n",
    "    targets_real = scaler_y.inverse_transform(test_targets)\n",
    "    \n",
    "    # Inizializza la classe per l'analisi delle probabilità\n",
    "    prob = ProbabilityFunctions()\n",
    "    \n",
    "    # Analizza ogni target\n",
    "    target_names = ['olive_prod', 'min_oil_prod', 'max_oil_prod', 'avg_oil_prod', 'total_water_need']\n",
    "    \n",
    "    for i, target in enumerate(target_names):\n",
    "        print(f\"\\nAnalisi per {target}\")\n",
    "        print(\"-\" * 50)\n",
    "        \n",
    "        # Calcola errori\n",
    "        errors = predictions_real[:, i] - targets_real[:, i]\n",
    "        \n",
    "        # Calcola statistiche degli errori\n",
    "        error_stats = prob.calculate_statistics(errors)\n",
    "        print(\"\\nStatistiche degli Errori:\")\n",
    "        for key, value in error_stats.items():\n",
    "            print(f\"{key}: {value:.3f}\")\n",
    "        \n",
    "        # Calcola PMF e CMF degli errori\n",
    "        bin_edges, pmf = prob.calculate_pmf(errors, bins=50)\n",
    "        cmf = prob.calculate_cmf(pmf)\n",
    "        \n",
    "        # Visualizza le distribuzioni degli errori\n",
    "        prob.plot_distributions(bin_edges, pmf, cmf, \n",
    "                              f\"Distribuzione degli Errori - {target}\")\n",
    "        \n",
    "        # Calcola intervalli di confidenza\n",
    "        confidence_levels = [0.68, 0.95, 0.99]  # 1σ, 2σ, 3σ\n",
    "        for level in confidence_levels:\n",
    "            lower_idx = np.searchsorted(cmf, (1 - level) / 2)\n",
    "            upper_idx = np.searchsorted(cmf, (1 + level) / 2)\n",
    "            \n",
    "            print(f\"\\nIntervallo di Confidenza {level*100}%:\")\n",
    "            print(f\"Range: [{bin_edges[lower_idx]:.2f}, {bin_edges[upper_idx]:.2f}]\")\n",
    "            \n",
    "def analyze_feature_importance(model, test_data, feature_names):\n",
    "    \"\"\"\n",
    "    Analizza l'importanza delle feature usando perturbazione.\n",
    "    \n",
    "    Args:\n",
    "        model: Modello TensorFlow addestrato\n",
    "        test_data: Dati di test\n",
    "        feature_names: Lista dei nomi delle feature\n",
    "    \"\"\"\n",
    "    base_prediction = model.predict(test_data)\n",
    "    feature_importance = {}\n",
    "    \n",
    "    # Per ogni feature\n",
    "    for i, feature in enumerate(feature_names):\n",
    "        # Crea copia perturbata dei dati\n",
    "        perturbed_data = test_data.copy()\n",
    "        noise = tf.random.normal(shape=perturbed_data[feature].shape,\n",
    "                               mean=0, stddev=0.1)\n",
    "        perturbed_data[feature] = perturbed_data[feature] + noise\n",
    "        \n",
    "        # Calcola nuova predizione\n",
    "        perturbed_prediction = model.predict(perturbed_data)\n",
    "        \n",
    "        # Calcola impatto della perturbazione\n",
    "        impact = tf.reduce_mean(tf.abs(perturbed_prediction - base_prediction))\n",
    "        feature_importance[feature] = impact.numpy()\n",
    "    \n",
    "    # Normalizza e ordina le importanze\n",
    "    total_importance = sum(feature_importance.values())\n",
    "    feature_importance = {k: v/total_importance for k, v in feature_importance.items()}\n",
    "    feature_importance = dict(sorted(feature_importance.items(), \n",
    "                                   key=lambda x: x[1], reverse=True))\n",
    "    \n",
    "    return feature_importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1150506-d853-4d00-a27c-266ac9c51f01",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_comprehensive_analysis(retrained_model, test_data, test_targets, scaler_y):\n",
    "    \"\"\"\n",
    "    Esegue un'analisi completa del modello includendo errori,\n",
    "    importanza delle feature e distribuzioni.\n",
    "    \"\"\"\n",
    "    print(\"=== ANALISI COMPLETA DEL MODELLO ===\")\n",
    "    \n",
    "    # 1. Analisi degli errori\n",
    "    print(\"\\n1. ANALISI DEGLI ERRORI\")\n",
    "    print(\"-\" * 50)\n",
    "    analyze_model_predictions(retrained_model, test_data, test_targets, scaler_y)\n",
    "    \n",
    "    # 2. Analisi dell'importanza delle feature\n",
    "    print(\"\\n2. IMPORTANZA DELLE FEATURE\")\n",
    "    print(\"-\" * 50)\n",
    "    \n",
    "    # Definisci i nomi delle feature\n",
    "    temporal_features = ['temp_mean', 'precip_sum', 'solar_energy_sum']\n",
    "    static_features = ['ha']  # Aggiungi le tue feature statiche\n",
    "    \n",
    "    all_features = temporal_features + static_features\n",
    "    importance = analyze_feature_importance(retrained_model, test_data, all_features)\n",
    "    \n",
    "    print(\"\\nImportanza relativa delle feature:\")\n",
    "    for feature, imp in sorted(importance.items(), key=lambda x: x[1], reverse=True):\n",
    "        print(f\"{feature}: {imp:.4f}\")\n",
    "        \n",
    "    # 3. Analisi distribuzionale\n",
    "    print(\"\\n3. ANALISI DISTRIBUZIONALE\")\n",
    "    print(\"-\" * 50)\n",
    "    \n",
    "    prob = ProbabilityFunctions()\n",
    "    predictions = retrained_model.predict(test_data)\n",
    "    predictions_real = scaler_y.inverse_transform(predictions)\n",
    "    targets_real = scaler_y.inverse_transform(test_targets)\n",
    "    \n",
    "    target_names = ['olive_prod', 'min_oil_prod', 'max_oil_prod', \n",
    "                    'avg_oil_prod', 'total_water_need']\n",
    "    \n",
    "    for i, target in enumerate(target_names):\n",
    "        print(f\"\\nAnalisi distribuzionale per {target}\")\n",
    "        \n",
    "        # Analizza predizioni\n",
    "        bin_edges_pred, pmf_pred = prob.calculate_pmf(predictions_real[:, i])\n",
    "        cmf_pred = prob.calculate_cmf(pmf_pred)\n",
    "        \n",
    "        # Analizza target reali\n",
    "        bin_edges_true, pmf_true = prob.calculate_pmf(targets_real[:, i])\n",
    "        cmf_true = prob.calculate_cmf(pmf_true)\n",
    "        \n",
    "        # Statistiche\n",
    "        stats_pred = prob.calculate_statistics(predictions_real[:, i])\n",
    "        stats_true = prob.calculate_statistics(targets_real[:, i])\n",
    "        \n",
    "        print(\"\\nStatistiche Predizioni:\")\n",
    "        for key, value in stats_pred.items():\n",
    "            print(f\"{key}: {value:.3f}\")\n",
    "            \n",
    "        print(\"\\nStatistiche Target Reali:\")\n",
    "        for key, value in stats_true.items():\n",
    "            print(f\"{key}: {value:.3f}\")\n",
    "        \n",
    "        # Visualizza distribuzioni\n",
    "        prob.plot_distributions(bin_edges_pred, pmf_pred, cmf_pred,\n",
    "                              f\"Distribuzione Predizioni - {target}\")\n",
    "        prob.plot_distributions(bin_edges_true, pmf_true, cmf_true,\n",
    "                              f\"Distribuzione Target Reali - {target}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b98f2a45-ba6f-4519-acaa-0138b4ee7812",
   "metadata": {},
   "outputs": [],
   "source": [
    "run_comprehensive_analysis(retrained_model, test_data, test_targets, scaler_y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
