{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Get:1 http://archive.ubuntu.com/ubuntu jammy InRelease [270 kB]\n",
      "Get:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease [1581 B]\n",
      "Get:3 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB]      \n",
      "Get:4 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  Packages [1192 kB]\n",
      "Get:5 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB]        \n",
      "Get:6 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB]      \n",
      "Get:7 http://archive.ubuntu.com/ubuntu jammy/restricted amd64 Packages [164 kB]\n",
      "Get:8 http://archive.ubuntu.com/ubuntu jammy/universe amd64 Packages [17.5 MB]\n",
      "Get:9 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [45.2 kB]\n",
      "Get:10 http://archive.ubuntu.com/ubuntu jammy/multiverse amd64 Packages [266 kB]\n",
      "Get:11 http://archive.ubuntu.com/ubuntu jammy/main amd64 Packages [1792 kB]    \n",
      "Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2738 kB]\n",
      "Get:13 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [3323 kB]\n",
      "Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [3446 kB]\n",
      "Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1514 kB]\n",
      "Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [53.3 kB]\n",
      "Get:17 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [33.8 kB]\n",
      "Get:18 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [81.4 kB]\n",
      "Get:19 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [2454 kB]\n",
      "Get:20 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1225 kB]\n",
      "Fetched 36.5 MB in 2s (18.2 MB/s)                           \n",
      "Reading package lists... Done\n",
      "Reading package lists... Done\n",
      "Building dependency tree... Done\n",
      "Reading state information... Done\n",
      "The following additional packages will be installed:\n",
      "  fontconfig fonts-liberation libann0 libcairo2 libcdt5 libcgraph6 libdatrie1\n",
      "  libfribidi0 libgraphite2-3 libgts-0.7-5 libgts-bin libgvc6 libgvpr2\n",
      "  libharfbuzz0b libice6 liblab-gamut1 libltdl7 libpango-1.0-0\n",
      "  libpangocairo-1.0-0 libpangoft2-1.0-0 libpathplan4 libpixman-1-0 libsm6\n",
      "  libthai-data libthai0 libxaw7 libxcb-render0 libxmu6 libxrender1 libxt6\n",
      "  x11-common\n",
      "Suggested packages:\n",
      "  gsfonts graphviz-doc\n",
      "The following NEW packages will be installed:\n",
      "  fontconfig fonts-liberation graphviz libann0 libcairo2 libcdt5 libcgraph6\n",
      "  libdatrie1 libfribidi0 libgraphite2-3 libgts-0.7-5 libgts-bin libgvc6\n",
      "  libgvpr2 libharfbuzz0b libice6 liblab-gamut1 libltdl7 libpango-1.0-0\n",
      "  libpangocairo-1.0-0 libpangoft2-1.0-0 libpathplan4 libpixman-1-0 libsm6\n",
      "  libthai-data libthai0 libxaw7 libxcb-render0 libxmu6 libxrender1 libxt6\n",
      "  x11-common\n",
      "0 upgraded, 32 newly installed, 0 to remove and 121 not upgraded.\n",
      "Need to get 7298 kB of archives.\n",
      "After this operation, 18.3 MB of additional disk space will be used.\n",
      "Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libfribidi0 amd64 1.0.8-2ubuntu3.1 [26.1 kB]\n",
      "Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 fontconfig amd64 2.13.1-4.2ubuntu5 [177 kB]\n",
      "Get:3 http://archive.ubuntu.com/ubuntu jammy/main amd64 fonts-liberation all 1:1.07.4-11 [822 kB]\n",
      "Get:4 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libann0 amd64 1.1.2+doc-7build1 [26.0 kB]\n",
      "Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libcdt5 amd64 2.42.2-6ubuntu0.1 [21.1 kB]\n",
      "Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libcgraph6 amd64 2.42.2-6ubuntu0.1 [45.4 kB]\n",
      "Get:7 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libgts-0.7-5 amd64 0.7.6+darcs121130-5 [164 kB]\n",
      "Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpixman-1-0 amd64 0.40.0-1ubuntu0.22.04.1 [264 kB]\n",
      "Get:9 http://archive.ubuntu.com/ubuntu jammy/main amd64 libxcb-render0 amd64 1.14-3ubuntu3 [16.4 kB]\n",
      "Get:10 http://archive.ubuntu.com/ubuntu jammy/main amd64 libxrender1 amd64 1:0.9.10-1build4 [19.7 kB]\n",
      "Get:11 http://archive.ubuntu.com/ubuntu jammy/main amd64 libcairo2 amd64 1.16.0-5ubuntu2 [628 kB]\n",
      "Get:12 http://archive.ubuntu.com/ubuntu jammy/main amd64 libltdl7 amd64 2.4.6-15build2 [39.6 kB]\n",
      "Get:13 http://archive.ubuntu.com/ubuntu jammy/main amd64 libgraphite2-3 amd64 1.3.14-1build2 [71.3 kB]\n",
      "Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libharfbuzz0b amd64 2.7.4-1ubuntu3.1 [352 kB]\n",
      "Get:15 http://archive.ubuntu.com/ubuntu jammy/main amd64 libthai-data all 0.1.29-1build1 [162 kB]\n",
      "Get:16 http://archive.ubuntu.com/ubuntu jammy/main amd64 libdatrie1 amd64 0.2.13-2 [19.9 kB]\n",
      "Get:17 http://archive.ubuntu.com/ubuntu jammy/main amd64 libthai0 amd64 0.1.29-1build1 [19.2 kB]\n",
      "Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpango-1.0-0 amd64 1.50.6+ds-2ubuntu1 [230 kB]\n",
      "Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpangoft2-1.0-0 amd64 1.50.6+ds-2ubuntu1 [54.0 kB]\n",
      "Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpangocairo-1.0-0 amd64 1.50.6+ds-2ubuntu1 [39.8 kB]\n",
      "Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libpathplan4 amd64 2.42.2-6ubuntu0.1 [23.4 kB]\n",
      "Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libgvc6 amd64 2.42.2-6ubuntu0.1 [724 kB]\n",
      "Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libgvpr2 amd64 2.42.2-6ubuntu0.1 [192 kB]\n",
      "Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 liblab-gamut1 amd64 2.42.2-6ubuntu0.1 [1965 kB]\n",
      "Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 x11-common all 1:7.7+23ubuntu2 [23.4 kB]\n",
      "Get:26 http://archive.ubuntu.com/ubuntu jammy/main amd64 libice6 amd64 2:1.0.10-1build2 [42.6 kB]\n",
      "Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 libsm6 amd64 2:1.2.3-1build2 [16.7 kB]\n",
      "Get:28 http://archive.ubuntu.com/ubuntu jammy/main amd64 libxt6 amd64 1:1.2.1-1 [177 kB]\n",
      "Get:29 http://archive.ubuntu.com/ubuntu jammy/main amd64 libxmu6 amd64 2:1.1.3-3 [49.6 kB]\n",
      "Get:30 http://archive.ubuntu.com/ubuntu jammy/main amd64 libxaw7 amd64 2:1.0.14-1 [191 kB]\n",
      "Get:31 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 graphviz amd64 2.42.2-6ubuntu0.1 [653 kB]\n",
      "Get:32 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libgts-bin amd64 0.7.6+darcs121130-5 [44.3 kB]\n",
      "Fetched 7298 kB in 2s (4771 kB/s)      \n",
      "debconf: delaying package configuration, since apt-utils is not installed\n",
      "Selecting previously unselected package libfribidi0:amd64.\n",
      "(Reading database ... 20752 files and directories currently installed.)\n",
      "Preparing to unpack .../00-libfribidi0_1.0.8-2ubuntu3.1_amd64.deb ...\n",
      "Unpacking libfribidi0:amd64 (1.0.8-2ubuntu3.1) ...\n",
      "Selecting previously unselected package fontconfig.\n",
      "Preparing to unpack .../01-fontconfig_2.13.1-4.2ubuntu5_amd64.deb ...\n",
      "Unpacking fontconfig (2.13.1-4.2ubuntu5) ...\n",
      "Selecting previously unselected package fonts-liberation.\n",
      "Preparing to unpack .../02-fonts-liberation_1%3a1.07.4-11_all.deb ...\n",
      "Unpacking fonts-liberation (1:1.07.4-11) ...\n",
      "Selecting previously unselected package libann0.\n",
      "Preparing to unpack .../03-libann0_1.1.2+doc-7build1_amd64.deb ...\n",
      "Unpacking libann0 (1.1.2+doc-7build1) ...\n",
      "Selecting previously unselected package libcdt5:amd64.\n",
      "Preparing to unpack .../04-libcdt5_2.42.2-6ubuntu0.1_amd64.deb ...\n",
      "Unpacking libcdt5:amd64 (2.42.2-6ubuntu0.1) ...\n",
      "Selecting previously unselected package libcgraph6:amd64.\n",
      "Preparing to unpack .../05-libcgraph6_2.42.2-6ubuntu0.1_amd64.deb ...\n",
      "Unpacking libcgraph6:amd64 (2.42.2-6ubuntu0.1) ...\n",
      "Selecting previously unselected package libgts-0.7-5:amd64.\n",
      "Preparing to unpack .../06-libgts-0.7-5_0.7.6+darcs121130-5_amd64.deb ...\n",
      "Unpacking libgts-0.7-5:amd64 (0.7.6+darcs121130-5) ...\n",
      "Selecting previously unselected package libpixman-1-0:amd64.\n",
      "Preparing to unpack .../07-libpixman-1-0_0.40.0-1ubuntu0.22.04.1_amd64.deb ...\n",
      "Unpacking libpixman-1-0:amd64 (0.40.0-1ubuntu0.22.04.1) ...\n",
      "Selecting previously unselected package libxcb-render0:amd64.\n",
      "Preparing to unpack .../08-libxcb-render0_1.14-3ubuntu3_amd64.deb ...\n",
      "Unpacking libxcb-render0:amd64 (1.14-3ubuntu3) ...\n",
      "Selecting previously unselected package libxrender1:amd64.\n",
      "Preparing to unpack .../09-libxrender1_1%3a0.9.10-1build4_amd64.deb ...\n",
      "Unpacking libxrender1:amd64 (1:0.9.10-1build4) ...\n",
      "Selecting previously unselected package libcairo2:amd64.\n",
      "Preparing to unpack .../10-libcairo2_1.16.0-5ubuntu2_amd64.deb ...\n",
      "Unpacking libcairo2:amd64 (1.16.0-5ubuntu2) ...\n",
      "Selecting previously unselected package libltdl7:amd64.\n",
      "Preparing to unpack .../11-libltdl7_2.4.6-15build2_amd64.deb ...\n",
      "Unpacking libltdl7:amd64 (2.4.6-15build2) ...\n",
      "Selecting previously unselected package libgraphite2-3:amd64.\n",
      "Preparing to unpack .../12-libgraphite2-3_1.3.14-1build2_amd64.deb ...\n",
      "Unpacking libgraphite2-3:amd64 (1.3.14-1build2) ...\n",
      "Selecting previously unselected package libharfbuzz0b:amd64.\n",
      "Preparing to unpack .../13-libharfbuzz0b_2.7.4-1ubuntu3.1_amd64.deb ...\n",
      "Unpacking libharfbuzz0b:amd64 (2.7.4-1ubuntu3.1) ...\n",
      "Selecting previously unselected package libthai-data.\n",
      "Preparing to unpack .../14-libthai-data_0.1.29-1build1_all.deb ...\n",
      "Unpacking libthai-data (0.1.29-1build1) ...\n",
      "Selecting previously unselected package libdatrie1:amd64.\n",
      "Preparing to unpack .../15-libdatrie1_0.2.13-2_amd64.deb ...\n",
      "Unpacking libdatrie1:amd64 (0.2.13-2) ...\n",
      "Selecting previously unselected package libthai0:amd64.\n",
      "Preparing to unpack .../16-libthai0_0.1.29-1build1_amd64.deb ...\n",
      "Unpacking libthai0:amd64 (0.1.29-1build1) ...\n",
      "Selecting previously unselected package libpango-1.0-0:amd64.\n",
      "Preparing to unpack .../17-libpango-1.0-0_1.50.6+ds-2ubuntu1_amd64.deb ...\n",
      "Unpacking libpango-1.0-0:amd64 (1.50.6+ds-2ubuntu1) ...\n",
      "Selecting previously unselected package libpangoft2-1.0-0:amd64.\n",
      "Preparing to unpack .../18-libpangoft2-1.0-0_1.50.6+ds-2ubuntu1_amd64.deb ...\n",
      "Unpacking libpangoft2-1.0-0:amd64 (1.50.6+ds-2ubuntu1) ...\n",
      "Selecting previously unselected package libpangocairo-1.0-0:amd64.\n",
      "Preparing to unpack .../19-libpangocairo-1.0-0_1.50.6+ds-2ubuntu1_amd64.deb ...\n",
      "Unpacking libpangocairo-1.0-0:amd64 (1.50.6+ds-2ubuntu1) ...\n",
      "Selecting previously unselected package libpathplan4:amd64.\n",
      "Preparing to unpack .../20-libpathplan4_2.42.2-6ubuntu0.1_amd64.deb ...\n",
      "Unpacking libpathplan4:amd64 (2.42.2-6ubuntu0.1) ...\n",
      "Selecting previously unselected package libgvc6.\n",
      "Preparing to unpack .../21-libgvc6_2.42.2-6ubuntu0.1_amd64.deb ...\n",
      "Unpacking libgvc6 (2.42.2-6ubuntu0.1) ...\n",
      "Selecting previously unselected package libgvpr2:amd64.\n",
      "Preparing to unpack .../22-libgvpr2_2.42.2-6ubuntu0.1_amd64.deb ...\n",
      "Unpacking libgvpr2:amd64 (2.42.2-6ubuntu0.1) ...\n",
      "Selecting previously unselected package liblab-gamut1:amd64.\n",
      "Preparing to unpack .../23-liblab-gamut1_2.42.2-6ubuntu0.1_amd64.deb ...\n",
      "Unpacking liblab-gamut1:amd64 (2.42.2-6ubuntu0.1) ...\n",
      "Selecting previously unselected package x11-common.\n",
      "Preparing to unpack .../24-x11-common_1%3a7.7+23ubuntu2_all.deb ...\n",
      "Unpacking x11-common (1:7.7+23ubuntu2) ...\n",
      "Selecting previously unselected package libice6:amd64.\n",
      "Preparing to unpack .../25-libice6_2%3a1.0.10-1build2_amd64.deb ...\n",
      "Unpacking libice6:amd64 (2:1.0.10-1build2) ...\n",
      "Selecting previously unselected package libsm6:amd64.\n",
      "Preparing to unpack .../26-libsm6_2%3a1.2.3-1build2_amd64.deb ...\n",
      "Unpacking libsm6:amd64 (2:1.2.3-1build2) ...\n",
      "Selecting previously unselected package libxt6:amd64.\n",
      "Preparing to unpack .../27-libxt6_1%3a1.2.1-1_amd64.deb ...\n",
      "Unpacking libxt6:amd64 (1:1.2.1-1) ...\n",
      "Selecting previously unselected package libxmu6:amd64.\n",
      "Preparing to unpack .../28-libxmu6_2%3a1.1.3-3_amd64.deb ...\n",
      "Unpacking libxmu6:amd64 (2:1.1.3-3) ...\n",
      "Selecting previously unselected package libxaw7:amd64.\n",
      "Preparing to unpack .../29-libxaw7_2%3a1.0.14-1_amd64.deb ...\n",
      "Unpacking libxaw7:amd64 (2:1.0.14-1) ...\n",
      "Selecting previously unselected package graphviz.\n",
      "Preparing to unpack .../30-graphviz_2.42.2-6ubuntu0.1_amd64.deb ...\n",
      "Unpacking graphviz (2.42.2-6ubuntu0.1) ...\n",
      "Selecting previously unselected package libgts-bin.\n",
      "Preparing to unpack .../31-libgts-bin_0.7.6+darcs121130-5_amd64.deb ...\n",
      "Unpacking libgts-bin (0.7.6+darcs121130-5) ...\n",
      "Setting up libgraphite2-3:amd64 (1.3.14-1build2) ...\n",
      "Setting up libpixman-1-0:amd64 (0.40.0-1ubuntu0.22.04.1) ...\n",
      "Setting up fontconfig (2.13.1-4.2ubuntu5) ...\n",
      "Regenerating fonts cache... done.\n",
      "Setting up libxrender1:amd64 (1:0.9.10-1build4) ...\n",
      "Setting up libdatrie1:amd64 (0.2.13-2) ...\n",
      "Setting up libxcb-render0:amd64 (1.14-3ubuntu3) ...\n",
      "Setting up liblab-gamut1:amd64 (2.42.2-6ubuntu0.1) ...\n",
      "Setting up x11-common (1:7.7+23ubuntu2) ...\n",
      "invoke-rc.d: could not determine current runlevel\n",
      "invoke-rc.d: policy-rc.d denied execution of start.\n",
      "Setting up libcairo2:amd64 (1.16.0-5ubuntu2) ...\n",
      "Setting up libgts-0.7-5:amd64 (0.7.6+darcs121130-5) ...\n",
      "Setting up libpathplan4:amd64 (2.42.2-6ubuntu0.1) ...\n",
      "Setting up libann0 (1.1.2+doc-7build1) ...\n",
      "Setting up libfribidi0:amd64 (1.0.8-2ubuntu3.1) ...\n",
      "Setting up libltdl7:amd64 (2.4.6-15build2) ...\n",
      "Setting up fonts-liberation (1:1.07.4-11) ...\n",
      "Setting up libharfbuzz0b:amd64 (2.7.4-1ubuntu3.1) ...\n",
      "Setting up libthai-data (0.1.29-1build1) ...\n",
      "Setting up libcdt5:amd64 (2.42.2-6ubuntu0.1) ...\n",
      "Setting up libcgraph6:amd64 (2.42.2-6ubuntu0.1) ...\n",
      "Setting up libgts-bin (0.7.6+darcs121130-5) ...\n",
      "Setting up libice6:amd64 (2:1.0.10-1build2) ...\n",
      "Setting up libthai0:amd64 (0.1.29-1build1) ...\n",
      "Setting up libgvpr2:amd64 (2.42.2-6ubuntu0.1) ...\n",
      "Setting up libsm6:amd64 (2:1.2.3-1build2) ...\n",
      "Setting up libpango-1.0-0:amd64 (1.50.6+ds-2ubuntu1) ...\n",
      "Setting up libxt6:amd64 (1:1.2.1-1) ...\n",
      "Setting up libpangoft2-1.0-0:amd64 (1.50.6+ds-2ubuntu1) ...\n",
      "Setting up libpangocairo-1.0-0:amd64 (1.50.6+ds-2ubuntu1) ...\n",
      "Setting up libxmu6:amd64 (2:1.1.3-3) ...\n",
      "Setting up libxaw7:amd64 (2:1.0.14-1) ...\n",
      "Setting up libgvc6 (2.42.2-6ubuntu0.1) ...\n",
      "Setting up graphviz (2.42.2-6ubuntu0.1) ...\n",
      "Processing triggers for libc-bin (2.35-0ubuntu3.3) ...\n",
      "Requirement already satisfied: tensorflow in /usr/local/lib/python3.11/dist-packages (2.14.0)\n",
      "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.0.0)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.6.3)\n",
      "Requirement already satisfied: flatbuffers>=23.5.26 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (23.5.26)\n",
      "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.5.4)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.2.0)\n",
      "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (3.9.0)\n",
      "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (16.0.6)\n",
      "Requirement already satisfied: ml-dtypes==0.2.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.2.0)\n",
      "Requirement already satisfied: numpy>=1.23.5 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.26.0)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (3.3.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from tensorflow) (23.1)\n",
      "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (4.24.3)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.11/dist-packages (from tensorflow) (68.2.2)\n",
      "Requirement already satisfied: six>=1.12.0 in /usr/lib/python3/dist-packages (from tensorflow) (1.16.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.3.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (4.8.0)\n",
      "Requirement already satisfied: wrapt<1.15,>=1.11.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.14.1)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.34.0)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.58.0)\n",
      "Requirement already satisfied: tensorboard<2.15,>=2.14 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.14.0)\n",
      "Requirement already satisfied: tensorflow-estimator<2.15,>=2.14.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.14.0)\n",
      "Requirement already satisfied: keras<2.15,>=2.14.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.14.0)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.11/dist-packages (from astunparse>=1.6.0->tensorflow) (0.41.2)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (2.23.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (1.0.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (3.4.4)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (2.31.0)\n",
      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (0.7.1)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (2.3.7)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (5.3.1)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (0.3.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (4.9)\n",
      "Requirement already satisfied: urllib3>=2.0.5 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (2.0.5)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard<2.15,>=2.14->tensorflow) (1.3.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.15,>=2.14->tensorflow) (3.2.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.15,>=2.14->tensorflow) (3.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.15,>=2.14->tensorflow) (2023.7.22)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.11/dist-packages (from werkzeug>=1.0.1->tensorboard<2.15,>=2.14->tensorflow) (2.1.3)\n",
      "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.11/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (0.5.0)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/lib/python3/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard<2.15,>=2.14->tensorflow) (3.2.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (1.26.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Collecting pandas\n",
      "  Obtaining dependency information for pandas from https://files.pythonhosted.org/packages/cd/5f/4dba1d39bb9c38d574a9a22548c540177f78ea47b32f99c0ff2ec499fac5/pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (89 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m89.9/89.9 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (1.26.0)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (2.8.2)\n",
      "Collecting pytz>=2020.1 (from pandas)\n",
      "  Obtaining dependency information for pytz>=2020.1 from https://files.pythonhosted.org/packages/11/c3/005fcca25ce078d2cc29fd559379817424e94885510568bc1bc53d7d5846/pytz-2024.2-py2.py3-none-any.whl.metadata\n",
      "  Downloading pytz-2024.2-py2.py3-none-any.whl.metadata (22 kB)\n",
      "Collecting tzdata>=2022.7 (from pandas)\n",
      "  Obtaining dependency information for tzdata>=2022.7 from https://files.pythonhosted.org/packages/a6/ab/7e5f53c3b9d14972843a647d8d7a853969a58aecc7559cb3267302c94774/tzdata-2024.2-py2.py3-none-any.whl.metadata\n",
      "  Downloading tzdata-2024.2-py2.py3-none-any.whl.metadata (1.4 kB)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
      "Downloading pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.1 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.1/13.1 MB\u001b[0m \u001b[31m74.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25hDownloading pytz-2024.2-py2.py3-none-any.whl (508 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m508.0/508.0 kB\u001b[0m \u001b[31m106.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading tzdata-2024.2-py2.py3-none-any.whl (346 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m346.6/346.6 kB\u001b[0m \u001b[31m103.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: pytz, tzdata, pandas\n",
      "Successfully installed pandas-2.2.3 pytz-2024.2 tzdata-2024.2\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: keras in /usr/local/lib/python3.11/dist-packages (2.14.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Collecting scikit-learn\n",
      "  Obtaining dependency information for scikit-learn from https://files.pythonhosted.org/packages/49/21/3723de321531c9745e40f1badafd821e029d346155b6c79704e0b7197552/scikit_learn-1.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading scikit_learn-1.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (13 kB)\n",
      "Requirement already satisfied: numpy>=1.19.5 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.26.0)\n",
      "Collecting scipy>=1.6.0 (from scikit-learn)\n",
      "  Obtaining dependency information for scipy>=1.6.0 from https://files.pythonhosted.org/packages/93/6b/701776d4bd6bdd9b629c387b5140f006185bd8ddea16788a44434376b98f/scipy-1.14.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading scipy-1.14.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (60 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.8/60.8 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting joblib>=1.2.0 (from scikit-learn)\n",
      "  Obtaining dependency information for joblib>=1.2.0 from https://files.pythonhosted.org/packages/91/29/df4b9b42f2be0b623cbd5e2140cafcaa2bef0759a00b7b70104dcfe2fb51/joblib-1.4.2-py3-none-any.whl.metadata\n",
      "  Downloading joblib-1.4.2-py3-none-any.whl.metadata (5.4 kB)\n",
      "Collecting threadpoolctl>=3.1.0 (from scikit-learn)\n",
      "  Obtaining dependency information for threadpoolctl>=3.1.0 from https://files.pythonhosted.org/packages/4b/2c/ffbf7a134b9ab11a67b0cf0726453cedd9c5043a4fe7a35d1cefa9a1bcfb/threadpoolctl-3.5.0-py3-none-any.whl.metadata\n",
      "  Downloading threadpoolctl-3.5.0-py3-none-any.whl.metadata (13 kB)\n",
      "Downloading scikit_learn-1.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.3 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.3/13.3 MB\u001b[0m \u001b[31m78.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading joblib-1.4.2-py3-none-any.whl (301 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m301.8/301.8 kB\u001b[0m \u001b[31m104.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading scipy-1.14.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (41.2 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.2/41.2 MB\u001b[0m \u001b[31m55.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hDownloading threadpoolctl-3.5.0-py3-none-any.whl (18 kB)\n",
      "Installing collected packages: threadpoolctl, scipy, joblib, scikit-learn\n",
      "Successfully installed joblib-1.4.2 scikit-learn-1.5.2 scipy-1.14.1 threadpoolctl-3.5.0\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: matplotlib in /usr/local/lib/python3.11/dist-packages (3.8.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (4.42.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.4.5)\n",
      "Requirement already satisfied: numpy<2,>=1.21 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.26.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (23.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (10.0.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3/dist-packages (from matplotlib) (2.4.7)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (2.8.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.11/dist-packages (1.4.2)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Collecting pyarrow\n",
      "  Obtaining dependency information for pyarrow from https://files.pythonhosted.org/packages/5e/b5/9e14e9f7590e0eaa435ecea84dabb137284a4dbba7b3c337b58b65b76d95/pyarrow-18.1.0-cp311-cp311-manylinux_2_28_x86_64.whl.metadata\n",
      "  Downloading pyarrow-18.1.0-cp311-cp311-manylinux_2_28_x86_64.whl.metadata (3.3 kB)\n",
      "Downloading pyarrow-18.1.0-cp311-cp311-manylinux_2_28_x86_64.whl (40.1 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.1/40.1 MB\u001b[0m \u001b[31m58.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: pyarrow\n",
      "Successfully installed pyarrow-18.1.0\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Collecting fastparquet\n",
      "  Obtaining dependency information for fastparquet from https://files.pythonhosted.org/packages/8d/e8/e1ede861bea68394a755d8be1aa2e2d60a3b9f6b551bfd56aeca74987e2e/fastparquet-2024.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading fastparquet-2024.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.2 kB)\n",
      "Requirement already satisfied: pandas>=1.5.0 in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2.2.3)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (from fastparquet) (1.26.0)\n",
      "Collecting cramjam>=2.3 (from fastparquet)\n",
      "  Obtaining dependency information for cramjam>=2.3 from https://files.pythonhosted.org/packages/79/1d/180f2ca168625073f0df80b16c795926deed91b7e89dbfc045263ba7444b/cramjam-2.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading cramjam-2.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.9 kB)\n",
      "Collecting fsspec (from fastparquet)\n",
      "  Obtaining dependency information for fsspec from https://files.pythonhosted.org/packages/c6/b2/454d6e7f0158951d8a78c2e1eb4f69ae81beb8dca5fee9809c6c99e9d0d0/fsspec-2024.10.0-py3-none-any.whl.metadata\n",
      "  Downloading fsspec-2024.10.0-py3-none-any.whl.metadata (11 kB)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from fastparquet) (23.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2024.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas>=1.5.0->fastparquet) (1.16.0)\n",
      "Downloading fastparquet-2024.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.8 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m16.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading cramjam-2.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.4 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m66.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hDownloading fsspec-2024.10.0-py3-none-any.whl (179 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m179.6/179.6 kB\u001b[0m \u001b[31m37.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: fsspec, cramjam, fastparquet\n",
      "Successfully installed cramjam-2.9.0 fastparquet-2024.11.0 fsspec-2024.10.0\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.11/dist-packages (1.14.1)\n",
      "Requirement already satisfied: numpy<2.3,>=1.23.5 in /usr/local/lib/python3.11/dist-packages (from scipy) (1.26.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Collecting seaborn\n",
      "  Obtaining dependency information for seaborn from https://files.pythonhosted.org/packages/83/11/00d3c3dfc25ad54e731d91449895a79e4bf2384dc3ac01809010ba88f6d5/seaborn-0.13.2-py3-none-any.whl.metadata\n",
      "  Downloading seaborn-0.13.2-py3-none-any.whl.metadata (5.4 kB)\n",
      "Requirement already satisfied: numpy!=1.24.0,>=1.20 in /usr/local/lib/python3.11/dist-packages (from seaborn) (1.26.0)\n",
      "Requirement already satisfied: pandas>=1.2 in /usr/local/lib/python3.11/dist-packages (from seaborn) (2.2.3)\n",
      "Requirement already satisfied: matplotlib!=3.6.1,>=3.4 in /usr/local/lib/python3.11/dist-packages (from seaborn) (3.8.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (4.42.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.4.5)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (23.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (10.0.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (2.4.7)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.2->seaborn) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.2->seaborn) (2024.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib!=3.6.1,>=3.4->seaborn) (1.16.0)\n",
      "Downloading seaborn-0.13.2-py3-none-any.whl (294 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m294.9/294.9 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: seaborn\n",
      "Successfully installed seaborn-0.13.2\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Collecting tqdm\n",
      "  Obtaining dependency information for tqdm from https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl.metadata\n",
      "  Downloading tqdm-4.67.1-py3-none-any.whl.metadata (57 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.7/57.7 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading tqdm-4.67.1-py3-none-any.whl (78 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.5/78.5 kB\u001b[0m \u001b[31m13.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: tqdm\n",
      "Successfully installed tqdm-4.67.1\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Collecting pydot\n",
      "  Obtaining dependency information for pydot from https://files.pythonhosted.org/packages/3e/1b/ef569ac44598b6b24bc0f80d5ac4f811af59d3f0d0d23b0216e014c0ec33/pydot-3.0.3-py3-none-any.whl.metadata\n",
      "  Downloading pydot-3.0.3-py3-none-any.whl.metadata (10 kB)\n",
      "Collecting pyparsing>=3.0.9 (from pydot)\n",
      "  Obtaining dependency information for pyparsing>=3.0.9 from https://files.pythonhosted.org/packages/be/ec/2eb3cd785efd67806c46c13a17339708ddc346cbb684eade7a6e6f79536a/pyparsing-3.2.0-py3-none-any.whl.metadata\n",
      "  Downloading pyparsing-3.2.0-py3-none-any.whl.metadata (5.0 kB)\n",
      "Downloading pydot-3.0.3-py3-none-any.whl (35 kB)\n",
      "Downloading pyparsing-3.2.0-py3-none-any.whl (106 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m106.9/106.9 kB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: pyparsing, pydot\n",
      "  Attempting uninstall: pyparsing\n",
      "    Found existing installation: pyparsing 2.4.7\n",
      "    Uninstalling pyparsing-2.4.7:\n",
      "      Successfully uninstalled pyparsing-2.4.7\n",
      "Successfully installed pydot-3.0.3 pyparsing-3.2.0\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Collecting tensorflow-io\n",
      "  Obtaining dependency information for tensorflow-io from https://files.pythonhosted.org/packages/f0/5e/f47443a14a00816fab54caf74599e2fcb34c05d6059e91f82126f8f4c68d/tensorflow_io-0.37.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading tensorflow_io-0.37.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (14 kB)\n",
      "Collecting tensorflow-io-gcs-filesystem==0.37.1 (from tensorflow-io)\n",
      "  Obtaining dependency information for tensorflow-io-gcs-filesystem==0.37.1 from https://files.pythonhosted.org/packages/66/7f/e36ae148c2f03d61ca1bff24bc13a0fef6d6825c966abef73fc6f880a23b/tensorflow_io_gcs_filesystem-0.37.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading tensorflow_io_gcs_filesystem-0.37.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (14 kB)\n",
      "Downloading tensorflow_io-0.37.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (49.6 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.6/49.6 MB\u001b[0m \u001b[31m22.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading tensorflow_io_gcs_filesystem-0.37.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.1 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.1/5.1 MB\u001b[0m \u001b[31m61.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: tensorflow-io-gcs-filesystem, tensorflow-io\n",
      "  Attempting uninstall: tensorflow-io-gcs-filesystem\n",
      "    Found existing installation: tensorflow-io-gcs-filesystem 0.34.0\n",
      "    Uninstalling tensorflow-io-gcs-filesystem-0.34.0:\n",
      "      Successfully uninstalled tensorflow-io-gcs-filesystem-0.34.0\n",
      "Successfully installed tensorflow-io-0.37.1 tensorflow-io-gcs-filesystem-0.37.1\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Collecting tensorflow-addons\n",
      "  Obtaining dependency information for tensorflow-addons from https://files.pythonhosted.org/packages/24/94/80165946ec4986505cbfac29b5ae79544bfe2200d9d7883e1ad7c7342a55/tensorflow_addons-0.23.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading tensorflow_addons-0.23.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.8 kB)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from tensorflow-addons) (23.1)\n",
      "Collecting typeguard<3.0.0,>=2.7 (from tensorflow-addons)\n",
      "  Obtaining dependency information for typeguard<3.0.0,>=2.7 from https://files.pythonhosted.org/packages/9a/bb/d43e5c75054e53efce310e79d63df0ac3f25e34c926be5dffb7d283fb2a8/typeguard-2.13.3-py3-none-any.whl.metadata\n",
      "  Downloading typeguard-2.13.3-py3-none-any.whl.metadata (3.6 kB)\n",
      "Downloading tensorflow_addons-0.23.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (611 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m611.8/611.8 kB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m0:01\u001b[0m\n",
      "\u001b[?25hDownloading typeguard-2.13.3-py3-none-any.whl (17 kB)\n",
      "Installing collected packages: typeguard, tensorflow-addons\n",
      "Successfully installed tensorflow-addons-0.23.0 typeguard-2.13.3\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!apt-get update\n",
    "!apt-get install graphviz -y\n",
    "\n",
    "!pip install tensorflow\n",
    "!pip install numpy\n",
    "!pip install pandas\n",
    "\n",
    "!pip install keras\n",
    "!pip install scikit-learn\n",
    "!pip install matplotlib\n",
    "!pip install joblib\n",
    "!pip install pyarrow\n",
    "!pip install fastparquet\n",
    "!pip install scipy\n",
    "!pip install seaborn\n",
    "!pip install tqdm\n",
    "!pip install pydot\n",
    "!pip install tensorflow-io\n",
    "!pip install tensorflow-addons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a467d3f0dfd9beab",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-12-06 10:36:10.368632: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-12-06 10:36:10.368679: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-12-06 10:36:10.368726: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-12-06 10:36:10.377750: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Keras version: 2.14.0\n",
      "TensorFlow version: 2.14.0\n",
      "TensorFlow version: 2.14.0\n",
      "CUDA available: True\n",
      "GPU devices: [PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n",
      "1 Physical GPUs, 1 Logical GPUs\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-12-06 10:36:13.233242: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1886] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 43404 MB memory:  -> device: 0, name: NVIDIA L40, pci bus id: 0000:81:00.0, compute capability: 8.9\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import keras\n",
    "\n",
    "print(f\"Keras version: {keras.__version__}\")\n",
    "print(f\"TensorFlow version: {tf.__version__}\")\n",
    "print(f\"TensorFlow version: {tf.__version__}\")\n",
    "print(f\"CUDA available: {tf.test.is_built_with_cuda()}\")\n",
    "print(f\"GPU devices: {tf.config.list_physical_devices('GPU')}\")\n",
    "\n",
    "# GPU configuration\n",
    "import tensorflow as tf\n",
    "import os\n",
    "\n",
    "# Limita la crescita della memoria GPU\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "if gpus:\n",
    "    try:\n",
    "        # Imposta la crescita di memoria dinamica\n",
    "        for gpu in gpus:\n",
    "            tf.config.experimental.set_memory_growth(gpu, True)\n",
    "            \n",
    "        # Opzionalmente, limita la memoria GPU massima (uncomment se necessario)\n",
    "        # tf.config.experimental.set_virtual_device_configuration(\n",
    "        #     gpus[0],\n",
    "        #     [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=1024*4)]  # 4GB\n",
    "        # )\n",
    "        \n",
    "        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
    "        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n",
    "    except RuntimeError as e:\n",
    "        print(e)\n",
    "        \n",
    "# Imposta le opzioni di logging\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'  # Riduce i messaggi di log\n",
    "        \n",
    "# Configura la modalità mista di precisione\n",
    "tf.keras.mixed_precision.set_global_policy('float32')\n",
    "\n",
    "# Imposta il seed per la riproducibilità\n",
    "##tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c0155cde4740b0a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.11/dist-packages/tensorflow_addons/utils/tfa_eol_msg.py:23: UserWarning: \n",
      "\n",
      "TensorFlow Addons (TFA) has ended development and introduction of new features.\n",
      "TFA has entered a minimal maintenance and release mode until a planned end of life in May 2024.\n",
      "Please modify downstream libraries to take dependencies from other repositories in our TensorFlow community (e.g. Keras, Keras-CV, and Keras-NLP). \n",
      "\n",
      "For more information see: https://github.com/tensorflow/addons/issues/2807 \n",
      "\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import tensorflow_addons as tfa\n",
    "from datetime import datetime\n",
    "import os\n",
    "import joblib\n",
    "import re\n",
    "from typing import List\n",
    "\n",
    "random_state_value = None\n",
    "execute_name = datetime.now().strftime(\"%Y-%m-%d_%H-%M\")\n",
    "\n",
    "base_project_dir = './'\n",
    "data_dir = '../../sources/'\n",
    "models_project_dir = base_project_dir\n",
    "\n",
    "os.makedirs(base_project_dir, exist_ok=True)\n",
    "os.makedirs(models_project_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1347fb59-50cc-4aa8-b805-ca9403037af5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_column_name(name: str) -> str:\n",
    "    \"\"\"\n",
    "    Rimuove caratteri speciali e spazi, converte in snake_case e abbrevia.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    name : str\n",
    "        Nome della colonna da pulire\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    str\n",
    "        Nome della colonna pulito\n",
    "    \"\"\"\n",
    "    # Rimuove caratteri speciali\n",
    "    name = re.sub(r'[^a-zA-Z0-9\\s]', '', name)\n",
    "    # Converte in snake_case\n",
    "    name = name.lower().replace(' ', '_')\n",
    "\n",
    "    # Abbreviazioni comuni\n",
    "    abbreviations = {\n",
    "        'production': 'prod',\n",
    "        'percentage': 'pct',\n",
    "        'hectare': 'ha',\n",
    "        'tonnes': 't',\n",
    "        'litres': 'l',\n",
    "        'minimum': 'min',\n",
    "        'maximum': 'max',\n",
    "        'average': 'avg'\n",
    "    }\n",
    "\n",
    "    for full, abbr in abbreviations.items():\n",
    "        name = name.replace(full, abbr)\n",
    "\n",
    "    return name\n",
    "\n",
    "\n",
    "def clean_column_names(df: pd.DataFrame) -> List[str]:\n",
    "    \"\"\"\n",
    "    Pulisce tutti i nomi delle colonne in un DataFrame.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    df : pd.DataFrame\n",
    "        DataFrame con le colonne da pulire\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    list\n",
    "        Lista dei nuovi nomi delle colonne puliti\n",
    "    \"\"\"\n",
    "    new_columns = []\n",
    "\n",
    "    for col in df.columns:\n",
    "        # Usa regex per separare le varietà\n",
    "        varieties = re.findall(r'([a-z]+)_([a-z_]+)', col)\n",
    "        if varieties:\n",
    "            new_columns.append(f\"{varieties[0][0]}_{varieties[0][1]}\")\n",
    "        else:\n",
    "            new_columns.append(col)\n",
    "\n",
    "    return new_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4da1f1bb67343e3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_plot(plt, title, output_dir=f'{base_project_dir}/{execute_name}_plots'):\n",
    "    os.makedirs(output_dir, exist_ok=True)\n",
    "    filename = \"\".join(x for x in title if x.isalnum() or x in [' ', '-', '_']).rstrip()\n",
    "    filename = filename.replace(' ', '_').lower()\n",
    "    filepath = os.path.join(output_dir, f\"{filename}.png\")\n",
    "    plt.savefig(filepath, bbox_inches='tight', dpi=300)\n",
    "    print(f\"Plot salvato come: {filepath}\")\n",
    "\n",
    "\n",
    "def encode_techniques(df, mapping_path=f'{data_dir}technique_mapping.joblib'):\n",
    "    if not os.path.exists(mapping_path):\n",
    "        raise FileNotFoundError(f\"Mapping not found at {mapping_path}. Run create_technique_mapping first.\")\n",
    "\n",
    "    technique_mapping = joblib.load(mapping_path)\n",
    "\n",
    "    # Trova tutte le colonne delle tecniche\n",
    "    tech_columns = [col for col in df.columns if col.endswith('_tech')]\n",
    "\n",
    "    # Applica il mapping a tutte le colonne delle tecniche\n",
    "    for col in tech_columns:\n",
    "        df[col] = df[col].str.lower().map(technique_mapping).fillna(0).astype(int)\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def decode_single_technique(technique_value, mapping_path=f'{data_dir}technique_mapping.joblib'):\n",
    "    if not os.path.exists(mapping_path):\n",
    "        raise FileNotFoundError(f\"Mapping not found at {mapping_path}\")\n",
    "\n",
    "    technique_mapping = joblib.load(mapping_path)\n",
    "    reverse_mapping = {v: k for k, v in technique_mapping.items()}\n",
    "    reverse_mapping[0] = ''\n",
    "\n",
    "    return reverse_mapping.get(technique_value, '')\n",
    "\n",
    "\n",
    "def prepare_comparison_data(simulated_data, olive_varieties):\n",
    "    # Pulisci i nomi delle colonne\n",
    "    df = simulated_data.copy()\n",
    "\n",
    "    df.columns = clean_column_names(df)\n",
    "    df = encode_techniques(df)\n",
    "\n",
    "    all_varieties = olive_varieties['Varietà di Olive'].unique()\n",
    "    varieties = [clean_column_name(variety) for variety in all_varieties]\n",
    "    comparison_data = []\n",
    "\n",
    "    for variety in varieties:\n",
    "        olive_prod_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_olive_prod')), None)\n",
    "        oil_prod_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_avg_oil_prod')), None)\n",
    "        tech_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_tech')), None)\n",
    "        water_need_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_water_need')), None)\n",
    "\n",
    "        if olive_prod_col and oil_prod_col and tech_col and water_need_col:\n",
    "            variety_data = df[[olive_prod_col, oil_prod_col, tech_col, water_need_col]]\n",
    "            variety_data = variety_data[variety_data[tech_col] != 0]  # Esclude le righe dove la tecnica è 0\n",
    "\n",
    "            if not variety_data.empty:\n",
    "                avg_olive_prod = pd.to_numeric(variety_data[olive_prod_col], errors='coerce').mean()\n",
    "                avg_oil_prod = pd.to_numeric(variety_data[oil_prod_col], errors='coerce').mean()\n",
    "                avg_water_need = pd.to_numeric(variety_data[water_need_col], errors='coerce').mean()\n",
    "                efficiency = avg_oil_prod / avg_olive_prod if avg_olive_prod > 0 else 0\n",
    "                water_efficiency = avg_oil_prod / avg_water_need if avg_water_need > 0 else 0\n",
    "\n",
    "                comparison_data.append({\n",
    "                    'Variety': variety,\n",
    "                    'Avg Olive Production (kg/ha)': avg_olive_prod,\n",
    "                    'Avg Oil Production (L/ha)': avg_oil_prod,\n",
    "                    'Avg Water Need (m³/ha)': avg_water_need,\n",
    "                    'Oil Efficiency (L/kg)': efficiency,\n",
    "                    'Water Efficiency (L oil/m³ water)': water_efficiency\n",
    "                })\n",
    "\n",
    "    return pd.DataFrame(comparison_data)\n",
    "\n",
    "\n",
    "def plot_variety_comparison(comparison_data, metric):\n",
    "    plt.figure(figsize=(12, 6))\n",
    "    bars = plt.bar(comparison_data['Variety'], comparison_data[metric])\n",
    "    plt.title(f'Comparison of {metric} across Olive Varieties')\n",
    "    plt.xlabel('Variety')\n",
    "    plt.ylabel(metric)\n",
    "    plt.xticks(rotation=45, ha='right')\n",
    "\n",
    "    for bar in bars:\n",
    "        height = bar.get_height()\n",
    "        plt.text(bar.get_x() + bar.get_width() / 2., height,\n",
    "                 f'{height:.2f}',\n",
    "                 ha='center', va='bottom')\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "    save_plot(plt, f'variety_comparison_{metric.lower().replace(\" \", \"_\").replace(\"/\", \"_\").replace(\"(\", \"\").replace(\")\", \"\")}')\n",
    "    plt.close()\n",
    "\n",
    "\n",
    "def plot_efficiency_vs_production(comparison_data):\n",
    "    plt.figure(figsize=(10, 6))\n",
    "\n",
    "    plt.scatter(comparison_data['Avg Olive Production (kg/ha)'],\n",
    "                comparison_data['Oil Efficiency (L/kg)'],\n",
    "                s=100)\n",
    "\n",
    "    for i, row in comparison_data.iterrows():\n",
    "        plt.annotate(row['Variety'],\n",
    "                     (row['Avg Olive Production (kg/ha)'], row['Oil Efficiency (L/kg)']),\n",
    "                     xytext=(5, 5), textcoords='offset points')\n",
    "\n",
    "    plt.title('Oil Efficiency vs Olive Production by Variety')\n",
    "    plt.xlabel('Average Olive Production (kg/ha)')\n",
    "    plt.ylabel('Oil Efficiency (L oil / kg olives)')\n",
    "    plt.tight_layout()\n",
    "    save_plot(plt, 'efficiency_vs_production')\n",
    "    plt.close()\n",
    "\n",
    "\n",
    "def plot_water_efficiency_vs_production(comparison_data):\n",
    "    plt.figure(figsize=(10, 6))\n",
    "\n",
    "    plt.scatter(comparison_data['Avg Olive Production (kg/ha)'],\n",
    "                comparison_data['Water Efficiency (L oil/m³ water)'],\n",
    "                s=100)\n",
    "\n",
    "    for i, row in comparison_data.iterrows():\n",
    "        plt.annotate(row['Variety'],\n",
    "                     (row['Avg Olive Production (kg/ha)'], row['Water Efficiency (L oil/m³ water)']),\n",
    "                     xytext=(5, 5), textcoords='offset points')\n",
    "\n",
    "    plt.title('Water Efficiency vs Olive Production by Variety')\n",
    "    plt.xlabel('Average Olive Production (kg/ha)')\n",
    "    plt.ylabel('Water Efficiency (L oil / m³ water)')\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "    save_plot(plt, 'water_efficiency_vs_production')\n",
    "    plt.close()\n",
    "\n",
    "\n",
    "def plot_water_need_vs_oil_production(comparison_data):\n",
    "    plt.figure(figsize=(10, 6))\n",
    "\n",
    "    plt.scatter(comparison_data['Avg Water Need (m³/ha)'],\n",
    "                comparison_data['Avg Oil Production (L/ha)'],\n",
    "                s=100)\n",
    "\n",
    "    for i, row in comparison_data.iterrows():\n",
    "        plt.annotate(row['Variety'],\n",
    "                     (row['Avg Water Need (m³/ha)'], row['Avg Oil Production (L/ha)']),\n",
    "                     xytext=(5, 5), textcoords='offset points')\n",
    "\n",
    "    plt.title('Oil Production vs Water Need by Variety')\n",
    "    plt.xlabel('Average Water Need (m³/ha)')\n",
    "    plt.ylabel('Average Oil Production (L/ha)')\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "    save_plot(plt, 'water_need_vs_oil_production')\n",
    "    plt.close()\n",
    "\n",
    "\n",
    "def analyze_by_technique(simulated_data, olive_varieties):\n",
    "    # Pulisci i nomi delle colonne\n",
    "    df = simulated_data.copy()\n",
    "\n",
    "    df.columns = clean_column_names(df)\n",
    "    df = encode_techniques(df)\n",
    "    all_varieties = olive_varieties['Varietà di Olive'].unique()\n",
    "    varieties = [clean_column_name(variety) for variety in all_varieties]\n",
    "\n",
    "    technique_data = []\n",
    "\n",
    "    for variety in varieties:\n",
    "        olive_prod_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_olive_prod')), None)\n",
    "        oil_prod_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_avg_oil_prod')), None)\n",
    "        tech_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_tech')), None)\n",
    "        water_need_col = next((col for col in df.columns if col.startswith(f'{variety}_') and col.endswith('_water_need')), None)\n",
    "\n",
    "        if olive_prod_col and oil_prod_col and tech_col and water_need_col:\n",
    "            variety_data = df[[olive_prod_col, oil_prod_col, tech_col, water_need_col]]\n",
    "            variety_data = variety_data[variety_data[tech_col] != 0]\n",
    "\n",
    "            if not variety_data.empty:\n",
    "                for tech in variety_data[tech_col].unique():\n",
    "                    tech_data = variety_data[variety_data[tech_col] == tech]\n",
    "\n",
    "                    avg_olive_prod = pd.to_numeric(tech_data[olive_prod_col], errors='coerce').mean()\n",
    "                    avg_oil_prod = pd.to_numeric(tech_data[oil_prod_col], errors='coerce').mean()\n",
    "                    avg_water_need = pd.to_numeric(tech_data[water_need_col], errors='coerce').mean()\n",
    "\n",
    "                    efficiency = avg_oil_prod / avg_olive_prod if avg_olive_prod > 0 else 0\n",
    "                    water_efficiency = avg_oil_prod / avg_water_need if avg_water_need > 0 else 0\n",
    "\n",
    "                    technique_data.append({\n",
    "                        'Variety': variety,\n",
    "                        'Technique': tech,\n",
    "                        'Technique String': decode_single_technique(tech),\n",
    "                        'Avg Olive Production (kg/ha)': avg_olive_prod,\n",
    "                        'Avg Oil Production (L/ha)': avg_oil_prod,\n",
    "                        'Avg Water Need (m³/ha)': avg_water_need,\n",
    "                        'Oil Efficiency (L/kg)': efficiency,\n",
    "                        'Water Efficiency (L oil/m³ water)': water_efficiency\n",
    "                    })\n",
    "\n",
    "    return pd.DataFrame(technique_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9aa4bf176c4affb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_real_error(model, test_data, test_targets, scaler_y):\n",
    "    # Fare predizioni\n",
    "    predictions = model.predict(test_data)\n",
    "\n",
    "    # Denormalizzare predizioni e target\n",
    "    predictions_real = scaler_y.inverse_transform(predictions)\n",
    "    targets_real = scaler_y.inverse_transform(test_targets)\n",
    "\n",
    "    # Calcolare errore percentuale per ogni target\n",
    "    percentage_errors = []\n",
    "    absolute_errors = []\n",
    "\n",
    "    for i in range(predictions_real.shape[1]):\n",
    "        mae = np.mean(np.abs(predictions_real[:, i] - targets_real[:, i]))\n",
    "        mape = np.mean(np.abs((predictions_real[:, i] - targets_real[:, i]) / targets_real[:, i])) * 100\n",
    "        percentage_errors.append(mape)\n",
    "        absolute_errors.append(mae)\n",
    "\n",
    "    # Stampa risultati per ogni target\n",
    "    target_names = ['olive_prod', 'min_oil_prod', 'max_oil_prod', 'avg_oil_prod', 'total_water_need']\n",
    "\n",
    "    print(\"\\nErrori per target:\")\n",
    "    print(\"-\" * 50)\n",
    "    for i, target in enumerate(target_names):\n",
    "        print(f\"{target}:\")\n",
    "        print(f\"MAE assoluto: {absolute_errors[i]:.2f}\")\n",
    "        print(f\"Errore percentuale medio: {percentage_errors[i]:.2f}%\")\n",
    "        print(f\"Precisione: {100 - percentage_errors[i]:.2f}%\")\n",
    "        print(\"-\" * 50)\n",
    "\n",
    "    return percentage_errors, absolute_errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b3ba2b96ba678389",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 73488 characters]",
      "text/plain": [
       "<Figure size 1200x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plot salvato come: .//2024-12-06_10-36_plots/variety_comparison_avg_olive_production_kg_ha.png\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 66612 characters]",
      "text/plain": [
       "<Figure size 1200x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plot salvato come: .//2024-12-06_10-36_plots/variety_comparison_avg_oil_production_l_ha.png\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 74092 characters]",
      "text/plain": [
       "<Figure size 1200x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plot salvato come: .//2024-12-06_10-36_plots/variety_comparison_avg_water_need_m³_ha.png\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 56556 characters]",
      "text/plain": [
       "<Figure size 1200x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plot salvato come: .//2024-12-06_10-36_plots/variety_comparison_oil_efficiency_l_kg.png\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 62988 characters]",
      "text/plain": [
       "<Figure size 1200x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plot salvato come: .//2024-12-06_10-36_plots/variety_comparison_water_efficiency_l_oil_m³_water.png\n",
      "Plot salvato come: .//2024-12-06_10-36_plots/efficiency_vs_production.png\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 54396 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plot salvato come: .//2024-12-06_10-36_plots/water_efficiency_vs_production.png\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 51700 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plot salvato come: .//2024-12-06_10-36_plots/water_need_vs_oil_production.png\n",
      "               Variety  Technique Technique String  \\\n",
      "0   nocellara_delletna          3     tradizionale   \n",
      "1   nocellara_delletna          1        intensiva   \n",
      "2   nocellara_delletna          2   superintensiva   \n",
      "3              leccino          1        intensiva   \n",
      "4              leccino          2   superintensiva   \n",
      "5              leccino          3     tradizionale   \n",
      "6             frantoio          2   superintensiva   \n",
      "7             frantoio          3     tradizionale   \n",
      "8             frantoio          1        intensiva   \n",
      "9             coratina          1        intensiva   \n",
      "10            coratina          2   superintensiva   \n",
      "11            coratina          3     tradizionale   \n",
      "12           taggiasca          3     tradizionale   \n",
      "13           taggiasca          2   superintensiva   \n",
      "14           taggiasca          1        intensiva   \n",
      "15           pendolino          1        intensiva   \n",
      "16           pendolino          2   superintensiva   \n",
      "17           pendolino          3     tradizionale   \n",
      "18            moraiolo          2   superintensiva   \n",
      "19            moraiolo          1        intensiva   \n",
      "20            moraiolo          3     tradizionale   \n",
      "\n",
      "    Avg Olive Production (kg/ha)  Avg Oil Production (L/ha)  \\\n",
      "0                    9564.638687                2088.362004   \n",
      "1                   13699.079622                2991.183032   \n",
      "2                   17826.710664                3892.059753   \n",
      "3                   16432.379678                3229.053194   \n",
      "4                   20528.499013                4033.942398   \n",
      "5                   10937.982122                2149.449585   \n",
      "6                   24621.040119                6047.876212   \n",
      "7                   13740.739760                3375.103688   \n",
      "8                   20550.900635                5047.942655   \n",
      "9                   16429.706879                4215.265516   \n",
      "10                  19164.700743                4916.649709   \n",
      "11                  12318.510310                3160.037128   \n",
      "12                   6839.506230                1381.247995   \n",
      "13                  16433.741502                3319.210170   \n",
      "14                  10968.603159                2215.371493   \n",
      "15                  13705.431414                2468.678455   \n",
      "16                  19183.689269                3455.879324   \n",
      "17                  10960.549241                1974.357984   \n",
      "18                  17793.971752                3885.415851   \n",
      "19                  13144.222436                2870.020002   \n",
      "20                   8765.195655                1913.745255   \n",
      "\n",
      "    Avg Water Need (m³/ha)  Oil Efficiency (L/kg)  \\\n",
      "0             32997.227891               0.218342   \n",
      "1             33079.012125               0.218349   \n",
      "2             33118.708645               0.218327   \n",
      "3             25013.303736               0.196506   \n",
      "4             24989.459147               0.196504   \n",
      "5             24981.219100               0.196512   \n",
      "6             28874.473543               0.245639   \n",
      "7             29003.452741               0.245628   \n",
      "8             28921.261327               0.245631   \n",
      "9             38270.638622               0.256564   \n",
      "10            38264.650562               0.256547   \n",
      "11            38253.676395               0.256528   \n",
      "12            26219.134374               0.201951   \n",
      "13            26253.317778               0.201975   \n",
      "14            26284.027794               0.201974   \n",
      "15            26154.359691               0.180124   \n",
      "16            26153.199618               0.180147   \n",
      "17            26152.823801               0.180133   \n",
      "18            32561.911109               0.218356   \n",
      "19            32577.899255               0.218348   \n",
      "20            32594.860153               0.218335   \n",
      "\n",
      "    Water Efficiency (L oil/m³ water)  \n",
      "0                            0.063289  \n",
      "1                            0.090425  \n",
      "2                            0.117518  \n",
      "3                            0.129093  \n",
      "4                            0.161426  \n",
      "5                            0.086043  \n",
      "6                            0.209454  \n",
      "7                            0.116369  \n",
      "8                            0.174541  \n",
      "9                            0.110144  \n",
      "10                           0.128491  \n",
      "11                           0.082607  \n",
      "12                           0.052681  \n",
      "13                           0.126430  \n",
      "14                           0.084286  \n",
      "15                           0.094389  \n",
      "16                           0.132140  \n",
      "17                           0.075493  \n",
      "18                           0.119324  \n",
      "19                           0.088097  \n",
      "20                           0.058713  \n",
      "Comparison by Variety:\n",
      "                    Avg Olive Production (kg/ha)  Avg Oil Production (L/ha)  \\\n",
      "Variety                                                                       \n",
      "nocellara_delletna                  13696.683690                2990.507461   \n",
      "leccino                             15971.162702                3138.439782   \n",
      "frantoio                            19648.631813                4826.360700   \n",
      "coratina                            15974.164423                4098.136472   \n",
      "taggiasca                           11412.636779                2305.011278   \n",
      "pendolino                           14617.432649                2633.129635   \n",
      "moraiolo                            13232.961913                2889.399172   \n",
      "\n",
      "                    Avg Water Need (m³/ha)  Oil Efficiency (L/kg)  \\\n",
      "Variety                                                             \n",
      "nocellara_delletna            33064.983905               0.218338   \n",
      "leccino                       24994.676451               0.196507   \n",
      "frantoio                      28932.932409               0.245633   \n",
      "coratina                      38262.995517               0.256548   \n",
      "taggiasca                     26252.184893               0.201970   \n",
      "pendolino                     26153.461822               0.180136   \n",
      "moraiolo                      32578.228327               0.218349   \n",
      "\n",
      "                    Water Efficiency (L oil/m³ water)  \n",
      "Variety                                                \n",
      "nocellara_delletna                           0.090443  \n",
      "leccino                                      0.125564  \n",
      "frantoio                                     0.166812  \n",
      "coratina                                     0.107104  \n",
      "taggiasca                                    0.087803  \n",
      "pendolino                                    0.100680  \n",
      "moraiolo                                     0.088691  \n",
      "\n",
      "Best Varieties by Water Efficiency:\n",
      "              Variety  Avg Olive Production (kg/ha)  \\\n",
      "2            frantoio                  19648.631813   \n",
      "1             leccino                  15971.162702   \n",
      "3            coratina                  15974.164423   \n",
      "5           pendolino                  14617.432649   \n",
      "0  nocellara_delletna                  13696.683690   \n",
      "\n",
      "   Avg Oil Production (L/ha)  Avg Water Need (m³/ha)  Oil Efficiency (L/kg)  \\\n",
      "2                4826.360700            28932.932409               0.245633   \n",
      "1                3138.439782            24994.676451               0.196507   \n",
      "3                4098.136472            38262.995517               0.256548   \n",
      "5                2633.129635            26153.461822               0.180136   \n",
      "0                2990.507461            33064.983905               0.218338   \n",
      "\n",
      "   Water Efficiency (L oil/m³ water)  \n",
      "2                           0.166812  \n",
      "1                           0.125564  \n",
      "3                           0.107104  \n",
      "5                           0.100680  \n",
      "0                           0.090443  \n"
     ]
    }
   ],
   "source": [
    "simulated_data = pd.read_parquet(f\"{data_dir}olive_training_dataset.parquet\")\n",
    "olive_varieties = pd.read_parquet(f\"{data_dir}olive_varieties.parquet\")\n",
    "# Esecuzione dell'analisi\n",
    "comparison_data = prepare_comparison_data(simulated_data, olive_varieties)\n",
    "\n",
    "# Genera i grafici\n",
    "plot_variety_comparison(comparison_data, 'Avg Olive Production (kg/ha)')\n",
    "plot_variety_comparison(comparison_data, 'Avg Oil Production (L/ha)')\n",
    "plot_variety_comparison(comparison_data, 'Avg Water Need (m³/ha)')\n",
    "plot_variety_comparison(comparison_data, 'Oil Efficiency (L/kg)')\n",
    "plot_variety_comparison(comparison_data, 'Water Efficiency (L oil/m³ water)')\n",
    "plot_efficiency_vs_production(comparison_data)\n",
    "plot_water_efficiency_vs_production(comparison_data)\n",
    "plot_water_need_vs_oil_production(comparison_data)\n",
    "\n",
    "# Analisi per tecnica\n",
    "technique_data = analyze_by_technique(simulated_data, olive_varieties)\n",
    "\n",
    "print(technique_data)\n",
    "\n",
    "# Stampa un sommario statistico\n",
    "print(\"Comparison by Variety:\")\n",
    "print(comparison_data.set_index('Variety'))\n",
    "print(\"\\nBest Varieties by Water Efficiency:\")\n",
    "print(comparison_data.sort_values('Water Efficiency (L oil/m³ water)', ascending=False).head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bbe87b415168368",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_transformer_data(df, olive_varieties_df):\n",
    "    # Crea una copia del DataFrame per evitare modifiche all'originale\n",
    "    df = df.copy()\n",
    "\n",
    "    # Ordina per zona e anno\n",
    "    df = df.sort_values(['zone', 'year'])\n",
    "\n",
    "    # Definisci le feature\n",
    "    temporal_features = ['temp_mean', 'precip_sum', 'solar_energy_sum']\n",
    "    static_features = ['ha']  # Feature statiche base\n",
    "    target_features = ['olive_prod', 'min_oil_prod', 'max_oil_prod', 'avg_oil_prod', 'total_water_need']\n",
    "\n",
    "    # Ottieni le varietà pulite\n",
    "    all_varieties = olive_varieties_df['Varietà di Olive'].unique()\n",
    "    varieties = [clean_column_name(variety) for variety in all_varieties]\n",
    "\n",
    "    # Crea la struttura delle feature per ogni varietà\n",
    "    variety_features = [\n",
    "        'tech', 'pct', 'prod_t_ha', 'oil_prod_t_ha', 'oil_prod_l_ha',\n",
    "        'min_yield_pct', 'max_yield_pct', 'min_oil_prod_l_ha', 'max_oil_prod_l_ha',\n",
    "        'avg_oil_prod_l_ha', 'l_per_t', 'min_l_per_t', 'max_l_per_t', 'avg_l_per_t'\n",
    "    ]\n",
    "\n",
    "    # Prepara dizionari per le nuove colonne\n",
    "    new_columns = {}\n",
    "\n",
    "    # Prepara le feature per ogni varietà\n",
    "    for variety in varieties:\n",
    "        # Feature esistenti\n",
    "        for feature in variety_features:\n",
    "            col_name = f\"{variety}_{feature}\"\n",
    "            if col_name in df.columns:\n",
    "                if feature != 'tech':  # Non includere la colonna tech direttamente\n",
    "                    static_features.append(col_name)\n",
    "\n",
    "        # Feature binarie per le tecniche di coltivazione\n",
    "        for technique in ['tradizionale', 'intensiva', 'superintensiva']:\n",
    "            col_name = f\"{variety}_{technique}\"\n",
    "            new_columns[col_name] = df[f\"{variety}_tech\"].notna() & (\n",
    "                    df[f\"{variety}_tech\"].str.lower() == technique\n",
    "            ).fillna(False)\n",
    "            static_features.append(col_name)\n",
    "\n",
    "    # Aggiungi tutte le nuove colonne in una volta sola\n",
    "    new_df = pd.concat([df] + [pd.Series(v, name=k) for k, v in new_columns.items()], axis=1)\n",
    "\n",
    "    # Ordiniamo per zona e anno per mantenere la continuità temporale\n",
    "    df_sorted = new_df.sort_values(['zone', 'year'])\n",
    "\n",
    "    # Definiamo la dimensione della finestra temporale\n",
    "    window_size = 41\n",
    "\n",
    "    # Liste per raccogliere i dati\n",
    "    temporal_sequences = []\n",
    "    static_features_list = []\n",
    "    targets_list = []\n",
    "\n",
    "    # Iteriamo per ogni zona\n",
    "    for zone in df_sorted['zone'].unique():\n",
    "        zone_data = df_sorted[df_sorted['zone'] == zone].reset_index(drop=True)\n",
    "\n",
    "        if len(zone_data) >= window_size:  # Verifichiamo che ci siano abbastanza dati\n",
    "            # Creiamo sequenze temporali scorrevoli\n",
    "            for i in range(len(zone_data) - window_size + 1):\n",
    "                # Sequenza temporale\n",
    "                temporal_window = zone_data.iloc[i:i + window_size][temporal_features].values\n",
    "                # Verifichiamo che non ci siano valori NaN\n",
    "                if not np.isnan(temporal_window).any():\n",
    "                    temporal_sequences.append(temporal_window)\n",
    "\n",
    "                    # Feature statiche (prendiamo quelle dell'ultimo timestep della finestra)\n",
    "                    static_features_list.append(zone_data.iloc[i + window_size - 1][static_features].values)\n",
    "\n",
    "                    # Target (prendiamo quelli dell'ultimo timestep della finestra)\n",
    "                    targets_list.append(zone_data.iloc[i + window_size - 1][target_features].values)\n",
    "\n",
    "    # Convertiamo in array numpy\n",
    "    X_temporal = np.array(temporal_sequences)\n",
    "    X_static = np.array(static_features_list)\n",
    "    y = np.array(targets_list)\n",
    "\n",
    "    print(f\"Dataset completo - Temporal: {X_temporal.shape}, Static: {X_static.shape}, Target: {y.shape}\")\n",
    "\n",
    "    # Split dei dati (usando indici casuali per una migliore distribuzione)\n",
    "    indices = np.random.permutation(len(X_temporal))\n",
    "\n",
    "    #train_idx = int(len(indices) * 0.7)        # 70% training\n",
    "    #val_idx = int(len(indices) * 0.85)         # 15% validation\n",
    "    # Il resto rimane 15% test\n",
    "\n",
    "    train_idx = int(len(indices) * 0.65)        # 65% training\n",
    "    val_idx = int(len(indices) * 0.85)          # 20% validation\n",
    "    # Il resto rimane 15% test\n",
    "\n",
    "    #train_idx = int(len(indices) * 0.60)       # 60% training\n",
    "    #val_idx = int(len(indices) * 0.85)         # 25% validation\n",
    "    # Il resto rimane 15% test\n",
    "\n",
    "    train_indices = indices[:train_idx]\n",
    "    val_indices = indices[train_idx:val_idx]\n",
    "    test_indices = indices[val_idx:]\n",
    "\n",
    "    # Split dei dati\n",
    "    X_temporal_train = X_temporal[train_indices]\n",
    "    X_temporal_val = X_temporal[val_indices]\n",
    "    X_temporal_test = X_temporal[test_indices]\n",
    "\n",
    "    X_static_train = X_static[train_indices]\n",
    "    X_static_val = X_static[val_indices]\n",
    "    X_static_test = X_static[test_indices]\n",
    "\n",
    "    y_train = y[train_indices]\n",
    "    y_val = y[val_indices]\n",
    "    y_test = y[test_indices]\n",
    "\n",
    "    # Standardizzazione\n",
    "    scaler_temporal = StandardScaler()\n",
    "    scaler_static = StandardScaler()\n",
    "    scaler_y = StandardScaler()\n",
    "\n",
    "    # Standardizzazione dei dati temporali\n",
    "    X_temporal_train = scaler_temporal.fit_transform(X_temporal_train.reshape(-1, len(temporal_features))).reshape(X_temporal_train.shape)\n",
    "    X_temporal_val = scaler_temporal.transform(X_temporal_val.reshape(-1, len(temporal_features))).reshape(X_temporal_val.shape)\n",
    "    X_temporal_test = scaler_temporal.transform(X_temporal_test.reshape(-1, len(temporal_features))).reshape(X_temporal_test.shape)\n",
    "\n",
    "    # Standardizzazione dei dati statici\n",
    "    X_static_train = scaler_static.fit_transform(X_static_train)\n",
    "    X_static_val = scaler_static.transform(X_static_val)\n",
    "    X_static_test = scaler_static.transform(X_static_test)\n",
    "\n",
    "    # Standardizzazione dei target\n",
    "    y_train = scaler_y.fit_transform(y_train)\n",
    "    y_val = scaler_y.transform(y_val)\n",
    "    y_test = scaler_y.transform(y_test)\n",
    "\n",
    "    print(\"\\nShape dopo lo split e standardizzazione:\")\n",
    "    print(f\"Train - Temporal: {X_temporal_train.shape}, Static: {X_static_train.shape}, Target: {y_train.shape}\")\n",
    "    print(f\"Val - Temporal: {X_temporal_val.shape}, Static: {X_static_val.shape}, Target: {y_val.shape}\")\n",
    "    print(f\"Test - Temporal: {X_temporal_test.shape}, Static: {X_static_test.shape}, Target: {y_test.shape}\")\n",
    "\n",
    "    # Prepara i dizionari di input\n",
    "    train_data = {'temporal': X_temporal_train, 'static': X_static_train}\n",
    "    val_data = {'temporal': X_temporal_val, 'static': X_static_val}\n",
    "    test_data = {'temporal': X_temporal_test, 'static': X_static_test}\n",
    "\n",
    "    joblib.dump(scaler_temporal, os.path.join(base_project_dir, f'{execute_name}_scaler_temporal.joblib'))\n",
    "    joblib.dump(scaler_static, os.path.join(base_project_dir, f'{execute_name}_scaler_static.joblib'))\n",
    "    joblib.dump(scaler_y, os.path.join(base_project_dir, f'{execute_name}_scaler_y.joblib'))\n",
    "\n",
    "    return (train_data, y_train), (val_data, y_val), (test_data, y_test), (scaler_temporal, scaler_static, scaler_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9c4d5f0f3fafdc2d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset completo - Temporal: (3920000, 41, 3), Static: (3920000, 113), Target: (3920000, 5)\n",
      "\n",
      "Shape dopo lo split e standardizzazione:\n",
      "Train - Temporal: (2548000, 41, 3), Static: (2548000, 113), Target: (2548000, 5)\n",
      "Val - Temporal: (784000, 41, 3), Static: (784000, 113), Target: (784000, 5)\n",
      "Test - Temporal: (588000, 41, 3), Static: (588000, 113), Target: (588000, 5)\n",
      "Temporal data shape: (2548000, 41, 3)\n",
      "Static data shape: (2548000, 113)\n",
      "Target shape: (2548000, 5)\n"
     ]
    }
   ],
   "source": [
    "simulated_data = pd.read_parquet(f\"{data_dir}olive_training_dataset.parquet\")\n",
    "olive_varieties = pd.read_parquet(f\"{data_dir}olive_varieties.parquet\")\n",
    "\n",
    "(train_data, train_targets), (val_data, val_targets), (test_data, test_targets), scalers = prepare_transformer_data(simulated_data, olive_varieties)\n",
    "\n",
    "scaler_temporal, scaler_static, scaler_y = scalers\n",
    "\n",
    "print(\"Temporal data shape:\", train_data['temporal'].shape)\n",
    "print(\"Static data shape:\", train_data['static'].shape)\n",
    "print(\"Target shape:\", train_targets.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "604c952c7195f40c",
   "metadata": {},
   "outputs": [],
   "source": [
    "@keras.saving.register_keras_serializable()\n",
    "class DataAugmentation(tf.keras.layers.Layer):\n",
    "    \"\"\"Custom layer per l'augmentation dei dati\"\"\"\n",
    "\n",
    "    def __init__(self, noise_stddev=0.03, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.noise_stddev = noise_stddev\n",
    "\n",
    "    def call(self, inputs, training=None):\n",
    "        if training:\n",
    "            return inputs + tf.random.normal(\n",
    "                shape=tf.shape(inputs),\n",
    "                mean=0.0,\n",
    "                stddev=self.noise_stddev\n",
    "            )\n",
    "        return inputs\n",
    "\n",
    "    def get_config(self):\n",
    "        config = super().get_config()\n",
    "        config.update({\"noise_stddev\": self.noise_stddev})\n",
    "        return config\n",
    "\n",
    "\n",
    "@keras.saving.register_keras_serializable()\n",
    "class PositionalEncoding(tf.keras.layers.Layer):\n",
    "    \"\"\"Custom layer per l'encoding posizionale\"\"\"\n",
    "\n",
    "    def __init__(self, d_model, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.d_model = d_model\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        _, seq_length, _ = input_shape\n",
    "\n",
    "        # Crea la matrice di encoding posizionale\n",
    "        position = tf.range(seq_length, dtype=tf.float32)[:, tf.newaxis]\n",
    "        div_term = tf.exp(\n",
    "            tf.range(0, self.d_model, 2, dtype=tf.float32) *\n",
    "            (-tf.math.log(10000.0) / self.d_model)\n",
    "        )\n",
    "\n",
    "        # Calcola sin e cos\n",
    "        pos_encoding = tf.zeros((1, seq_length, self.d_model))\n",
    "        pos_encoding_even = tf.sin(position * div_term)\n",
    "        pos_encoding_odd = tf.cos(position * div_term)\n",
    "\n",
    "        # Assegna i valori alle posizioni pari e dispari\n",
    "        pos_encoding = tf.concat(\n",
    "            [tf.expand_dims(pos_encoding_even, -1),\n",
    "             tf.expand_dims(pos_encoding_odd, -1)],\n",
    "            axis=-1\n",
    "        )\n",
    "        pos_encoding = tf.reshape(pos_encoding, (1, seq_length, -1))\n",
    "        pos_encoding = pos_encoding[:, :, :self.d_model]\n",
    "\n",
    "        # Salva l'encoding come peso non trainabile\n",
    "        self.pos_encoding = self.add_weight(\n",
    "            shape=(1, seq_length, self.d_model),\n",
    "            initializer=tf.keras.initializers.Constant(pos_encoding),\n",
    "            trainable=False,\n",
    "            name='positional_encoding'\n",
    "        )\n",
    "\n",
    "        super().build(input_shape)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        # Broadcast l'encoding posizionale sul batch\n",
    "        batch_size = tf.shape(inputs)[0]\n",
    "        pos_encoding_tiled = tf.tile(self.pos_encoding, [batch_size, 1, 1])\n",
    "        return inputs + pos_encoding_tiled\n",
    "\n",
    "    def get_config(self):\n",
    "        config = super().get_config()\n",
    "        config.update({\"d_model\": self.d_model})\n",
    "        return config\n",
    "\n",
    "\n",
    "@keras.saving.register_keras_serializable()\n",
    "class WarmUpLearningRateSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
    "    \"\"\"Custom learning rate schedule with linear warmup and exponential decay.\"\"\"\n",
    "\n",
    "    def __init__(self, initial_learning_rate=1e-3, warmup_steps=500, decay_steps=5000):\n",
    "        super().__init__()\n",
    "        self.initial_learning_rate = initial_learning_rate\n",
    "        self.warmup_steps = warmup_steps\n",
    "        self.decay_steps = decay_steps\n",
    "\n",
    "    def __call__(self, step):\n",
    "        warmup_pct = tf.cast(step, tf.float32) / self.warmup_steps\n",
    "        warmup_lr = self.initial_learning_rate * warmup_pct\n",
    "        decay_factor = tf.pow(0.1, tf.cast(step, tf.float32) / self.decay_steps)\n",
    "        decayed_lr = self.initial_learning_rate * decay_factor\n",
    "        return tf.where(step < self.warmup_steps, warmup_lr, decayed_lr)\n",
    "\n",
    "    def get_config(self):\n",
    "        return {\n",
    "            'initial_learning_rate': self.initial_learning_rate,\n",
    "            'warmup_steps': self.warmup_steps,\n",
    "            'decay_steps': self.decay_steps\n",
    "        }\n",
    "\n",
    "\n",
    "def create_olive_oil_transformer(temporal_shape, static_shape, num_outputs,\n",
    "                                 d_model=128, num_heads=8, ff_dim=256,\n",
    "                                 num_transformer_blocks=4, mlp_units=None,\n",
    "                                 dropout=0.2):\n",
    "    \"\"\"\n",
    "    Crea un transformer per la predizione della produzione di olio d'oliva.\n",
    "    \"\"\"\n",
    "    # Input layers\n",
    "    if mlp_units is None:\n",
    "        mlp_units = [256, 128, 64]\n",
    "\n",
    "    temporal_input = tf.keras.layers.Input(shape=temporal_shape, name='temporal')\n",
    "    static_input = tf.keras.layers.Input(shape=static_shape, name='static')\n",
    "\n",
    "    # === TEMPORAL PATH ===\n",
    "    x = tf.keras.layers.LayerNormalization(epsilon=1e-6)(temporal_input)\n",
    "    x = DataAugmentation()(x)\n",
    "\n",
    "    # Temporal projection\n",
    "    x = tf.keras.layers.Dense(\n",
    "        d_model // 2,\n",
    "        activation='swish',\n",
    "        kernel_regularizer=tf.keras.regularizers.l2(1e-5)\n",
    "    )(x)\n",
    "    x = tf.keras.layers.Dropout(dropout)(x)\n",
    "    x = tf.keras.layers.Dense(\n",
    "        d_model,\n",
    "        activation='swish',\n",
    "        kernel_regularizer=tf.keras.regularizers.l2(1e-5)\n",
    "    )(x)\n",
    "\n",
    "    # Positional encoding\n",
    "    x = PositionalEncoding(d_model)(x)\n",
    "\n",
    "    # Transformer blocks\n",
    "    skip_connection = x\n",
    "    for _ in range(num_transformer_blocks):\n",
    "        # Self-attention\n",
    "        attention_output = tf.keras.layers.MultiHeadAttention(\n",
    "            num_heads=num_heads,\n",
    "            key_dim=d_model // num_heads,\n",
    "            value_dim=d_model // num_heads\n",
    "        )(x, x)\n",
    "        attention_output = tf.keras.layers.Dropout(dropout)(attention_output)\n",
    "\n",
    "        # Residual connection con pesi addestrabili\n",
    "        residual_weights = tf.keras.layers.Dense(d_model, activation='sigmoid')(x)\n",
    "        x = tfa.layers.StochasticDepth(survival_probability=0.3)([x, residual_weights * attention_output])\n",
    "        x = tf.keras.layers.LayerNormalization(epsilon=1e-6)(x)\n",
    "\n",
    "        # Feed-forward network\n",
    "        ffn = tf.keras.layers.Dense(ff_dim, activation=\"swish\")(x)\n",
    "        ffn = tf.keras.layers.Dropout(dropout)(ffn)\n",
    "        ffn = tf.keras.layers.Dense(d_model)(ffn)\n",
    "        ffn = tf.keras.layers.Dropout(dropout)(ffn)\n",
    "\n",
    "        # Second residual connection\n",
    "        x = tfa.layers.StochasticDepth()([x, ffn])\n",
    "        x = tf.keras.layers.LayerNormalization(epsilon=1e-6)(x)\n",
    "\n",
    "    # Add final skip connection\n",
    "    x = tfa.layers.StochasticDepth(survival_probability=0.5)([x, skip_connection])\n",
    "\n",
    "    # Temporal pooling\n",
    "    attention_pooled = tf.keras.layers.MultiHeadAttention(\n",
    "        num_heads=num_heads,\n",
    "        key_dim=d_model // 4\n",
    "    )(x, x)\n",
    "    attention_pooled = tf.keras.layers.GlobalAveragePooling1D()(attention_pooled)\n",
    "\n",
    "    # Additional pooling operations\n",
    "    avg_pooled = tf.keras.layers.GlobalAveragePooling1D()(x)\n",
    "    max_pooled = tf.keras.layers.GlobalMaxPooling1D()(x)\n",
    "\n",
    "    # Combine pooling results\n",
    "    temporal_features = tf.keras.layers.Concatenate()(\n",
    "        [attention_pooled, avg_pooled, max_pooled]\n",
    "    )\n",
    "\n",
    "    # === STATIC PATH ===\n",
    "    static_features = tf.keras.layers.LayerNormalization(epsilon=1e-6)(static_input)\n",
    "    for units in [256, 128, 64]:\n",
    "        static_features = tf.keras.layers.Dense(\n",
    "            units,\n",
    "            activation='swish',\n",
    "            kernel_regularizer=tf.keras.regularizers.l2(1e-5)\n",
    "        )(static_features)\n",
    "        static_features = tf.keras.layers.Dropout(dropout)(static_features)\n",
    "\n",
    "    # === FEATURE FUSION ===\n",
    "    combined = tf.keras.layers.Concatenate()([temporal_features, static_features])\n",
    "\n",
    "    # === MLP HEAD ===\n",
    "    x = combined\n",
    "    for units in mlp_units:\n",
    "        x = tf.keras.layers.BatchNormalization()(x)\n",
    "        x = tf.keras.layers.Dense(\n",
    "            units,\n",
    "            activation=\"swish\",\n",
    "            kernel_regularizer=tf.keras.regularizers.l2(1e-5)\n",
    "        )(x)\n",
    "        x = tf.keras.layers.Dropout(dropout)(x)\n",
    "\n",
    "    # Output layer\n",
    "    outputs = tf.keras.layers.Dense(\n",
    "        num_outputs,\n",
    "        activation='linear',\n",
    "        kernel_regularizer=tf.keras.regularizers.l2(1e-5)\n",
    "    )(x)\n",
    "\n",
    "    # Create model\n",
    "    model = tf.keras.Model(\n",
    "        inputs={'temporal': temporal_input, 'static': static_input},\n",
    "        outputs=outputs,\n",
    "        name='OilTransformer'\n",
    "    )\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "def create_transformer_callbacks(target_names, val_data, val_targets):\n",
    "    \"\"\"\n",
    "    Crea i callbacks per il training del modello.\n",
    "    \n",
    "    Parameters:\n",
    "    -----------\n",
    "    target_names : list\n",
    "        Lista dei nomi dei target per il monitoraggio specifico\n",
    "    val_data : dict\n",
    "        Dati di validazione\n",
    "    val_targets : array\n",
    "        Target di validazione\n",
    "    \n",
    "    Returns:\n",
    "    --------\n",
    "    list\n",
    "        Lista dei callbacks configurati\n",
    "    \"\"\"\n",
    "\n",
    "    # Custom Metric per target specifici\n",
    "    class TargetSpecificMetric(tf.keras.callbacks.Callback):\n",
    "        def __init__(self, validation_data, target_names):\n",
    "            super().__init__()\n",
    "            self.validation_data = validation_data\n",
    "            self.target_names = target_names\n",
    "\n",
    "        def on_epoch_end(self, epoch, logs={}):\n",
    "            x_val, y_val = self.validation_data\n",
    "            y_pred = self.model.predict(x_val, verbose=0)\n",
    "\n",
    "            for i, name in enumerate(self.target_names):\n",
    "                mae = np.mean(np.abs(y_val[:, i] - y_pred[:, i]))\n",
    "                logs[f'val_{name}_mae'] = mae\n",
    "\n",
    "\n",
    "    callbacks = [\n",
    "        # Early Stopping\n",
    "        tf.keras.callbacks.EarlyStopping(\n",
    "            monitor='val_loss',\n",
    "            patience=20,\n",
    "            restore_best_weights=True,\n",
    "            min_delta=0.0005,\n",
    "            mode='min'\n",
    "        ),\n",
    "\n",
    "        # Model Checkpoint\n",
    "        tf.keras.callbacks.ModelCheckpoint(\n",
    "            filepath=f'{execute_name}_best_oil_model.h5',\n",
    "            monitor='val_loss',\n",
    "            save_best_only=True,\n",
    "            mode='min',\n",
    "            save_weights_only=True\n",
    "        ),\n",
    "\n",
    "        # Metric per target specifici\n",
    "        TargetSpecificMetric(\n",
    "            validation_data=(val_data, val_targets),\n",
    "            target_names=target_names\n",
    "        ),\n",
    "\n",
    "        # Reduce LR on Plateau\n",
    "        tf.keras.callbacks.ReduceLROnPlateau(\n",
    "            monitor='val_loss',\n",
    "            factor=0.5,\n",
    "            patience=10,\n",
    "            min_lr=1e-6,\n",
    "            verbose=1\n",
    "        ),\n",
    "\n",
    "        # TensorBoard logging\n",
    "        tf.keras.callbacks.TensorBoard(\n",
    "            log_dir=f'./logs_{execute_name}',\n",
    "            histogram_freq=1,\n",
    "            write_graph=True,\n",
    "            update_freq='epoch'\n",
    "        )\n",
    "    ]\n",
    "\n",
    "    return callbacks\n",
    "\n",
    "\n",
    "def compile_model(model, learning_rate=1e-3):\n",
    "    \"\"\"\n",
    "    Compila il modello con le impostazioni standard.\n",
    "    \"\"\"\n",
    "    lr_schedule = WarmUpLearningRateSchedule(\n",
    "        initial_learning_rate=learning_rate,\n",
    "        warmup_steps=500,\n",
    "        decay_steps=5000\n",
    "    )\n",
    "\n",
    "    model.compile(\n",
    "        optimizer=tf.keras.optimizers.AdamW(\n",
    "            learning_rate=lr_schedule,\n",
    "            weight_decay=0.01\n",
    "        ),\n",
    "        loss=tf.keras.losses.Huber(),\n",
    "        metrics=['mae']\n",
    "    )\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "def setup_transformer_training(train_data, train_targets, val_data, val_targets):\n",
    "    \"\"\"\n",
    "    Configura e prepara il transformer con dimensioni dinamiche basate sui dati.\n",
    "    \"\"\"\n",
    "    # Estrai le shape dai dati\n",
    "    temporal_shape = (train_data['temporal'].shape[1], train_data['temporal'].shape[2])\n",
    "    static_shape = (train_data['static'].shape[1],)\n",
    "    num_outputs = train_targets.shape[1]\n",
    "\n",
    "    print(f\"Shape rilevate:\")\n",
    "    print(f\"- Temporal shape: {temporal_shape}\")\n",
    "    print(f\"- Static shape: {static_shape}\")\n",
    "    print(f\"- Numero di output: {num_outputs}\")\n",
    "\n",
    "    # Target names basati sul numero di output\n",
    "    target_names = ['olive_prod', 'min_oil_prod', 'max_oil_prod', 'avg_oil_prod', 'total_water_need']\n",
    "\n",
    "    # Assicurati che il numero di target names corrisponda al numero di output\n",
    "    assert len(target_names) == num_outputs, \\\n",
    "        f\"Il numero di target names ({len(target_names)}) non corrisponde al numero di output ({num_outputs})\"\n",
    "\n",
    "    # Crea il modello con le dimensioni rilevate\n",
    "    model = create_olive_oil_transformer(\n",
    "        temporal_shape=temporal_shape,\n",
    "        static_shape=static_shape,\n",
    "        num_outputs=num_outputs\n",
    "    )\n",
    "\n",
    "    # Compila il modello\n",
    "    model = compile_model(model)\n",
    "\n",
    "    # Crea i callbacks\n",
    "    callbacks = create_transformer_callbacks(target_names, val_data, val_targets)\n",
    "\n",
    "    return model, callbacks, target_names\n",
    "\n",
    "\n",
    "def train_transformer(train_data, train_targets, val_data, val_targets, epochs=150, batch_size=64, save_name='final_model'):\n",
    "    \"\"\"\n",
    "    Funzione principale per l'addestramento del transformer con ottimizzazioni.\n",
    "    \"\"\"\n",
    "    # Conversione dei dati in tf.data.Dataset per una gestione più efficiente della memoria\n",
    "    train_dataset = tf.data.Dataset.from_tensor_slices((train_data, train_targets))\\\n",
    "        .cache()\\\n",
    "        .shuffle(buffer_size=1024)\\\n",
    "        .batch(batch_size)\\\n",
    "        .prefetch(tf.data.AUTOTUNE)\n",
    "\n",
    "    val_dataset = tf.data.Dataset.from_tensor_slices((val_data, val_targets))\\\n",
    "        .cache()\\\n",
    "        .batch(batch_size)\\\n",
    "        .prefetch(tf.data.AUTOTUNE)\n",
    "\n",
    "    # Setup del modello\n",
    "    strategy = tf.distribute.MirroredStrategy() if len(tf.config.list_physical_devices('GPU')) > 1 else tf.distribute.get_strategy()\n",
    "    \n",
    "    with strategy.scope():\n",
    "        model, callbacks, target_names = setup_transformer_training(\n",
    "            train_data, train_targets, val_data, val_targets\n",
    "        )\n",
    "\n",
    "    # Mostra il summary del modello\n",
    "    model.summary()\n",
    "    \n",
    "    try:\n",
    "        keras.utils.plot_model(model, f\"{execute_name}_{save_name}.png\", show_shapes=True)\n",
    "    except Exception as e:\n",
    "        print(f\"Warning: Could not create model plot: {e}\")\n",
    "\n",
    "    # Training con gestione degli errori\n",
    "    try:\n",
    "        history = model.fit(\n",
    "            train_dataset,\n",
    "            validation_data=val_dataset,\n",
    "            epochs=epochs,\n",
    "            callbacks=callbacks,\n",
    "            verbose=1,\n",
    "            workers=4,\n",
    "            use_multiprocessing=True\n",
    "        )\n",
    "    except tf.errors.ResourceExhaustedError:\n",
    "        print(\"Memoria GPU esaurita, riprovo con batch size più piccolo...\")\n",
    "        # Riprova con batch size più piccolo\n",
    "        batch_size = batch_size // 2\n",
    "        train_dataset = train_dataset.unbatch().batch(batch_size)\n",
    "        val_dataset = val_dataset.unbatch().batch(batch_size)\n",
    "        history = model.fit(\n",
    "            train_dataset,\n",
    "            validation_data=val_dataset,\n",
    "            epochs=epochs,\n",
    "            callbacks=callbacks,\n",
    "            verbose=1\n",
    "        )\n",
    "\n",
    "    # Salva il modello finale\n",
    "    try:\n",
    "        save_path = f'{execute_name}_{save_name}.keras'\n",
    "        model.save(save_path, save_format='keras')\n",
    "        \n",
    "        os.makedirs(f'{execute_name}/weights', exist_ok=True)\n",
    "        model.save_weights(f'{execute_name}/weights')\n",
    "        print(f\"\\nModello salvato in: {save_path}\")\n",
    "    except Exception as e:\n",
    "        print(f\"Warning: Could not save model: {e}\")\n",
    "\n",
    "    return model, history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "35490e902e494c4a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape rilevate:\n",
      "- Temporal shape: (41, 3)\n",
      "- Static shape: (113,)\n",
      "- Numero di output: 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-12-06 11:43:09.026945: I tensorflow/tsl/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"OilTransformer\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                Output Shape                 Param #   Connected to                  \n",
      "==================================================================================================\n",
      " temporal (InputLayer)       [(None, 41, 3)]              0         []                            \n",
      "                                                                                                  \n",
      " layer_normalization (Layer  (None, 41, 3)                6         ['temporal[0][0]']            \n",
      " Normalization)                                                                                   \n",
      "                                                                                                  \n",
      " data_augmentation (DataAug  (None, 41, 3)                0         ['layer_normalization[0][0]'] \n",
      " mentation)                                                                                       \n",
      "                                                                                                  \n",
      " dense (Dense)               (None, 41, 64)               256       ['data_augmentation[0][0]']   \n",
      "                                                                                                  \n",
      " dropout (Dropout)           (None, 41, 64)               0         ['dense[0][0]']               \n",
      "                                                                                                  \n",
      " dense_1 (Dense)             (None, 41, 128)              8320      ['dropout[0][0]']             \n",
      "                                                                                                  \n",
      " positional_encoding (Posit  (None, 41, 128)              5248      ['dense_1[0][0]']             \n",
      " ionalEncoding)                                                                                   \n",
      "                                                                                                  \n",
      " multi_head_attention (Mult  (None, 41, 128)              66048     ['positional_encoding[0][0]', \n",
      " iHeadAttention)                                                     'positional_encoding[0][0]'] \n",
      "                                                                                                  \n",
      " dense_2 (Dense)             (None, 41, 128)              16512     ['positional_encoding[0][0]'] \n",
      "                                                                                                  \n",
      " dropout_1 (Dropout)         (None, 41, 128)              0         ['multi_head_attention[0][0]']\n",
      "                                                                                                  \n",
      " tf.math.multiply (TFOpLamb  (None, 41, 128)              0         ['dense_2[0][0]',             \n",
      " da)                                                                 'dropout_1[0][0]']           \n",
      "                                                                                                  \n",
      " stochastic_depth (Stochast  (None, 41, 128)              0         ['positional_encoding[0][0]', \n",
      " icDepth)                                                            'tf.math.multiply[0][0]']    \n",
      "                                                                                                  \n",
      " layer_normalization_1 (Lay  (None, 41, 128)              256       ['stochastic_depth[0][0]']    \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dense_3 (Dense)             (None, 41, 256)              33024     ['layer_normalization_1[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dropout_2 (Dropout)         (None, 41, 256)              0         ['dense_3[0][0]']             \n",
      "                                                                                                  \n",
      " dense_4 (Dense)             (None, 41, 128)              32896     ['dropout_2[0][0]']           \n",
      "                                                                                                  \n",
      " dropout_3 (Dropout)         (None, 41, 128)              0         ['dense_4[0][0]']             \n",
      "                                                                                                  \n",
      " stochastic_depth_1 (Stocha  (None, 41, 128)              0         ['layer_normalization_1[0][0]'\n",
      " sticDepth)                                                         , 'dropout_3[0][0]']          \n",
      "                                                                                                  \n",
      " layer_normalization_2 (Lay  (None, 41, 128)              256       ['stochastic_depth_1[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " multi_head_attention_1 (Mu  (None, 41, 128)              66048     ['layer_normalization_2[0][0]'\n",
      " ltiHeadAttention)                                                  , 'layer_normalization_2[0][0]\n",
      "                                                                    ']                            \n",
      "                                                                                                  \n",
      " dense_5 (Dense)             (None, 41, 128)              16512     ['layer_normalization_2[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dropout_4 (Dropout)         (None, 41, 128)              0         ['multi_head_attention_1[0][0]\n",
      "                                                                    ']                            \n",
      "                                                                                                  \n",
      " tf.math.multiply_1 (TFOpLa  (None, 41, 128)              0         ['dense_5[0][0]',             \n",
      " mbda)                                                               'dropout_4[0][0]']           \n",
      "                                                                                                  \n",
      " stochastic_depth_2 (Stocha  (None, 41, 128)              0         ['layer_normalization_2[0][0]'\n",
      " sticDepth)                                                         , 'tf.math.multiply_1[0][0]'] \n",
      "                                                                                                  \n",
      " layer_normalization_3 (Lay  (None, 41, 128)              256       ['stochastic_depth_2[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dense_6 (Dense)             (None, 41, 256)              33024     ['layer_normalization_3[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dropout_5 (Dropout)         (None, 41, 256)              0         ['dense_6[0][0]']             \n",
      "                                                                                                  \n",
      " dense_7 (Dense)             (None, 41, 128)              32896     ['dropout_5[0][0]']           \n",
      "                                                                                                  \n",
      " dropout_6 (Dropout)         (None, 41, 128)              0         ['dense_7[0][0]']             \n",
      "                                                                                                  \n",
      " stochastic_depth_3 (Stocha  (None, 41, 128)              0         ['layer_normalization_3[0][0]'\n",
      " sticDepth)                                                         , 'dropout_6[0][0]']          \n",
      "                                                                                                  \n",
      " layer_normalization_4 (Lay  (None, 41, 128)              256       ['stochastic_depth_3[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " multi_head_attention_2 (Mu  (None, 41, 128)              66048     ['layer_normalization_4[0][0]'\n",
      " ltiHeadAttention)                                                  , 'layer_normalization_4[0][0]\n",
      "                                                                    ']                            \n",
      "                                                                                                  \n",
      " dense_8 (Dense)             (None, 41, 128)              16512     ['layer_normalization_4[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dropout_7 (Dropout)         (None, 41, 128)              0         ['multi_head_attention_2[0][0]\n",
      "                                                                    ']                            \n",
      "                                                                                                  \n",
      " tf.math.multiply_2 (TFOpLa  (None, 41, 128)              0         ['dense_8[0][0]',             \n",
      " mbda)                                                               'dropout_7[0][0]']           \n",
      "                                                                                                  \n",
      " stochastic_depth_4 (Stocha  (None, 41, 128)              0         ['layer_normalization_4[0][0]'\n",
      " sticDepth)                                                         , 'tf.math.multiply_2[0][0]'] \n",
      "                                                                                                  \n",
      " layer_normalization_5 (Lay  (None, 41, 128)              256       ['stochastic_depth_4[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dense_9 (Dense)             (None, 41, 256)              33024     ['layer_normalization_5[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dropout_8 (Dropout)         (None, 41, 256)              0         ['dense_9[0][0]']             \n",
      "                                                                                                  \n",
      " dense_10 (Dense)            (None, 41, 128)              32896     ['dropout_8[0][0]']           \n",
      "                                                                                                  \n",
      " dropout_9 (Dropout)         (None, 41, 128)              0         ['dense_10[0][0]']            \n",
      "                                                                                                  \n",
      " stochastic_depth_5 (Stocha  (None, 41, 128)              0         ['layer_normalization_5[0][0]'\n",
      " sticDepth)                                                         , 'dropout_9[0][0]']          \n",
      "                                                                                                  \n",
      " layer_normalization_6 (Lay  (None, 41, 128)              256       ['stochastic_depth_5[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " multi_head_attention_3 (Mu  (None, 41, 128)              66048     ['layer_normalization_6[0][0]'\n",
      " ltiHeadAttention)                                                  , 'layer_normalization_6[0][0]\n",
      "                                                                    ']                            \n",
      "                                                                                                  \n",
      " dense_11 (Dense)            (None, 41, 128)              16512     ['layer_normalization_6[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dropout_10 (Dropout)        (None, 41, 128)              0         ['multi_head_attention_3[0][0]\n",
      "                                                                    ']                            \n",
      "                                                                                                  \n",
      " tf.math.multiply_3 (TFOpLa  (None, 41, 128)              0         ['dense_11[0][0]',            \n",
      " mbda)                                                               'dropout_10[0][0]']          \n",
      "                                                                                                  \n",
      " stochastic_depth_6 (Stocha  (None, 41, 128)              0         ['layer_normalization_6[0][0]'\n",
      " sticDepth)                                                         , 'tf.math.multiply_3[0][0]'] \n",
      "                                                                                                  \n",
      " layer_normalization_7 (Lay  (None, 41, 128)              256       ['stochastic_depth_6[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dense_12 (Dense)            (None, 41, 256)              33024     ['layer_normalization_7[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dropout_11 (Dropout)        (None, 41, 256)              0         ['dense_12[0][0]']            \n",
      "                                                                                                  \n",
      " dense_13 (Dense)            (None, 41, 128)              32896     ['dropout_11[0][0]']          \n",
      "                                                                                                  \n",
      " static (InputLayer)         [(None, 113)]                0         []                            \n",
      "                                                                                                  \n",
      " dropout_12 (Dropout)        (None, 41, 128)              0         ['dense_13[0][0]']            \n",
      "                                                                                                  \n",
      " layer_normalization_9 (Lay  (None, 113)                  226       ['static[0][0]']              \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " stochastic_depth_7 (Stocha  (None, 41, 128)              0         ['layer_normalization_7[0][0]'\n",
      " sticDepth)                                                         , 'dropout_12[0][0]']         \n",
      "                                                                                                  \n",
      " dense_14 (Dense)            (None, 256)                  29184     ['layer_normalization_9[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " layer_normalization_8 (Lay  (None, 41, 128)              256       ['stochastic_depth_7[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dropout_13 (Dropout)        (None, 256)                  0         ['dense_14[0][0]']            \n",
      "                                                                                                  \n",
      " stochastic_depth_8 (Stocha  (None, 41, 128)              0         ['layer_normalization_8[0][0]'\n",
      " sticDepth)                                                         , 'positional_encoding[0][0]']\n",
      "                                                                                                  \n",
      " dense_15 (Dense)            (None, 128)                  32896     ['dropout_13[0][0]']          \n",
      "                                                                                                  \n",
      " multi_head_attention_4 (Mu  (None, 41, 128)              131968    ['stochastic_depth_8[0][0]',  \n",
      " ltiHeadAttention)                                                   'stochastic_depth_8[0][0]']  \n",
      "                                                                                                  \n",
      " dropout_14 (Dropout)        (None, 128)                  0         ['dense_15[0][0]']            \n",
      "                                                                                                  \n",
      " global_average_pooling1d (  (None, 128)                  0         ['multi_head_attention_4[0][0]\n",
      " GlobalAveragePooling1D)                                            ']                            \n",
      "                                                                                                  \n",
      " global_average_pooling1d_1  (None, 128)                  0         ['stochastic_depth_8[0][0]']  \n",
      "  (GlobalAveragePooling1D)                                                                        \n",
      "                                                                                                  \n",
      " global_max_pooling1d (Glob  (None, 128)                  0         ['stochastic_depth_8[0][0]']  \n",
      " alMaxPooling1D)                                                                                  \n",
      "                                                                                                  \n",
      " dense_16 (Dense)            (None, 64)                   8256      ['dropout_14[0][0]']          \n",
      "                                                                                                  \n",
      " concatenate (Concatenate)   (None, 384)                  0         ['global_average_pooling1d[0][\n",
      "                                                                    0]',                          \n",
      "                                                                     'global_average_pooling1d_1[0\n",
      "                                                                    ][0]',                        \n",
      "                                                                     'global_max_pooling1d[0][0]']\n",
      "                                                                                                  \n",
      " dropout_15 (Dropout)        (None, 64)                   0         ['dense_16[0][0]']            \n",
      "                                                                                                  \n",
      " concatenate_1 (Concatenate  (None, 448)                  0         ['concatenate[0][0]',         \n",
      " )                                                                   'dropout_15[0][0]']          \n",
      "                                                                                                  \n",
      " batch_normalization (Batch  (None, 448)                  1792      ['concatenate_1[0][0]']       \n",
      " Normalization)                                                                                   \n",
      "                                                                                                  \n",
      " dense_17 (Dense)            (None, 256)                  114944    ['batch_normalization[0][0]'] \n",
      "                                                                                                  \n",
      " dropout_16 (Dropout)        (None, 256)                  0         ['dense_17[0][0]']            \n",
      "                                                                                                  \n",
      " batch_normalization_1 (Bat  (None, 256)                  1024      ['dropout_16[0][0]']          \n",
      " chNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dense_18 (Dense)            (None, 128)                  32896     ['batch_normalization_1[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dropout_17 (Dropout)        (None, 128)                  0         ['dense_18[0][0]']            \n",
      "                                                                                                  \n",
      " batch_normalization_2 (Bat  (None, 128)                  512       ['dropout_17[0][0]']          \n",
      " chNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dense_19 (Dense)            (None, 64)                   8256      ['batch_normalization_2[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dropout_18 (Dropout)        (None, 64)                   0         ['dense_19[0][0]']            \n",
      "                                                                                                  \n",
      " dense_20 (Dense)            (None, 5)                    325       ['dropout_18[0][0]']          \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 972077 (3.71 MB)\n",
      "Trainable params: 965165 (3.68 MB)\n",
      "Non-trainable params: 6912 (27.00 KB)\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/150\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-12-06 11:43:25.651745: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x7d7e70d1ce40 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2024-12-06 11:43:25.651778: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): NVIDIA L40, Compute Capability 8.9\n",
      "2024-12-06 11:43:25.659099: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:269] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n",
      "2024-12-06 11:43:25.722749: I tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:442] Loaded cuDNN version 8905\n",
      "2024-12-06 11:43:25.861911: I ./tensorflow/compiler/jit/device_compiler.h:186] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9954/9954 [==============================] - 481s 46ms/step - loss: 0.0460 - mae: 0.1872 - val_loss: 0.0145 - val_mae: 0.0865 - val_olive_prod_mae: 0.0964 - val_min_oil_prod_mae: 0.0935 - val_max_oil_prod_mae: 0.0936 - val_avg_oil_prod_mae: 0.0894 - val_total_water_need_mae: 0.0598 - lr: 1.0219e-05\n",
      "Epoch 2/150\n",
      "9954/9954 [==============================] - 473s 47ms/step - loss: 0.0273 - mae: 0.1505 - val_loss: 0.0143 - val_mae: 0.0863 - val_olive_prod_mae: 0.0963 - val_min_oil_prod_mae: 0.0931 - val_max_oil_prod_mae: 0.0929 - val_avg_oil_prod_mae: 0.0889 - val_total_water_need_mae: 0.0603 - lr: 1.0438e-07\n",
      "Epoch 3/150\n",
      "9954/9954 [==============================] - 477s 48ms/step - loss: 0.0273 - mae: 0.1506 - val_loss: 0.0143 - val_mae: 0.0861 - val_olive_prod_mae: 0.0964 - val_min_oil_prod_mae: 0.0929 - val_max_oil_prod_mae: 0.0927 - val_avg_oil_prod_mae: 0.0886 - val_total_water_need_mae: 0.0602 - lr: 1.0661e-09\n",
      "Epoch 4/150\n",
      "9954/9954 [==============================] - 508s 51ms/step - loss: 0.0272 - mae: 0.1505 - val_loss: 0.0143 - val_mae: 0.0867 - val_olive_prod_mae: 0.0967 - val_min_oil_prod_mae: 0.0932 - val_max_oil_prod_mae: 0.0930 - val_avg_oil_prod_mae: 0.0889 - val_total_water_need_mae: 0.0616 - lr: 1.0889e-11\n",
      "Epoch 5/150\n",
      "9954/9954 [==============================] - 431s 43ms/step - loss: 0.0273 - mae: 0.1507 - val_loss: 0.0143 - val_mae: 0.0865 - val_olive_prod_mae: 0.0965 - val_min_oil_prod_mae: 0.0931 - val_max_oil_prod_mae: 0.0929 - val_avg_oil_prod_mae: 0.0889 - val_total_water_need_mae: 0.0612 - lr: 1.1122e-13\n",
      "Epoch 6/150\n",
      "9954/9954 [==============================] - 438s 44ms/step - loss: 0.0273 - mae: 0.1506 - val_loss: 0.0143 - val_mae: 0.0863 - val_olive_prod_mae: 0.0965 - val_min_oil_prod_mae: 0.0931 - val_max_oil_prod_mae: 0.0929 - val_avg_oil_prod_mae: 0.0889 - val_total_water_need_mae: 0.0598 - lr: 1.1361e-15\n",
      "Epoch 7/150\n",
      "9954/9954 [==============================] - 413s 41ms/step - loss: 0.0273 - mae: 0.1506 - val_loss: 0.0143 - val_mae: 0.0868 - val_olive_prod_mae: 0.0967 - val_min_oil_prod_mae: 0.0932 - val_max_oil_prod_mae: 0.0930 - val_avg_oil_prod_mae: 0.0890 - val_total_water_need_mae: 0.0620 - lr: 1.1604e-17\n",
      "Epoch 8/150\n",
      "9954/9954 [==============================] - 433s 43ms/step - loss: 0.0272 - mae: 0.1505 - val_loss: 0.0143 - val_mae: 0.0865 - val_olive_prod_mae: 0.0966 - val_min_oil_prod_mae: 0.0931 - val_max_oil_prod_mae: 0.0929 - val_avg_oil_prod_mae: 0.0888 - val_total_water_need_mae: 0.0611 - lr: 1.1852e-19\n",
      "Epoch 9/150\n",
      "9954/9954 [==============================] - 413s 41ms/step - loss: 0.0273 - mae: 0.1507 - val_loss: 0.0143 - val_mae: 0.0865 - val_olive_prod_mae: 0.0967 - val_min_oil_prod_mae: 0.0933 - val_max_oil_prod_mae: 0.0930 - val_avg_oil_prod_mae: 0.0890 - val_total_water_need_mae: 0.0608 - lr: 1.2106e-21\n",
      "Epoch 10/150\n",
      "9954/9954 [==============================] - 430s 43ms/step - loss: 0.0273 - mae: 0.1508 - val_loss: 0.0143 - val_mae: 0.0864 - val_olive_prod_mae: 0.0965 - val_min_oil_prod_mae: 0.0931 - val_max_oil_prod_mae: 0.0929 - val_avg_oil_prod_mae: 0.0889 - val_total_water_need_mae: 0.0607 - lr: 1.2365e-23\n",
      "Epoch 11/150\n",
      "9954/9954 [==============================] - 438s 44ms/step - loss: 0.0273 - mae: 0.1507 - val_loss: 0.0143 - val_mae: 0.0863 - val_olive_prod_mae: 0.0965 - val_min_oil_prod_mae: 0.0930 - val_max_oil_prod_mae: 0.0928 - val_avg_oil_prod_mae: 0.0887 - val_total_water_need_mae: 0.0604 - lr: 1.2630e-25\n",
      "Epoch 12/150\n",
      "9954/9954 [==============================] - 430s 43ms/step - loss: 0.0273 - mae: 0.1505 - val_loss: 0.0144 - val_mae: 0.0866 - val_olive_prod_mae: 0.0968 - val_min_oil_prod_mae: 0.0933 - val_max_oil_prod_mae: 0.0932 - val_avg_oil_prod_mae: 0.0891 - val_total_water_need_mae: 0.0606 - lr: 1.2900e-27\n",
      "Epoch 13/150\n",
      "9954/9954 [==============================] - 425s 43ms/step - loss: 0.0273 - mae: 0.1507 - val_loss: 0.0144 - val_mae: 0.0868 - val_olive_prod_mae: 0.0966 - val_min_oil_prod_mae: 0.0932 - val_max_oil_prod_mae: 0.0930 - val_avg_oil_prod_mae: 0.0890 - val_total_water_need_mae: 0.0619 - lr: 1.3177e-29\n",
      "Epoch 14/150\n",
      "9954/9954 [==============================] - 409s 41ms/step - loss: 0.0272 - mae: 0.1504 - val_loss: 0.0144 - val_mae: 0.0865 - val_olive_prod_mae: 0.0967 - val_min_oil_prod_mae: 0.0933 - val_max_oil_prod_mae: 0.0932 - val_avg_oil_prod_mae: 0.0891 - val_total_water_need_mae: 0.0605 - lr: 1.3459e-31\n",
      "Epoch 15/150\n",
      "9954/9954 [==============================] - 439s 44ms/step - loss: 0.0273 - mae: 0.1509 - val_loss: 0.0143 - val_mae: 0.0863 - val_olive_prod_mae: 0.0964 - val_min_oil_prod_mae: 0.0929 - val_max_oil_prod_mae: 0.0926 - val_avg_oil_prod_mae: 0.0886 - val_total_water_need_mae: 0.0609 - lr: 1.3747e-33\n",
      "Epoch 16/150\n",
      "9954/9954 [==============================] - 421s 42ms/step - loss: 0.0273 - mae: 0.1508 - val_loss: 0.0143 - val_mae: 0.0862 - val_olive_prod_mae: 0.0963 - val_min_oil_prod_mae: 0.0930 - val_max_oil_prod_mae: 0.0928 - val_avg_oil_prod_mae: 0.0887 - val_total_water_need_mae: 0.0604 - lr: 1.4041e-35\n",
      "Epoch 17/150\n",
      "9954/9954 [==============================] - 429s 43ms/step - loss: 0.0272 - mae: 0.1505 - val_loss: 0.0143 - val_mae: 0.0863 - val_olive_prod_mae: 0.0966 - val_min_oil_prod_mae: 0.0931 - val_max_oil_prod_mae: 0.0929 - val_avg_oil_prod_mae: 0.0888 - val_total_water_need_mae: 0.0600 - lr: 1.4342e-37\n",
      "Epoch 18/150\n",
      "9954/9954 [==============================] - 414s 41ms/step - loss: 0.0272 - mae: 0.1505 - val_loss: 0.0144 - val_mae: 0.0865 - val_olive_prod_mae: 0.0967 - val_min_oil_prod_mae: 0.0933 - val_max_oil_prod_mae: 0.0931 - val_avg_oil_prod_mae: 0.0890 - val_total_water_need_mae: 0.0602 - lr: 0.0000e+00\n",
      "Epoch 19/150\n",
      "9954/9954 [==============================] - 441s 44ms/step - loss: 0.0272 - mae: 0.1506 - val_loss: 0.0143 - val_mae: 0.0864 - val_olive_prod_mae: 0.0965 - val_min_oil_prod_mae: 0.0930 - val_max_oil_prod_mae: 0.0928 - val_avg_oil_prod_mae: 0.0888 - val_total_water_need_mae: 0.0608 - lr: 0.0000e+00\n",
      "Epoch 20/150\n",
      "9954/9954 [==============================] - 440s 44ms/step - loss: 0.0272 - mae: 0.1505 - val_loss: 0.0143 - val_mae: 0.0862 - val_olive_prod_mae: 0.0963 - val_min_oil_prod_mae: 0.0930 - val_max_oil_prod_mae: 0.0929 - val_avg_oil_prod_mae: 0.0888 - val_total_water_need_mae: 0.0601 - lr: 0.0000e+00\n",
      "Epoch 21/150\n",
      "9954/9954 [==============================] - 448s 45ms/step - loss: 0.0273 - mae: 0.1508 - val_loss: 0.0143 - val_mae: 0.0862 - val_olive_prod_mae: 0.0964 - val_min_oil_prod_mae: 0.0935 - val_max_oil_prod_mae: 0.0936 - val_avg_oil_prod_mae: 0.0894 - val_total_water_need_mae: 0.0598 - lr: 0.0000e+00\n",
      "\n",
      "Modello salvato in: 2024-12-06_10-36_final_model.keras\n"
     ]
    }
   ],
   "source": [
    "model, history = train_transformer(train_data, train_targets, val_data, val_targets, 150, 512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3e2fb5a5341dac92",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24500/24500 [==============================] - 102s 4ms/step\n",
      "\n",
      "Errori per target:\n",
      "--------------------------------------------------\n",
      "olive_prod:\n",
      "MAE assoluto: 1585.45\n",
      "Errore percentuale medio: 6.91%\n",
      "Precisione: 93.09%\n",
      "--------------------------------------------------\n",
      "min_oil_prod:\n",
      "MAE assoluto: 319.12\n",
      "Errore percentuale medio: 6.61%\n",
      "Precisione: 93.39%\n",
      "--------------------------------------------------\n",
      "max_oil_prod:\n",
      "MAE assoluto: 387.31\n",
      "Errore percentuale medio: 6.74%\n",
      "Precisione: 93.26%\n",
      "--------------------------------------------------\n",
      "avg_oil_prod:\n",
      "MAE assoluto: 337.11\n",
      "Errore percentuale medio: 6.46%\n",
      "Precisione: 93.54%\n",
      "--------------------------------------------------\n",
      "total_water_need:\n",
      "MAE assoluto: 1775.48\n",
      "Errore percentuale medio: 4.24%\n",
      "Precisione: 95.76%\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "percentage_errors, absolute_errors = calculate_real_error(model, val_data, val_targets, scaler_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4af58aa9bbc156f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_model_performance(model, data, targets, set_name=\"\"):\n",
    "    \"\"\"\n",
    "    Valuta le performance del modello su un set di dati specifico.\n",
    "    \"\"\"\n",
    "    predictions = model.predict(data, verbose=0)\n",
    "\n",
    "    target_names = ['olive_prod', 'min_oil_prod', 'max_oil_prod', 'avg_oil_prod', 'total_water_need']\n",
    "    metrics = {}\n",
    "\n",
    "    for i, name in enumerate(target_names):\n",
    "        mae = np.mean(np.abs(targets[:, i] - predictions[:, i]))\n",
    "        mse = np.mean(np.square(targets[:, i] - predictions[:, i]))\n",
    "        rmse = np.sqrt(mse)\n",
    "        mape = np.mean(np.abs((targets[:, i] - predictions[:, i]) / (targets[:, i] + 1e-7))) * 100\n",
    "\n",
    "        metrics[f\"{name}_mae\"] = mae\n",
    "        metrics[f\"{name}_rmse\"] = rmse\n",
    "        metrics[f\"{name}_mape\"] = mape\n",
    "\n",
    "    if set_name:\n",
    "        print(f\"\\nPerformance sul set {set_name}:\")\n",
    "        for metric, value in metrics.items():\n",
    "            print(f\"{metric}: {value:.4f}\")\n",
    "\n",
    "    return metrics\n",
    "\n",
    "\n",
    "def retrain_model(base_model, train_data, train_targets,\n",
    "                  val_data, val_targets,\n",
    "                  test_data, test_targets,\n",
    "                  epochs=50, batch_size=128):\n",
    "    \"\"\"\n",
    "    Implementa il retraining del modello con i dati combinati.\n",
    "    \"\"\"\n",
    "    print(\"Valutazione performance iniziali del modello...\")\n",
    "    initial_metrics = {\n",
    "        'train': evaluate_model_performance(base_model, train_data, train_targets, \"training\"),\n",
    "        'val': evaluate_model_performance(base_model, val_data, val_targets, \"validazione\"),\n",
    "        'test': evaluate_model_performance(base_model, test_data, test_targets, \"test\")\n",
    "    }\n",
    "\n",
    "    # Combina i dati per il retraining\n",
    "    combined_data = {\n",
    "        'temporal': np.concatenate([train_data['temporal'], val_data['temporal'], test_data['temporal']]),\n",
    "        'static': np.concatenate([train_data['static'], val_data['static'], test_data['static']])\n",
    "    }\n",
    "    combined_targets = np.concatenate([train_targets, val_targets, test_targets])\n",
    "\n",
    "    # Crea una nuova suddivisione per la validazione\n",
    "    indices = np.arange(len(combined_targets))\n",
    "    np.random.shuffle(indices)\n",
    "\n",
    "    split_idx = int(len(indices) * 0.9)\n",
    "    train_idx, val_idx = indices[:split_idx], indices[split_idx:]\n",
    "\n",
    "    # Prepara i dati per il retraining\n",
    "    retrain_data = {k: v[train_idx] for k, v in combined_data.items()}\n",
    "    retrain_targets = combined_targets[train_idx]\n",
    "    retrain_val_data = {k: v[val_idx] for k, v in combined_data.items()}\n",
    "    retrain_val_targets = combined_targets[val_idx]\n",
    "\n",
    "    # Configura callbacks\n",
    "    callbacks = [\n",
    "        tf.keras.callbacks.EarlyStopping(\n",
    "            monitor='val_loss',\n",
    "            patience=10,\n",
    "            restore_best_weights=True,\n",
    "            min_delta=0.0001\n",
    "        ),\n",
    "        tf.keras.callbacks.ReduceLROnPlateau(\n",
    "            monitor='val_loss',\n",
    "            factor=0.2,\n",
    "            patience=5,\n",
    "            min_lr=1e-6,\n",
    "            verbose=1\n",
    "        ),\n",
    "        tf.keras.callbacks.ModelCheckpoint(\n",
    "            filepath=f'{execute_name}_retrained_best_oil_model.h5',\n",
    "            monitor='val_loss',\n",
    "            save_best_only=True,\n",
    "            mode='min',\n",
    "            save_weights_only=True\n",
    "        )\n",
    "    ]\n",
    "\n",
    "    # Imposta learning rate per il fine-tuning\n",
    "    optimizer = tf.keras.optimizers.AdamW(\n",
    "        learning_rate=tf.keras.optimizers.schedules.ExponentialDecay(\n",
    "            initial_learning_rate=1e-4,\n",
    "            decay_steps=1000,\n",
    "            decay_rate=0.9\n",
    "        ),\n",
    "        weight_decay=0.01\n",
    "    )\n",
    "\n",
    "    # Ricompila il modello con il nuovo optimizer\n",
    "    base_model.compile(\n",
    "        optimizer=optimizer,\n",
    "        loss=tf.keras.losses.Huber(),\n",
    "        metrics=['mae']\n",
    "    )\n",
    "\n",
    "    print(\"\\nAvvio retraining...\")\n",
    "    history = base_model.fit(\n",
    "        retrain_data,\n",
    "        retrain_targets,\n",
    "        validation_data=(retrain_val_data, retrain_val_targets),\n",
    "        epochs=epochs,\n",
    "        batch_size=batch_size,\n",
    "        callbacks=callbacks,\n",
    "        verbose=1\n",
    "    )\n",
    "\n",
    "    print(\"\\nValutazione performance finali...\")\n",
    "    final_metrics = {\n",
    "        'train': evaluate_model_performance(base_model, train_data, train_targets, \"training\"),\n",
    "        'val': evaluate_model_performance(base_model, val_data, val_targets, \"validazione\"),\n",
    "        'test': evaluate_model_performance(base_model, test_data, test_targets, \"test\")\n",
    "    }\n",
    "\n",
    "    # Salva il modello finale\n",
    "    save_path = f'{execute_name}_retrained_model.keras'\n",
    "    os.makedirs(f'{execute_name}_retrained/weights', exist_ok=True)\n",
    "    \n",
    "    base_model.save_weights(f'{execute_name}_retrained/weights')\n",
    "    base_model.save(save_path, save_format='keras')\n",
    "    print(f\"\\nModello riaddestrato salvato in: {save_path}\")\n",
    "\n",
    "    # Report miglioramenti\n",
    "    print(\"\\nMiglioramenti delle performance:\")\n",
    "    for dataset in ['train', 'val', 'test']:\n",
    "        print(f\"\\nSet {dataset}:\")\n",
    "        for metric in initial_metrics[dataset].keys():\n",
    "            initial = initial_metrics[dataset][metric]\n",
    "            final = final_metrics[dataset][metric]\n",
    "            improvement = ((initial - final) / initial) * 100\n",
    "            print(f\"{metric}: {improvement:.2f}% di miglioramento\")\n",
    "\n",
    "    return base_model, history, final_metrics\n",
    "\n",
    "\n",
    "def start_retraining(model_path, train_data, train_targets,\n",
    "                     val_data, val_targets,\n",
    "                     test_data, test_targets,\n",
    "                     epochs=50, batch_size=128):\n",
    "    \"\"\"\n",
    "    Avvia il processo di retraining in modo sicuro.\n",
    "    \"\"\"\n",
    "    try:\n",
    "        print(\"Caricamento del modello...\")\n",
    "        base_model = tf.keras.models.load_model(model_path, compile=False)\n",
    "        print(\"Modello caricato con successo!\")\n",
    "\n",
    "        return retrain_model(\n",
    "            base_model=base_model,\n",
    "            train_data=train_data,\n",
    "            train_targets=train_targets,\n",
    "            val_data=val_data,\n",
    "            val_targets=val_targets,\n",
    "            test_data=test_data,\n",
    "            test_targets=test_targets,\n",
    "            epochs=epochs,\n",
    "            batch_size=batch_size\n",
    "        )\n",
    "    except Exception as e:\n",
    "        print(f\"Errore durante il retraining: {str(e)}\")\n",
    "        raise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "588c7e49371f4a0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Caricamento del modello...\n",
      "Modello caricato con successo!\n",
      "Valutazione performance iniziali del modello...\n",
      "\n",
      "Performance sul set training:\n",
      "olive_prod_mae: 0.0963\n",
      "olive_prod_rmse: 0.1300\n",
      "olive_prod_mape: 77.2491\n",
      "min_oil_prod_mae: 0.0936\n",
      "min_oil_prod_rmse: 0.1312\n",
      "min_oil_prod_mape: 91.4612\n",
      "max_oil_prod_mae: 0.0936\n",
      "max_oil_prod_rmse: 0.1304\n",
      "max_oil_prod_mape: 88.9396\n",
      "avg_oil_prod_mae: 0.0895\n",
      "avg_oil_prod_rmse: 0.1238\n",
      "avg_oil_prod_mape: 89.5317\n",
      "total_water_need_mae: 0.0598\n",
      "total_water_need_rmse: 0.0808\n",
      "total_water_need_mape: 44.4531\n",
      "\n",
      "Performance sul set validazione:\n",
      "olive_prod_mae: 0.0964\n",
      "olive_prod_rmse: 0.1301\n",
      "olive_prod_mape: 133.2427\n",
      "min_oil_prod_mae: 0.0935\n",
      "min_oil_prod_rmse: 0.1310\n",
      "min_oil_prod_mape: 120.7693\n",
      "max_oil_prod_mae: 0.0936\n",
      "max_oil_prod_rmse: 0.1304\n",
      "max_oil_prod_mape: 86.2224\n",
      "avg_oil_prod_mae: 0.0894\n",
      "avg_oil_prod_rmse: 0.1237\n",
      "avg_oil_prod_mape: 83.8138\n",
      "total_water_need_mae: 0.0598\n",
      "total_water_need_rmse: 0.0809\n",
      "total_water_need_mape: 53.9347\n",
      "\n",
      "Performance sul set test:\n",
      "olive_prod_mae: 0.0962\n",
      "olive_prod_rmse: 0.1298\n",
      "olive_prod_mape: 77.9806\n",
      "min_oil_prod_mae: 0.0935\n",
      "min_oil_prod_rmse: 0.1312\n",
      "min_oil_prod_mape: 95.5886\n",
      "max_oil_prod_mae: 0.0934\n",
      "max_oil_prod_rmse: 0.1301\n",
      "max_oil_prod_mape: 76.3217\n",
      "avg_oil_prod_mae: 0.0893\n",
      "avg_oil_prod_rmse: 0.1237\n",
      "avg_oil_prod_mape: 111.2211\n",
      "total_water_need_mae: 0.0596\n",
      "total_water_need_rmse: 0.0806\n",
      "total_water_need_mape: 38.1699\n",
      "\n",
      "Avvio retraining...\n",
      "Epoch 1/50\n",
      "27563/27563 [==============================] - 851s 30ms/step - loss: 0.0261 - mae: 0.1520 - val_loss: 0.0118 - val_mae: 0.0804 - lr: 5.4806e-06\n",
      "Epoch 2/50\n",
      "27563/27563 [==============================] - 852s 31ms/step - loss: 0.0245 - mae: 0.1478 - val_loss: 0.0117 - val_mae: 0.0803 - lr: 3.0034e-07\n",
      "Epoch 3/50\n",
      "27563/27563 [==============================] - 836s 30ms/step - loss: 0.0244 - mae: 0.1476 - val_loss: 0.0117 - val_mae: 0.0807 - lr: 1.6459e-08\n",
      "Epoch 4/50\n",
      "27563/27563 [==============================] - 863s 31ms/step - loss: 0.0244 - mae: 0.1476 - val_loss: 0.0118 - val_mae: 0.0807 - lr: 9.0196e-10\n",
      "Epoch 5/50\n",
      "27563/27563 [==============================] - 854s 31ms/step - loss: 0.0243 - mae: 0.1474 - val_loss: 0.0119 - val_mae: 0.0812 - lr: 4.9428e-11\n",
      "Epoch 6/50\n",
      "27563/27563 [==============================] - 869s 32ms/step - loss: 0.0244 - mae: 0.1475 - val_loss: 0.0118 - val_mae: 0.0807 - lr: 2.7087e-12\n",
      "Epoch 7/50\n",
      "27563/27563 [==============================] - 867s 31ms/step - loss: 0.0244 - mae: 0.1475 - val_loss: 0.0118 - val_mae: 0.0806 - lr: 1.4844e-13\n",
      "Epoch 8/50\n",
      "27563/27563 [==============================] - 899s 33ms/step - loss: 0.0244 - mae: 0.1475 - val_loss: 0.0117 - val_mae: 0.0803 - lr: 8.1345e-15\n",
      "Epoch 9/50\n",
      "27563/27563 [==============================] - 966s 35ms/step - loss: 0.0244 - mae: 0.1475 - val_loss: 0.0117 - val_mae: 0.0804 - lr: 4.4578e-16\n",
      "Epoch 10/50\n",
      "27563/27563 [==============================] - 930s 34ms/step - loss: 0.0244 - mae: 0.1474 - val_loss: 0.0118 - val_mae: 0.0807 - lr: 2.4429e-17\n",
      "Epoch 11/50\n",
      "27563/27563 [==============================] - 921s 33ms/step - loss: 0.0244 - mae: 0.1475 - val_loss: 0.0118 - val_mae: 0.0809 - lr: 1.3387e-18\n",
      "\n",
      "Valutazione performance finali...\n",
      "\n",
      "Performance sul set training:\n",
      "olive_prod_mae: 0.0901\n",
      "olive_prod_rmse: 0.1222\n",
      "olive_prod_mape: 75.7735\n",
      "min_oil_prod_mae: 0.0886\n",
      "min_oil_prod_rmse: 0.1245\n",
      "min_oil_prod_mape: 91.0646\n",
      "max_oil_prod_mae: 0.0888\n",
      "max_oil_prod_rmse: 0.1243\n",
      "max_oil_prod_mape: 89.5375\n",
      "avg_oil_prod_mae: 0.0845\n",
      "avg_oil_prod_rmse: 0.1171\n",
      "avg_oil_prod_mape: 86.3355\n",
      "total_water_need_mae: 0.0495\n",
      "total_water_need_rmse: 0.0678\n",
      "total_water_need_mape: 41.0436\n",
      "\n",
      "Performance sul set validazione:\n",
      "olive_prod_mae: 0.0901\n",
      "olive_prod_rmse: 0.1222\n",
      "olive_prod_mape: 138.3196\n",
      "min_oil_prod_mae: 0.0885\n",
      "min_oil_prod_rmse: 0.1243\n",
      "min_oil_prod_mape: 126.9523\n",
      "max_oil_prod_mae: 0.0888\n",
      "max_oil_prod_rmse: 0.1243\n",
      "max_oil_prod_mape: 82.7593\n",
      "avg_oil_prod_mae: 0.0843\n",
      "avg_oil_prod_rmse: 0.1169\n",
      "avg_oil_prod_mape: 84.3605\n",
      "total_water_need_mae: 0.0495\n",
      "total_water_need_rmse: 0.0679\n",
      "total_water_need_mape: 48.6941\n",
      "\n",
      "Performance sul set test:\n",
      "olive_prod_mae: 0.0899\n",
      "olive_prod_rmse: 0.1219\n",
      "olive_prod_mape: 77.0356\n",
      "min_oil_prod_mae: 0.0886\n",
      "min_oil_prod_rmse: 0.1243\n",
      "min_oil_prod_mape: 96.3498\n",
      "max_oil_prod_mae: 0.0885\n",
      "max_oil_prod_rmse: 0.1238\n",
      "max_oil_prod_mape: 76.4509\n",
      "avg_oil_prod_mae: 0.0843\n",
      "avg_oil_prod_rmse: 0.1167\n",
      "avg_oil_prod_mape: 87.8912\n",
      "total_water_need_mae: 0.0494\n",
      "total_water_need_rmse: 0.0677\n",
      "total_water_need_mape: 30.6997\n",
      "\n",
      "Modello riaddestrato salvato in: 2024-12-06_10-36_retrained_model.keras\n",
      "\n",
      "Miglioramenti delle performance:\n",
      "\n",
      "Set train:\n",
      "olive_prod_mae: 6.48% di miglioramento\n",
      "olive_prod_rmse: 6.00% di miglioramento\n",
      "olive_prod_mape: 1.91% di miglioramento\n",
      "min_oil_prod_mae: 5.29% di miglioramento\n",
      "min_oil_prod_rmse: 5.12% di miglioramento\n",
      "min_oil_prod_mape: 0.43% di miglioramento\n",
      "max_oil_prod_mae: 5.11% di miglioramento\n",
      "max_oil_prod_rmse: 4.70% di miglioramento\n",
      "max_oil_prod_mape: -0.67% di miglioramento\n",
      "avg_oil_prod_mae: 5.58% di miglioramento\n",
      "avg_oil_prod_rmse: 5.45% di miglioramento\n",
      "avg_oil_prod_mape: 3.57% di miglioramento\n",
      "total_water_need_mae: 17.16% di miglioramento\n",
      "total_water_need_rmse: 15.99% di miglioramento\n",
      "total_water_need_mape: 7.67% di miglioramento\n",
      "\n",
      "Set val:\n",
      "olive_prod_mae: 6.51% di miglioramento\n",
      "olive_prod_rmse: 6.04% di miglioramento\n",
      "olive_prod_mape: -3.81% di miglioramento\n",
      "min_oil_prod_mae: 5.33% di miglioramento\n",
      "min_oil_prod_rmse: 5.16% di miglioramento\n",
      "min_oil_prod_mape: -5.12% di miglioramento\n",
      "max_oil_prod_mae: 5.13% di miglioramento\n",
      "max_oil_prod_rmse: 4.70% di miglioramento\n",
      "max_oil_prod_mape: 4.02% di miglioramento\n",
      "avg_oil_prod_mae: 5.62% di miglioramento\n",
      "avg_oil_prod_rmse: 5.48% di miglioramento\n",
      "avg_oil_prod_mape: -0.65% di miglioramento\n",
      "total_water_need_mae: 17.23% di miglioramento\n",
      "total_water_need_rmse: 16.08% di miglioramento\n",
      "total_water_need_mape: 9.72% di miglioramento\n",
      "\n",
      "Set test:\n",
      "olive_prod_mae: 6.52% di miglioramento\n",
      "olive_prod_rmse: 6.09% di miglioramento\n",
      "olive_prod_mape: 1.21% di miglioramento\n",
      "min_oil_prod_mae: 5.32% di miglioramento\n",
      "min_oil_prod_rmse: 5.22% di miglioramento\n",
      "min_oil_prod_mape: -0.80% di miglioramento\n",
      "max_oil_prod_mae: 5.22% di miglioramento\n",
      "max_oil_prod_rmse: 4.83% di miglioramento\n",
      "max_oil_prod_mape: -0.17% di miglioramento\n",
      "avg_oil_prod_mae: 5.64% di miglioramento\n",
      "avg_oil_prod_rmse: 5.59% di miglioramento\n",
      "avg_oil_prod_mape: 20.98% di miglioramento\n",
      "total_water_need_mae: 17.22% di miglioramento\n",
      "total_water_need_rmse: 16.03% di miglioramento\n",
      "total_water_need_mape: 19.57% di miglioramento\n"
     ]
    }
   ],
   "source": [
    "model_path = f'{execute_name}_final_model.keras'\n",
    "\n",
    "retrained_model, retrain_history, final_metrics = start_retraining(\n",
    "    model_path=model_path,\n",
    "    train_data=train_data,\n",
    "    train_targets=train_targets,\n",
    "    val_data=val_data,\n",
    "    val_targets=val_targets,\n",
    "    test_data=test_data,\n",
    "    test_targets=test_targets,\n",
    "    epochs=50,\n",
    "    batch_size=128\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a95606bc-c4bc-418a-acdb-2e24c30dfa81",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24500/24500 [==============================] - 137s 6ms/step\n",
      "\n",
      "Errori per target:\n",
      "--------------------------------------------------\n",
      "olive_prod:\n",
      "MAE assoluto: 1482.22\n",
      "Errore percentuale medio: 5.77%\n",
      "Precisione: 94.23%\n",
      "--------------------------------------------------\n",
      "min_oil_prod:\n",
      "MAE assoluto: 302.12\n",
      "Errore percentuale medio: 5.68%\n",
      "Precisione: 94.32%\n",
      "--------------------------------------------------\n",
      "max_oil_prod:\n",
      "MAE assoluto: 367.45\n",
      "Errore percentuale medio: 5.78%\n",
      "Precisione: 94.22%\n",
      "--------------------------------------------------\n",
      "avg_oil_prod:\n",
      "MAE assoluto: 318.15\n",
      "Errore percentuale medio: 5.49%\n",
      "Precisione: 94.51%\n",
      "--------------------------------------------------\n",
      "total_water_need:\n",
      "MAE assoluto: 1469.51\n",
      "Errore percentuale medio: 3.31%\n",
      "Precisione: 96.69%\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "percentage_errors, absolute_errors = calculate_real_error(retrained_model, val_data, val_targets, scaler_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "e6f357cb-56a4-4f19-a4e8-77b73a28329d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from typing import List, Dict, Tuple, Union\n",
    "\n",
    "def analyze_feature_importance(model: tf.keras.Model, \n",
    "                             test_data: dict, \n",
    "                             feature_names: List[str]) -> Dict[str, float]:\n",
    "    \"\"\"\n",
    "    Analizza l'importanza delle feature usando perturbazione.\n",
    "    \n",
    "    Args:\n",
    "        model: Modello TensorFlow addestrato\n",
    "        test_data: Dizionario con chiavi 'temporal' e 'static' contenenti i dati\n",
    "        feature_names: Lista dei nomi delle feature\n",
    "        \n",
    "    Returns:\n",
    "        dict: Dizionario con l'importanza relativa di ogni feature\n",
    "    \"\"\"\n",
    "    # Estrai i dati temporali e statici\n",
    "    temporal_data = test_data['temporal']\n",
    "    static_data = test_data['static']\n",
    "    \n",
    "    # Ottieni la predizione base\n",
    "    base_prediction = model.predict(test_data)\n",
    "    feature_importance = {}\n",
    "    \n",
    "    # Per ogni feature temporale\n",
    "    for i, feature in enumerate(feature_names):\n",
    "        if feature in ['temp_mean', 'precip_sum', 'solar_energy_sum']:\n",
    "            # Crea copia perturbata dei dati\n",
    "            perturbed_data = {\n",
    "                'temporal': temporal_data.copy(),\n",
    "                'static': static_data.copy()\n",
    "            }\n",
    "            \n",
    "            # Trova l'indice della feature temporale\n",
    "            temp_idx = ['temp_mean', 'precip_sum', 'solar_energy_sum'].index(feature)\n",
    "            \n",
    "            # Crea rumore per la feature temporale\n",
    "            feature_values = temporal_data[..., temp_idx]\n",
    "            noise = np.random.normal(0, np.std(feature_values) * 0.1, \n",
    "                                   size=feature_values.shape)\n",
    "            \n",
    "            # Applica il rumore alla feature temporale\n",
    "            perturbed_temporal = perturbed_data['temporal'].copy()\n",
    "            perturbed_temporal[..., temp_idx] = feature_values + noise\n",
    "            perturbed_data['temporal'] = perturbed_temporal\n",
    "            \n",
    "        else:  # Feature statiche\n",
    "            # Crea copia perturbata dei dati\n",
    "            perturbed_data = {\n",
    "                'temporal': temporal_data.copy(),\n",
    "                'static': static_data.copy()\n",
    "            }\n",
    "            \n",
    "            # Trova l'indice della feature statica\n",
    "            static_idx = ['ha'].index(feature)\n",
    "            \n",
    "            # Crea rumore per la feature statica\n",
    "            feature_values = static_data[..., static_idx]\n",
    "            noise = np.random.normal(0, np.std(feature_values) * 0.1, \n",
    "                                   size=feature_values.shape)\n",
    "            \n",
    "            # Applica il rumore alla feature statica\n",
    "            perturbed_static = perturbed_data['static'].copy()\n",
    "            perturbed_static[..., static_idx] = feature_values + noise\n",
    "            perturbed_data['static'] = perturbed_static\n",
    "        \n",
    "        # Calcola nuova predizione\n",
    "        perturbed_prediction = model.predict(perturbed_data)\n",
    "        \n",
    "        # Calcola impatto della perturbazione\n",
    "        impact = np.mean(np.abs(perturbed_prediction - base_prediction))\n",
    "        feature_importance[feature] = float(impact)\n",
    "    \n",
    "    # Normalizza le importanze\n",
    "    total_importance = sum(feature_importance.values())\n",
    "    feature_importance = {k: v/total_importance \n",
    "                         for k, v in feature_importance.items()}\n",
    "    \n",
    "    return feature_importance\n",
    "\n",
    "class ProbabilityFunctions:\n",
    "    @staticmethod\n",
    "    def calculate_statistics(data: Union[np.ndarray, tf.Tensor]) -> Dict[str, float]:\n",
    "        \"\"\"\n",
    "        Calcola statistiche di base usando TensorFlow.\n",
    "        \n",
    "        Args:\n",
    "            data: Tensor o array dei dati\n",
    "            \n",
    "        Returns:\n",
    "            dict: Dizionario con le statistiche\n",
    "        \"\"\"\n",
    "        if not isinstance(data, tf.Tensor):\n",
    "            data = tf.convert_to_tensor(data, dtype=tf.float32)\n",
    "            \n",
    "        mean = tf.reduce_mean(data)\n",
    "        # Calcola varianza manualmente\n",
    "        squared_deviations = tf.square(data - mean)\n",
    "        variance = tf.reduce_mean(squared_deviations)\n",
    "        std = tf.sqrt(variance)\n",
    "        \n",
    "        # Ordina il tensor per il calcolo della mediana\n",
    "        sorted_data = tf.sort(data)\n",
    "        size = tf.size(data)\n",
    "        mid_index = size // 2\n",
    "        median = sorted_data[mid_index]\n",
    "        \n",
    "        return {\n",
    "            'mean': mean.numpy(),\n",
    "            'variance': variance.numpy(),\n",
    "            'std': std.numpy(),\n",
    "            'min': tf.reduce_min(data).numpy(),\n",
    "            'max': tf.reduce_max(data).numpy(),\n",
    "            'median': median.numpy()\n",
    "        }\n",
    "\n",
    "    @staticmethod\n",
    "    def calculate_pmf(data: np.ndarray, bins: int = 50) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n",
    "        \"\"\"\n",
    "        Calcola la Probability Mass Function (PMF) dei dati.\n",
    "        \n",
    "        Args:\n",
    "            data: Array di dati\n",
    "            bins: Numero di bin per l'istogramma\n",
    "            \n",
    "        Returns:\n",
    "            tuple: (bin_centers, pmf, bin_edges)\n",
    "        \"\"\"\n",
    "        # Calcola l'istogramma\n",
    "        hist, bin_edges = np.histogram(data, bins=bins, density=True)\n",
    "        \n",
    "        # Calcola i centri dei bin\n",
    "        bin_centers = (bin_edges[:-1] + bin_edges[1:]) / 2\n",
    "        \n",
    "        # Normalizza per ottenere la PMF\n",
    "        pmf = hist / np.sum(hist)\n",
    "        \n",
    "        return bin_centers, pmf, bin_edges\n",
    "\n",
    "    @staticmethod\n",
    "    def calculate_cmf(pmf: np.ndarray) -> np.ndarray:\n",
    "        \"\"\"\n",
    "        Calcola la Cumulative Mass Function (CMF) dalla PMF.\n",
    "        \n",
    "        Args:\n",
    "            pmf: Probability Mass Function\n",
    "            \n",
    "        Returns:\n",
    "            array: Cumulative Mass Function\n",
    "        \"\"\"\n",
    "        return np.cumsum(pmf)\n",
    "\n",
    "    def plot_distributions(self, data: np.ndarray, \n",
    "                         bins: int = 50, \n",
    "                         title: str = \"Distribuzione\") -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n",
    "        \"\"\"\n",
    "        Calcola e visualizza PMF e CMF delle distribuzioni.\n",
    "        \n",
    "        Args:\n",
    "            data: Array di dati da analizzare\n",
    "            bins: Numero di bin per l'istogramma\n",
    "            title: Titolo del grafico\n",
    "            \n",
    "        Returns:\n",
    "            tuple: (bin_centers, pmf, cmf)\n",
    "        \"\"\"\n",
    "        # Calcola PMF e CMF\n",
    "        bin_centers, pmf, bin_edges = self.calculate_pmf(data, bins)\n",
    "        cmf = self.calculate_cmf(pmf)\n",
    "        \n",
    "        # Crea il plot\n",
    "        fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10, 8))\n",
    "        \n",
    "        # Plot PMF\n",
    "        width = np.diff(bin_edges)\n",
    "        ax1.bar(bin_centers, pmf, width=width, alpha=0.5, label='PMF')\n",
    "        ax1.set_title('Probability Mass Function')\n",
    "        ax1.set_ylabel('Probability')\n",
    "        ax1.grid(True, alpha=0.3)\n",
    "        ax1.legend()\n",
    "        \n",
    "        # Plot CMF\n",
    "        ax2.plot(bin_centers, cmf, 'r-', label='CMF')\n",
    "        ax2.set_title('Cumulative Mass Function')\n",
    "        ax2.set_xlabel('Value')\n",
    "        ax2.set_ylabel('Cumulative Probability')\n",
    "        ax2.grid(True, alpha=0.3)\n",
    "        ax2.legend()\n",
    "        \n",
    "        # Imposta il titolo generale\n",
    "        fig.suptitle(title, y=1.02)\n",
    "        plt.tight_layout()\n",
    "        plt.show()\n",
    "        \n",
    "        return bin_centers, pmf, cmf\n",
    "\n",
    "def analyze_model_predictions(model: tf.keras.Model, \n",
    "                            test_data: np.ndarray,\n",
    "                            test_targets: np.ndarray,\n",
    "                            scaler_y) -> None:\n",
    "    \"\"\"\n",
    "    Analizza le distribuzioni di probabilità delle predizioni del modello.\n",
    "    \n",
    "    Args:\n",
    "        model: Modello TensorFlow addestrato\n",
    "        test_data: Dati di test\n",
    "        test_targets: Target di test\n",
    "        scaler_y: Scaler usato per denormalizzare i target\n",
    "    \"\"\"\n",
    "    # Ottieni le predizioni\n",
    "    predictions = model.predict(test_data)\n",
    "    \n",
    "    # Denormalizza predizioni e target\n",
    "    predictions_real = scaler_y.inverse_transform(predictions)\n",
    "    targets_real = scaler_y.inverse_transform(test_targets)\n",
    "    \n",
    "    # Inizializza la classe per l'analisi delle probabilità\n",
    "    prob = ProbabilityFunctions()\n",
    "    \n",
    "    # Analizza ogni target\n",
    "    target_names = ['olive_prod', 'min_oil_prod', 'max_oil_prod', \n",
    "                   'avg_oil_prod', 'total_water_need']\n",
    "    \n",
    "    for i, target in enumerate(target_names):\n",
    "        print(f\"\\nAnalisi per {target}\")\n",
    "        print(\"-\" * 50)\n",
    "        \n",
    "        # Calcola errori\n",
    "        errors = predictions_real[:, i] - targets_real[:, i]\n",
    "        \n",
    "        # Calcola statistiche degli errori\n",
    "        error_stats = prob.calculate_statistics(errors)\n",
    "        print(\"\\nStatistiche degli Errori:\")\n",
    "        for key, value in error_stats.items():\n",
    "            print(f\"{key}: {value:.3f}\")\n",
    "        \n",
    "        # Visualizza le distribuzioni degli errori\n",
    "        bin_centers, pmf, cmf = prob.plot_distributions(\n",
    "            errors, \n",
    "            bins=50,\n",
    "            title=f\"Distribuzione degli Errori - {target}\"\n",
    "        )\n",
    "        \n",
    "        # Calcola intervalli di confidenza\n",
    "        confidence_levels = [0.68, 0.95, 0.99]  # 1σ, 2σ, 3σ\n",
    "        for level in confidence_levels:\n",
    "            lower_idx = np.searchsorted(cmf, (1 - level) / 2)\n",
    "            upper_idx = np.searchsorted(cmf, (1 + level) / 2)\n",
    "            \n",
    "            print(f\"\\nIntervallo di Confidenza {level*100}%:\")\n",
    "            print(f\"Range: [{bin_centers[lower_idx]:.2f}, {bin_centers[upper_idx]:.2f}]\")\n",
    "\n",
    "def run_comprehensive_analysis(retrained_model, test_data, test_targets, scaler_y):\n",
    "    \"\"\"\n",
    "    Esegue un'analisi completa del modello includendo errori,\n",
    "    importanza delle feature e distribuzioni.\n",
    "    \"\"\"\n",
    "    print(\"=== ANALISI COMPLETA DEL MODELLO ===\")\n",
    "    \n",
    "    # 1. Analisi degli errori\n",
    "    print(\"\\n1. ANALISI DEGLI ERRORI\")\n",
    "    print(\"-\" * 50)\n",
    "    analyze_model_predictions(retrained_model, test_data, test_targets, scaler_y)\n",
    "    \n",
    "    # 2. Analisi dell'importanza delle feature\n",
    "    print(\"\\n2. IMPORTANZA DELLE FEATURE\")\n",
    "    print(\"-\" * 50)\n",
    "    \n",
    "    # Definisci i nomi delle feature\n",
    "    temporal_features = ['temp_mean', 'precip_sum', 'solar_energy_sum']\n",
    "    static_features = ['ha']\n",
    "    \n",
    "    all_features = temporal_features + static_features\n",
    "    importance = analyze_feature_importance(retrained_model, test_data, all_features)\n",
    "    \n",
    "    print(\"\\nImportanza relativa delle feature:\")\n",
    "    for feature, imp in sorted(importance.items(), key=lambda x: x[1], reverse=True):\n",
    "        print(f\"{feature}: {imp:.4f}\")\n",
    "        \n",
    "    # 3. Analisi distribuzionale\n",
    "    print(\"\\n3. ANALISI DISTRIBUZIONALE\")\n",
    "    print(\"-\" * 50)\n",
    "    \n",
    "    prob = ProbabilityFunctions()\n",
    "    predictions = retrained_model.predict(test_data)\n",
    "    predictions_real = scaler_y.inverse_transform(predictions)\n",
    "    targets_real = scaler_y.inverse_transform(test_targets)\n",
    "    \n",
    "    target_names = ['olive_prod', 'min_oil_prod', 'max_oil_prod', \n",
    "                    'avg_oil_prod', 'total_water_need']\n",
    "    \n",
    "    for i, target in enumerate(target_names):\n",
    "        print(f\"\\nAnalisi distribuzionale per {target}\")\n",
    "        \n",
    "        # Statistiche\n",
    "        stats_pred = prob.calculate_statistics(predictions_real[:, i])\n",
    "        stats_true = prob.calculate_statistics(targets_real[:, i])\n",
    "        \n",
    "        print(\"\\nStatistiche Predizioni:\")\n",
    "        for key, value in stats_pred.items():\n",
    "            print(f\"{key}: {value:.3f}\")\n",
    "            \n",
    "        print(\"\\nStatistiche Target Reali:\")\n",
    "        for key, value in stats_true.items():\n",
    "            print(f\"{key}: {value:.3f}\")\n",
    "        \n",
    "        # Visualizza distribuzioni\n",
    "        prob.plot_distributions(predictions_real[:, i], bins=50,\n",
    "                              title=f\"Distribuzione Predizioni - {target}\")\n",
    "        prob.plot_distributions(targets_real[:, i], bins=50,\n",
    "                              title=f\"Distribuzione Target Reali - {target}\")\n",
    "\n",
    "def analyze_model_predictions(model, test_data, test_targets, scaler_y):\n",
    "    \"\"\"\n",
    "    Analizza le distribuzioni di probabilità delle predizioni del modello.\n",
    "    \n",
    "    Args:\n",
    "        model: Modello TensorFlow addestrato\n",
    "        test_data: Dati di test\n",
    "        test_targets: Target di test\n",
    "        scaler_y: Scaler usato per denormalizzare i target\n",
    "    \"\"\"\n",
    "    # Ottieni le predizioni\n",
    "    predictions = model.predict(test_data)\n",
    "    \n",
    "    # Denormalizza predizioni e target\n",
    "    predictions_real = scaler_y.inverse_transform(predictions)\n",
    "    targets_real = scaler_y.inverse_transform(test_targets)\n",
    "    \n",
    "    # Inizializza la classe per l'analisi delle probabilità\n",
    "    prob = ProbabilityFunctions()\n",
    "    \n",
    "    # Analizza ogni target\n",
    "    target_names = ['olive_prod', 'min_oil_prod', 'max_oil_prod', \n",
    "                   'avg_oil_prod', 'total_water_need']\n",
    "    \n",
    "    for i, target in enumerate(target_names):\n",
    "        print(f\"\\nAnalisi per {target}\")\n",
    "        print(\"-\" * 50)\n",
    "        \n",
    "        # Calcola errori\n",
    "        errors = predictions_real[:, i] - targets_real[:, i]\n",
    "        \n",
    "        # Calcola statistiche degli errori\n",
    "        error_stats = prob.calculate_statistics(errors)\n",
    "        print(\"\\nStatistiche degli Errori:\")\n",
    "        for key, value in error_stats.items():\n",
    "            print(f\"{key}: {value:.3f}\")\n",
    "        \n",
    "        # Visualizza le distribuzioni degli errori\n",
    "        bin_centers, pmf, cmf = prob.plot_distributions(\n",
    "            errors, \n",
    "            bins=50,\n",
    "            title=f\"Distribuzione degli Errori - {target}\"\n",
    "        )\n",
    "        \n",
    "        # Calcola intervalli di confidenza\n",
    "        confidence_levels = [0.80,0.85, 0.90, 0.95, 0.99]  # 1σ, 2σ, 3σ\n",
    "        for level in confidence_levels:\n",
    "            lower_idx = np.searchsorted(cmf, (1 - level) / 2)\n",
    "            upper_idx = np.searchsorted(cmf, (1 + level) / 2)\n",
    "            \n",
    "            print(f\"\\nIntervallo di Confidenza {level*100}%:\")\n",
    "            print(f\"Range: [{bin_centers[lower_idx]:.2f}, {bin_centers[upper_idx]:.2f}]\")\n",
    "\n",
    "class ProbabilityFunctions:\n",
    "    @staticmethod\n",
    "    def calculate_statistics(data):\n",
    "        \"\"\"\n",
    "        Calcola statistiche di base usando TensorFlow.\n",
    "        \n",
    "        Args:\n",
    "            data: Tensor dei dati\n",
    "            \n",
    "        Returns:\n",
    "            dict: Dizionario con le statistiche\n",
    "        \"\"\"\n",
    "        if not isinstance(data, tf.Tensor):\n",
    "            data = tf.convert_to_tensor(data, dtype=tf.float32)\n",
    "            \n",
    "        mean = tf.reduce_mean(data)\n",
    "        # Calculate variance manually\n",
    "        squared_deviations = tf.square(data - mean)\n",
    "        variance = tf.reduce_mean(squared_deviations)\n",
    "        std = tf.sqrt(variance)\n",
    "        \n",
    "        # Sort the tensor for median calculation\n",
    "        sorted_data = tf.sort(data)\n",
    "        size = tf.size(data)\n",
    "        mid_index = size // 2\n",
    "        median = sorted_data[mid_index]\n",
    "        \n",
    "        return {\n",
    "            'mean': mean.numpy(),\n",
    "            'variance': variance.numpy(),\n",
    "            'std': std.numpy(),\n",
    "            'min': tf.reduce_min(data).numpy(),\n",
    "            'max': tf.reduce_max(data).numpy(),\n",
    "            'median': median.numpy()\n",
    "        }\n",
    "\n",
    "    @staticmethod\n",
    "    def calculate_pmf(data, bins=50):\n",
    "        \"\"\"\n",
    "        Calcola la Probability Mass Function (PMF) dei dati.\n",
    "        \n",
    "        Args:\n",
    "            data: Array di dati\n",
    "            bins: Numero di bin per l'istogramma\n",
    "            \n",
    "        Returns:\n",
    "            tuple: (bin_centers, pmf, bin_edges)\n",
    "        \"\"\"\n",
    "        # Calcola l'istogramma\n",
    "        hist, bin_edges = np.histogram(data, bins=bins, density=True)\n",
    "        \n",
    "        # Calcola i centri dei bin\n",
    "        bin_centers = (bin_edges[:-1] + bin_edges[1:]) / 2\n",
    "        \n",
    "        # Normalizza per ottenere la PMF\n",
    "        pmf = hist / np.sum(hist)\n",
    "        \n",
    "        return bin_centers, pmf, bin_edges\n",
    "\n",
    "    @staticmethod\n",
    "    def calculate_cmf(pmf):\n",
    "        \"\"\"\n",
    "        Calcola la Cumulative Mass Function (CMF) dalla PMF.\n",
    "        \n",
    "        Args:\n",
    "            pmf: Probability Mass Function\n",
    "            \n",
    "        Returns:\n",
    "            array: Cumulative Mass Function\n",
    "        \"\"\"\n",
    "        return np.cumsum(pmf)\n",
    "\n",
    "    def plot_distributions(self, data, bins=50, title=\"Distribuzione\"):\n",
    "        \"\"\"\n",
    "        Calcola e visualizza PMF e CMF delle distribuzioni.\n",
    "        \n",
    "        Args:\n",
    "            data: Array di dati da analizzare\n",
    "            bins: Numero di bin per l'istogramma\n",
    "            title: Titolo del grafico\n",
    "            \n",
    "        Returns:\n",
    "            tuple: (bin_centers, pmf, cmf)\n",
    "        \"\"\"\n",
    "        # Calcola PMF e CMF\n",
    "        bin_centers, pmf, bin_edges = self.calculate_pmf(data, bins)\n",
    "        cmf = self.calculate_cmf(pmf)\n",
    "        \n",
    "        # Crea il plot\n",
    "        fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10, 8))\n",
    "        \n",
    "        # Plot PMF\n",
    "        width = np.diff(bin_edges)\n",
    "        ax1.bar(bin_centers, pmf, width=width, alpha=0.5, label='PMF')\n",
    "        ax1.set_title('Probability Mass Function')\n",
    "        ax1.set_ylabel('Probability')\n",
    "        ax1.grid(True, alpha=0.3)\n",
    "        ax1.legend()\n",
    "        \n",
    "        # Plot CMF\n",
    "        ax2.plot(bin_centers, cmf, 'r-', label='CMF')\n",
    "        ax2.set_title('Cumulative Mass Function')\n",
    "        ax2.set_xlabel('Value')\n",
    "        ax2.set_ylabel('Cumulative Probability')\n",
    "        ax2.grid(True, alpha=0.3)\n",
    "        ax2.legend()\n",
    "        \n",
    "        # Set overall title\n",
    "        fig.suptitle(title, y=1.02)\n",
    "        plt.tight_layout()\n",
    "        plt.show()\n",
    "        \n",
    "        return bin_centers, pmf, cmf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b98f2a45-ba6f-4519-acaa-0138b4ee7812",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== ANALISI COMPLETA DEL MODELLO ===\n",
      "\n",
      "1. ANALISI DEGLI ERRORI\n",
      "--------------------------------------------------\n",
      "18375/18375 [==============================] - 78s 4ms/step\n",
      "\n",
      "Analisi per olive_prod\n",
      "--------------------------------------------------\n",
      "\n",
      "Statistiche degli Errori:\n",
      "mean: -71.944\n",
      "variance: 4009595.000\n",
      "std: 2002.397\n",
      "min: -18637.889\n",
      "max: 12871.579\n",
      "median: 48.672\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 67064 characters]",
      "text/plain": [
       "<Figure size 1000x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Intervallo di Confidenza 68.0%:\n",
      "Range: [-1937.87, 1843.27]\n",
      "\n",
      "Intervallo di Confidenza 95.0%:\n",
      "Range: [-4458.63, 3733.83]\n",
      "\n",
      "Intervallo di Confidenza 99.0%:\n",
      "Range: [-6979.39, 5624.40]\n",
      "\n",
      "Analisi per min_oil_prod\n",
      "--------------------------------------------------\n",
      "\n",
      "Statistiche degli Errori:\n",
      "mean: -32.785\n",
      "variance: 179026.016\n",
      "std: 423.115\n",
      "min: -4439.664\n",
      "max: 3453.714\n",
      "median: -12.655\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 70264 characters]",
      "text/plain": [
       "<Figure size 1000x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Intervallo di Confidenza 68.0%:\n",
      "Range: [-414.04, 375.30]\n",
      "\n",
      "Intervallo di Confidenza 95.0%:\n",
      "Range: [-1045.51, 848.90]\n",
      "\n",
      "Intervallo di Confidenza 99.0%:\n",
      "Range: [-1519.11, 1164.63]\n",
      "\n",
      "Analisi per max_oil_prod\n",
      "--------------------------------------------------\n",
      "\n",
      "Statistiche degli Errori:\n",
      "mean: -34.971\n",
      "variance: 261409.344\n",
      "std: 511.282\n",
      "min: -5732.709\n",
      "max: 4274.197\n",
      "median: -11.391\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 68672 characters]",
      "text/plain": [
       "<Figure size 1000x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Intervallo di Confidenza 68.0%:\n",
      "Range: [-429.05, 371.50]\n",
      "\n",
      "Intervallo di Confidenza 95.0%:\n",
      "Range: [-1229.60, 971.92]\n",
      "\n",
      "Intervallo di Confidenza 99.0%:\n",
      "Range: [-1830.02, 1572.33]\n",
      "\n",
      "Analisi per avg_oil_prod\n",
      "--------------------------------------------------\n",
      "\n",
      "Statistiche degli Errori:\n",
      "mean: -33.549\n",
      "variance: 192810.531\n",
      "std: 439.102\n",
      "min: -4876.229\n",
      "max: 3813.953\n",
      "median: -13.710\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 67168 characters]",
      "text/plain": [
       "<Figure size 1000x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Intervallo di Confidenza 68.0%:\n",
      "Range: [-444.24, 250.98]\n",
      "\n",
      "Intervallo di Confidenza 95.0%:\n",
      "Range: [-965.65, 772.39]\n",
      "\n",
      "Intervallo di Confidenza 99.0%:\n",
      "Range: [-1487.06, 1293.80]\n",
      "\n",
      "Analisi per total_water_need\n",
      "--------------------------------------------------\n",
      "\n",
      "Statistiche degli Errori:\n",
      "mean: -216.226\n",
      "variance: 3987062.750\n",
      "std: 1996.763\n",
      "min: -22812.350\n",
      "max: 13374.520\n",
      "median: -119.823\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 69492 characters]",
      "text/plain": [
       "<Figure size 1000x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Intervallo di Confidenza 68.0%:\n",
      "Range: [-2185.83, 1432.85]\n",
      "\n",
      "Intervallo di Confidenza 95.0%:\n",
      "Range: [-4357.05, 3604.06]\n",
      "\n",
      "Intervallo di Confidenza 99.0%:\n",
      "Range: [-7252.00, 5051.54]\n",
      "\n",
      "2. IMPORTANZA DELLE FEATURE\n",
      "--------------------------------------------------\n",
      "18375/18375 [==============================] - 79s 4ms/step\n",
      "18375/18375 [==============================] - 80s 4ms/step\n",
      "18375/18375 [==============================] - 99s 5ms/step\n",
      "18375/18375 [==============================] - 96s 5ms/step\n",
      "13976/18375 [=====================>........] - ETA: 21s"
     ]
    }
   ],
   "source": [
    "run_comprehensive_analysis(retrained_model, test_data, test_targets, scaler_y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
