{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8adcbe0819b88578",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease\n",
      "Hit:2 http://security.ubuntu.com/ubuntu jammy-security InRelease\n",
      "Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease\n",
      "Hit:4 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease\n",
      "Hit:5 http://archive.ubuntu.com/ubuntu jammy-backports InRelease\n",
      "Reading package lists... Done\n",
      "Reading package lists... Done\n",
      "Building dependency tree... Done\n",
      "Reading state information... Done\n",
      "graphviz is already the newest version (2.42.2-6ubuntu0.1).\n",
      "0 upgraded, 0 newly installed, 0 to remove and 121 not upgraded.\n",
      "Requirement already satisfied: tensorflow in /usr/local/lib/python3.11/dist-packages (2.13.0)\n",
      "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.0.0)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.6.3)\n",
      "Requirement already satisfied: flatbuffers>=23.1.21 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (23.5.26)\n",
      "Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.4.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.2.0)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.58.0)\n",
      "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (3.9.0)\n",
      "Requirement already satisfied: keras<2.14,>=2.13.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.13.1)\n",
      "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (16.0.6)\n",
      "Requirement already satisfied: numpy<=1.24.3,>=1.22 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.24.3)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (3.3.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from tensorflow) (23.1)\n",
      "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (4.24.3)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.11/dist-packages (from tensorflow) (68.2.2)\n",
      "Requirement already satisfied: six>=1.12.0 in /usr/lib/python3/dist-packages (from tensorflow) (1.16.0)\n",
      "Requirement already satisfied: tensorboard<2.14,>=2.13 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.13.0)\n",
      "Requirement already satisfied: tensorflow-estimator<2.14,>=2.13.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.13.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.3.0)\n",
      "Requirement already satisfied: typing-extensions<4.6.0,>=3.6.6 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (4.5.0)\n",
      "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.14.1)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.37.1)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.11/dist-packages (from astunparse>=1.6.0->tensorflow) (0.41.2)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow) (2.23.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow) (1.0.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow) (3.4.4)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow) (2.31.0)\n",
      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow) (0.7.1)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow) (2.3.7)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow) (5.3.1)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow) (0.3.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow) (4.9)\n",
      "Requirement already satisfied: urllib3>=2.0.5 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow) (2.0.5)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard<2.14,>=2.13->tensorflow) (1.3.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.14,>=2.13->tensorflow) (3.2.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.14,>=2.13->tensorflow) (3.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.14,>=2.13->tensorflow) (2023.7.22)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.11/dist-packages (from werkzeug>=1.0.1->tensorboard<2.14,>=2.13->tensorflow) (2.1.3)\n",
      "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.11/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow) (0.5.0)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/lib/python3/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard<2.14,>=2.13->tensorflow) (3.2.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (1.24.3)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (2.2.3)\n",
      "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (1.24.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: keras in /usr/local/lib/python3.11/dist-packages (2.13.1)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.11/dist-packages (1.5.2)\n",
      "Requirement already satisfied: numpy>=1.19.5 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.24.3)\n",
      "Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.14.1)\n",
      "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.4.2)\n",
      "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (3.5.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: matplotlib in /usr/local/lib/python3.11/dist-packages (3.8.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (4.42.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.4.5)\n",
      "Requirement already satisfied: numpy<2,>=1.21 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.24.3)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (23.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (10.0.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (3.2.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (2.8.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.11/dist-packages (1.4.2)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: pyarrow in /usr/local/lib/python3.11/dist-packages (18.0.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: fastparquet in /usr/local/lib/python3.11/dist-packages (2024.11.0)\n",
      "Requirement already satisfied: pandas>=1.5.0 in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2.2.3)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (from fastparquet) (1.24.3)\n",
      "Requirement already satisfied: cramjam>=2.3 in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2.9.0)\n",
      "Requirement already satisfied: fsspec in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2024.10.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from fastparquet) (23.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2024.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas>=1.5.0->fastparquet) (1.16.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.11/dist-packages (1.14.1)\n",
      "Requirement already satisfied: numpy<2.3,>=1.23.5 in /usr/local/lib/python3.11/dist-packages (from scipy) (1.24.3)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: seaborn in /usr/local/lib/python3.11/dist-packages (0.13.2)\n",
      "Requirement already satisfied: numpy!=1.24.0,>=1.20 in /usr/local/lib/python3.11/dist-packages (from seaborn) (1.24.3)\n",
      "Requirement already satisfied: pandas>=1.2 in /usr/local/lib/python3.11/dist-packages (from seaborn) (2.2.3)\n",
      "Requirement already satisfied: matplotlib!=3.6.1,>=3.4 in /usr/local/lib/python3.11/dist-packages (from seaborn) (3.8.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (4.42.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.4.5)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (23.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (10.0.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (3.2.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.2->seaborn) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.2->seaborn) (2024.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib!=3.6.1,>=3.4->seaborn) (1.16.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (4.67.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: pydot in /usr/local/lib/python3.11/dist-packages (3.0.2)\n",
      "Requirement already satisfied: pyparsing>=3.0.9 in /usr/local/lib/python3.11/dist-packages (from pydot) (3.2.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: tensorflow-io in /usr/local/lib/python3.11/dist-packages (0.37.1)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem==0.37.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow-io) (0.37.1)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.11/dist-packages (0.23.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from tensorflow-addons) (23.1)\n",
      "Requirement already satisfied: typeguard<3.0.0,>=2.7 in /usr/local/lib/python3.11/dist-packages (from tensorflow-addons) (2.13.3)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n"
     ]
    }
   ],
   "source": [
    "from opt_einsum.paths import branch_1\n",
    "!apt-get update\n",
    "!apt-get install graphviz -y\n",
    "\n",
    "!pip install tensorflow\n",
    "!pip install numpy\n",
    "!pip install pandas\n",
    "\n",
    "!pip install keras\n",
    "!pip install scikit-learn\n",
    "!pip install matplotlib\n",
    "!pip install joblib\n",
    "!pip install pyarrow\n",
    "!pip install fastparquet\n",
    "!pip install scipy\n",
    "!pip install seaborn\n",
    "!pip install tqdm\n",
    "!pip install pydot\n",
    "!pip install tensorflow-io\n",
    "!pip install tensorflow-addons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7a813e3cbca057b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-20 18:26:10.290667: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "/usr/local/lib/python3.11/dist-packages/tensorflow_addons/utils/tfa_eol_msg.py:23: UserWarning: \n",
      "\n",
      "TensorFlow Addons (TFA) has ended development and introduction of new features.\n",
      "TFA has entered a minimal maintenance and release mode until a planned end of life in May 2024.\n",
      "Please modify downstream libraries to take dependencies from other repositories in our TensorFlow community (e.g. Keras, Keras-CV, and Keras-NLP). \n",
      "\n",
      "For more information see: https://github.com/tensorflow/addons/issues/2807 \n",
      "\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Dense, LSTM, MultiHeadAttention, Dropout, BatchNormalization, LayerNormalization, Input, Activation, Lambda, Bidirectional, Add, MaxPooling1D, SpatialDropout1D, GlobalAveragePooling1D, \\\n",
    "    GlobalMaxPooling1D, Concatenate\n",
    "from tensorflow.keras import regularizers\n",
    "from tensorflow.keras.models import Model\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\n",
    "from tensorflow.keras.optimizers import AdamW\n",
    "import json\n",
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.keras.utils import plot_model\n",
    "import tensorflow_addons as tfa\n",
    "import os\n",
    "import joblib\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score, confusion_matrix\n",
    "\n",
    "folder_name = datetime.now().strftime(\"%Y-%m-%d_%H-%M\")\n",
    "\n",
    "random_state_value = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b3f525e19f78a1da",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_season(date):\n",
    "    month = date.month\n",
    "    day = date.day\n",
    "    if (month == 12 and day >= 21) or (month <= 3 and day < 20):\n",
    "        return 'Winter'\n",
    "    elif (month == 3 and day >= 20) or (month <= 6 and day < 21):\n",
    "        return 'Spring'\n",
    "    elif (month == 6 and day >= 21) or (month <= 9 and day < 23):\n",
    "        return 'Summer'\n",
    "    elif (month == 9 and day >= 23) or (month <= 12 and day < 21):\n",
    "        return 'Autumn'\n",
    "    else:\n",
    "        return 'Unknown'\n",
    "\n",
    "\n",
    "def get_time_period(hour):\n",
    "    if 5 <= hour < 12:\n",
    "        return 'Morning'\n",
    "    elif 12 <= hour < 17:\n",
    "        return 'Afternoon'\n",
    "    elif 17 <= hour < 21:\n",
    "        return 'Evening'\n",
    "    else:\n",
    "        return 'Night'\n",
    "\n",
    "\n",
    "def add_time_features(df):\n",
    "    \"\"\"\n",
    "    Add time-based features to the DataFrame.\n",
    "    Works with both 'datetime' as column or index.\n",
    "    \"\"\"\n",
    "    # Se datetime è l'indice, lo usiamo direttamente\n",
    "    if isinstance(df.index, pd.DatetimeIndex):\n",
    "        datetime_col = df.index\n",
    "    else:\n",
    "        # Se datetime è una colonna, la convertiamo\n",
    "        if 'datetime' in df.columns:\n",
    "            datetime_col = pd.to_datetime(df['datetime'])\n",
    "        else:\n",
    "            raise ValueError(\"No datetime column or index found in DataFrame\")\n",
    "\n",
    "    # Creazione delle feature temporali\n",
    "    df['timestamp'] = datetime_col.astype(np.int64) // 10 ** 9\n",
    "    df['year'] = datetime_col.year\n",
    "    df['month'] = datetime_col.month\n",
    "    df['day'] = datetime_col.day\n",
    "    df['hour'] = datetime_col.hour\n",
    "    df['minute'] = datetime_col.minute\n",
    "    df['hour_sin'] = np.sin(datetime_col.hour * (2 * np.pi / 24))\n",
    "    df['hour_cos'] = np.cos(datetime_col.hour * (2 * np.pi / 24))\n",
    "    df['day_of_week'] = datetime_col.dayofweek\n",
    "    df['day_of_year'] = datetime_col.dayofyear\n",
    "    df['week_of_year'] = datetime_col.isocalendar().week.astype(int)\n",
    "    df['quarter'] = datetime_col.quarter\n",
    "    df['is_month_end'] = datetime_col.is_month_end.astype(int)\n",
    "    df['is_quarter_end'] = datetime_col.is_quarter_end.astype(int)\n",
    "    df['is_year_end'] = datetime_col.is_year_end.astype(int)\n",
    "    df['month_sin'] = np.sin(datetime_col.month * (2 * np.pi / 12))\n",
    "    df['month_cos'] = np.cos(datetime_col.month * (2 * np.pi / 12))\n",
    "    df['day_of_year_sin'] = np.sin(datetime_col.dayofyear * (2 * np.pi / 365.25))\n",
    "    df['day_of_year_cos'] = np.cos(datetime_col.dayofyear * (2 * np.pi / 365.25))\n",
    "    df['season'] = datetime_col.map(get_season)\n",
    "    df['time_period'] = datetime_col.hour.map(get_time_period)\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_solar_features(df):\n",
    "    # Solar angle calculation\n",
    "    df['solar_angle'] = np.sin(df['day_of_year'] * (2 * np.pi / 365.25)) * np.sin(df['hour'] * (2 * np.pi / 24))\n",
    "\n",
    "    # Interactions between relevant features\n",
    "    df['cloud_temp_interaction'] = df['cloudcover'] * df['temp']\n",
    "    df['visibility_cloud_interaction'] = df['visibility'] * (100 - df['cloudcover'])\n",
    "\n",
    "    # Derived features\n",
    "    df['clear_sky_index'] = (100 - df['cloudcover']) / 100\n",
    "    df['temp_gradient'] = df['temp'] - df['tempmin']\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_solar_specific_features(df):\n",
    "    \"\"\"\n",
    "    Aggiunge feature specifiche per la predizione della radiazione solare\n",
    "    combinando caratteristiche astronomiche e meteorologiche\n",
    "    \"\"\"\n",
    "    # Caratteristiche astronomiche\n",
    "    df['day_length'] = 12 + 3 * np.sin(2 * np.pi * (df['day_of_year'] - 81) / 365.25)\n",
    "    df['solar_noon'] = 12 - df['hour']\n",
    "    df['solar_elevation'] = np.sin(2 * np.pi * df['day_of_year'] / 365.25) * np.cos(2 * np.pi * df['solar_noon'] / 24)\n",
    "\n",
    "    # Angolo solare teorico\n",
    "    df['solar_angle'] = np.sin(df['hour_sin']) * np.sin(df['day_of_year_sin'])\n",
    "\n",
    "    # Interazioni con condizioni atmosferiche\n",
    "    df['cloud_elevation'] = df['cloudcover'] * df['solar_elevation']\n",
    "    df['visibility_elevation'] = df['visibility'] * df['solar_elevation']\n",
    "    df['uv_cloud_interaction'] = df['uvindex'] * (100 - df['cloudcover']) / 100\n",
    "\n",
    "    # Indici di chiarezza e trasmissione\n",
    "    df['clearness_index'] = (100 - df['cloudcover']) * df['visibility'] / 10000\n",
    "    df['atmospheric_attenuation'] = (df['pressure'] / 1013.25) * (1 - (df['humidity'] / 100) * 0.6)\n",
    "\n",
    "    # Radiazione teorica e attenuazione\n",
    "    df['theoretical_radiation'] = df['solar_angle'].clip(0, 1) * 1000\n",
    "    df['expected_radiation'] = df['theoretical_radiation'] * df['clearness_index']\n",
    "\n",
    "    # Rolling features\n",
    "    df['cloud_rolling_12h'] = df['cloudcover'].rolling(window=12).mean()\n",
    "    df['temp_rolling_12h'] = df['temp'].rolling(window=12).mean()\n",
    "    df['uv_rolling_12h'] = df['uvindex'].rolling(window=12).mean()\n",
    "\n",
    "    # Interazioni temperatura-radiazione\n",
    "    df['temp_radiation_potential'] = df['temp'] * df['solar_elevation']\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_radiation_energy_features(df):\n",
    "    \"\"\"Adds specific features based on solarenergy and uvindex\"\"\"\n",
    "\n",
    "    # Assicuriamoci che l'indice sia di tipo datetime\n",
    "    if not isinstance(df.index, pd.DatetimeIndex):\n",
    "        df.index = pd.to_datetime(df['datetime'])\n",
    "\n",
    "    # Solar energy to UV ratio (independent from solarradiation)\n",
    "    df['energy_uv_ratio'] = df['solarenergy'] / (df['uvindex'] + 1e-6)\n",
    "\n",
    "    # Time aggregations\n",
    "    # Moving averages\n",
    "    windows = [3, 6, 12, 24]  # hours\n",
    "    for w in windows:\n",
    "        df[f'energy_rolling_mean_{w}h'] = df['solarenergy'].rolling(window=w).mean()\n",
    "        df[f'uv_rolling_mean_{w}h'] = df['uvindex'].rolling(window=w).mean()\n",
    "\n",
    "    # Daily aggregations utilizzando datetime\n",
    "    df['energy_daily_sum'] = df.groupby(df.index.date)['solarenergy'].transform('sum')\n",
    "    df['uv_daily_max'] = df.groupby(df.index.date)['uvindex'].transform('max')\n",
    "\n",
    "    # Changes\n",
    "    df['energy_change'] = df['solarenergy'].diff()\n",
    "    df['uv_change'] = df['uvindex'].diff()\n",
    "\n",
    "    # Lag features\n",
    "    lags = [1, 2, 3, 6, 12, 24]  # hours\n",
    "    for lag in lags:\n",
    "        df[f'energy_lag_{lag}h'] = df['solarenergy'].shift(lag)\n",
    "        df[f'uv_lag_{lag}h'] = df['uvindex'].shift(lag)\n",
    "\n",
    "    # Peak indicators\n",
    "    df['is_energy_peak'] = (df['solarenergy'] > df['energy_rolling_mean_6h'] * 1.2).astype(int)\n",
    "    df['is_uv_peak'] = (df['uvindex'] > df['uv_rolling_mean_6h'] * 1.2).astype(int)\n",
    "\n",
    "    # Aggiungiamo alcune metriche di volatilità\n",
    "    df['energy_volatility'] = df['energy_change'].rolling(window=24).std()\n",
    "    df['uv_volatility'] = df['uv_change'].rolling(window=24).std()\n",
    "\n",
    "    # Indice di intensità solare composito\n",
    "    df['solar_intensity_index'] = (df['solarenergy'] * df['uvindex']) / (df['cloudcover'] + 1e-6)\n",
    "\n",
    "    # Interazioni\n",
    "    df['uv_cloud_interaction'] = df['uvindex'] * (100 - df['cloudcover']) / 100\n",
    "    df['energy_temp_interaction'] = df['solarenergy'] * df['temp']\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_advanced_features(df):\n",
    "    \"\"\"\n",
    "    Add all advanced features to the DataFrame\n",
    "    Assumes df has a DatetimeIndex\n",
    "    \"\"\"\n",
    "    # Verifichiamo che abbiamo un DatetimeIndex\n",
    "    if not isinstance(df.index, pd.DatetimeIndex):\n",
    "        raise ValueError(\"DataFrame must have a DatetimeIndex\")\n",
    "\n",
    "    # Existing features\n",
    "    df = add_time_features(df)\n",
    "    df = add_solar_features(df)\n",
    "    df = add_solar_specific_features(df)\n",
    "    df = add_radiation_energy_features(df)\n",
    "\n",
    "    # Weather variable interactions\n",
    "    df['temp_humidity'] = df['temp'] * df['humidity']\n",
    "    df['temp_cloudcover'] = df['temp'] * df['cloudcover']\n",
    "    df['visibility_cloudcover'] = df['visibility'] * df['cloudcover']\n",
    "\n",
    "    # Derived features\n",
    "    df['clear_sky_factor'] = (100 - df['cloudcover']) / 100\n",
    "    df['temp_humidity_interaction'] = df['temp'] * df['humidity'] / 100\n",
    "    df['atmospheric_transparency'] = (100 - df['cloudcover']) * (df['visibility'] / 10)\n",
    "\n",
    "    # Rolling means\n",
    "    df['temp_rolling_mean_6h'] = df['temp'].rolling(window=6).mean()\n",
    "    df['cloudcover_rolling_mean_6h'] = df['cloudcover'].rolling(window=6).mean()\n",
    "\n",
    "    # Lag features\n",
    "    df['temp_1h_lag'] = df['temp'].shift(1)\n",
    "    df['cloudcover_1h_lag'] = df['cloudcover'].shift(1)\n",
    "    df['humidity_1h_lag'] = df['humidity'].shift(1)\n",
    "\n",
    "    # Extreme conditions indicator\n",
    "    df['extreme_conditions'] = ((df['temp'] > df['temp'].quantile(0.75)) &\n",
    "                                (df['humidity'] < df['humidity'].quantile(0.25))).astype(int)\n",
    "\n",
    "    # One-hot encoding for categorical features\n",
    "    df = pd.get_dummies(df, columns=['season', 'time_period'])\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def prepare_advanced_data(df):\n",
    "    \"\"\"\n",
    "    Prepare data for advanced modeling with proper datetime handling\n",
    "    \"\"\"\n",
    "    # Assicuriamoci che abbiamo una copia del DataFrame\n",
    "    df = df.copy()\n",
    "\n",
    "    # Verifichiamo se datetime è già l'indice\n",
    "    if not isinstance(df.index, pd.DatetimeIndex):\n",
    "        if 'datetime' in df.columns:\n",
    "            df['datetime'] = pd.to_datetime(df['datetime'])\n",
    "            df.set_index('datetime', inplace=True)\n",
    "        else:\n",
    "            raise ValueError(\"No datetime column or index found in DataFrame\")\n",
    "\n",
    "    # Ordiniamo il DataFrame per datetime\n",
    "    df = df.sort_index()\n",
    "\n",
    "    # Apply feature engineering functions\n",
    "    df = add_advanced_features(df)\n",
    "\n",
    "    #all_columns = list(df.columns)\n",
    "    #print(all_columns)\n",
    "\n",
    "    features = {\n",
    "        # Primary Features (strong direct correlation)\n",
    "        'primary_features': [\n",
    "            'uvindex',  # Direct radiation indicator\n",
    "            'cloudcover',  # Cloud coverage\n",
    "            'visibility',  # Atmospheric transparency\n",
    "            'temp',  # Temperature\n",
    "            'pressure',  # Atmospheric pressure\n",
    "            'humidity',  # Humidity\n",
    "        ],\n",
    "\n",
    "        # Astronomical and Temporal Features\n",
    "        'astronomical_features': [\n",
    "            'solar_elevation',  # Solar elevation\n",
    "            'solar_angle',  # Solar angle\n",
    "            'day_length',  # Day length\n",
    "            'hour_sin',  # Daily cycle\n",
    "            'hour_cos',\n",
    "            'day_of_year_sin',  # Annual cycle\n",
    "            'day_of_year_cos',\n",
    "            'month_sin',  # Monthly cycle\n",
    "            'month_cos',\n",
    "        ],\n",
    "\n",
    "        # Key Indices and Interactions\n",
    "        'key_interactions': [\n",
    "            'clear_sky_index',  # Clear sky index\n",
    "            'atmospheric_attenuation',  # Atmospheric attenuation\n",
    "            'theoretical_radiation',  # Theoretical radiation\n",
    "            'expected_radiation',  # Expected radiation\n",
    "            'cloud_elevation',  # Cloud-elevation interaction\n",
    "            'visibility_elevation',  # Visibility-elevation interaction\n",
    "            'uv_cloud_interaction',  # UV-cloud interaction\n",
    "            'temp_radiation_potential',  # Temperature-radiation potential\n",
    "        ],\n",
    "\n",
    "        # Rolling Features (temporal trends)\n",
    "        'rolling_features': [\n",
    "            'cloud_rolling_12h',  # Cloud coverage moving average\n",
    "            'temp_rolling_12h',  # Temperature moving average\n",
    "            'uv_rolling_12h',  # UV moving average\n",
    "            'cloudcover_rolling_mean_6h',\n",
    "            'temp_rolling_mean_6h',\n",
    "        ],\n",
    "\n",
    "        # Lag Features (most recent)\n",
    "        'lag_features': [\n",
    "            'temp_1h_lag',  # 1-hour temperature lag\n",
    "            'cloudcover_1h_lag',  # 1-hour cloud coverage lag\n",
    "            'humidity_1h_lag',  # 1-hour humidity lag\n",
    "            'uv_lag_1h',  # 1-hour UV lag\n",
    "        ],\n",
    "\n",
    "        # Categorical Features\n",
    "        'categorical_features': [\n",
    "            'season_Spring',  # Seasons\n",
    "            'season_Summer',\n",
    "            'season_Autumn',\n",
    "            'season_Winter',\n",
    "            'time_period_Morning',  # Time periods\n",
    "            'time_period_Afternoon',\n",
    "            'time_period_Evening',\n",
    "            'time_period_Night',\n",
    "        ]\n",
    "    }\n",
    "\n",
    "    final_features = [feature for group in features.values() for feature in group]\n",
    "\n",
    "    # Handle missing values\n",
    "    target_variables = ['solarradiation', 'solarenergy', 'uvindex']\n",
    "    for column in final_features + target_variables:\n",
    "        if column in df.columns:\n",
    "            df[column] = df[column].interpolate(method='time')\n",
    "    df.fillna(0, inplace=True)\n",
    "\n",
    "    # Temporal split\n",
    "    data_after_2010 = df[df['year'] >= 2010].copy()\n",
    "    data_before_2010 = df[df['year'] < 2010].copy()\n",
    "\n",
    "    X = data_after_2010[final_features]\n",
    "    y = data_after_2010['solarradiation']\n",
    "    X_to_predict = data_before_2010[final_features]\n",
    "\n",
    "    # Train-test split\n",
    "    X_train, X_test, y_train, y_test = train_test_split(\n",
    "        X, y, test_size=0.2, random_state=42, shuffle=False\n",
    "    )\n",
    "\n",
    "    # Scaling\n",
    "    feature_scaler = RobustScaler()\n",
    "    X_train_scaled = feature_scaler.fit_transform(X_train)\n",
    "    X_test_scaled = feature_scaler.transform(X_test)\n",
    "    X_to_predict_scaled = feature_scaler.transform(X_to_predict)\n",
    "\n",
    "    target_scaler = RobustScaler()\n",
    "    y_train_scaled = target_scaler.fit_transform(y_train.values.reshape(-1, 1))\n",
    "    y_test_scaled = target_scaler.transform(y_test.values.reshape(-1, 1))\n",
    "\n",
    "    # Print info about selected features\n",
    "    print(\"\\nSelected features:\")\n",
    "    print(f\"Number of features: {len(final_features)}\")\n",
    "    print(\"Features list:\", final_features)\n",
    "\n",
    "    return X_train_scaled, X_test_scaled, y_train_scaled, y_test_scaled, feature_scaler, target_scaler, final_features, X_to_predict_scaled\n",
    "\n",
    "\n",
    "def create_sequence_data(X, sequence_length=24):\n",
    "    \"\"\"\n",
    "    Converts data into sequences for LSTM input\n",
    "    sequence_length represents how many previous hours to consider\n",
    "    \"\"\"\n",
    "    sequences = []\n",
    "    for i in range(len(X) - sequence_length + 1):\n",
    "        sequences.append(X[i:i + sequence_length])\n",
    "    return np.array(sequences)\n",
    "\n",
    "\n",
    "def prepare_hybrid_data(df):\n",
    "    X_train_scaled, X_test_scaled, y_train_scaled, y_test_scaled, feature_scaler, target_scaler, features, X_to_predict_scaled = prepare_advanced_data(df)\n",
    "\n",
    "    # Convert data into sequences\n",
    "    sequence_length = 24  # 24 hours of historical data\n",
    "\n",
    "    X_train_seq = create_sequence_data(X_train_scaled, sequence_length)\n",
    "    X_test_seq = create_sequence_data(X_test_scaled, sequence_length)\n",
    "\n",
    "    # Adjust y by removing the first (sequence_length-1) elements\n",
    "    y_train = y_train_scaled[sequence_length - 1:]\n",
    "    y_test = y_test_scaled[sequence_length - 1:]\n",
    "\n",
    "    X_to_predict_seq = create_sequence_data(X_to_predict_scaled, sequence_length)\n",
    "\n",
    "    return X_train_seq, X_test_seq, y_train, y_test, feature_scaler, target_scaler, features, X_to_predict_seq"
   ]
  },
  {
   "cell_type": "code",
   "id": "9dff3259-b376-4cfc-89d8-ab2ea18aaa5e",
   "metadata": {
    "jupyter": {
     "is_executing": true
    }
   },
   "source": [
    "def create_residual_lstm_layer(x, units, dropout_rate, l2_reg=0.01, return_sequences=True, survival_probability=0.8):\n",
    "    \"\"\"\n",
    "    Creates a bidirectional LSTM layer with residual connections and regularization.\n",
    "\n",
    "    Parameters:\n",
    "        x: Input tensor\n",
    "        units: Number of LSTM units\n",
    "        dropout_rate: Dropout rate for regularization\n",
    "        l2_reg: L2 regularization factor\n",
    "        return_sequences: Whether to return sequences or just the last output\n",
    "        survival_probability: Probability of layer survival for stochastic depth\n",
    "    \"\"\"\n",
    "    residual = x\n",
    "    x = Bidirectional(LSTM(units, return_sequences=return_sequences, kernel_regularizer=regularizers.l2(l2_reg)))(x)\n",
    "    x = LayerNormalization()(x)\n",
    "    x = Dropout(dropout_rate)(x)\n",
    "\n",
    "    if return_sequences:\n",
    "        if int(residual.shape[-1]) != 2 * units:\n",
    "            residual = Dense(2 * units, activation='linear')(residual)\n",
    "        x = tfa.layers.StochasticDepth(survival_probability)([x, residual])\n",
    "    return x\n",
    "\n",
    "\n",
    "def attention_block(x, units, num_heads=8, survival_probability=0.8):\n",
    "    \"\"\"\n",
    "    Creates a multi-head attention block with residual connections.\n",
    "\n",
    "    Parameters:\n",
    "        x: Input tensor\n",
    "        units: Dimension of the key space\n",
    "        num_heads: Number of attention heads\n",
    "        survival_probability: Probability of layer survival for stochastic depth\n",
    "    \"\"\"\n",
    "    attention = MultiHeadAttention(num_heads=num_heads, key_dim=units)(x, x)\n",
    "    x = tfa.layers.StochasticDepth(survival_probability)([x, attention])\n",
    "    x = LayerNormalization()(x)\n",
    "    return x\n",
    "\n",
    "\n",
    "def create_solarradiation_model(input_shape, folder_name, l2_lambda=0.005, min_output=0, max_output=1):\n",
    "    \"\"\"\n",
    "    Creates a deep learning model for solar radiation prediction using LSTM and attention mechanisms.\n",
    "\n",
    "    Parameters:\n",
    "        input_shape: Shape of input data\n",
    "        folder_name: Directory to save model architecture visualization\n",
    "        l2_lambda: L2 regularization factor\n",
    "    \"\"\"\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    # Progressive hyperparameters for model architecture\n",
    "    survival_probs = [0.9, 0.8, 0.7, 0.6]  # Decreasing survival probabilities for deeper layers\n",
    "    attention_survival_probs = [0.85, 0.75, 0.65, 0.55]  # Survival probabilities for attention blocks\n",
    "    lstm_units = [256, 128, 64, 32]  # Decreasing number of units for LSTM layers\n",
    "    dropout_rates = [0.4, 0.3, 0.2, 0.2]  # Decreasing dropout rates\n",
    "    attention_heads = [32, 24, 16, 8]  # Decreasing number of attention heads\n",
    "\n",
    "    lstm_blocks = 4\n",
    "    # Main network architecture\n",
    "    x = inputs\n",
    "    for i in range(lstm_blocks):\n",
    "        # LSTM layer with residual connections\n",
    "        x = create_residual_lstm_layer(\n",
    "            x,\n",
    "            units=lstm_units[i],\n",
    "            dropout_rate=dropout_rates[i],\n",
    "            l2_reg=l2_lambda,\n",
    "            return_sequences=True,\n",
    "            survival_probability=survival_probs[i]\n",
    "        )\n",
    "        # Attention block\n",
    "        x = attention_block(\n",
    "            x,\n",
    "            units=lstm_units[i],\n",
    "            num_heads=attention_heads[i],\n",
    "            survival_probability=attention_survival_probs[i]\n",
    "        )\n",
    "        if i < lstm_blocks - 1:  # No pooling after last LSTM layer\n",
    "            x = MaxPooling1D()(x)\n",
    "\n",
    "    # Final LSTM layer for sequence aggregation\n",
    "    x = create_residual_lstm_layer(\n",
    "        x,\n",
    "        units=32,\n",
    "        dropout_rate=0.1,\n",
    "        l2_reg=l2_lambda,\n",
    "        return_sequences=False,\n",
    "        survival_probability=0.6\n",
    "    )\n",
    "\n",
    "    # Dense layers for final prediction\n",
    "    dense_units = [128, 64, 32]\n",
    "    dense_dropout = [0.2, 0.1, 0.05]\n",
    "\n",
    "    for units, dropout in zip(dense_units, dense_dropout):\n",
    "        x = Dense(units, kernel_regularizer=regularizers.l2(l2_lambda))(x)\n",
    "        x = BatchNormalization()(x)\n",
    "        x = Activation('swish')(x)\n",
    "        x = Dropout(dropout)(x)\n",
    "\n",
    "    # Output layer with value clipping\n",
    "    outputs = Dense(1)(x)\n",
    "    outputs = Lambda(lambda x: tf.clip_by_value(x, min_output, max_output))(outputs)\n",
    "\n",
    "    # Model compilation\n",
    "    model = Model(inputs=inputs, outputs=outputs, name=\"SolarRadiationModel\")\n",
    "\n",
    "    # Improved loss function\n",
    "    def hybrid_focal_loss(y_true, y_pred):\n",
    "        # MSE with focal weighting\n",
    "        mse = tf.square(y_true - y_pred)\n",
    "        error_ratio = tf.abs(y_true - y_pred) / (tf.abs(y_true) + 1.0)\n",
    "        focal_weight = tf.pow(error_ratio, 2)\n",
    "        weighted_mse = focal_weight * mse\n",
    "\n",
    "        # MAE component\n",
    "        mae = tf.abs(y_true - y_pred)\n",
    "\n",
    "        return tf.reduce_mean(0.7 * weighted_mse + 0.3 * mae)\n",
    "\n",
    "    # Custom metrics\n",
    "    def rmse(y_true, y_pred):\n",
    "        return tf.sqrt(tf.reduce_mean(tf.square(y_true - y_pred)))\n",
    "\n",
    "    def custom_mape(y_true, y_pred):\n",
    "        epsilon = 1e-7\n",
    "        diff = tf.abs((y_true - y_pred) / (y_true + epsilon))\n",
    "        diff = tf.clip_by_value(diff, 0, 1)\n",
    "        return tf.reduce_mean(diff) * 100\n",
    "\n",
    "    # Learning rate schedule\n",
    "    lr_schedule = tf.keras.optimizers.schedules.CosineDecayRestarts(\n",
    "        initial_learning_rate=0.0003,\n",
    "        first_decay_steps=1000,\n",
    "        t_mul=2.0,\n",
    "        m_mul=0.9,\n",
    "        alpha=1e-5\n",
    "    )\n",
    "\n",
    "    # Optimizer\n",
    "    optimizer = AdamW(\n",
    "        learning_rate=lr_schedule,\n",
    "        beta_1=0.9,\n",
    "        beta_2=0.999,\n",
    "        epsilon=1e-7,\n",
    "        weight_decay=0.001,\n",
    "        amsgrad=True\n",
    "    )\n",
    "\n",
    "    model.compile(\n",
    "        optimizer=optimizer,\n",
    "        loss=hybrid_focal_loss,\n",
    "        metrics=['mse', 'mae', rmse, custom_mape]\n",
    "    )\n",
    "\n",
    "    model.summary()\n",
    "\n",
    "    plot_model(model,\n",
    "               to_file=f'{folder_name}_model_architecture.png',\n",
    "               show_shapes=True,\n",
    "               show_layer_names=True,\n",
    "               dpi=150,\n",
    "               show_layer_activations=True)\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "def evaluate_solarradiation_predictions(y_true, y_pred, hour=None, folder_name=None):\n",
    "    \"\"\"\n",
    "    Comprehensive evaluation of solar radiation predictions with detailed analysis and visualizations.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    y_true : array-like\n",
    "        Actual solar radiation values (W/m²)\n",
    "    y_pred : array-like\n",
    "        Predicted solar radiation values (W/m²)\n",
    "    hour : array-like, optional\n",
    "        Array of hours corresponding to predictions, for temporal analysis\n",
    "    folder_name : str, optional\n",
    "        Directory to save analysis plots\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    dict\n",
    "        Dictionary containing all calculated metrics\n",
    "    \"\"\"\n",
    "\n",
    "    # Data preparation\n",
    "    y_true = np.array(y_true).ravel()\n",
    "    y_pred = np.array(y_pred).ravel()\n",
    "    errors = y_pred - y_true\n",
    "\n",
    "    # Basic metrics calculation\n",
    "    mae_raw = mean_absolute_error(y_true, y_pred)\n",
    "    rmse_raw = np.sqrt(mean_squared_error(y_true, y_pred))\n",
    "    r2_raw = r2_score(y_true, y_pred)\n",
    "    mape = np.mean(np.abs((y_true - y_pred) / (y_true + 1e-7))) * 100\n",
    "\n",
    "    # Error margin accuracy\n",
    "    within_5_percent = np.mean(np.abs((y_pred - y_true) / (y_true + 1e-7)) <= 0.05)\n",
    "    within_10_percent = np.mean(np.abs((y_pred - y_true) / (y_true + 1e-7)) <= 0.10)\n",
    "    within_20_percent = np.mean(np.abs((y_pred - y_true) / (y_true + 1e-7)) <= 0.20)\n",
    "\n",
    "    # Radiation level classification\n",
    "    def get_radiation_level(value):\n",
    "        if value <= 200:\n",
    "            return 'Very Low'\n",
    "        elif value <= 400:\n",
    "            return 'Low'\n",
    "        elif value <= 600:\n",
    "            return 'Moderate'\n",
    "        elif value <= 800:\n",
    "            return 'High'\n",
    "        elif value <= 1000:\n",
    "            return 'Very High'\n",
    "        else:\n",
    "            return 'Extreme'\n",
    "\n",
    "    # Calculate radiation levels\n",
    "    y_true_levels = [get_radiation_level(v) for v in y_true]\n",
    "    y_pred_levels = [get_radiation_level(v) for v in y_pred]\n",
    "    level_accuracy = np.mean([t == p for t, p in zip(y_true_levels, y_pred_levels)])\n",
    "\n",
    "    # Print main metrics\n",
    "    print(\"\\nSolar Radiation Prediction Metrics:\")\n",
    "    print(\"\\nAbsolute Metrics:\")\n",
    "    print(f\"MAE: {mae_raw:.2f} W/m²\")\n",
    "    print(f\"RMSE: {rmse_raw:.2f} W/m²\")\n",
    "    print(f\"R² Score: {r2_raw:.3f}\")\n",
    "    print(f\"MAPE: {mape:.2f}%\")\n",
    "\n",
    "    print(\"\\nPercentage Accuracy:\")\n",
    "    print(f\"Within ±5%: {within_5_percent * 100:.1f}%\")\n",
    "    print(f\"Within ±10%: {within_10_percent * 100:.1f}%\")\n",
    "    print(f\"Within ±20%: {within_20_percent * 100:.1f}%\")\n",
    "\n",
    "    print(\"\\nLevel Accuracy:\")\n",
    "    print(f\"Level Accuracy: {level_accuracy * 100:.1f}%\")\n",
    "\n",
    "    # Confusion matrix for radiation levels\n",
    "    cm = confusion_matrix(y_true_levels, y_pred_levels)\n",
    "    print(\"\\nConfusion Matrix for Radiation Levels:\")\n",
    "    cm_df = pd.DataFrame(\n",
    "        cm,\n",
    "        columns=['Very Low', 'Low', 'Moderate', 'High', 'Very High', 'Extreme'],\n",
    "        index=['Very Low', 'Low', 'Moderate', 'High', 'Very High', 'Extreme']\n",
    "    )\n",
    "    print(cm_df)\n",
    "\n",
    "    # Time period analysis\n",
    "    if hour is not None:\n",
    "        day_periods = {\n",
    "            'Morning (5-11)': (5, 11),\n",
    "            'Noon (11-13)': (11, 13),\n",
    "            'Afternoon (13-17)': (13, 17),\n",
    "            'Evening (17-21)': (17, 21),\n",
    "            'Night (21-5)': (21, 5)\n",
    "        }\n",
    "\n",
    "        print(\"\\nAnalysis by Time Period:\")\n",
    "        for period, (start, end) in day_periods.items():\n",
    "            if start < end:\n",
    "                mask = (hour >= start) & (hour < end)\n",
    "            else:\n",
    "                mask = (hour >= start) | (hour < end)\n",
    "\n",
    "            if np.any(mask):\n",
    "                period_mae = mean_absolute_error(y_true[mask], y_pred[mask])\n",
    "                period_mape = np.mean(np.abs((y_true[mask] - y_pred[mask]) / (y_true[mask] + 1e-7))) * 100\n",
    "                print(f\"\\n{period}:\")\n",
    "                print(f\"MAE: {period_mae:.2f} W/m²\")\n",
    "                print(f\"MAPE: {period_mape:.2f}%\")\n",
    "\n",
    "    # Visualizations\n",
    "    if folder_name is not None:\n",
    "        try:\n",
    "\n",
    "            # Figure 1: Main analysis plots\n",
    "            plt.figure(figsize=(20, 15))\n",
    "\n",
    "            # Plot 1: Scatter plot of actual vs predicted values\n",
    "            plt.subplot(3, 2, 1)\n",
    "            plt.scatter(y_true, y_pred, alpha=0.5)\n",
    "            plt.plot([y_true.min(), y_true.max()], [y_true.min(), y_true.max()], 'r--', lw=2)\n",
    "            plt.xlabel('Actual Radiation (W/m²)')\n",
    "            plt.ylabel('Predicted Radiation (W/m²)')\n",
    "            plt.title('Actual vs Predicted Values')\n",
    "            plt.grid(True)\n",
    "\n",
    "            # Plot 2: Absolute error distribution\n",
    "            plt.subplot(3, 2, 2)\n",
    "            plt.hist(errors, bins=50, alpha=0.7)\n",
    "            plt.xlabel('Prediction Error (W/m²)')\n",
    "            plt.ylabel('Frequency')\n",
    "            plt.title('Error Distribution')\n",
    "            plt.grid(True)\n",
    "\n",
    "            # Plot 3: Percentage error distribution\n",
    "            plt.subplot(3, 2, 3)\n",
    "            percentage_errors = ((y_pred - y_true) / (y_true + 1e-7)) * 100\n",
    "            plt.hist(np.clip(percentage_errors, -100, 100), bins=50, alpha=0.7)\n",
    "            plt.xlabel('Percentage Error (%)')\n",
    "            plt.ylabel('Frequency')\n",
    "            plt.title('Percentage Error Distribution')\n",
    "            plt.grid(True)\n",
    "\n",
    "            # Plot 4: Errors vs actual values\n",
    "            plt.subplot(3, 2, 4)\n",
    "            plt.scatter(y_true, errors, alpha=0.5)\n",
    "            plt.axhline(y=0, color='r', linestyle='--')\n",
    "            plt.xlabel('Actual Radiation (W/m²)')\n",
    "            plt.ylabel('Error (W/m²)')\n",
    "            plt.title('Errors vs Actual Values')\n",
    "            plt.grid(True)\n",
    "\n",
    "            # Plot 5: Error boxplot by radiation level\n",
    "            plt.subplot(3, 2, 5)\n",
    "            sns.boxplot(x=[get_radiation_level(v) for v in y_true], y=errors)\n",
    "            plt.xticks(rotation=45)\n",
    "            plt.xlabel('Radiation Level')\n",
    "            plt.ylabel('Error (W/m²)')\n",
    "            plt.title('Error Distribution by Level')\n",
    "\n",
    "            # Plot 6: Confusion matrix heatmap\n",
    "            plt.subplot(3, 2, 6)\n",
    "            sns.heatmap(cm_df, annot=True, fmt='d', cmap='Blues')\n",
    "            plt.title('Confusion Matrix')\n",
    "            plt.xticks(rotation=45)\n",
    "            plt.yticks(rotation=45)\n",
    "\n",
    "            plt.tight_layout()\n",
    "            filename = f'{folder_name}_radiation_analysis.png'\n",
    "            plt.savefig(filename, dpi=300, bbox_inches='tight')\n",
    "            print(f\"\\nPlot saved as: {filename}\")\n",
    "            plt.close()\n",
    "\n",
    "        except Exception as e:\n",
    "            print(f\"\\nError saving plots: {str(e)}\")\n",
    "\n",
    "    # Additional error statistics\n",
    "    print(\"\\nError Statistics:\")\n",
    "    print(f\"Mean error: {np.mean(errors):.3f}\")\n",
    "    print(f\"Error standard deviation: {np.std(errors):.3f}\")\n",
    "    print(f\"Median error: {np.median(errors):.3f}\")\n",
    "    print(f\"95th percentile absolute error: {np.percentile(np.abs(errors), 95):.3f}\")\n",
    "\n",
    "    # Return structured metrics\n",
    "    metrics = {\n",
    "        'absolute': {\n",
    "            'mae': mae_raw,\n",
    "            'rmse': rmse_raw,\n",
    "            'r2': r2_raw,\n",
    "            'mape': mape\n",
    "        },\n",
    "        'percentage_accuracy': {\n",
    "            'within_5_percent': within_5_percent,\n",
    "            'within_10_percent': within_10_percent,\n",
    "            'within_20_percent': within_20_percent\n",
    "        },\n",
    "        'categorical': {\n",
    "            'level_accuracy': level_accuracy\n",
    "        },\n",
    "        'error_stats': {\n",
    "            'mean': float(np.mean(errors)),\n",
    "            'std': float(np.std(errors)),\n",
    "            'median': float(np.median(errors)),\n",
    "            'p95_abs': float(np.percentile(np.abs(errors), 95))\n",
    "        }\n",
    "    }\n",
    "\n",
    "    return metrics\n",
    "\n",
    "\n",
    "def plot_training_history(history, folder_name=None):\n",
    "    \"\"\"\n",
    "    Visualize and save training loss and metrics plots\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    history : tensorflow.keras.callbacks.History\n",
    "        History object returned by model training\n",
    "    folder_name : str\n",
    "        Directory to save the plots and metrics\n",
    "    \"\"\"\n",
    "\n",
    "    try:\n",
    "        # Create figure\n",
    "        plt.figure(figsize=(12, 4))\n",
    "\n",
    "        # Loss plot\n",
    "        plt.subplot(1, 2, 1)\n",
    "        plt.plot(history.history['loss'], label='Training Loss')\n",
    "        plt.plot(history.history['val_loss'], label='Validation Loss')\n",
    "        plt.title('Model Loss')\n",
    "        plt.xlabel('Epoch')\n",
    "        plt.ylabel('Loss')\n",
    "        plt.legend()\n",
    "        plt.grid(True)\n",
    "\n",
    "        # MAE plot\n",
    "        plt.subplot(1, 2, 2)\n",
    "        plt.plot(history.history['mae'], label='Training MAE')\n",
    "        plt.plot(history.history['val_mae'], label='Validation MAE')\n",
    "        plt.title('Model MAE')\n",
    "        plt.xlabel('Epoch')\n",
    "        plt.ylabel('MAE')\n",
    "        plt.legend()\n",
    "        plt.grid(True)\n",
    "\n",
    "        plt.tight_layout()\n",
    "\n",
    "        if folder_name is not None:\n",
    "            # Generate filename with timestamp\n",
    "            filename = f'{folder_name}_training_history.png'  # Rimossa parentesi extra\n",
    "\n",
    "            # Save figure\n",
    "            plt.savefig(filename, dpi=300, bbox_inches='tight')\n",
    "            print(f\"\\nTraining history plot saved as: {filename}\")\n",
    "\n",
    "        # Save numerical data in CSV format\n",
    "        history_df = pd.DataFrame({\n",
    "            'epoch': range(1, len(history.history['loss']) + 1),\n",
    "            'training_loss': history.history['loss'],\n",
    "            'validation_loss': history.history['val_loss'],\n",
    "            'training_mae': history.history['mae'],\n",
    "            'validation_mae': history.history['val_mae']})\n",
    "\n",
    "        if folder_name is not None:\n",
    "            csv_filename = f'{folder_name}_training_history.csv'  # Rimossa parentesi extra\n",
    "            history_df.to_csv(csv_filename, index=False)\n",
    "            print(f\"Training history data saved as: {csv_filename}\")\n",
    "\n",
    "        # Calculate and save final statistics\n",
    "        final_stats = {\n",
    "            'final_training_loss': history.history['loss'][-1],\n",
    "            'final_validation_loss': history.history['val_loss'][-1],\n",
    "            'final_training_mae': history.history['mae'][-1],\n",
    "            'final_validation_mae': history.history['val_mae'][-1],\n",
    "            'best_validation_loss': min(history.history['val_loss']),\n",
    "            'best_validation_mae': min(history.history['val_mae']),\n",
    "            'epochs': len(history.history['loss']),\n",
    "        }\n",
    "\n",
    "        if folder_name is not None:\n",
    "            # Save statistics in JSON format\n",
    "            stats_filename = f'{folder_name}_training_stats.json'  # Rimossa parentesi extra\n",
    "            with open(stats_filename, 'w') as f:\n",
    "                json.dump(final_stats, f, indent=4)\n",
    "            print(f\"Final statistics saved as: {stats_filename}\")\n",
    "\n",
    "        # Print main statistics\n",
    "        print(\"\\nFinal Training Statistics:\")\n",
    "        print(f\"Final Loss (train/val): {final_stats['final_training_loss']:.4f}/{final_stats['final_validation_loss']:.4f}\")\n",
    "        print(f\"Final MAE (train/val): {final_stats['final_training_mae']:.4f}/{final_stats['final_validation_mae']:.4f}\")\n",
    "        print(f\"Best validation loss: {final_stats['best_validation_loss']:.4f}\")\n",
    "        print(f\"Best validation MAE: {final_stats['best_validation_mae']:.4f}\")\n",
    "\n",
    "        plt.show()\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"\\nError during plot creation or saving: {str(e)}\")\n",
    "\n",
    "\n",
    "def train_hybrid_model(model, X_train, y_train, X_test, y_test, epochs=100, batch_size=32, folder_name='solarradiation'):\n",
    "    \"\"\"\n",
    "    Advanced training function for the hybrid solar radiation model\n",
    "    \"\"\"\n",
    "\n",
    "    def calculate_metrics(y_true, y_pred):\n",
    "        \"\"\"Helper function to calculate metrics safely\"\"\"\n",
    "        y_true = np.array(y_true).flatten()\n",
    "        y_pred = np.array(y_pred).flatten()\n",
    "\n",
    "        # Count out of range predictions\n",
    "        out_of_range = np.sum((y_pred < 0) | (y_pred > 1500))\n",
    "\n",
    "        # Calculate MAPE with clipping to avoid extreme values\n",
    "        diff = np.abs((y_true - y_pred) / (y_true + 1e-7))\n",
    "        diff = np.clip(diff, 0, 1)  # Clip to maximum 100% error\n",
    "        mape = np.mean(diff) * 100\n",
    "\n",
    "        # Calculate accuracy within 10%\n",
    "        within_10_percent = np.mean(diff <= 0.10) * 100\n",
    "\n",
    "        # Calculate MAE and RMSE\n",
    "        mae = np.mean(np.abs(y_true - y_pred))\n",
    "        rmse = np.sqrt(np.mean(np.square(y_true - y_pred)))\n",
    "\n",
    "        return out_of_range, mape, within_10_percent, mae, rmse\n",
    "\n",
    "    callbacks = [\n",
    "        EarlyStopping(\n",
    "            monitor='val_loss',\n",
    "            patience=15,\n",
    "            restore_best_weights=True,\n",
    "            mode='min',\n",
    "            verbose=1,\n",
    "            min_delta=1e-4\n",
    "        ),\n",
    "        ReduceLROnPlateau(\n",
    "            monitor='val_loss',\n",
    "            factor=0.2,\n",
    "            patience=5,\n",
    "            verbose=1,\n",
    "            mode='min',\n",
    "            min_delta=1e-4,\n",
    "            cooldown=3,\n",
    "            min_lr=1e-7\n",
    "        ),\n",
    "        tf.keras.callbacks.ModelCheckpoint(\n",
    "            filepath=f'{folder_name}_best_model.h5',\n",
    "            monitor='val_loss',\n",
    "            save_best_only=True,\n",
    "            mode='min',\n",
    "            save_weights_only=False\n",
    "        ),\n",
    "        tf.keras.callbacks.TensorBoard(\n",
    "            log_dir=f'./{folder_name}_logs',\n",
    "            histogram_freq=1,\n",
    "            write_graph=True,\n",
    "            update_freq='epoch'\n",
    "        ),\n",
    "        tf.keras.callbacks.LambdaCallback(\n",
    "            on_epoch_end=lambda epoch, logs: (\n",
    "                    print(f\"\\nEpoch {epoch + 1} Detailed Metrics:\") and\n",
    "                    (lambda: (\n",
    "                        y_pred := model.predict(X_test, verbose=0),\n",
    "                        metrics := calculate_metrics(y_test, y_pred),\n",
    "                        print(f\"Out of range: {metrics[0]} predictions\"),\n",
    "                        print(f\"MAPE: {metrics[1]:.2f}%\"),\n",
    "                        print(f\"Within ±10%: {metrics[2]:.2f}%\"),\n",
    "                        print(f\"MAE: {metrics[3]:.2f}\"),\n",
    "                        print(f\"RMSE: {metrics[4]:.2f}\")\n",
    "                    ))()\n",
    "            ) if epoch % 5 == 0 else None\n",
    "        )\n",
    "    ]\n",
    "\n",
    "    try:\n",
    "        history = model.fit(\n",
    "            X_train, y_train,\n",
    "            validation_data=(X_test, y_test),\n",
    "            epochs=epochs,\n",
    "            batch_size=batch_size,\n",
    "            callbacks=callbacks,\n",
    "            verbose=1,\n",
    "            shuffle=False\n",
    "        )\n",
    "\n",
    "        print(\"\\nTraining completed successfully!\")\n",
    "\n",
    "        # Final evaluation\n",
    "        final_pred = model.predict(X_test, verbose=0)\n",
    "        metrics = calculate_metrics(y_test, final_pred)\n",
    "\n",
    "        print(\"\\nFinal Model Performance:\")\n",
    "        print(f\"Out of range predictions: {metrics[0]} ({metrics[0] / len(y_test) * 100:.2f}%)\")\n",
    "        print(f\"MAPE: {metrics[1]:.2f}%\")\n",
    "        print(f\"Predictions within ±10%: {metrics[2]:.2f}%\")\n",
    "        print(f\"MAE: {metrics[3]:.2f}\")\n",
    "        print(f\"RMSE: {metrics[4]:.2f}\")\n",
    "\n",
    "        plot_training_history(history, folder_name=folder_name)\n",
    "\n",
    "        return history\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"\\nError during training: {str(e)}\")\n",
    "        raise\n",
    "\n",
    "    finally:\n",
    "        tf.keras.backend.clear_session()\n",
    "\n",
    "\n",
    "def integrate_predictions(df, predictions, sequence_length=24):\n",
    "    \"\"\"\n",
    "    Integrates solar radiation predictions into the original dataset for pre-2010 data.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    df : pandas.DataFrame\n",
    "        Original dataset\n",
    "    predictions : numpy.ndarray\n",
    "        Array of solar radiation predictions\n",
    "    sequence_length : int\n",
    "        Sequence length used for predictions\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    pandas.DataFrame\n",
    "        Updated dataset with solar radiation predictions\n",
    "    \"\"\"\n",
    "    # Convert datetime to datetime format if not already\n",
    "    df['datetime'] = pd.to_datetime(df['datetime'])\n",
    "\n",
    "    # Identify pre-2010 rows\n",
    "    mask_pre_2010 = df['datetime'].dt.year < 2010\n",
    "\n",
    "    # Create temporary DataFrame with predictions\n",
    "    dates_pre_2010 = df[mask_pre_2010]['datetime'].iloc[sequence_length - 1:]\n",
    "    predictions_df = pd.DataFrame({\n",
    "        'datetime': dates_pre_2010,\n",
    "        'solarradiation_predicted': predictions.flatten()})\n",
    "\n",
    "    # Merge with original dataset\n",
    "    df = df.merge(predictions_df, on='datetime', how='left')\n",
    "\n",
    "    # Update solar radiation column where missing\n",
    "    df['solarradiation'] = df['solarradiation'].fillna(df['solarradiation_predicted'])\n",
    "\n",
    "    # Remove temporary column\n",
    "    df = df.drop('solarradiation_predicted', axis=1)\n",
    "\n",
    "    print(f\"Added {len(predictions)} predictions to dataset\")\n",
    "    print(f\"Rows with solar radiation after integration: {df['solarradiation'].notna().sum()}\")\n",
    "\n",
    "    return df"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "initial_id",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing solar radiation model training...\n",
      "\n",
      "1. Preparing data...\n",
      "\n",
      "Selected features:\n",
      "Number of features: 40\n",
      "Features list: ['uvindex', 'cloudcover', 'visibility', 'temp', 'pressure', 'humidity', 'solar_elevation', 'solar_angle', 'day_length', 'hour_sin', 'hour_cos', 'day_of_year_sin', 'day_of_year_cos', 'month_sin', 'month_cos', 'clear_sky_index', 'atmospheric_attenuation', 'theoretical_radiation', 'expected_radiation', 'cloud_elevation', 'visibility_elevation', 'uv_cloud_interaction', 'temp_radiation_potential', 'cloud_rolling_12h', 'temp_rolling_12h', 'uv_rolling_12h', 'cloudcover_rolling_mean_6h', 'temp_rolling_mean_6h', 'temp_1h_lag', 'cloudcover_1h_lag', 'humidity_1h_lag', 'uv_lag_1h', 'season_Spring', 'season_Summer', 'season_Autumn', 'season_Winter', 'time_period_Morning', 'time_period_Afternoon', 'time_period_Evening', 'time_period_Night']\n",
      "Training data shape: (103798, 24, 40)\n",
      "Test data shape: (25933, 24, 40)\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_parquet('../../sources/weather_data_uvindex.parquet')\n",
    "\n",
    "print(\"Initializing solar radiation model training...\")\n",
    "\n",
    "# Data preparation\n",
    "print(\"\\n1. Preparing data...\")\n",
    "X_train_seq, X_test_seq, y_train, y_test, feature_scaler, target_scaler, features, X_to_predict_seq = prepare_hybrid_data(df)\n",
    "\n",
    "print(f\"Training data shape: {X_train_seq.shape}\")\n",
    "print(f\"Test data shape: {X_test_seq.shape}\")\n",
    "\n",
    "# Save or load scaler and features\n",
    "feature_scaler_path = f'{folder_name}_scale_X.joblib'\n",
    "target_scaler_path = f'{folder_name}_scale_y.joblib'\n",
    "features_path = f'{folder_name}_features.json'\n",
    "model_path = f'{folder_name}_best_model.h5'\n",
    "history_path = f'{folder_name}_training_history.json'\n",
    "\n",
    "if os.path.exists(feature_scaler_path):\n",
    "    print(f\"Loading existing scaler X from: {feature_scaler_path}\")\n",
    "    scaler = joblib.load(feature_scaler_path)\n",
    "else:\n",
    "    print(f\"Saving scaler X to: {feature_scaler_path}\")\n",
    "    joblib.dump(feature_scaler, feature_scaler_path)\n",
    "\n",
    "if os.path.exists(target_scaler_path):\n",
    "    print(f\"Loading existing scaler X from: {target_scaler_path}\")\n",
    "    scaler = joblib.load(target_scaler_path)\n",
    "else:\n",
    "    print(f\"Saving scaler X to: {target_scaler_path}\")\n",
    "    joblib.dump(target_scaler, target_scaler_path)\n",
    "\n",
    "if os.path.exists(features_path):\n",
    "    print(f\"Loading existing features from: {features_path}\")\n",
    "    with open(features_path, 'r') as f:\n",
    "        features = json.load(f)\n",
    "else:\n",
    "    print(f\"Saving features to: {features_path}\")\n",
    "    with open(features_path, 'w') as f:\n",
    "        json.dump(features, f)\n",
    "\n",
    "# Data quality verification\n",
    "if np.isnan(X_train_seq).any() or np.isnan(y_train).any():\n",
    "    raise ValueError(\"Found NaN values in training data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "44749f6a-8941-41e8-8105-c36417245ed5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "2. Creating model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-20 18:26:58.091461: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 43404 MB memory:  -> device: 0, name: NVIDIA L40, pci bus id: 0000:a1:00.0, compute capability: 8.9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"SolarRadiationModel\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                Output Shape                 Param #   Connected to                  \n",
      "==================================================================================================\n",
      " input_1 (InputLayer)        [(None, 24, 40)]             0         []                            \n",
      "                                                                                                  \n",
      " bidirectional (Bidirection  (None, 24, 512)              608256    ['input_1[0][0]']             \n",
      " al)                                                                                              \n",
      "                                                                                                  \n",
      " layer_normalization (Layer  (None, 24, 512)              1024      ['bidirectional[0][0]']       \n",
      " Normalization)                                                                                   \n",
      "                                                                                                  \n",
      " dropout (Dropout)           (None, 24, 512)              0         ['layer_normalization[0][0]'] \n",
      "                                                                                                  \n",
      " dense (Dense)               (None, 24, 512)              20992     ['input_1[0][0]']             \n",
      "                                                                                                  \n",
      " stochastic_depth (Stochast  (None, 24, 512)              0         ['dropout[0][0]',             \n",
      " icDepth)                                                            'dense[0][0]']               \n",
      "                                                                                                  \n",
      " multi_head_attention (Mult  (None, 24, 512)              1680230   ['stochastic_depth[0][0]',    \n",
      " iHeadAttention)                                          4          'stochastic_depth[0][0]']    \n",
      "                                                                                                  \n",
      " stochastic_depth_1 (Stocha  (None, 24, 512)              0         ['stochastic_depth[0][0]',    \n",
      " sticDepth)                                                          'multi_head_attention[0][0]']\n",
      "                                                                                                  \n",
      " layer_normalization_1 (Lay  (None, 24, 512)              1024      ['stochastic_depth_1[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " max_pooling1d (MaxPooling1  (None, 12, 512)              0         ['layer_normalization_1[0][0]'\n",
      " D)                                                                 ]                             \n",
      "                                                                                                  \n",
      " bidirectional_1 (Bidirecti  (None, 12, 256)              656384    ['max_pooling1d[0][0]']       \n",
      " onal)                                                                                            \n",
      "                                                                                                  \n",
      " layer_normalization_2 (Lay  (None, 12, 256)              512       ['bidirectional_1[0][0]']     \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dropout_1 (Dropout)         (None, 12, 256)              0         ['layer_normalization_2[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_1 (Dense)             (None, 12, 256)              131328    ['max_pooling1d[0][0]']       \n",
      "                                                                                                  \n",
      " stochastic_depth_2 (Stocha  (None, 12, 256)              0         ['dropout_1[0][0]',           \n",
      " sticDepth)                                                          'dense_1[0][0]']             \n",
      "                                                                                                  \n",
      " multi_head_attention_1 (Mu  (None, 12, 256)              3155200   ['stochastic_depth_2[0][0]',  \n",
      " ltiHeadAttention)                                                   'stochastic_depth_2[0][0]']  \n",
      "                                                                                                  \n",
      " stochastic_depth_3 (Stocha  (None, 12, 256)              0         ['stochastic_depth_2[0][0]',  \n",
      " sticDepth)                                                          'multi_head_attention_1[0][0]\n",
      "                                                                    ']                            \n",
      "                                                                                                  \n",
      " layer_normalization_3 (Lay  (None, 12, 256)              512       ['stochastic_depth_3[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " max_pooling1d_1 (MaxPoolin  (None, 6, 256)               0         ['layer_normalization_3[0][0]'\n",
      " g1D)                                                               ]                             \n",
      "                                                                                                  \n",
      " bidirectional_2 (Bidirecti  (None, 6, 128)               164352    ['max_pooling1d_1[0][0]']     \n",
      " onal)                                                                                            \n",
      "                                                                                                  \n",
      " layer_normalization_4 (Lay  (None, 6, 128)               256       ['bidirectional_2[0][0]']     \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dropout_2 (Dropout)         (None, 6, 128)               0         ['layer_normalization_4[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_2 (Dense)             (None, 6, 128)               32896     ['max_pooling1d_1[0][0]']     \n",
      "                                                                                                  \n",
      " stochastic_depth_4 (Stocha  (None, 6, 128)               0         ['dropout_2[0][0]',           \n",
      " sticDepth)                                                          'dense_2[0][0]']             \n",
      "                                                                                                  \n",
      " multi_head_attention_2 (Mu  (None, 6, 128)               527488    ['stochastic_depth_4[0][0]',  \n",
      " ltiHeadAttention)                                                   'stochastic_depth_4[0][0]']  \n",
      "                                                                                                  \n",
      " stochastic_depth_5 (Stocha  (None, 6, 128)               0         ['stochastic_depth_4[0][0]',  \n",
      " sticDepth)                                                          'multi_head_attention_2[0][0]\n",
      "                                                                    ']                            \n",
      "                                                                                                  \n",
      " layer_normalization_5 (Lay  (None, 6, 128)               256       ['stochastic_depth_5[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " bidirectional_3 (Bidirecti  (None, 64)                   41216     ['layer_normalization_5[0][0]'\n",
      " onal)                                                              ]                             \n",
      "                                                                                                  \n",
      " layer_normalization_6 (Lay  (None, 64)                   128       ['bidirectional_3[0][0]']     \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dropout_3 (Dropout)         (None, 64)                   0         ['layer_normalization_6[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_3 (Dense)             (None, 64)                   4160      ['dropout_3[0][0]']           \n",
      "                                                                                                  \n",
      " batch_normalization (Batch  (None, 64)                   256       ['dense_3[0][0]']             \n",
      " Normalization)                                                                                   \n",
      "                                                                                                  \n",
      " activation (Activation)     (None, 64)                   0         ['batch_normalization[0][0]'] \n",
      "                                                                                                  \n",
      " dropout_4 (Dropout)         (None, 64)                   0         ['activation[0][0]']          \n",
      "                                                                                                  \n",
      " dense_4 (Dense)             (None, 32)                   2080      ['dropout_4[0][0]']           \n",
      "                                                                                                  \n",
      " batch_normalization_1 (Bat  (None, 32)                   128       ['dense_4[0][0]']             \n",
      " chNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " activation_1 (Activation)   (None, 32)                   0         ['batch_normalization_1[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dropout_5 (Dropout)         (None, 32)                   0         ['activation_1[0][0]']        \n",
      "                                                                                                  \n",
      " dense_5 (Dense)             (None, 1)                    33        ['dropout_5[0][0]']           \n",
      "                                                                                                  \n",
      " lambda (Lambda)             (None, 1)                    0         ['dense_5[0][0]']             \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 22150785 (84.50 MB)\n",
      "Trainable params: 22150593 (84.50 MB)\n",
      "Non-trainable params: 192 (768.00 Byte)\n",
      "__________________________________________________________________________________________________\n",
      "\n",
      "4. Starting training...\n",
      "Epoch 1/100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-20 18:28:08.462396: I tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:606] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n",
      "2024-11-20 18:28:08.551588: I tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:432] Loaded cuDNN version 8905\n",
      "2024-11-20 18:28:08.771296: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x7494b4d65840 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2024-11-20 18:28:08.771333: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): NVIDIA L40, Compute Capability 8.9\n",
      "2024-11-20 18:28:08.779091: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:255] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n",
      "2024-11-20 18:28:08.849938: I tensorflow/tsl/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory\n",
      "2024-11-20 18:28:08.914097: I ./tensorflow/compiler/jit/device_compiler.h:186] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "810/811 [============================>.] - ETA: 0s - loss: 72528.1172 - mse: 106644.4453 - mae: 187.8485 - rmse: 303.3813 - custom_mape: 54.8740"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.11/dist-packages/keras/src/engine/training.py:3000: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n",
      "  saving_api.save_model(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 1 Detailed Metrics:\n",
      "811/811 [==============================] - 59s 47ms/step - loss: 72456.4219 - mse: 106540.3906 - mae: 187.7001 - rmse: 303.1588 - custom_mape: 54.8602 - val_loss: 65954.8828 - val_mse: 98078.7031 - val_mae: 177.0511 - val_rmse: 289.9380 - val_custom_mape: 52.7771 - lr: 2.5941e-05\n",
      "Epoch 2/100\n",
      "811/811 [==============================] - 35s 43ms/step - loss: 69535.9297 - mse: 104347.7812 - mae: 185.7435 - rmse: 299.4631 - custom_mape: 53.3868 - val_loss: 63015.7383 - val_mse: 95844.1562 - val_mae: 175.0045 - val_rmse: 286.3577 - val_custom_mape: 52.6138 - lr: 2.1071e-04\n",
      "Epoch 3/100\n",
      "811/811 [==============================] - 34s 42ms/step - loss: 65040.6328 - mse: 100867.1875 - mae: 182.4250 - rmse: 293.5602 - custom_mape: 52.5410 - val_loss: 56593.6914 - val_mse: 90806.5000 - val_mae: 170.5536 - val_rmse: 278.2007 - val_custom_mape: 51.3653 - lr: 5.0264e-05\n",
      "Epoch 4/100\n",
      "811/811 [==============================] - 35s 43ms/step - loss: 63179.4570 - mse: 99390.6328 - mae: 181.0240 - rmse: 291.0518 - custom_mape: 52.2108 - val_loss: 54896.5859 - val_mse: 89427.9453 - val_mae: 169.4865 - val_rmse: 276.2857 - val_custom_mape: 51.1932 - lr: 2.4079e-04\n",
      "Epoch 5/100\n",
      "811/811 [==============================] - 36s 44ms/step - loss: 58537.2383 - mse: 95561.4453 - mae: 177.0757 - rmse: 284.3125 - custom_mape: 51.0609 - val_loss: 48058.6758 - val_mse: 83616.4297 - val_mae: 162.7125 - val_rmse: 266.4749 - val_custom_mape: 48.8749 - lr: 2.0369e-04\n",
      "Epoch 6/100\n",
      "810/811 [============================>.] - ETA: 0s - loss: 52993.6406 - mse: 90827.1172 - mae: 172.1619 - rmse: 275.9435 - custom_mape: 49.6688\n",
      "Epoch 6 Detailed Metrics:\n",
      "811/811 [==============================] - 35s 43ms/step - loss: 52937.2539 - mse: 90734.0391 - mae: 172.0164 - rmse: 275.7198 - custom_mape: 49.6529 - val_loss: 44455.3203 - val_mse: 80384.6250 - val_mae: 160.1475 - val_rmse: 260.5288 - val_custom_mape: 48.5323 - lr: 1.3436e-04\n",
      "Epoch 7/100\n",
      "811/811 [==============================] - 34s 42ms/step - loss: 49213.3164 - mse: 87367.3594 - mae: 168.3520 - rmse: 269.5628 - custom_mape: 48.4534 - val_loss: 40941.0117 - val_mse: 77138.5859 - val_mae: 156.7561 - val_rmse: 255.2473 - val_custom_mape: 47.8560 - lr: 5.9983e-05\n",
      "Epoch 8/100\n",
      "811/811 [==============================] - 33s 41ms/step - loss: 47519.0391 - mse: 85789.4453 - mae: 166.6271 - rmse: 266.6346 - custom_mape: 48.0879 - val_loss: 41349.0312 - val_mse: 77484.2734 - val_mae: 156.9061 - val_rmse: 255.2714 - val_custom_mape: 47.6252 - lr: 9.7321e-06\n",
      "Epoch 9/100\n",
      "811/811 [==============================] - 34s 42ms/step - loss: 46780.1719 - mse: 85090.1484 - mae: 165.8883 - rmse: 265.3610 - custom_mape: 47.9995 - val_loss: 40748.4062 - val_mse: 76847.7734 - val_mae: 156.1971 - val_rmse: 253.6026 - val_custom_mape: 48.0470 - lr: 2.1795e-04\n",
      "Epoch 10/100\n",
      "811/811 [==============================] - 34s 42ms/step - loss: 42443.9609 - mse: 80814.5703 - mae: 160.8849 - rmse: 257.1220 - custom_mape: 46.6063 - val_loss: 35794.1914 - val_mse: 71881.1797 - val_mae: 149.8344 - val_rmse: 244.4584 - val_custom_mape: 45.0777 - lr: 2.0849e-04\n",
      "Epoch 11/100\n",
      "810/811 [============================>.] - ETA: 0s - loss: 37364.0820 - mse: 75557.8281 - mae: 154.7132 - rmse: 246.8283 - custom_mape: 45.6366\n",
      "Epoch 11 Detailed Metrics:\n",
      "811/811 [==============================] - 35s 43ms/step - loss: 37322.7383 - mse: 75477.1875 - mae: 154.5700 - rmse: 246.6078 - custom_mape: 45.6219 - val_loss: 29906.3047 - val_mse: 65430.3633 - val_mae: 141.8013 - val_rmse: 231.2519 - val_custom_mape: 42.4489 - lr: 1.8906e-04\n",
      "Epoch 12/100\n",
      "811/811 [==============================] - 35s 43ms/step - loss: 32976.7734 - mse: 70634.2344 - mae: 148.5551 - rmse: 236.7462 - custom_mape: 44.1733 - val_loss: 29356.2773 - val_mse: 64463.9414 - val_mae: 141.1121 - val_rmse: 229.5340 - val_custom_mape: 42.9936 - lr: 1.6162e-04\n",
      "Epoch 13/100\n",
      "811/811 [==============================] - 37s 46ms/step - loss: 29592.6035 - mse: 66591.3828 - mae: 143.4485 - rmse: 228.2309 - custom_mape: 43.5998 - val_loss: 22979.4648 - val_mse: 56865.0938 - val_mae: 131.7958 - val_rmse: 214.5151 - val_custom_mape: 39.4901 - lr: 1.2891e-04\n",
      "Epoch 14/100\n",
      "811/811 [==============================] - 34s 42ms/step - loss: 26951.7656 - mse: 63337.9648 - mae: 139.0988 - rmse: 221.0572 - custom_mape: 41.7504 - val_loss: 25277.5273 - val_mse: 59844.8008 - val_mae: 134.7617 - val_rmse: 219.8931 - val_custom_mape: 40.0160 - lr: 9.4242e-05\n",
      "Epoch 15/100\n",
      "811/811 [==============================] - 32s 39ms/step - loss: 25191.9902 - mse: 61055.3008 - mae: 135.9996 - rmse: 215.9450 - custom_mape: 40.7117 - val_loss: 20492.2637 - val_mse: 53286.4492 - val_mae: 126.0936 - val_rmse: 205.3292 - val_custom_mape: 37.5140 - lr: 6.1091e-05\n",
      "Epoch 16/100\n",
      "810/811 [============================>.] - ETA: 0s - loss: 24165.8750 - mse: 59707.5938 - mae: 134.1485 - rmse: 212.9445 - custom_mape: 40.2365\n",
      "Epoch 16 Detailed Metrics:\n",
      "811/811 [==============================] - 32s 39ms/step - loss: 24138.5391 - mse: 59641.1523 - mae: 134.0126 - rmse: 212.7257 - custom_mape: 40.2135 - val_loss: 18706.9590 - val_mse: 51048.4062 - val_mae: 124.1925 - val_rmse: 201.2298 - val_custom_mape: 37.6503 - lr: 3.2793e-05\n",
      "Epoch 17/100\n",
      "811/811 [==============================] - 32s 39ms/step - loss: 23539.9023 - mse: 58823.9805 - mae: 132.9010 - rmse: 210.9564 - custom_mape: 40.2574 - val_loss: 18301.4316 - val_mse: 49967.2422 - val_mae: 121.6120 - val_rmse: 197.4946 - val_custom_mape: 36.3558 - lr: 1.2196e-05\n",
      "Epoch 18/100\n",
      "811/811 [==============================] - 31s 38ms/step - loss: 23317.7012 - mse: 58529.7852 - mae: 132.4304 - rmse: 210.2198 - custom_mape: 39.6456 - val_loss: 18765.3145 - val_mse: 50811.3203 - val_mae: 123.1689 - val_rmse: 199.7548 - val_custom_mape: 37.1078 - lr: 1.3695e-06\n",
      "Epoch 19/100\n",
      "811/811 [==============================] - 30s 37ms/step - loss: 22958.7871 - mse: 58021.4062 - mae: 131.7025 - rmse: 209.0721 - custom_mape: 39.5251 - val_loss: 22003.3301 - val_mse: 53933.9766 - val_mae: 125.5596 - val_rmse: 204.8436 - val_custom_mape: 36.6993 - lr: 1.9652e-04\n",
      "Epoch 20/100\n",
      "811/811 [==============================] - 31s 39ms/step - loss: 20586.0566 - mse: 54516.5820 - mae: 126.9349 - rmse: 201.2817 - custom_mape: 40.1354 - val_loss: 19601.9219 - val_mse: 51482.2773 - val_mae: 123.8536 - val_rmse: 202.0632 - val_custom_mape: 37.4784 - lr: 1.9403e-04\n",
      "Epoch 21/100\n",
      "810/811 [============================>.] - ETA: 0s - loss: 18059.2461 - mse: 50707.9844 - mae: 121.4836 - rmse: 192.6480 - custom_mape: 37.9434\n",
      "Epoch 21 Detailed Metrics:\n",
      "811/811 [==============================] - 30s 37ms/step - loss: 18040.1738 - mse: 50651.9258 - mae: 121.3665 - rmse: 192.4565 - custom_mape: 37.9314 - val_loss: 27332.0293 - val_mse: 61814.3359 - val_mae: 138.1345 - val_rmse: 223.8045 - val_custom_mape: 41.7334 - lr: 1.8912e-04\n",
      "Epoch 22/100\n",
      "811/811 [==============================] - 32s 39ms/step - loss: 15813.7461 - mse: 47093.9961 - mae: 116.1009 - rmse: 183.7298 - custom_mape: 36.6234 - val_loss: 15547.1533 - val_mse: 25035.1250 - val_mae: 83.4982 - val_rmse: 138.2488 - val_custom_mape: 26.6810 - lr: 1.8191e-04\n",
      "Epoch 23/100\n",
      "811/811 [==============================] - 30s 37ms/step - loss: 14064.2656 - mse: 44066.1055 - mae: 111.5707 - rmse: 176.2691 - custom_mape: 35.6786 - val_loss: 15586.9551 - val_mse: 33592.5234 - val_mae: 96.0797 - val_rmse: 154.1422 - val_custom_mape: 30.0333 - lr: 1.7260e-04\n",
      "Epoch 24/100\n",
      "811/811 [==============================] - 30s 37ms/step - loss: 13045.5820 - mse: 41777.5039 - mae: 108.8138 - rmse: 172.3399 - custom_mape: 35.9103 - val_loss: 21892.5098 - val_mse: 38512.5820 - val_mae: 102.0757 - val_rmse: 169.1723 - val_custom_mape: 30.7781 - lr: 1.6140e-04\n",
      "Epoch 25/100\n",
      "811/811 [==============================] - 30s 38ms/step - loss: 11775.0107 - mse: 39650.0430 - mae: 105.5830 - rmse: 166.6329 - custom_mape: 34.5414 - val_loss: 15617.9189 - val_mse: 24887.7910 - val_mae: 81.2674 - val_rmse: 134.0242 - val_custom_mape: 27.7191 - lr: 1.4861e-04\n",
      "Epoch 26/100\n",
      "810/811 [============================>.] - ETA: 0s - loss: 10885.3691 - mse: 37987.7930 - mae: 103.5588 - rmse: 163.0795 - custom_mape: 34.6734\n",
      "Epoch 26 Detailed Metrics:\n",
      "811/811 [==============================] - 32s 40ms/step - loss: 10874.6182 - mse: 37946.5391 - mae: 103.4657 - rmse: 162.9293 - custom_mape: 34.6890 - val_loss: 11223.3662 - val_mse: 30266.1914 - val_mae: 92.8420 - val_rmse: 151.5255 - val_custom_mape: 29.9186 - lr: 1.3456e-04\n",
      "Epoch 27/100\n",
      "811/811 [==============================] - 31s 38ms/step - loss: 10069.0938 - mse: 36349.3711 - mae: 100.6019 - rmse: 158.6973 - custom_mape: 33.1503 - val_loss: 12460.3320 - val_mse: 29399.4160 - val_mae: 91.2526 - val_rmse: 146.5061 - val_custom_mape: 29.6366 - lr: 1.1958e-04\n",
      "Epoch 28/100\n",
      "811/811 [==============================] - 31s 38ms/step - loss: 9476.7852 - mse: 35165.4648 - mae: 99.0255 - rmse: 156.0713 - custom_mape: 32.9813 - val_loss: 13038.0391 - val_mse: 25782.1328 - val_mae: 83.4327 - val_rmse: 134.0097 - val_custom_mape: 27.6063 - lr: 1.0408e-04\n",
      "Epoch 29/100\n",
      "811/811 [==============================] - 31s 38ms/step - loss: 9004.3877 - mse: 34155.9062 - mae: 97.4144 - rmse: 153.6510 - custom_mape: 32.3526 - val_loss: 14318.9199 - val_mse: 27135.3652 - val_mae: 85.2753 - val_rmse: 138.5686 - val_custom_mape: 27.5048 - lr: 8.8425e-05\n",
      "Epoch 30/100\n",
      "811/811 [==============================] - 31s 38ms/step - loss: 8699.8916 - mse: 33452.5859 - mae: 96.3121 - rmse: 152.1849 - custom_mape: 31.9938 - val_loss: 14860.2725 - val_mse: 26410.0449 - val_mae: 84.2833 - val_rmse: 135.7974 - val_custom_mape: 27.4504 - lr: 7.3026e-05\n",
      "Epoch 31/100\n",
      "810/811 [============================>.] - ETA: 0s - loss: 8397.3701 - mse: 32822.3984 - mae: 95.5427 - rmse: 150.7910 - custom_mape: 32.0949\n",
      "Epoch 31 Detailed Metrics:\n",
      "811/811 [==============================] - 32s 39ms/step - loss: 8389.3984 - mse: 32788.4531 - mae: 95.4662 - rmse: 150.6722 - custom_mape: 32.1090 - val_loss: 10881.0303 - val_mse: 21581.2246 - val_mae: 76.6546 - val_rmse: 122.9109 - val_custom_mape: 26.1594 - lr: 5.8271e-05\n",
      "Epoch 32/100\n",
      "811/811 [==============================] - 31s 38ms/step - loss: 8211.7285 - mse: 32299.0742 - mae: 94.7628 - rmse: 149.5487 - custom_mape: 31.8022 - val_loss: 10990.5146 - val_mse: 22554.5059 - val_mae: 78.0447 - val_rmse: 125.7095 - val_custom_mape: 26.2527 - lr: 4.4531e-05\n",
      "Epoch 33/100\n",
      "811/811 [==============================] - 31s 38ms/step - loss: 8059.6475 - mse: 31937.1406 - mae: 94.2104 - rmse: 148.7915 - custom_mape: 31.6591 - val_loss: 11965.4951 - val_mse: 19633.5859 - val_mae: 71.8940 - val_rmse: 115.6991 - val_custom_mape: 25.1729 - lr: 3.2155e-05\n",
      "Epoch 34/100\n",
      "811/811 [==============================] - 32s 39ms/step - loss: 7951.0010 - mse: 31709.4746 - mae: 93.8289 - rmse: 148.3072 - custom_mape: 31.4630 - val_loss: 9540.8096 - val_mse: 22911.6699 - val_mae: 78.2621 - val_rmse: 125.1414 - val_custom_mape: 26.4563 - lr: 2.1455e-05\n",
      "Epoch 35/100\n",
      "811/811 [==============================] - 32s 39ms/step - loss: 7870.2021 - mse: 31553.3457 - mae: 93.6221 - rmse: 147.9300 - custom_mape: 31.4443 - val_loss: 9422.6035 - val_mse: 21434.2637 - val_mae: 75.5224 - val_rmse: 120.7919 - val_custom_mape: 25.9513 - lr: 1.2703e-05\n",
      "Epoch 36/100\n",
      "810/811 [============================>.] - ETA: 0s - loss: 7791.0630 - mse: 31464.9941 - mae: 93.5448 - rmse: 147.9198 - custom_mape: 31.4216\n",
      "Epoch 36 Detailed Metrics:\n",
      "811/811 [==============================] - 32s 39ms/step - loss: 7783.9956 - mse: 31433.4629 - mae: 93.4737 - rmse: 147.8126 - custom_mape: 31.4435 - val_loss: 9728.4277 - val_mse: 23398.5430 - val_mae: 78.7421 - val_rmse: 126.2052 - val_custom_mape: 26.5261 - lr: 6.1193e-06\n",
      "Epoch 37/100\n",
      "811/811 [==============================] - 31s 38ms/step - loss: 7782.7549 - mse: 31406.9688 - mae: 93.3864 - rmse: 147.6053 - custom_mape: 31.3862 - val_loss: 9560.6436 - val_mse: 23029.3730 - val_mae: 78.1451 - val_rmse: 125.1727 - val_custom_mape: 26.4319 - lr: 1.8714e-06\n",
      "Epoch 38/100\n",
      "811/811 [==============================] - 31s 38ms/step - loss: 7777.0708 - mse: 31382.8203 - mae: 93.4302 - rmse: 147.6423 - custom_mape: 31.4434 - val_loss: 9647.4736 - val_mse: 23123.4121 - val_mae: 78.2223 - val_rmse: 125.4518 - val_custom_mape: 26.4217 - lr: 6.6522e-08\n",
      "Epoch 39/100\n",
      "811/811 [==============================] - 31s 38ms/step - loss: 7725.5112 - mse: 31040.5840 - mae: 93.0916 - rmse: 147.1082 - custom_mape: 31.8782 - val_loss: 9582.8750 - val_mse: 20723.7246 - val_mae: 74.9037 - val_rmse: 119.4773 - val_custom_mape: 26.3673 - lr: 1.7698e-04\n",
      "Epoch 40/100\n",
      "810/811 [============================>.] - ETA: 0s - loss: 7263.8110 - mse: 30007.5918 - mae: 91.4642 - rmse: 144.8820 - custom_mape: 31.5126\n",
      "Epoch 40: ReduceLROnPlateau reducing learning rate to 3.52531613316387e-05.\n",
      "811/811 [==============================] - 31s 38ms/step - loss: 7262.0361 - mse: 29977.5000 - mae: 91.3955 - rmse: 144.7766 - custom_mape: 31.5394 - val_loss: 14648.0088 - val_mse: 20503.3223 - val_mae: 72.9793 - val_rmse: 118.3558 - val_custom_mape: 25.1001 - lr: 1.7627e-04\n",
      "Epoch 41/100\n",
      "810/811 [============================>.] - ETA: 0s - loss: 6973.3042 - mse: 28997.6914 - mae: 89.9694 - rmse: 142.9392 - custom_mape: 30.8228\n",
      "Epoch 41 Detailed Metrics:\n",
      "811/811 [==============================] - 31s 38ms/step - loss: 6969.2603 - mse: 28970.1289 - mae: 89.9065 - rmse: 142.8480 - custom_mape: 30.8349 - val_loss: 11942.8125 - val_mse: 15700.4219 - val_mae: 64.9745 - val_rmse: 107.5340 - val_custom_mape: 24.5915 - lr: 1.7500e-04\n",
      "Epoch 42/100\n",
      "811/811 [==============================] - 31s 39ms/step - loss: 6722.7314 - mse: 28121.0098 - mae: 88.7315 - rmse: 141.0947 - custom_mape: 30.9093 - val_loss: 18087.9434 - val_mse: 23490.2754 - val_mae: 77.7477 - val_rmse: 127.0767 - val_custom_mape: 26.3092 - lr: 1.7318e-04\n",
      "Epoch 43/100\n",
      "811/811 [==============================] - 31s 38ms/step - loss: 6352.7544 - mse: 27314.2441 - mae: 87.3129 - rmse: 139.6047 - custom_mape: 30.1676 - val_loss: 12681.6553 - val_mse: 17494.8359 - val_mae: 67.6718 - val_rmse: 111.1969 - val_custom_mape: 24.9519 - lr: 1.7083e-04\n",
      "Epoch 44/100\n",
      "811/811 [==============================] - 30s 37ms/step - loss: 6174.1055 - mse: 26661.8184 - mae: 86.5631 - rmse: 138.3531 - custom_mape: 30.2715 - val_loss: 9485.2715 - val_mse: 17080.8105 - val_mae: 68.2397 - val_rmse: 109.1803 - val_custom_mape: 24.5443 - lr: 1.6795e-04\n",
      "Epoch 45/100\n",
      "811/811 [==============================] - 31s 38ms/step - loss: 5928.2930 - mse: 25945.6465 - mae: 85.3553 - rmse: 136.5068 - custom_mape: 30.0199 - val_loss: 10805.3350 - val_mse: 12679.4893 - val_mae: 59.6128 - val_rmse: 95.3850 - val_custom_mape: 23.4863 - lr: 1.6457e-04\n",
      "Epoch 46/100\n",
      "810/811 [============================>.] - ETA: 0s - loss: 5771.2266 - mse: 25396.0840 - mae: 84.5208 - rmse: 135.4825 - custom_mape: 29.6018\n",
      "Epoch 46 Detailed Metrics:\n",
      "811/811 [==============================] - 31s 38ms/step - loss: 5771.6826 - mse: 25375.7852 - mae: 84.4771 - rmse: 135.4225 - custom_mape: 29.6105 - val_loss: 16886.9219 - val_mse: 15556.0742 - val_mae: 64.1053 - val_rmse: 103.2760 - val_custom_mape: 23.5403 - lr: 1.6071e-04\n",
      "Epoch 47/100\n",
      "811/811 [==============================] - ETA: 0s - loss: 5708.6367 - mse: 24972.6895 - mae: 83.7521 - rmse: 134.5005 - custom_mape: 29.4252\n",
      "Epoch 47: ReduceLROnPlateau reducing learning rate to 3.127976669929922e-05.\n",
      "811/811 [==============================] - 32s 39ms/step - loss: 5708.6367 - mse: 24972.6895 - mae: 83.7521 - rmse: 134.5005 - custom_mape: 29.4252 - val_loss: 19152.4121 - val_mse: 20032.2090 - val_mae: 72.7711 - val_rmse: 117.7244 - val_custom_mape: 24.8048 - lr: 1.5640e-04\n",
      "Epoch 48/100\n",
      "811/811 [==============================] - 31s 39ms/step - loss: 5571.7061 - mse: 24644.9062 - mae: 83.0004 - rmse: 133.9219 - custom_mape: 29.2346 - val_loss: 17331.9844 - val_mse: 16389.8301 - val_mae: 66.4118 - val_rmse: 105.8761 - val_custom_mape: 23.9444 - lr: 1.5165e-04\n",
      "Epoch 49/100\n",
      "811/811 [==============================] - 32s 39ms/step - loss: 5616.5620 - mse: 24360.8047 - mae: 82.9673 - rmse: 133.5276 - custom_mape: 29.4230 - val_loss: 35736.7578 - val_mse: 19179.0918 - val_mae: 70.1682 - val_rmse: 114.7201 - val_custom_mape: 25.1097 - lr: 1.4651e-04\n",
      "Epoch 50/100\n",
      "811/811 [==============================] - ETA: 0s - loss: 5390.3022 - mse: 24032.5449 - mae: 82.3451 - rmse: 132.8925 - custom_mape: 29.1358Restoring model weights from the end of the best epoch: 35.\n",
      "811/811 [==============================] - 31s 38ms/step - loss: 5390.3022 - mse: 24032.5449 - mae: 82.3451 - rmse: 132.8925 - custom_mape: 29.1358 - val_loss: 13423.8428 - val_mse: 18049.5527 - val_mae: 67.9114 - val_rmse: 110.8442 - val_custom_mape: 23.8173 - lr: 1.4100e-04\n",
      "Epoch 50: early stopping\n",
      "\n",
      "Training completed successfully!\n",
      "\n",
      "Final Model Performance:\n",
      "Out of range predictions: 0 (0.00%)\n",
      "MAPE: 25.97%\n",
      "Predictions within ±10%: 50.91%\n",
      "MAE: 75.52\n",
      "RMSE: 146.40\n",
      "\n",
      "Training history plot saved as: 2024-11-20_18-26_training_history.png\n",
      "Training history data saved as: 2024-11-20_18-26_training_history.csv\n",
      "Final statistics saved as: 2024-11-20_18-26_training_stats.json\n",
      "\n",
      "Final Training Statistics:\n",
      "Final Loss (train/val): 5390.3022/13423.8428\n",
      "Final MAE (train/val): 82.3451/67.9114\n",
      "Best validation loss: 9422.6035\n",
      "Best validation MAE: 59.6128\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 88140 characters]",
      "text/plain": [
       "<Figure size 1200x400 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Model creation\n",
    "print(\"\\n2. Creating model...\")\n",
    "input_shape = (X_train_seq.shape[1], X_train_seq.shape[2])\n",
    "\n",
    "MAX_RADIATION = 885\n",
    "\n",
    "min_val_scaled = target_scaler.transform([[0]])[0][0]\n",
    "max_val_scaled = target_scaler.transform([[MAX_RADIATION]])[0][0]\n",
    "\n",
    "model = create_solarradiation_model(input_shape=input_shape, folder_name=folder_name, min_output=min_val_scaled, max_output=max_val_scaled)\n",
    "\n",
    "print(\"\\n4. Starting training...\")\n",
    "history = train_hybrid_model(\n",
    "    model=model,\n",
    "    X_train=X_train_seq,\n",
    "    y_train=y_train,\n",
    "    X_test=X_test_seq,\n",
    "    y_test=y_test,\n",
    "    epochs=100,\n",
    "    batch_size=192,\n",
    "    folder_name=folder_name\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ad6226ea-ab01-47aa-9571-52ea9e654c01",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "5. Generating predictions...\n",
      "811/811 [==============================] - 5s 6ms/step\n",
      "\n",
      "6. Evaluating model...\n",
      "\n",
      "Solar Radiation Prediction Metrics:\n",
      "\n",
      "Absolute Metrics:\n",
      "MAE: 175.39 W/m²\n",
      "RMSE: 313.03 W/m²\n",
      "R² Score: -0.413\n",
      "MAPE: 9983339.55%\n",
      "\n",
      "Percentage Accuracy:\n",
      "Within ±5%: 46.6%\n",
      "Within ±10%: 46.6%\n",
      "Within ±20%: 46.6%\n",
      "\n",
      "Level Accuracy:\n",
      "Level Accuracy: 68.7%\n",
      "\n",
      "Confusion Matrix for Radiation Levels:\n",
      "           Very Low  Low  Moderate  High  Very High  Extreme\n",
      "Very Low          0    0         0     0          0       10\n",
      "Low               0    0         0     0          0     1827\n",
      "Moderate          0    0         0     0          0     2861\n",
      "High              0    0         0     0          0     2296\n",
      "Very High         0    0         0     0          0     1131\n",
      "Extreme           0    0         0     0          0    17808\n",
      "\n",
      "Plot saved as: 2024-11-20_18-26_radiation_analysis.png\n",
      "\n",
      "Error Statistics:\n",
      "Mean error: -175.370\n",
      "Error standard deviation: 259.295\n",
      "Median error: -12.000\n",
      "95th percentile absolute error: 773.000\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n5. Generating predictions...\")\n",
    "predictions = model.predict(X_test_seq)\n",
    "predictions = np.clip(predictions, 0, 11)\n",
    "\n",
    "predictions_original = target_scaler.inverse_transform(predictions)\n",
    "y_test_original = target_scaler.inverse_transform(y_test)\n",
    "\n",
    "print(\"\\n6. Evaluating model...\")\n",
    "metrics = evaluate_solarradiation_predictions(y_test_original, predictions_original, folder_name=folder_name)\n",
    "\n",
    "# Create results dictionary\n",
    "training_results = {\n",
    "    'model_params': {\n",
    "        'input_shape': input_shape,\n",
    "        'n_features': len(features),\n",
    "        'sequence_length': X_train_seq.shape[1]\n",
    "    },\n",
    "    'training_params': {\n",
    "        'batch_size': 32,\n",
    "        'total_epochs': len(history.history['loss']),\n",
    "        'best_epoch': np.argmin(history.history['val_loss']) + 1\n",
    "    },\n",
    "    'performance_metrics': {\n",
    "        'final_loss': float(history.history['val_loss'][-1]),\n",
    "        'final_mae': float(history.history['val_mae'][-1]),\n",
    "        'best_val_loss': float(min(history.history['val_loss'])),\n",
    "        'out_of_range_predictions': int(np.sum((predictions < 0) | (predictions > 11)))\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2c42461d-b189-4dc0-81da-4eb4879b9135",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "7. Predicting missing data...\n",
      "7122/7122 [==============================] - 42s 6ms/step\n",
      "\n",
      "8. Integrating predictions into original dataset...\n",
      "Added 227879 predictions to dataset\n",
      "Rows with solar radiation after integration: 357615\n",
      "\n",
      "Training completed successfully!\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n7. Predicting missing data...\")\n",
    "to_predict_predictions = model.predict(X_to_predict_seq)\n",
    "to_predict_predictions = np.clip(to_predict_predictions, 0, 11)\n",
    "\n",
    "print(\"\\n8. Integrating predictions into original dataset...\")\n",
    "df_updated = integrate_predictions(df.copy(), to_predict_predictions)\n",
    "\n",
    "df_updated.to_parquet('../../sources/weather_data_solarradiation.parquet')\n",
    "\n",
    "# Add prediction statistics to training_results\n",
    "training_results['prediction_stats'] = {\n",
    "    'n_predictions_added': len(to_predict_predictions),\n",
    "    'mean_predicted_solarradiation': float(to_predict_predictions.mean()),\n",
    "    'min_predicted_solarradiation': float(to_predict_predictions.min()),\n",
    "    'max_predicted_solarradiation': float(to_predict_predictions.max()),\n",
    "}\n",
    "\n",
    "print(\"\\nTraining completed successfully!\")\n",
    "\n",
    "tf.keras.backend.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "08fd4208-0afb-4bf1-bdef-b10b4065fe55",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Plot saved as: 2024-11-20_18-26_error_analysis.png\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 71932 characters]",
      "text/plain": [
       "<Figure size 1500x500 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Error Statistics:\n",
      "MAE: 175.3912\n",
      "MSE: 97988.2659\n",
      "RMSE: 313.0308\n",
      "Mean error: -175.3699\n",
      "Error std: 259.2946\n",
      "Predictions within ±0.5: 46.6%\n",
      "Predictions within ±1.0: 47.0%\n",
      "Predictions within ±1.5: 47.0%\n",
      "Predictions within ±2.0: 47.2%\n"
     ]
    }
   ],
   "source": [
    "def plot_error_analysis(y_true, y_pred, folder_name=None):\n",
    "    \"\"\"\n",
    "    Function to visualize prediction error analysis\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    y_true : array-like\n",
    "        Actual values\n",
    "    y_pred : array-like\n",
    "        Predicted values\n",
    "    folder_name : str, optional\n",
    "        Directory to save plots. If None, plots are only displayed\n",
    "\n",
    "    Generates:\n",
    "    ----------\n",
    "    - Error distribution histogram\n",
    "    - Actual vs Predicted scatter plot\n",
    "    - Errors vs Actual Values scatter plot\n",
    "    - Comprehensive error statistics\n",
    "    \"\"\"\n",
    "\n",
    "    # Convert to 1D numpy arrays if needed\n",
    "    if isinstance(y_true, pd.Series):\n",
    "        y_true = y_true.values\n",
    "    if isinstance(y_pred, pd.Series):\n",
    "        y_pred = y_pred.values\n",
    "\n",
    "    y_true = y_true.ravel()\n",
    "    y_pred = y_pred.ravel()\n",
    "\n",
    "    # Calculate errors\n",
    "    errors = y_pred - y_true\n",
    "\n",
    "    # Create main figure\n",
    "    fig = plt.figure(figsize=(15, 5))\n",
    "\n",
    "    # Plot 1: Error Distribution\n",
    "    plt.subplot(1, 3, 1)\n",
    "    plt.hist(errors, bins=50, alpha=0.7)\n",
    "    plt.title('Prediction Error Distribution')\n",
    "    plt.xlabel('Error')\n",
    "    plt.ylabel('Frequency')\n",
    "\n",
    "    # Plot 2: Actual vs Predicted\n",
    "    plt.subplot(1, 3, 2)\n",
    "    plt.scatter(y_true, y_pred, alpha=0.5)\n",
    "    plt.plot([y_true.min(), y_true.max()], [y_true.min(), y_true.max()], 'r--', lw=2)\n",
    "    plt.title('Actual vs Predicted Values')\n",
    "    plt.xlabel('Actual Values')\n",
    "    plt.ylabel('Predicted Values')\n",
    "\n",
    "    # Plot 3: Errors vs Actual Values\n",
    "    plt.subplot(1, 3, 3)\n",
    "    plt.scatter(y_true, errors, alpha=0.5)\n",
    "    plt.axhline(y=0, color='r', linestyle='--')\n",
    "    plt.title('Errors vs Actual Values')\n",
    "    plt.xlabel('Actual Values')\n",
    "    plt.ylabel('Error')\n",
    "\n",
    "    plt.tight_layout()\n",
    "\n",
    "    # Save plot if directory is specified\n",
    "    if folder_name is not None:\n",
    "        try:\n",
    "            # Create directory if it doesn't exist\n",
    "            filename = f'{folder_name}_error_analysis.png'\n",
    "\n",
    "            # Save figure\n",
    "            plt.savefig(filename, dpi=300, bbox_inches='tight')\n",
    "            print(f\"\\nPlot saved as: {filename}\")\n",
    "        except Exception as e:\n",
    "            print(f\"\\nError saving plot: {str(e)}\")\n",
    "\n",
    "    plt.show()\n",
    "\n",
    "    # Print error statistics\n",
    "    print(\"\\nError Statistics:\")\n",
    "    print(f\"MAE: {np.mean(np.abs(errors)):.4f}\")\n",
    "    print(f\"MSE: {np.mean(errors ** 2):.4f}\")\n",
    "    print(f\"RMSE: {np.sqrt(np.mean(errors ** 2)):.4f}\")\n",
    "    print(f\"Mean error: {np.mean(errors):.4f}\")\n",
    "    print(f\"Error std: {np.std(errors):.4f}\")\n",
    "\n",
    "    # Calculate percentage of errors within thresholds\n",
    "    thresholds = [0.5, 1.0, 1.5, 2.0]\n",
    "    for threshold in thresholds:\n",
    "        within_threshold = np.mean(np.abs(errors) <= threshold) * 100\n",
    "        print(f\"Predictions within ±{threshold}: {within_threshold:.1f}%\")\n",
    "\n",
    "\n",
    "# Example usage\n",
    "plot_error_analysis(y_test, predictions, folder_name=folder_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f982c92c-ba99-4df6-b3c8-df92426679db",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
