{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8adcbe0819b88578",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-20T00:55:22.066729Z",
     "start_time": "2024-11-20T00:54:13.878615Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hit:1 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease\n",
      "Hit:2 http://security.ubuntu.com/ubuntu jammy-security InRelease\n",
      "Hit:3 http://archive.ubuntu.com/ubuntu jammy InRelease\n",
      "Hit:4 http://archive.ubuntu.com/ubuntu jammy-updates InRelease\n",
      "Hit:5 http://archive.ubuntu.com/ubuntu jammy-backports InRelease\n",
      "Reading package lists... Done\n",
      "Reading package lists... Done\n",
      "Building dependency tree... Done\n",
      "Reading state information... Done\n",
      "graphviz is already the newest version (2.42.2-6ubuntu0.1).\n",
      "0 upgraded, 0 newly installed, 0 to remove and 121 not upgraded.\n",
      "Requirement already satisfied: tensorflow==2.13.0 in /usr/local/lib/python3.11/dist-packages (2.13.0)\n",
      "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (2.0.0)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (1.6.3)\n",
      "Requirement already satisfied: flatbuffers>=23.1.21 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (23.5.26)\n",
      "Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (0.4.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (0.2.0)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (1.58.0)\n",
      "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (3.9.0)\n",
      "Requirement already satisfied: keras<2.14,>=2.13.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (2.13.1)\n",
      "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (16.0.6)\n",
      "Requirement already satisfied: numpy<=1.24.3,>=1.22 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (1.24.3)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (3.3.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (23.1)\n",
      "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (4.24.3)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (68.2.2)\n",
      "Requirement already satisfied: six>=1.12.0 in /usr/lib/python3/dist-packages (from tensorflow==2.13.0) (1.16.0)\n",
      "Requirement already satisfied: tensorboard<2.14,>=2.13 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (2.13.0)\n",
      "Requirement already satisfied: tensorflow-estimator<2.14,>=2.13.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (2.13.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (2.3.0)\n",
      "Requirement already satisfied: typing-extensions<4.6.0,>=3.6.6 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (4.5.0)\n",
      "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (1.14.1)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (0.37.1)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.11/dist-packages (from astunparse>=1.6.0->tensorflow==2.13.0) (0.41.2)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2.23.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (1.0.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (3.4.4)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2.31.0)\n",
      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (0.7.1)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2.3.7)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (5.3.1)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (0.3.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (4.9)\n",
      "Requirement already satisfied: urllib3>=2.0.5 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2.0.5)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (1.3.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (3.2.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (3.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2023.7.22)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.11/dist-packages (from werkzeug>=1.0.1->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2.1.3)\n",
      "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.11/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (0.5.0)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/lib/python3/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (3.2.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (1.24.3)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (2.2.3)\n",
      "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (1.24.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: keras==2.13.1 in /usr/local/lib/python3.11/dist-packages (2.13.1)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.11/dist-packages (1.5.2)\n",
      "Requirement already satisfied: numpy>=1.19.5 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.24.3)\n",
      "Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.14.1)\n",
      "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.4.2)\n",
      "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (3.5.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: matplotlib in /usr/local/lib/python3.11/dist-packages (3.8.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (4.42.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.4.5)\n",
      "Requirement already satisfied: numpy<2,>=1.21 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.24.3)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (23.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (10.0.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (3.2.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (2.8.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.11/dist-packages (1.4.2)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: pyarrow in /usr/local/lib/python3.11/dist-packages (18.0.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: fastparquet in /usr/local/lib/python3.11/dist-packages (2024.11.0)\n",
      "Requirement already satisfied: pandas>=1.5.0 in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2.2.3)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (from fastparquet) (1.24.3)\n",
      "Requirement already satisfied: cramjam>=2.3 in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2.9.0)\n",
      "Requirement already satisfied: fsspec in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2024.10.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from fastparquet) (23.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2024.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas>=1.5.0->fastparquet) (1.16.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.11/dist-packages (1.14.1)\n",
      "Requirement already satisfied: numpy<2.3,>=1.23.5 in /usr/local/lib/python3.11/dist-packages (from scipy) (1.24.3)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: seaborn in /usr/local/lib/python3.11/dist-packages (0.13.2)\n",
      "Requirement already satisfied: numpy!=1.24.0,>=1.20 in /usr/local/lib/python3.11/dist-packages (from seaborn) (1.24.3)\n",
      "Requirement already satisfied: pandas>=1.2 in /usr/local/lib/python3.11/dist-packages (from seaborn) (2.2.3)\n",
      "Requirement already satisfied: matplotlib!=3.6.1,>=3.4 in /usr/local/lib/python3.11/dist-packages (from seaborn) (3.8.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (4.42.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.4.5)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (23.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (10.0.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (3.2.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.2->seaborn) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.2->seaborn) (2024.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib!=3.6.1,>=3.4->seaborn) (1.16.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (4.67.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: pydot in /usr/local/lib/python3.11/dist-packages (3.0.2)\n",
      "Requirement already satisfied: pyparsing>=3.0.9 in /usr/local/lib/python3.11/dist-packages (from pydot) (3.2.0)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: tensorflow-io in /usr/local/lib/python3.11/dist-packages (0.37.1)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem==0.37.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow-io) (0.37.1)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
      "Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.11/dist-packages (0.23.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from tensorflow-addons) (23.1)\n",
      "Requirement already satisfied: typeguard<3.0.0,>=2.7 in /usr/local/lib/python3.11/dist-packages (from tensorflow-addons) (2.13.3)\n",
      "\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
      "\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n"
     ]
    }
   ],
   "source": [
    "from opt_einsum.paths import branch_1\n",
    "!apt-get update\n",
    "!apt-get install graphviz -y\n",
    "\n",
    "!pip install tensorflow==2.13.0\n",
    "!pip install numpy\n",
    "!pip install pandas\n",
    "!pip install keras==2.13.1\n",
    "!pip install scikit-learn\n",
    "!pip install matplotlib\n",
    "!pip install joblib\n",
    "!pip install pyarrow\n",
    "!pip install fastparquet\n",
    "!pip install scipy\n",
    "!pip install seaborn\n",
    "!pip install tqdm\n",
    "!pip install pydot\n",
    "!pip install tensorflow-io\n",
    "!pip install tensorflow-addons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7a813e3cbca057b7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-20T00:55:22.782689Z",
     "start_time": "2024-11-20T00:55:22.089165Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-20 11:04:20.516922: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "/usr/local/lib/python3.11/dist-packages/tensorflow_addons/utils/tfa_eol_msg.py:23: UserWarning: \n",
      "\n",
      "TensorFlow Addons (TFA) has ended development and introduction of new features.\n",
      "TFA has entered a minimal maintenance and release mode until a planned end of life in May 2024.\n",
      "Please modify downstream libraries to take dependencies from other repositories in our TensorFlow community (e.g. Keras, Keras-CV, and Keras-NLP). \n",
      "\n",
      "For more information see: https://github.com/tensorflow/addons/issues/2807 \n",
      "\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Dense, LSTM, MultiHeadAttention, Dropout, BatchNormalization, LayerNormalization, Input, Activation, Lambda, Bidirectional, Add, MaxPooling1D, Conv1D, GlobalAveragePooling1D\n",
    "from tensorflow.keras import regularizers\n",
    "from tensorflow.keras.models import Model\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\n",
    "from tensorflow.keras.optimizers import AdamW\n",
    "import json\n",
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.keras.utils import plot_model\n",
    "import tensorflow_addons as tfa\n",
    "import os\n",
    "import joblib\n",
    "\n",
    "folder_name = datetime.now().strftime(\"%Y-%m-%d_%H-%M\")\n",
    "random_state_value = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b3f525e19f78a1da",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_season(date):\n",
    "    month = date.month\n",
    "    day = date.day\n",
    "    if (month == 12 and day >= 21) or (month <= 3 and day < 20):\n",
    "        return 'Winter'\n",
    "    elif (month == 3 and day >= 20) or (month <= 6 and day < 21):\n",
    "        return 'Spring'\n",
    "    elif (month == 6 and day >= 21) or (month <= 9 and day < 23):\n",
    "        return 'Summer'\n",
    "    elif (month == 9 and day >= 23) or (month <= 12 and day < 21):\n",
    "        return 'Autumn'\n",
    "    else:\n",
    "        return 'Unknown'\n",
    "\n",
    "\n",
    "def get_time_period(hour):\n",
    "    if 5 <= hour < 12:\n",
    "        return 'Morning'\n",
    "    elif 12 <= hour < 17:\n",
    "        return 'Afternoon'\n",
    "    elif 17 <= hour < 21:\n",
    "        return 'Evening'\n",
    "    else:\n",
    "        return 'Night'\n",
    "\n",
    "\n",
    "def add_time_features(df):\n",
    "    df['datetime'] = pd.to_datetime(df['datetime'])\n",
    "    df['timestamp'] = df['datetime'].astype(np.int64) // 10 ** 9\n",
    "    df['year'] = df['datetime'].dt.year\n",
    "    df['month'] = df['datetime'].dt.month\n",
    "    df['day'] = df['datetime'].dt.day\n",
    "    df['hour'] = df['datetime'].dt.hour\n",
    "    df['minute'] = df['datetime'].dt.minute\n",
    "    df['hour_sin'] = np.sin(df['hour'] * (2 * np.pi / 24))\n",
    "    df['hour_cos'] = np.cos(df['hour'] * (2 * np.pi / 24))\n",
    "    df['day_of_week'] = df['datetime'].dt.dayofweek\n",
    "    df['day_of_year'] = df['datetime'].dt.dayofyear\n",
    "    df['week_of_year'] = df['datetime'].dt.isocalendar().week.astype(int)\n",
    "    df['quarter'] = df['datetime'].dt.quarter\n",
    "    df['is_month_end'] = df['datetime'].dt.is_month_end.astype(int)\n",
    "    df['is_quarter_end'] = df['datetime'].dt.is_quarter_end.astype(int)\n",
    "    df['is_year_end'] = df['datetime'].dt.is_year_end.astype(int)\n",
    "    df['month_sin'] = np.sin(df['month'] * (2 * np.pi / 12))\n",
    "    df['month_cos'] = np.cos(df['month'] * (2 * np.pi / 12))\n",
    "    df['day_of_year_sin'] = np.sin(df['day_of_year'] * (2 * np.pi / 365.25))\n",
    "    df['day_of_year_cos'] = np.cos(df['day_of_year'] * (2 * np.pi / 365.25))\n",
    "    df['season'] = df['datetime'].apply(get_season)\n",
    "    df['time_period'] = df['hour'].apply(get_time_period)\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_solar_features(df):\n",
    "    # Calculate solar angle\n",
    "    df['solar_angle'] = np.sin(df['day_of_year'] * (2 * np.pi / 365.25)) * np.sin(df['hour'] * (2 * np.pi / 24))\n",
    "\n",
    "    # Interactions between relevant features\n",
    "    df['cloud_temp_interaction'] = df['cloudcover'] * df['temp']\n",
    "    df['visibility_cloud_interaction'] = df['visibility'] * (100 - df['cloudcover'])\n",
    "\n",
    "    # Derived features\n",
    "    df['clear_sky_index'] = (100 - df['cloudcover']) / 100\n",
    "    df['temp_gradient'] = df['temp'] - df['tempmin']\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_solar_specific_features(df):\n",
    "    # Solar angle and day length calculations\n",
    "    df['day_length'] = 12 + 3 * np.sin(2 * np.pi * (df['day_of_year'] - 81) / 365.25)\n",
    "    df['solar_noon'] = 12 - df['hour']\n",
    "    df['solar_elevation'] = np.sin(2 * np.pi * df['day_of_year'] / 365.25) * np.cos(2 * np.pi * df['solar_noon'] / 24)\n",
    "\n",
    "    # Feature interactions\n",
    "    df['cloud_elevation'] = df['cloudcover'] * df['solar_elevation']\n",
    "    df['visibility_elevation'] = df['visibility'] * df['solar_elevation']\n",
    "\n",
    "    # Extended window rolling features\n",
    "    df['cloud_rolling_12h'] = df['cloudcover'].rolling(window=12).mean()\n",
    "    df['temp_rolling_12h'] = df['temp'].rolling(window=12).mean()\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_uv_specific_features(df):\n",
    "    # Solar zenith angle calculation\n",
    "    lat = 41.9  # assuming constant latitude for the dataset - Rome's latitude\n",
    "    df['solar_zenith'] = 90 - np.degrees(\n",
    "        np.arcsin(\n",
    "            np.sin(np.radians(lat)) * np.sin(df['solar_elevation']) +\n",
    "            np.cos(np.radians(lat)) * np.cos(df['solar_elevation']) * np.cos(df['hour'] * 15)\n",
    "        )\n",
    "    )\n",
    "\n",
    "    # UV peak hours indicator (10:00-16:00)\n",
    "    df['is_uv_peak_hours'] = ((df['hour'] >= 10) & (df['hour'] <= 16)).astype(int)\n",
    "\n",
    "    # Atmospheric attenuation factor\n",
    "    df['atmospheric_attenuation'] = (100 - df['cloudcover']) * (df['visibility'] / 100) * (1 - df['humidity'] / 200)\n",
    "\n",
    "    # Seasonal UV factor\n",
    "    df['uv_seasonal_factor'] = np.where(df['season_Summer'], 1.0,\n",
    "                                        np.where(df['season_Spring'], 0.7,\n",
    "                                                 np.where(df['season_Autumn'], 0.5, 0.3)))\n",
    "\n",
    "    # Solar elevation and atmospheric transparency interaction\n",
    "    df['solar_clarity_index'] = df['solar_elevation'] * df['atmospheric_attenuation'] / 100\n",
    "\n",
    "    # UV-specific rolling features\n",
    "    df['clarity_rolling_3h'] = df['atmospheric_attenuation'].rolling(window=3).mean()\n",
    "    df['temp_uv_interaction'] = df['temp'] * df['solar_clarity_index']\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_advanced_features(df):\n",
    "    \"\"\"\n",
    "    Add all advanced features in the correct order\n",
    "    \"\"\"\n",
    "    # 1. First add basic time features\n",
    "    df = add_time_features(df)\n",
    "\n",
    "    # 2. One-hot encoding for categorical features\n",
    "    df = pd.get_dummies(df, columns=['season', 'time_period'])\n",
    "\n",
    "    # 3. Add solar and specific features\n",
    "    df = add_solar_features(df)\n",
    "    df = add_solar_specific_features(df)\n",
    "\n",
    "    # 4. Ensure datetime index\n",
    "    if not isinstance(df.index, pd.DatetimeIndex):\n",
    "        df.index = pd.to_datetime(df.index)\n",
    "\n",
    "    # 5. Add weather variable interactions\n",
    "    df['temp_humidity'] = df['temp'] * df['humidity']\n",
    "    df['temp_cloudcover'] = df['temp'] * df['cloudcover']\n",
    "    df['visibility_cloudcover'] = df['visibility'] * df['cloudcover']\n",
    "\n",
    "    # 6. Add solar radiation derived features\n",
    "    df['clear_sky_factor'] = (100 - df['cloudcover']) / 100\n",
    "    df['day_length'] = np.sin(df['day_of_year_sin']) * 12 + 12\n",
    "\n",
    "    # 7. Add lag features\n",
    "    df['temp_1h_lag'] = df['temp'].shift(1)\n",
    "    df['cloudcover_1h_lag'] = df['cloudcover'].shift(1)\n",
    "    df['humidity_1h_lag'] = df['humidity'].shift(1)\n",
    "\n",
    "    # 8. Add rolling means\n",
    "    df['temp_rolling_mean_6h'] = df['temp'].rolling(window=6).mean()\n",
    "    df['cloudcover_rolling_mean_6h'] = df['cloudcover'].rolling(window=6).mean()\n",
    "    df['temp_humidity_interaction'] = df['temp'] * df['humidity'] / 100\n",
    "\n",
    "    # 9. Add atmospheric stability\n",
    "    df['atmospheric_stability'] = df.groupby(df.index.date)['pressure'].transform(\n",
    "        lambda x: x.std()\n",
    "    ).fillna(0)\n",
    "\n",
    "    # 10. Add extreme conditions indicator\n",
    "    df['extreme_conditions'] = ((df['temp'] > df['temp'].quantile(0.75)) &\n",
    "                                (df['humidity'] < df['humidity'].quantile(0.25))).astype(int)\n",
    "\n",
    "    # 11. Add atmospheric transparency\n",
    "    df['atmospheric_transparency'] = (100 - df['cloudcover']) * (df['visibility'] / 10)\n",
    "\n",
    "    # 12. Add transitional seasons indicator\n",
    "    df['is_transition_season'] = ((df['season_Spring'] | df['season_Autumn'])).astype(int)\n",
    "\n",
    "    # 13. Add solar cloud effect\n",
    "    if 'solar_elevation' in df.columns:\n",
    "        df['solar_cloud_effect'] = df['solar_elevation'] * (100 - df['cloudcover']) / 100\n",
    "\n",
    "    # 14. Finally add UV specific features\n",
    "    df = add_uv_specific_features(df)\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def prepare_advanced_data(df):\n",
    "    \"\"\"\n",
    "    Prepares data for UV index prediction model with advanced feature engineering\n",
    "    and optimized preprocessing.\n",
    "\n",
    "    Args:\n",
    "        df: DataFrame with meteorological data\n",
    "\n",
    "    Returns:\n",
    "        tuple: (X_train_scaled, X_test_scaled, y_train, y_test, scaler, final_features, X_to_predict_scaled)\n",
    "    \"\"\"\n",
    "    # Apply feature engineering functions\n",
    "    df = add_advanced_features(df)\n",
    "\n",
    "    # Optimized feature selection for UV index\n",
    "    selected_features = {\n",
    "        # Primary meteorological features\n",
    "        'atmospheric': [\n",
    "            'temp', 'humidity', 'cloudcover', 'visibility',\n",
    "            'clear_sky_index', 'atmospheric_transparency'\n",
    "        ],\n",
    "\n",
    "        # Essential temporal features\n",
    "        'temporal': [\n",
    "            'hour_sin', 'hour_cos',\n",
    "            'day_of_year_sin', 'day_of_year_cos'\n",
    "        ],\n",
    "\n",
    "        # Solar features\n",
    "        'solar': [\n",
    "            'solar_angle', 'solar_elevation',\n",
    "            'day_length', 'solar_noon',\n",
    "            'solar_cloud_effect'\n",
    "        ],\n",
    "\n",
    "        # Key interactions\n",
    "        'interactions': [\n",
    "            'cloud_temp_interaction',\n",
    "            'visibility_cloud_interaction',\n",
    "            'temp_humidity_interaction',\n",
    "            'solar_clarity_index'\n",
    "        ],\n",
    "\n",
    "        # Rolling features\n",
    "        'rolling': [\n",
    "            'cloud_rolling_12h',\n",
    "            'temp_rolling_mean_6h'\n",
    "        ]\n",
    "    }\n",
    "\n",
    "    # Flatten feature list\n",
    "    base_features = [item for sublist in selected_features.values() for item in sublist]\n",
    "\n",
    "    # Add categorical features (one-hot encoded)\n",
    "    categorical_columns = [col for col in df.columns if col.startswith(('season_', 'time_period_'))]\n",
    "    final_features = base_features + categorical_columns\n",
    "\n",
    "    # Temporal preprocessing\n",
    "    df = df.sort_values('datetime')\n",
    "    df.set_index('datetime', inplace=True)\n",
    "\n",
    "    # Advanced interpolation for missing values\n",
    "    for column in final_features:\n",
    "        if column in df.columns:\n",
    "            if df[column].isnull().any():\n",
    "                if column in selected_features['rolling']:\n",
    "                    df[column] = df[column].ffill().bfill()\n",
    "                else:\n",
    "                    df[column] = df[column].interpolate(method='time', limit_direction='both')\n",
    "\n",
    "    # Temporal data split\n",
    "    data_after_2010 = df[df.index.year >= 2010].copy()\n",
    "    data_before_2010 = df[df.index.year < 2010].copy()\n",
    "\n",
    "    print(f\"\\nTemporal distribution of data:\")\n",
    "    print(f\"Records after 2010: {len(data_after_2010):,}\")\n",
    "    print(f\"Records before 2010: {len(data_before_2010):,}\")\n",
    "\n",
    "    # Feature and target preparation\n",
    "    X = data_after_2010[final_features]\n",
    "    y = data_after_2010['uvindex']\n",
    "    X_to_predict = data_before_2010[final_features]\n",
    "\n",
    "    # Data validation\n",
    "    if X.isnull().any().any() or y.isnull().any():\n",
    "        print(\"\\nWarning: Found missing values after preprocessing\")\n",
    "        print(\"Features with missing values:\", X.columns[X.isnull().any()].tolist())\n",
    "        X = X.fillna(X.mean())\n",
    "        y = y.fillna(y.mean())\n",
    "\n",
    "    # Stratified data split\n",
    "    X_train, X_test, y_train, y_test = train_test_split(\n",
    "        X, y,\n",
    "        test_size=0.5,\n",
    "        random_state=random_state_value,\n",
    "        stratify=pd.qcut(y, q=5, duplicates='drop', labels=False)\n",
    "    )\n",
    "\n",
    "    # Robust feature scaling\n",
    "    feature_scaler = RobustScaler()\n",
    "    X_train_scaled = feature_scaler.fit_transform(X_train)\n",
    "    X_test_scaled = feature_scaler.transform(X_test)\n",
    "    X_to_predict_scaled = feature_scaler.transform(X_to_predict)\n",
    "\n",
    "    target_scaler = RobustScaler()\n",
    "    y_train_scaled = target_scaler.fit_transform(y_train.values.reshape(-1, 1)).ravel()\n",
    "    y_test_scaled = target_scaler.transform(y_test.values.reshape(-1, 1)).ravel()\n",
    "\n",
    "    # Final validation\n",
    "    assert not np.isnan(X_train_scaled).any(), \"Found NaN in X_train_scaled\"\n",
    "    assert not np.isnan(X_test_scaled).any(), \"Found NaN in X_test_scaled\"\n",
    "    assert not np.isnan(X_to_predict_scaled).any(), \"Found NaN in X_to_predict_scaled\"\n",
    "\n",
    "    # Print feature information\n",
    "    print(\"\\nNumber of features used:\", len(final_features))\n",
    "    print(\"\\nFeature categories:\")\n",
    "    for category, features in selected_features.items():\n",
    "        print(f\"{category}: {len(features)} features\")\n",
    "    print(f\"Categorical: {len(categorical_columns)} features\")\n",
    "\n",
    "    return (X_train_scaled, X_test_scaled, y_train_scaled, y_test_scaled,\n",
    "            feature_scaler, target_scaler, final_features, X_to_predict_scaled)\n",
    "\n",
    "\n",
    "def create_sequence_data(X, sequence_length=24):\n",
    "    \"\"\"\n",
    "    Converts data into sequences for LSTM input\n",
    "    sequence_length represents how many previous hours to consider\n",
    "    \"\"\"\n",
    "    sequences = []\n",
    "    for i in range(len(X) - sequence_length + 1):\n",
    "        sequences.append(X[i:i + sequence_length])\n",
    "    return np.array(sequences)\n",
    "\n",
    "\n",
    "def prepare_hybrid_data(df):\n",
    "    # Use existing data preparation\n",
    "    X_train_scaled, X_test_scaled, y_train, y_test, feature_scaler, target_scaler, features, X_to_predict_scaled = prepare_advanced_data(df)\n",
    "\n",
    "    # Convert data to sequences\n",
    "    sequence_length = 24  # 24 hours of historical data\n",
    "\n",
    "    X_train_seq = create_sequence_data(X_train_scaled, sequence_length)\n",
    "    X_test_seq = create_sequence_data(X_test_scaled, sequence_length)\n",
    "\n",
    "    # Adjust y by removing the first (sequence_length-1) elements\n",
    "    y_train = y_train[sequence_length - 1:]\n",
    "    y_test = y_test[sequence_length - 1:]\n",
    "\n",
    "    X_to_predict_seq = create_sequence_data(X_to_predict_scaled, sequence_length)\n",
    "\n",
    "    return X_train_seq, X_test_seq, y_train, y_test, feature_scaler, target_scaler, features, X_to_predict_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9dff3259-b376-4cfc-89d8-ab2ea18aaa5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_residual_lstm_layer(x, units, dropout_rate, l2_reg=0.01,\n",
    "                               survival_probability=0.8, return_sequences=True):\n",
    "    \"\"\"LSTM layer with stochastic depth\"\"\"\n",
    "    residual = x\n",
    "\n",
    "    # Main path\n",
    "    x = Bidirectional(LSTM(units, return_sequences=return_sequences,\n",
    "                           kernel_regularizer=regularizers.l2(l2_reg)))(x)\n",
    "    x = LayerNormalization()(x)\n",
    "    x = Dropout(dropout_rate)(x)\n",
    "\n",
    "    # Adjust residual dimension if needed\n",
    "    if return_sequences:\n",
    "        # For Bidirectional LSTM, the output dimension is 2 * units\n",
    "        target_dim = 2 * units\n",
    "        if int(residual.shape[-1]) != target_dim:\n",
    "            # Use Dense layer instead of Conv1D for better dimension matching\n",
    "            residual = Dense(target_dim)(residual)\n",
    "\n",
    "    # Apply stochastic depth only if dimensions match\n",
    "    if x.shape[-1] == residual.shape[-1]:\n",
    "        x = tfa.layers.StochasticDepth(survival_probability)([x, residual])\n",
    "    else:\n",
    "        print(f\"Warning: Dimension mismatch - x: {x.shape}, residual: {residual.shape}\")\n",
    "        # Skip residual connection if dimensions don't match\n",
    "        pass\n",
    "\n",
    "    return x\n",
    "\n",
    "\n",
    "def attention_block(x, units, num_heads=8, survival_probability=0.8):\n",
    "    \"\"\"\n",
    "    Attention block with stochastic depth.\n",
    "    \"\"\"\n",
    "    original_x = x\n",
    "\n",
    "    # Compute self-attention\n",
    "    attention = MultiHeadAttention(num_heads=num_heads, key_dim=units)(x, x)\n",
    "\n",
    "    # Ensure dimensions match before applying stochastic depth\n",
    "    if attention.shape[-1] != original_x.shape[-1]:\n",
    "        original_x = Dense(attention.shape[-1])(original_x)\n",
    "\n",
    "    # Apply stochastic depth to the attention path\n",
    "    x = tfa.layers.StochasticDepth(survival_probability)([attention, original_x])\n",
    "    x = LayerNormalization()(x)\n",
    "\n",
    "    # Store the input to the FFN\n",
    "    ffn_input = x\n",
    "\n",
    "    # FFN block\n",
    "    x = Dense(units * 4, activation='swish')(x)\n",
    "    x = Dense(ffn_input.shape[-1])(x)  # Match the input dimension\n",
    "\n",
    "    # Apply stochastic depth to the FFN\n",
    "    x = tfa.layers.StochasticDepth(survival_probability)([x, ffn_input])\n",
    "    x = LayerNormalization()(x)\n",
    "\n",
    "    return x\n",
    "\n",
    "\n",
    "def create_uv_index_model(input_shape, folder_name, l2_lambda=0.005):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    # Further adjusted hyperparameters\n",
    "    survival_probs = [0.98, 0.95, 0.92]  # Even higher survival probabilities\n",
    "    attention_survival_probs = [0.95, 0.92, 0.9]\n",
    "\n",
    "    # First LSTM block\n",
    "    x = create_residual_lstm_layer(\n",
    "        inputs, 64, dropout_rate=0.2,  # Further reduced dropout\n",
    "        l2_reg=l2_lambda,\n",
    "        survival_probability=survival_probs[0],\n",
    "        return_sequences=True\n",
    "    )\n",
    "    x = attention_block(x, 128, num_heads=2,  # Reduced heads\n",
    "                        survival_probability=attention_survival_probs[0])\n",
    "\n",
    "    # Second LSTM block\n",
    "    x = create_residual_lstm_layer(\n",
    "        x, 32, dropout_rate=0.15,\n",
    "        l2_reg=l2_lambda,\n",
    "        survival_probability=survival_probs[1],\n",
    "        return_sequences=True\n",
    "    )\n",
    "    x = attention_block(x, 64, num_heads=2,\n",
    "                        survival_probability=attention_survival_probs[1])\n",
    "\n",
    "    # Third LSTM block\n",
    "    x = create_residual_lstm_layer(\n",
    "        x, 16, dropout_rate=0.1,\n",
    "        l2_reg=l2_lambda,\n",
    "        survival_probability=survival_probs[2],\n",
    "        return_sequences=True\n",
    "    )\n",
    "    x = attention_block(x, 32, num_heads=2,\n",
    "                        survival_probability=attention_survival_probs[2])\n",
    "\n",
    "    # Global attention with reduced complexity\n",
    "    x_input = x\n",
    "    x = MultiHeadAttention(num_heads=2, key_dim=32)(x, x)\n",
    "\n",
    "    if x.shape[-1] != x_input.shape[-1]:\n",
    "        x_input = Dense(x.shape[-1])(x_input)\n",
    "\n",
    "    x = tfa.layers.StochasticDepth(survival_probability=0.95)([x, x_input])\n",
    "    x = LayerNormalization()(x)\n",
    "\n",
    "    # Simplified dense layers\n",
    "    x = GlobalAveragePooling1D()(x)\n",
    "\n",
    "    # Gradual dimension reduction\n",
    "    x = Dense(32, activation='swish', kernel_regularizer=regularizers.l2(l2_lambda / 2), kernel_constraint=tf.keras.constraints.MaxNorm(3))(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dropout(0.05)(x)  # Minimal dropout\n",
    "\n",
    "    x = Dense(16, activation='swish',\n",
    "              kernel_regularizer=regularizers.l2(l2_lambda / 2))(x)\n",
    "    x = BatchNormalization()(x)\n",
    "\n",
    "    # Modified output layer\n",
    "    x = Dense(8, activation='swish')(x)\n",
    "    outputs = Dense(1, activation='sigmoid')(x)  # Sigmoid activation\n",
    "    outputs = Lambda(lambda x: x * 11.0)(outputs)  # Scale to [0, 11] range\n",
    "\n",
    "    model = Model(inputs=inputs, outputs=outputs, name=\"UvModel\")\n",
    "\n",
    "    # More stable learning rate schedule\n",
    "    initial_learning_rate = 0.0001  # Further reduced\n",
    "    warmup_steps = 1000\n",
    "    decay_steps = 5000\n",
    "\n",
    "    # Corretto learning rate schedule\n",
    "    class CustomLRSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
    "        def __init__(self, initial_lr=0.0001, warmup_steps=1000, decay_steps=5000):\n",
    "            super().__init__()\n",
    "            self.initial_lr = initial_lr\n",
    "            self.warmup_steps = warmup_steps\n",
    "            self.decay_steps = decay_steps\n",
    "\n",
    "        def __call__(self, step):\n",
    "            # Convert to float32\n",
    "            step_f = tf.cast(step, tf.float32)\n",
    "            warmup_steps_f = tf.cast(self.warmup_steps, tf.float32)\n",
    "            decay_steps_f = tf.cast(self.decay_steps, tf.float32)\n",
    "\n",
    "            # Warmup phase\n",
    "            warmup_progress = step_f / warmup_steps_f\n",
    "            warmup_lr = self.initial_lr * warmup_progress\n",
    "\n",
    "            # Decay phase\n",
    "            decay_progress = (step_f - warmup_steps_f) / decay_steps_f\n",
    "            decay_factor = 0.5 * (1.0 + tf.cos(tf.constant(np.pi) * decay_progress))\n",
    "            decay_lr = self.initial_lr * decay_factor\n",
    "\n",
    "            # Combine phases\n",
    "            lr = tf.where(step_f < warmup_steps_f, warmup_lr, decay_lr)\n",
    "            return lr\n",
    "\n",
    "        def get_config(self):\n",
    "            return {\n",
    "                \"initial_lr\": self.initial_lr,\n",
    "                \"warmup_steps\": self.warmup_steps,\n",
    "                \"decay_steps\": self.decay_steps\n",
    "            }\n",
    "\n",
    "    # Utilizzo dello schedule corretto\n",
    "    lr_schedule = CustomLRSchedule(\n",
    "        initial_lr=0.0001,\n",
    "        warmup_steps=1000,\n",
    "        decay_steps=5000\n",
    "    )\n",
    "\n",
    "    optimizer = AdamW(\n",
    "        learning_rate=lr_schedule,\n",
    "        weight_decay=0.0005,\n",
    "        beta_1=0.9,\n",
    "        beta_2=0.999,\n",
    "        epsilon=1e-7\n",
    "    )\n",
    "\n",
    "    # Improved loss function\n",
    "    def smooth_uv_loss(y_true, y_pred):\n",
    "        # Basic MSE with smoothing\n",
    "        mse = tf.square(y_true - y_pred)\n",
    "\n",
    "        # Smooth L1 component for better stability\n",
    "        abs_diff = tf.abs(y_true - y_pred)\n",
    "        smooth_l1 = tf.where(abs_diff < 1.0,\n",
    "                             0.5 * tf.square(abs_diff),\n",
    "                             abs_diff - 0.5)\n",
    "\n",
    "        # Combined loss with dynamic weighting\n",
    "        combined_loss = 0.7 * mse + 0.3 * smooth_l1\n",
    "\n",
    "        # Gentle weighting for high UV values\n",
    "        high_uv_weight = tf.where(y_true >= 8.0, 1.2, 1.0)\n",
    "\n",
    "        # Smooth peak hours weight\n",
    "        time_of_day = tf.cast(tf.math.floormod(tf.range(tf.shape(y_true)[0]), 24),\n",
    "                              tf.float32)\n",
    "        peak_weight = 1.0 + 0.2 * tf.math.sigmoid((time_of_day - 10.0) * 0.5) * \\\n",
    "                      tf.math.sigmoid((16.0 - time_of_day) * 0.5)\n",
    "\n",
    "        total_weight = high_uv_weight * peak_weight\n",
    "\n",
    "        return tf.reduce_mean(combined_loss * total_weight)\n",
    "\n",
    "    # Improved MAPE metric\n",
    "    def smooth_mape(y_true, y_pred):\n",
    "        epsilon = 1e-7\n",
    "        diff = tf.abs(y_true - y_pred)\n",
    "        scale = tf.maximum(tf.abs(y_true) + epsilon, 0.5)  # Minimum scale of 0.5\n",
    "        return tf.reduce_mean(diff / scale) * 100\n",
    "\n",
    "    model.compile(\n",
    "        optimizer=optimizer,\n",
    "        loss=smooth_uv_loss,\n",
    "        metrics=[\n",
    "            'mae',\n",
    "            'mse',\n",
    "            tf.keras.metrics.RootMeanSquaredError(),\n",
    "            smooth_mape\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "def evaluate_uv_predictions(y_true, y_pred, folder_name=None):\n",
    "    \"\"\"\n",
    "    Comprehensive evaluation of UV index predictions with detailed analysis and visualizations.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    y_true : array-like\n",
    "        Actual UV index values\n",
    "    y_pred : array-like\n",
    "        Predicted UV index values\n",
    "    folder_name : str, optional\n",
    "        Folder to save analysis plots\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    dict\n",
    "        Dictionary containing all calculated metrics\n",
    "    \"\"\"\n",
    "    import os\n",
    "    from datetime import datetime\n",
    "    import seaborn as sns\n",
    "    from sklearn.metrics import confusion_matrix, mean_absolute_error, mean_squared_error, r2_score\n",
    "\n",
    "    # Initialize plot paths\n",
    "    main_plot_path = None\n",
    "    conf_matrix_path = None\n",
    "\n",
    "    # Data preprocessing\n",
    "    y_true = np.array(y_true).ravel()\n",
    "    y_pred = np.array(y_pred).ravel()\n",
    "\n",
    "    # Rounding and clipping predictions\n",
    "    y_pred_rounded = np.round(y_pred * 2) / 2  # Round to nearest 0.5\n",
    "    y_pred_clipped = np.clip(y_pred_rounded, 0, 11)\n",
    "\n",
    "    # Calculate errors\n",
    "    errors = y_pred - y_true\n",
    "    errors_rounded = y_pred_clipped - y_true\n",
    "\n",
    "    # Function to determine UV risk level\n",
    "    def get_uv_risk_level(values):\n",
    "        levels = np.full_like(values, 'Low', dtype=object)\n",
    "        levels[(values > 2) & (values <= 5)] = 'Moderate'\n",
    "        levels[(values > 5) & (values <= 7)] = 'High'\n",
    "        levels[(values > 7) & (values <= 10)] = 'Very High'\n",
    "        levels[values > 10] = 'Extreme'\n",
    "        return levels\n",
    "\n",
    "    # Calculate basic metrics\n",
    "    metrics = {\n",
    "        'raw': {\n",
    "            'mae': mean_absolute_error(y_true, y_pred),\n",
    "            'rmse': np.sqrt(mean_squared_error(y_true, y_pred)),\n",
    "            'r2': r2_score(y_true, y_pred),\n",
    "            'mean_error': np.mean(errors),\n",
    "            'std_error': np.std(errors),\n",
    "            'median_error': np.median(errors),\n",
    "            'p95_abs_error': np.percentile(np.abs(errors), 95)\n",
    "        },\n",
    "        'rounded': {\n",
    "            'mae': mean_absolute_error(y_true, y_pred_clipped),\n",
    "            'rmse': np.sqrt(mean_squared_error(y_true, y_pred_clipped)),\n",
    "            'r2': r2_score(y_true, y_pred_clipped)\n",
    "        }\n",
    "    }\n",
    "\n",
    "    # Calculate accuracies for different margins\n",
    "    for data_type, errors_data in [('raw', errors), ('rounded', errors_rounded)]:\n",
    "        metrics[data_type].update({\n",
    "            'within_05': np.mean(np.abs(errors_data) <= 0.5) * 100,\n",
    "            'within_1': np.mean(np.abs(errors_data) <= 1.0) * 100,\n",
    "            'within_15': np.mean(np.abs(errors_data) <= 1.5) * 100,\n",
    "            'within_2': np.mean(np.abs(errors_data) <= 2.0) * 100\n",
    "        })\n",
    "\n",
    "    # Analysis by UV risk level\n",
    "    y_true_risk = get_uv_risk_level(y_true)\n",
    "    y_pred_risk = get_uv_risk_level(y_pred_clipped)\n",
    "\n",
    "    # Calculate confusion matrix with handling for missing classes\n",
    "    risk_levels = ['Low', 'Moderate', 'High', 'Very High', 'Extreme']\n",
    "\n",
    "    # Get unique labels present in the data\n",
    "    present_labels = np.unique(np.concatenate([y_true_risk, y_pred_risk]))\n",
    "\n",
    "    # Calculate confusion matrix for present labels\n",
    "    cm = confusion_matrix(y_true_risk, y_pred_risk, labels=present_labels)\n",
    "\n",
    "    # Create full confusion matrix with zeros\n",
    "    full_cm = np.zeros((len(risk_levels), len(risk_levels)))\n",
    "\n",
    "    # Map present labels to their positions in the full matrix\n",
    "    label_positions = {label: i for i, label in enumerate(risk_levels)}\n",
    "    for i, true_label in enumerate(present_labels):\n",
    "        for j, pred_label in enumerate(present_labels):\n",
    "            full_cm[label_positions[true_label], label_positions[pred_label]] = cm[i, j]\n",
    "\n",
    "    # Create DataFrame with all risk levels\n",
    "    cm_df = pd.DataFrame(full_cm, columns=risk_levels, index=risk_levels)\n",
    "\n",
    "    # Analysis by UV range\n",
    "    uv_ranges = [\n",
    "        (0, 2, 'Low'),\n",
    "        (2, 5, 'Moderate'),\n",
    "        (5, 7, 'High'),\n",
    "        (7, 10, 'Very High'),\n",
    "        (10, 11, 'Extreme')\n",
    "    ]\n",
    "\n",
    "    range_analysis = {}\n",
    "    for low, high, label in uv_ranges:\n",
    "        mask = (y_true >= low) & (y_true < high)\n",
    "        if mask.any():\n",
    "            range_analysis[label] = {\n",
    "                'mae': mean_absolute_error(y_true[mask], y_pred[mask]),\n",
    "                'count': np.sum(mask),\n",
    "                'accuracy_within_05': np.mean(np.abs(errors[mask]) <= 0.5) * 100,\n",
    "                'accuracy_within_1': np.mean(np.abs(errors[mask]) <= 1.0) * 100\n",
    "            }\n",
    "\n",
    "    # Visualizations\n",
    "    if folder_name is not None:\n",
    "        try:\n",
    "            # Main figure with 4 subplots\n",
    "            fig = plt.figure(figsize=(20, 15))\n",
    "\n",
    "            # 1. Error distribution\n",
    "            plt.subplot(2, 2, 1)\n",
    "            plt.hist(errors, bins=50, alpha=0.7)\n",
    "            plt.title('Prediction Error Distribution')\n",
    "            plt.xlabel('Error')\n",
    "            plt.ylabel('Frequency')\n",
    "\n",
    "            # 2. Actual vs Predicted scatter plot\n",
    "            plt.subplot(2, 2, 2)\n",
    "            plt.scatter(y_true, y_pred, alpha=0.5)\n",
    "            plt.plot([0, 11], [0, 11], 'r--', lw=2)\n",
    "            plt.title('Actual vs Predicted Values')\n",
    "            plt.xlabel('Actual Values')\n",
    "            plt.ylabel('Predicted Values')\n",
    "\n",
    "            # 3. Errors vs Actual Values\n",
    "            plt.subplot(2, 2, 3)\n",
    "            plt.scatter(y_true, errors, alpha=0.5)\n",
    "            plt.axhline(y=0, color='r', linestyle='--')\n",
    "            plt.title('Errors vs Actual Values')\n",
    "            plt.xlabel('Actual Values')\n",
    "            plt.ylabel('Error')\n",
    "\n",
    "            # 4. Accuracy and MAE by range\n",
    "            ax = plt.subplot(2, 2, 4)\n",
    "            x_labels = [f\"{label}\\n({low}-{high})\" for low, high, label in uv_ranges]\n",
    "            accuracies = [range_analysis[label]['accuracy_within_05']\n",
    "                          for _, _, label in uv_ranges if label in range_analysis]\n",
    "            mae_values = [range_analysis[label]['mae']\n",
    "                          for _, _, label in uv_ranges if label in range_analysis]\n",
    "\n",
    "            bars = plt.bar(x_labels, accuracies, alpha=0.6)\n",
    "            plt.ylabel('Accuracy within ±0.5 (%)')\n",
    "            plt.title('Accuracy and MAE by UV Range')\n",
    "\n",
    "            # Add MAE as line\n",
    "            ax2 = ax.twinx()\n",
    "            ax2.plot(x_labels, mae_values, 'r-o', label='MAE')\n",
    "            ax2.set_ylabel('MAE', color='red')\n",
    "\n",
    "            plt.tight_layout()\n",
    "\n",
    "            # Save main figure\n",
    "            main_plot_path = f'{folder_name}_uv_analysis.png'\n",
    "            plt.savefig(main_plot_path, dpi=300, bbox_inches='tight')\n",
    "\n",
    "            # Confusion matrix as separate plot\n",
    "            plt.figure(figsize=(10, 8))\n",
    "            sns.heatmap(cm_df, annot=True, fmt='d', cmap='Blues')\n",
    "            plt.title('Confusion Matrix for UV Risk Levels')\n",
    "\n",
    "            conf_matrix_path = f'{folder_name}_confusion_matrix.png'\n",
    "            plt.savefig(conf_matrix_path, dpi=300, bbox_inches='tight')\n",
    "\n",
    "            plt.close('all')\n",
    "\n",
    "        except Exception as e:\n",
    "            print(f\"\\nError saving plots: {str(e)}\")\n",
    "            main_plot_path = None\n",
    "            conf_matrix_path = None\n",
    "\n",
    "    # Print detailed report\n",
    "    print(\"\\nUV Index Prediction Analysis:\")\n",
    "    print(\"\\nRaw Metrics:\")\n",
    "    for key, value in metrics['raw'].items():\n",
    "        print(f\"{key}: {value:.3f}\")\n",
    "\n",
    "    print(\"\\nRounded Metrics:\")\n",
    "    for key, value in metrics['rounded'].items():\n",
    "        print(f\"{key}: {value:.3f}\")\n",
    "\n",
    "    print(\"\\nAnalysis by UV Range:\")\n",
    "    for label, stats in range_analysis.items():\n",
    "        print(f\"\\n{label}:\")\n",
    "        for key, value in stats.items():\n",
    "            print(f\"  {key}: {value:.3f}\")\n",
    "\n",
    "    print(\"\\nConfusion Matrix:\")\n",
    "    print(cm_df)\n",
    "\n",
    "    # Add range analysis and confusion matrix to metrics dictionary\n",
    "    metrics.update({\n",
    "        'range_analysis': range_analysis,\n",
    "        'confusion_matrix': cm_df.to_dict(),\n",
    "        'plot_paths': {\n",
    "            'main_analysis': main_plot_path,\n",
    "            'confusion_matrix': conf_matrix_path\n",
    "        }\n",
    "    })\n",
    "\n",
    "    return metrics\n",
    "\n",
    "\n",
    "def plot_training_history(history, folder_name=None):\n",
    "    \"\"\"\n",
    "    Visualize and save the loss and metrics plots during training\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    history : tensorflow.keras.callbacks.History\n",
    "        The history object returned by model training\n",
    "    folder_name : str\n",
    "        Folder where to save the plot\n",
    "    \"\"\"\n",
    "    import os\n",
    "\n",
    "    try:\n",
    "        # Create the figure\n",
    "        plt.figure(figsize=(12, 4))\n",
    "\n",
    "        # Loss Plot\n",
    "        plt.subplot(1, 2, 1)\n",
    "        plt.plot(history.history['loss'], label='Training Loss')\n",
    "        plt.plot(history.history['val_loss'], label='Validation Loss')\n",
    "        plt.title('Model Loss')\n",
    "        plt.xlabel('Epoch')\n",
    "        plt.ylabel('Loss')\n",
    "        plt.legend()\n",
    "        plt.grid(True)\n",
    "\n",
    "        # MAE Plot\n",
    "        plt.subplot(1, 2, 2)\n",
    "        plt.plot(history.history['mae'], label='Training MAE')\n",
    "        plt.plot(history.history['val_mae'], label='Validation MAE')\n",
    "        plt.title('Model MAE')\n",
    "        plt.xlabel('Epoch')\n",
    "        plt.ylabel('MAE')\n",
    "        plt.legend()\n",
    "        plt.grid(True)\n",
    "\n",
    "        plt.tight_layout()\n",
    "\n",
    "        if folder_name is not None:\n",
    "            os.makedirs(folder_name, exist_ok=True)\n",
    "            # Generate filename with timestamp\n",
    "            filename = os.path.join(folder_name, 'training_history.png')\n",
    "\n",
    "            # Save the figure\n",
    "            plt.savefig(filename, dpi=300, bbox_inches='tight')\n",
    "            print(f\"\\nTraining history plot saved as: {filename}\")\n",
    "\n",
    "        # Also save numerical data in CSV format\n",
    "        history_df = pd.DataFrame({\n",
    "            'epoch': range(1, len(history.history['loss']) + 1),\n",
    "            'training_loss': history.history['loss'],\n",
    "            'validation_loss': history.history['val_loss'],\n",
    "            'training_mae': history.history['mae'],\n",
    "            'validation_mae': history.history['val_mae']\n",
    "        })\n",
    "\n",
    "        if folder_name is not None:\n",
    "            csv_filename = os.path.join(folder_name, 'training_history.csv')\n",
    "            history_df.to_csv(csv_filename, index=False)\n",
    "            print(f\"Training history data saved as: {csv_filename}\")\n",
    "\n",
    "        # Calculate and save final statistics\n",
    "        final_stats = {\n",
    "            'final_training_loss': history.history['loss'][-1],\n",
    "            'final_validation_loss': history.history['val_loss'][-1],\n",
    "            'final_training_mae': history.history['mae'][-1],\n",
    "            'final_validation_mae': history.history['val_mae'][-1],\n",
    "            'best_validation_loss': min(history.history['val_loss']),\n",
    "            'best_validation_mae': min(history.history['val_mae']),\n",
    "            'epochs': len(history.history['loss']),\n",
    "        }\n",
    "\n",
    "        if folder_name is not None:\n",
    "            # Save statistics in JSON format\n",
    "            stats_filename = os.path.join(folder_name, 'training_stats.json')\n",
    "            with open(stats_filename, 'w') as f:\n",
    "                json.dump(final_stats, f, indent=4)\n",
    "            print(f\"Final statistics saved as: {stats_filename}\")\n",
    "\n",
    "        # Print main statistics\n",
    "        print(\"\\nFinal training statistics:\")\n",
    "        print(f\"Final Loss (train/val): {final_stats['final_training_loss']:.4f}/{final_stats['final_validation_loss']:.4f}\")\n",
    "        print(f\"Final MAE (train/val): {final_stats['final_training_mae']:.4f}/{final_stats['final_validation_mae']:.4f}\")\n",
    "        print(f\"Best validation loss: {final_stats['best_validation_loss']:.4f}\")\n",
    "        print(f\"Best validation MAE: {final_stats['best_validation_mae']:.4f}\")\n",
    "\n",
    "        plt.show()\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"\\nError during plot creation or saving: {str(e)}\")\n",
    "\n",
    "\n",
    "def train_hybrid_model(model, X_train, y_train, X_test, y_test, epochs=100, batch_size=32, folder_name='uv_index'):\n",
    "    \"\"\"\n",
    "    Advanced training function for the hybrid UV index model with detailed monitoring\n",
    "    and training management.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    model : keras.Model\n",
    "        The compiled hybrid model\n",
    "    X_train : numpy.ndarray\n",
    "        Training data\n",
    "    y_train : numpy.ndarray\n",
    "        Training targets\n",
    "    X_test : numpy.ndarray\n",
    "        Validation data\n",
    "    y_test : numpy.ndarray\n",
    "        Validation targets\n",
    "    epochs : int, optional\n",
    "        Maximum number of training epochs\n",
    "    batch_size : int, optional\n",
    "        Batch size\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    history : keras.callbacks.History\n",
    "        Training history with all metrics\n",
    "    \"\"\"\n",
    "\n",
    "    # Advanced callbacks for training\n",
    "    callbacks = [\n",
    "        # Advanced Early Stopping\n",
    "        EarlyStopping(\n",
    "            monitor='mae',\n",
    "            patience=15,\n",
    "            restore_best_weights=True,\n",
    "            mode='min',\n",
    "            verbose=1,\n",
    "            min_delta=1e-6\n",
    "        ),\n",
    "        ReduceLROnPlateau(\n",
    "            monitor='mae',\n",
    "            factor=0.05,\n",
    "            patience=3,\n",
    "            verbose=1,\n",
    "            mode='min',\n",
    "            min_delta=1e-6,\n",
    "            cooldown=2,\n",
    "            min_lr=1e-7\n",
    "        ),\n",
    "        ReduceLROnPlateau(\n",
    "            monitor='val_loss',\n",
    "            factor=0.2,\n",
    "            patience=2,\n",
    "            verbose=1,\n",
    "            mode='min',\n",
    "            min_delta=1e-6,\n",
    "            cooldown=1,\n",
    "            min_lr=1e-7\n",
    "        ),\n",
    "        tf.keras.callbacks.ModelCheckpoint(\n",
    "            filepath=f'{folder_name}_best_uv_model.h5',\n",
    "            monitor='mae',\n",
    "            save_best_only=True,\n",
    "            mode='min'\n",
    "        ),\n",
    "        tf.keras.callbacks.TensorBoard(\n",
    "            log_dir=f'./{folder_name}_logs',\n",
    "            histogram_freq=1,\n",
    "            write_graph=True,\n",
    "            update_freq='epoch'\n",
    "        ),\n",
    "        tf.keras.callbacks.LambdaCallback(\n",
    "            on_epoch_end=lambda epoch, logs: print(\n",
    "                f\"\\nEpoch {epoch + 1}: Out of range predictions: \"\n",
    "                f\"{np.sum((model.predict(X_test) < 0) | (model.predict(X_test) > 11))}\"\n",
    "            ) if epoch % 20 == 0 else None\n",
    "        )\n",
    "    ]\n",
    "\n",
    "    try:\n",
    "        history = model.fit(\n",
    "            X_train, y_train,\n",
    "            validation_data=(X_test, y_test),\n",
    "            epochs=epochs,\n",
    "            batch_size=batch_size,\n",
    "            callbacks=callbacks,\n",
    "            verbose=1,\n",
    "            shuffle=False,\n",
    "            validation_freq=1,\n",
    "        )\n",
    "\n",
    "        # Post-training analysis\n",
    "        print(\"\\nTraining completed successfully!\")\n",
    "\n",
    "        return history\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"\\nError during training: {str(e)}\")\n",
    "        raise\n",
    "\n",
    "    finally:\n",
    "        # Memory cleanup\n",
    "        tf.keras.backend.clear_session()\n",
    "\n",
    "\n",
    "def integrate_predictions(df, predictions, sequence_length=24):\n",
    "    \"\"\"\n",
    "    Integrate UV index predictions into the original dataset for pre-2010 data.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    df : pandas.DataFrame\n",
    "        Original dataset\n",
    "    predictions : numpy.ndarray\n",
    "        Array of UV index predictions\n",
    "    sequence_length : int\n",
    "        Sequence length used for predictions\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    pandas.DataFrame\n",
    "        Updated dataset with UV index predictions\n",
    "    \"\"\"\n",
    "    # Convert datetime to datetime format if not already\n",
    "    df['datetime'] = pd.to_datetime(df['datetime'])\n",
    "\n",
    "    # Identify pre-2010 rows\n",
    "    mask_pre_2010 = df['datetime'].dt.year < 2010\n",
    "\n",
    "    # Create temporary DataFrame with predictions\n",
    "    dates_pre_2010 = df[mask_pre_2010]['datetime'].iloc[sequence_length - 1:]\n",
    "    predictions_df = pd.DataFrame({\n",
    "        'datetime': dates_pre_2010,\n",
    "        'uvindex_predicted': predictions.flatten()\n",
    "    })\n",
    "\n",
    "    # Merge with original dataset\n",
    "    df = df.merge(predictions_df, on='datetime', how='left')\n",
    "\n",
    "    # Update uvindex column where missing\n",
    "    df['uvindex'] = df['uvindex'].fillna(df['uvindex_predicted'])\n",
    "\n",
    "    # Remove temporary column\n",
    "    df = df.drop('uvindex_predicted', axis=1)\n",
    "\n",
    "    print(f\"Added {len(predictions)} predictions to dataset\")\n",
    "    print(f\"Rows with UV index after integration: {df['uvindex'].notna().sum()}\")\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "initial_id",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing UV index model training...\n",
      "\n",
      "1. Preparing data...\n",
      "\n",
      "Temporal distribution of data:\n",
      "Records after 2010: 129,777\n",
      "Records before 2010: 227,902\n",
      "\n",
      "Warning: Found missing values after preprocessing\n",
      "Features with missing values: []\n",
      "\n",
      "Number of features used: 30\n",
      "\n",
      "Feature categories:\n",
      "atmospheric: 6 features\n",
      "temporal: 4 features\n",
      "solar: 5 features\n",
      "interactions: 4 features\n",
      "rolling: 2 features\n",
      "Categorical: 9 features\n",
      "Training data shape: (64865, 24, 30)\n",
      "Test data shape: (64866, 24, 30)\n",
      "Saving scaler to: 2024-11-20_11-04_scaler.joblib\n",
      "Saving features to: 2024-11-20_11-04_features.json\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_parquet('../../sources/weather_data.parquet')\n",
    "\n",
    "print(\"Initializing UV index model training...\")\n",
    "\n",
    "# Data preparation\n",
    "print(\"\\n1. Preparing data...\")\n",
    "X_train_seq, X_test_seq, y_train, y_test, scaler, feature_scaler, target_scaler, features, X_to_predict_seq = prepare_hybrid_data(df)\n",
    "\n",
    "print(f\"Training data shape: {X_train_seq.shape}\")\n",
    "print(f\"Test data shape: {X_test_seq.shape}\")\n",
    "\n",
    "# Save or load scaler and features\n",
    "feature_scaler_path = f'{folder_name}_feature_scaler.joblib'\n",
    "target_scaler_path = f'{folder_name}_target_scaler.joblib'\n",
    "features_path = f'{folder_name}_features.json'\n",
    "model_path = f'{folder_name}_best_model.h5'\n",
    "history_path = f'{folder_name}_training_history.json'\n",
    "\n",
    "if os.path.exists(feature_scaler_path):\n",
    "    print(f\"Loading existing scaler from: {feature_scaler_path}\")\n",
    "    scaler = joblib.load(feature_scaler_path)\n",
    "else:\n",
    "    print(f\"Saving scaler to: {feature_scaler_path}\")\n",
    "    joblib.dump(feature_scaler, feature_scaler_path)\n",
    "\n",
    "if os.path.exists(target_scaler_path):\n",
    "    print(f\"Loading existing scaler from: {target_scaler_path}\")\n",
    "    scaler = joblib.load(target_scaler_path)\n",
    "else:\n",
    "    print(f\"Saving scaler to: {target_scaler_path}\")\n",
    "    joblib.dump(target_scaler, target_scaler_path)\n",
    "\n",
    "if os.path.exists(features_path):\n",
    "    print(f\"Loading existing features from: {features_path}\")\n",
    "    with open(features_path, 'r') as f:\n",
    "        features = json.load(f)\n",
    "else:\n",
    "    print(f\"Saving features to: {features_path}\")\n",
    "    with open(features_path, 'w') as f:\n",
    "        json.dump(features, f)\n",
    "\n",
    "# Data quality verification\n",
    "if np.isnan(X_train_seq).any() or np.isnan(y_train).any():\n",
    "    raise ValueError(\"Found NaN values in training data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "83771453-71db-4bb2-833d-7b81c022863d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "2. Model initialization...\n",
      "Creating new model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-20 11:04:26.029246: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 43404 MB memory:  -> device: 0, name: NVIDIA L40, pci bus id: 0000:a1:00.0, compute capability: 8.9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "3. Starting training...\n",
      "Epoch 1/100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-20 11:04:44.087739: I tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:606] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n",
      "2024-11-20 11:04:44.182119: I tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:432] Loaded cuDNN version 8905\n",
      "2024-11-20 11:04:44.340866: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x710c9c3f7ae0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2024-11-20 11:04:44.340894: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): NVIDIA L40, Compute Capability 8.9\n",
      "2024-11-20 11:04:44.347733: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:255] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n",
      "2024-11-20 11:04:44.423715: I tensorflow/tsl/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory\n",
      "2024-11-20 11:04:44.481054: I ./tensorflow/compiler/jit/device_compiler.h:186] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "506/507 [============================>.] - ETA: 0s - loss: 22.0229 - mae: 4.3730 - mse: 24.6210 - root_mean_squared_error: 4.9620 - smooth_mape: 683.3177"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.11/dist-packages/keras/src/engine/training.py:3000: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n",
      "  saving_api.save_model(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2028/2028 [==============================] - 19s 8ms/step\n",
      "2028/2028 [==============================] - 17s 8ms/step\n",
      "\n",
      "Epoch 1: Out of range predictions: 0\n",
      "507/507 [==============================] - 286s 522ms/step - loss: 22.0128 - mae: 4.3719 - mse: 24.6071 - root_mean_squared_error: 4.9606 - smooth_mape: 682.9322 - val_loss: 12.9149 - val_mae: 3.0398 - val_mse: 12.5725 - val_root_mean_squared_error: 3.5458 - val_smooth_mape: 420.2022 - lr: 5.0600e-05\n",
      "Epoch 2/100\n",
      "507/507 [==============================] - 22s 43ms/step - loss: 13.0204 - mae: 3.1727 - mse: 12.7635 - root_mean_squared_error: 3.5726 - smooth_mape: 465.3772 - val_loss: 10.2818 - val_mae: 2.5288 - val_mse: 9.1610 - val_root_mean_squared_error: 3.0267 - val_smooth_mape: 309.3470 - lr: 9.9998e-05\n",
      "Epoch 3/100\n",
      "507/507 [==============================] - 21s 41ms/step - loss: 10.2727 - mae: 2.6377 - mse: 9.3028 - root_mean_squared_error: 3.0500 - smooth_mape: 359.8687 - val_loss: 8.9749 - val_mae: 2.3628 - val_mse: 7.9181 - val_root_mean_squared_error: 2.8139 - val_smooth_mape: 331.3861 - lr: 9.7355e-05\n",
      "Epoch 4/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 7.0915 - mae: 1.9078 - mse: 5.5205 - root_mean_squared_error: 2.3496 - smooth_mape: 243.7966 - val_loss: 4.3336 - val_mae: 1.0649 - val_mse: 2.2310 - val_root_mean_squared_error: 1.4936 - val_smooth_mape: 93.4224 - lr: 8.9946e-05\n",
      "Epoch 5/100\n",
      "507/507 [==============================] - 19s 37ms/step - loss: 4.4735 - mae: 1.1798 - mse: 2.4518 - root_mean_squared_error: 1.5658 - smooth_mape: 123.4787 - val_loss: 3.6128 - val_mae: 0.7664 - val_mse: 1.4863 - val_root_mean_squared_error: 1.2191 - val_smooth_mape: 51.1476 - lr: 7.8518e-05\n",
      "Epoch 6/100\n",
      "507/507 [==============================] - 21s 41ms/step - loss: 3.8903 - mae: 0.9776 - mse: 1.8588 - root_mean_squared_error: 1.3634 - smooth_mape: 92.1401 - val_loss: 3.2720 - val_mae: 0.7516 - val_mse: 1.1635 - val_root_mean_squared_error: 1.0787 - val_smooth_mape: 61.2309 - lr: 6.4221e-05\n",
      "Epoch 7/100\n",
      "507/507 [==============================] - 19s 38ms/step - loss: 3.5464 - mae: 0.8645 - mse: 1.5388 - root_mean_squared_error: 1.2405 - smooth_mape: 76.1412 - val_loss: 3.1084 - val_mae: 0.6539 - val_mse: 1.0697 - val_root_mean_squared_error: 1.0343 - val_smooth_mape: 44.6303 - lr: 4.8492e-05\n",
      "Epoch 8/100\n",
      "507/507 [==============================] - 20s 40ms/step - loss: 3.3832 - mae: 0.8174 - mse: 1.4213 - root_mean_squared_error: 1.1922 - smooth_mape: 69.5612 - val_loss: 2.9920 - val_mae: 0.6508 - val_mse: 0.9838 - val_root_mean_squared_error: 0.9919 - val_smooth_mape: 46.8297 - lr: 3.2915e-05\n",
      "Epoch 9/100\n",
      "507/507 [==============================] - 20s 39ms/step - loss: 3.2616 - mae: 0.7830 - mse: 1.3409 - root_mean_squared_error: 1.1580 - smooth_mape: 64.8256 - val_loss: 2.9825 - val_mae: 0.6214 - val_mse: 1.0355 - val_root_mean_squared_error: 1.0176 - val_smooth_mape: 36.8223 - lr: 1.9058e-05\n",
      "Epoch 10/100\n",
      "507/507 [==============================] - 19s 37ms/step - loss: 3.1757 - mae: 0.7625 - mse: 1.2823 - root_mean_squared_error: 1.1324 - smooth_mape: 62.7514 - val_loss: 2.9305 - val_mae: 0.6223 - val_mse: 0.9985 - val_root_mean_squared_error: 0.9992 - val_smooth_mape: 38.1533 - lr: 8.3134e-06\n",
      "Epoch 11/100\n",
      "507/507 [==============================] - 18s 36ms/step - loss: 3.1505 - mae: 0.7591 - mse: 1.2741 - root_mean_squared_error: 1.1288 - smooth_mape: 62.2294 - val_loss: 2.9239 - val_mae: 0.6197 - val_mse: 1.0041 - val_root_mean_squared_error: 1.0021 - val_smooth_mape: 36.8731 - lr: 1.7639e-06\n",
      "Epoch 12/100\n",
      "507/507 [==============================] - 21s 41ms/step - loss: 3.1376 - mae: 0.7580 - mse: 1.2645 - root_mean_squared_error: 1.1245 - smooth_mape: 62.0938 - val_loss: 2.9252 - val_mae: 0.6223 - val_mse: 1.0063 - val_root_mean_squared_error: 1.0031 - val_smooth_mape: 37.1174 - lr: 6.7976e-08\n",
      "Epoch 13/100\n",
      "507/507 [==============================] - 19s 38ms/step - loss: 3.1254 - mae: 0.7533 - mse: 1.2526 - root_mean_squared_error: 1.1192 - smooth_mape: 61.8713 - val_loss: 2.8793 - val_mae: 0.6077 - val_mse: 0.9641 - val_root_mean_squared_error: 0.9819 - val_smooth_mape: 37.2706 - lr: 3.3964e-06\n",
      "Epoch 14/100\n",
      "507/507 [==============================] - 19s 37ms/step - loss: 3.1145 - mae: 0.7475 - mse: 1.2572 - root_mean_squared_error: 1.1213 - smooth_mape: 60.5547 - val_loss: 2.8106 - val_mae: 0.5856 - val_mse: 0.9232 - val_root_mean_squared_error: 0.9609 - val_smooth_mape: 36.9508 - lr: 1.1414e-05\n",
      "Epoch 15/100\n",
      "507/507 [==============================] - 19s 37ms/step - loss: 3.0425 - mae: 0.7360 - mse: 1.2261 - root_mean_squared_error: 1.1073 - smooth_mape: 59.0602 - val_loss: 2.7845 - val_mae: 0.6135 - val_mse: 0.9482 - val_root_mean_squared_error: 0.9738 - val_smooth_mape: 38.2734 - lr: 2.3315e-05\n",
      "Epoch 16/100\n",
      "507/507 [==============================] - 19s 38ms/step - loss: 2.9185 - mae: 0.7179 - mse: 1.1919 - root_mean_squared_error: 1.0917 - smooth_mape: 56.6800 - val_loss: 2.5441 - val_mae: 0.5651 - val_mse: 0.8265 - val_root_mean_squared_error: 0.9091 - val_smooth_mape: 37.8188 - lr: 3.7901e-05\n",
      "Epoch 17/100\n",
      "507/507 [==============================] - 19s 38ms/step - loss: 2.7362 - mae: 0.6923 - mse: 1.1508 - root_mean_squared_error: 1.0728 - smooth_mape: 52.4607 - val_loss: 2.3594 - val_mae: 0.5339 - val_mse: 0.8245 - val_root_mean_squared_error: 0.9080 - val_smooth_mape: 33.7305 - lr: 5.3704e-05\n",
      "Epoch 18/100\n",
      "507/507 [==============================] - 20s 40ms/step - loss: 2.4901 - mae: 0.6563 - mse: 1.0811 - root_mean_squared_error: 1.0398 - smooth_mape: 47.8445 - val_loss: 2.1830 - val_mae: 0.5437 - val_mse: 0.8250 - val_root_mean_squared_error: 0.9083 - val_smooth_mape: 30.7037 - lr: 6.9134e-05\n",
      "Epoch 19/100\n",
      "507/507 [==============================] - 20s 39ms/step - loss: 2.2579 - mae: 0.6335 - mse: 1.0428 - root_mean_squared_error: 1.0212 - smooth_mape: 44.4837 - val_loss: 1.8973 - val_mae: 0.5041 - val_mse: 0.7399 - val_root_mean_squared_error: 0.8602 - val_smooth_mape: 31.0347 - lr: 8.2639e-05\n",
      "Epoch 20/100\n",
      "507/507 [==============================] - 19s 38ms/step - loss: 2.0034 - mae: 0.5995 - mse: 0.9690 - root_mean_squared_error: 0.9844 - smooth_mape: 40.8604 - val_loss: 1.6954 - val_mae: 0.4815 - val_mse: 0.7209 - val_root_mean_squared_error: 0.8491 - val_smooth_mape: 27.7490 - lr: 9.2860e-05\n",
      "Epoch 21/100\n",
      "2028/2028 [==============================] - 18s 9ms/step\n",
      "2028/2028 [==============================] - 15s 7ms/step\n",
      "\n",
      "Epoch 21: Out of range predictions: 0\n",
      "507/507 [==============================] - 58s 114ms/step - loss: 1.8096 - mae: 0.5802 - mse: 0.9400 - root_mean_squared_error: 0.9695 - smooth_mape: 38.2496 - val_loss: 1.5649 - val_mae: 0.4797 - val_mse: 0.7443 - val_root_mean_squared_error: 0.8627 - val_smooth_mape: 24.1879 - lr: 9.8768e-05\n",
      "Epoch 22/100\n",
      "507/507 [==============================] - 21s 41ms/step - loss: 1.6151 - mae: 0.5557 - mse: 0.8778 - root_mean_squared_error: 0.9369 - smooth_mape: 35.6242 - val_loss: 1.3949 - val_mae: 0.4745 - val_mse: 0.6896 - val_root_mean_squared_error: 0.8305 - val_smooth_mape: 26.4508 - lr: 9.9769e-05\n",
      "Epoch 23/100\n",
      "507/507 [==============================] - 20s 39ms/step - loss: 1.4698 - mae: 0.5405 - mse: 0.8422 - root_mean_squared_error: 0.9177 - smooth_mape: 34.5647 - val_loss: 1.2868 - val_mae: 0.4511 - val_mse: 0.6922 - val_root_mean_squared_error: 0.8320 - val_smooth_mape: 22.6886 - lr: 9.5762e-05\n",
      "Epoch 24/100\n",
      "507/507 [==============================] - 22s 44ms/step - loss: 1.3615 - mae: 0.5265 - mse: 0.8222 - root_mean_squared_error: 0.9067 - smooth_mape: 32.6609 - val_loss: 1.2085 - val_mae: 0.4516 - val_mse: 0.7004 - val_root_mean_squared_error: 0.8369 - val_smooth_mape: 24.5215 - lr: 8.7150e-05\n",
      "Epoch 25/100\n",
      "507/507 [==============================] - 20s 40ms/step - loss: 1.2829 - mae: 0.5197 - mse: 0.8124 - root_mean_squared_error: 0.9013 - smooth_mape: 32.1299 - val_loss: 1.1158 - val_mae: 0.4417 - val_mse: 0.6551 - val_root_mean_squared_error: 0.8094 - val_smooth_mape: 23.7300 - lr: 7.4800e-05\n",
      "Epoch 26/100\n",
      "507/507 [==============================] - 18s 36ms/step - loss: 1.2053 - mae: 0.5059 - mse: 0.7807 - root_mean_squared_error: 0.8836 - smooth_mape: 30.7133 - val_loss: 1.0587 - val_mae: 0.4362 - val_mse: 0.6405 - val_root_mean_squared_error: 0.8003 - val_smooth_mape: 24.4506 - lr: 5.9955e-05\n",
      "Epoch 27/100\n",
      "507/507 [==============================] - 21s 41ms/step - loss: 1.1578 - mae: 0.5006 - mse: 0.7685 - root_mean_squared_error: 0.8767 - smooth_mape: 30.2353 - val_loss: 1.0600 - val_mae: 0.4425 - val_mse: 0.6752 - val_root_mean_squared_error: 0.8217 - val_smooth_mape: 21.5856 - lr: 4.4108e-05\n",
      "Epoch 28/100\n",
      "507/507 [==============================] - 20s 39ms/step - loss: 1.1243 - mae: 0.4957 - mse: 0.7584 - root_mean_squared_error: 0.8708 - smooth_mape: 29.8448 - val_loss: 1.0003 - val_mae: 0.4280 - val_mse: 0.6281 - val_root_mean_squared_error: 0.7926 - val_smooth_mape: 22.9052 - lr: 2.8853e-05\n",
      "Epoch 29/100\n",
      "507/507 [==============================] - 20s 39ms/step - loss: 1.0922 - mae: 0.4875 - mse: 0.7390 - root_mean_squared_error: 0.8597 - smooth_mape: 29.1903 - val_loss: 0.9880 - val_mae: 0.4282 - val_mse: 0.6307 - val_root_mean_squared_error: 0.7942 - val_smooth_mape: 23.7812 - lr: 1.5727e-05\n",
      "Epoch 30/100\n",
      "507/507 [==============================] - 19s 37ms/step - loss: 1.0761 - mae: 0.4838 - mse: 0.7305 - root_mean_squared_error: 0.8547 - smooth_mape: 28.9260 - val_loss: 0.9782 - val_mae: 0.4257 - val_mse: 0.6241 - val_root_mean_squared_error: 0.7900 - val_smooth_mape: 23.4881 - lr: 6.0491e-06\n",
      "Epoch 31/100\n",
      "507/507 [==============================] - 19s 37ms/step - loss: 1.0754 - mae: 0.4845 - mse: 0.7336 - root_mean_squared_error: 0.8565 - smooth_mape: 29.0048 - val_loss: 0.9815 - val_mae: 0.4236 - val_mse: 0.6312 - val_root_mean_squared_error: 0.7945 - val_smooth_mape: 22.2870 - lr: 7.9394e-07\n",
      "Epoch 32/100\n",
      "507/507 [==============================] - ETA: 0s - loss: 1.0757 - mae: 0.4848 - mse: 0.7354 - root_mean_squared_error: 0.8575 - smooth_mape: 29.0269\n",
      "Epoch 32: ReduceLROnPlateau reducing learning rate to 1e-07.\n",
      "507/507 [==============================] - 21s 41ms/step - loss: 1.0757 - mae: 0.4848 - mse: 0.7354 - root_mean_squared_error: 0.8575 - smooth_mape: 29.0269 - val_loss: 0.9825 - val_mae: 0.4256 - val_mse: 0.6304 - val_root_mean_squared_error: 0.7940 - val_smooth_mape: 22.1371 - lr: 4.9000e-07\n",
      "Epoch 33/100\n",
      "507/507 [==============================] - 22s 43ms/step - loss: 1.0689 - mae: 0.4828 - mse: 0.7273 - root_mean_squared_error: 0.8528 - smooth_mape: 28.7582 - val_loss: 0.9748 - val_mae: 0.4243 - val_mse: 0.6236 - val_root_mean_squared_error: 0.7897 - val_smooth_mape: 23.0276 - lr: 5.1679e-06\n",
      "Epoch 34/100\n",
      "507/507 [==============================] - 19s 37ms/step - loss: 1.0745 - mae: 0.4855 - mse: 0.7373 - root_mean_squared_error: 0.8587 - smooth_mape: 28.9500 - val_loss: 0.9708 - val_mae: 0.4240 - val_mse: 0.6260 - val_root_mean_squared_error: 0.7912 - val_smooth_mape: 23.0926 - lr: 1.4357e-05\n",
      "Epoch 35/100\n",
      "507/507 [==============================] - 20s 39ms/step - loss: 1.0620 - mae: 0.4851 - mse: 0.7316 - root_mean_squared_error: 0.8553 - smooth_mape: 28.9883 - val_loss: 0.9650 - val_mae: 0.4304 - val_mse: 0.6269 - val_root_mean_squared_error: 0.7918 - val_smooth_mape: 22.8981 - lr: 2.7133e-05\n",
      "Epoch 36/100\n",
      "507/507 [==============================] - ETA: 0s - loss: 1.0530 - mae: 0.4845 - mse: 0.7370 - root_mean_squared_error: 0.8585 - smooth_mape: 28.9197\n",
      "Epoch 36: ReduceLROnPlateau reducing learning rate to 2.1104648112668657e-06.\n",
      "507/507 [==============================] - 21s 41ms/step - loss: 1.0530 - mae: 0.4845 - mse: 0.7370 - root_mean_squared_error: 0.8585 - smooth_mape: 28.9197 - val_loss: 0.9438 - val_mae: 0.4280 - val_mse: 0.6211 - val_root_mean_squared_error: 0.7881 - val_smooth_mape: 23.1570 - lr: 2.1105e-06\n",
      "Epoch 37/100\n",
      "507/507 [==============================] - 21s 41ms/step - loss: 1.0339 - mae: 0.4831 - mse: 0.7352 - root_mean_squared_error: 0.8574 - smooth_mape: 28.5829 - val_loss: 0.9257 - val_mae: 0.4244 - val_mse: 0.6262 - val_root_mean_squared_error: 0.7913 - val_smooth_mape: 22.7749 - lr: 5.8070e-05\n",
      "Epoch 38/100\n",
      "507/507 [==============================] - 21s 41ms/step - loss: 1.0097 - mae: 0.4806 - mse: 0.7331 - root_mean_squared_error: 0.8562 - smooth_mape: 28.3004 - val_loss: 0.9042 - val_mae: 0.4311 - val_mse: 0.6269 - val_root_mean_squared_error: 0.7918 - val_smooth_mape: 25.1432 - lr: 7.3118e-05\n",
      "Epoch 39/100\n",
      "507/507 [==============================] - 19s 38ms/step - loss: 0.9923 - mae: 0.4829 - mse: 0.7402 - root_mean_squared_error: 0.8604 - smooth_mape: 28.2334 - val_loss: 0.8823 - val_mae: 0.4198 - val_mse: 0.6305 - val_root_mean_squared_error: 0.7940 - val_smooth_mape: 21.3908 - lr: 8.5841e-05\n",
      "Epoch 40/100\n",
      "507/507 [==============================] - 21s 42ms/step - loss: 0.9551 - mae: 0.4759 - mse: 0.7249 - root_mean_squared_error: 0.8514 - smooth_mape: 27.5520 - val_loss: 0.8642 - val_mae: 0.4233 - val_mse: 0.6429 - val_root_mean_squared_error: 0.8018 - val_smooth_mape: 23.5225 - lr: 9.4957e-05\n",
      "Epoch 41/100\n",
      "2028/2028 [==============================] - 18s 9ms/step\n",
      "2028/2028 [==============================] - 17s 8ms/step\n",
      "\n",
      "Epoch 41: Out of range predictions: 0\n",
      "507/507 [==============================] - 56s 110ms/step - loss: 0.9397 - mae: 0.4787 - mse: 0.7327 - root_mean_squared_error: 0.8560 - smooth_mape: 27.8895 - val_loss: 0.8242 - val_mae: 0.4152 - val_mse: 0.6168 - val_root_mean_squared_error: 0.7854 - val_smooth_mape: 21.9886 - lr: 9.9549e-05\n",
      "Epoch 42/100\n",
      "507/507 [==============================] - 21s 41ms/step - loss: 0.9064 - mae: 0.4719 - mse: 0.7185 - root_mean_squared_error: 0.8476 - smooth_mape: 27.2030 - val_loss: 0.8109 - val_mae: 0.4125 - val_mse: 0.6283 - val_root_mean_squared_error: 0.7927 - val_smooth_mape: 21.8665 - lr: 9.9155e-05\n",
      "Epoch 43/100\n",
      "507/507 [==============================] - 20s 40ms/step - loss: 0.8691 - mae: 0.4629 - mse: 0.6978 - root_mean_squared_error: 0.8354 - smooth_mape: 26.6493 - val_loss: 0.7991 - val_mae: 0.4117 - val_mse: 0.6371 - val_root_mean_squared_error: 0.7982 - val_smooth_mape: 21.4879 - lr: 9.3815e-05\n",
      "Epoch 44/100\n",
      "507/507 [==============================] - 19s 38ms/step - loss: 0.8550 - mae: 0.4635 - mse: 0.6999 - root_mean_squared_error: 0.8366 - smooth_mape: 26.7802 - val_loss: 0.7693 - val_mae: 0.4128 - val_mse: 0.6116 - val_root_mean_squared_error: 0.7820 - val_smooth_mape: 21.3808 - lr: 8.4067e-05\n",
      "Epoch 45/100\n",
      "507/507 [==============================] - 18s 36ms/step - loss: 0.8287 - mae: 0.4571 - mse: 0.6841 - root_mean_squared_error: 0.8271 - smooth_mape: 26.1963 - val_loss: 0.7851 - val_mae: 0.4167 - val_mse: 0.6530 - val_root_mean_squared_error: 0.8081 - val_smooth_mape: 22.6425 - lr: 7.0890e-05\n",
      "Epoch 46/100\n",
      "507/507 [==============================] - 19s 37ms/step - loss: 0.8183 - mae: 0.4563 - mse: 0.6844 - root_mean_squared_error: 0.8273 - smooth_mape: 26.1332 - val_loss: 0.7583 - val_mae: 0.4167 - val_mse: 0.6226 - val_root_mean_squared_error: 0.7890 - val_smooth_mape: 20.5157 - lr: 5.5612e-05\n",
      "Epoch 47/100\n",
      "507/507 [==============================] - 22s 44ms/step - loss: 0.7980 - mae: 0.4494 - mse: 0.6691 - root_mean_squared_error: 0.8180 - smooth_mape: 25.7179 - val_loss: 0.7370 - val_mae: 0.4110 - val_mse: 0.6051 - val_root_mean_squared_error: 0.7779 - val_smooth_mape: 21.5193 - lr: 3.9768e-05\n",
      "Epoch 48/100\n",
      "507/507 [==============================] - 22s 44ms/step - loss: 0.7870 - mae: 0.4474 - mse: 0.6626 - root_mean_squared_error: 0.8140 - smooth_mape: 25.5764 - val_loss: 0.7277 - val_mae: 0.4052 - val_mse: 0.6017 - val_root_mean_squared_error: 0.7757 - val_smooth_mape: 21.4695 - lr: 2.4955e-05\n",
      "Epoch 49/100\n",
      "507/507 [==============================] - 21s 41ms/step - loss: 0.7747 - mae: 0.4432 - mse: 0.6525 - root_mean_squared_error: 0.8078 - smooth_mape: 25.3902 - val_loss: 0.7257 - val_mae: 0.4020 - val_mse: 0.6028 - val_root_mean_squared_error: 0.7764 - val_smooth_mape: 20.7401 - lr: 1.2661e-05\n",
      "Epoch 50/100\n",
      "507/507 [==============================] - 22s 43ms/step - loss: 0.7725 - mae: 0.4430 - mse: 0.6518 - root_mean_squared_error: 0.8073 - smooth_mape: 25.4604 - val_loss: 0.7225 - val_mae: 0.4077 - val_mse: 0.5955 - val_root_mean_squared_error: 0.7717 - val_smooth_mape: 21.3071 - lr: 4.1248e-06\n",
      "Epoch 51/100\n",
      "507/507 [==============================] - 21s 40ms/step - loss: 0.7744 - mae: 0.4439 - mse: 0.6543 - root_mean_squared_error: 0.8089 - smooth_mape: 25.3972 - val_loss: 0.7287 - val_mae: 0.4031 - val_mse: 0.6063 - val_root_mean_squared_error: 0.7787 - val_smooth_mape: 19.8541 - lr: 2.0452e-07\n",
      "Epoch 52/100\n",
      "505/507 [============================>.] - ETA: 0s - loss: 0.7740 - mae: 0.4430 - mse: 0.6545 - root_mean_squared_error: 0.8090 - smooth_mape: 25.3371\n",
      "Epoch 52: ReduceLROnPlateau reducing learning rate to 2.589756149973255e-07.\n",
      "507/507 [==============================] - 21s 40ms/step - loss: 0.7748 - mae: 0.4432 - mse: 0.6554 - root_mean_squared_error: 0.8096 - smooth_mape: 25.3532 - val_loss: 0.7241 - val_mae: 0.4061 - val_mse: 0.5989 - val_root_mean_squared_error: 0.7739 - val_smooth_mape: 20.7611 - lr: 1.2949e-06\n",
      "Epoch 53/100\n",
      "507/507 [==============================] - 22s 44ms/step - loss: 0.7670 - mae: 0.4407 - mse: 0.6466 - root_mean_squared_error: 0.8041 - smooth_mape: 25.2363 - val_loss: 0.7210 - val_mae: 0.4034 - val_mse: 0.5974 - val_root_mean_squared_error: 0.7729 - val_smooth_mape: 20.8914 - lr: 7.2861e-06\n",
      "Epoch 54/100\n",
      "507/507 [==============================] - 20s 39ms/step - loss: 0.7692 - mae: 0.4415 - mse: 0.6502 - root_mean_squared_error: 0.8064 - smooth_mape: 25.2406 - val_loss: 0.7234 - val_mae: 0.4073 - val_mse: 0.6006 - val_root_mean_squared_error: 0.7750 - val_smooth_mape: 20.8373 - lr: 1.7575e-05\n",
      "Epoch 55/100\n",
      "507/507 [==============================] - 19s 37ms/step - loss: 0.7694 - mae: 0.4438 - mse: 0.6530 - root_mean_squared_error: 0.8081 - smooth_mape: 25.2635 - val_loss: 0.7208 - val_mae: 0.4075 - val_mse: 0.6020 - val_root_mean_squared_error: 0.7759 - val_smooth_mape: 20.9485 - lr: 3.1127e-05\n",
      "Epoch 56/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.7695 - mae: 0.4437 - mse: 0.6571 - root_mean_squared_error: 0.8106 - smooth_mape: 25.3100\n",
      "Epoch 56: ReduceLROnPlateau reducing learning rate to 2.3289143427973617e-06.\n",
      "507/507 [==============================] - 20s 40ms/step - loss: 0.7699 - mae: 0.4438 - mse: 0.6576 - root_mean_squared_error: 0.8109 - smooth_mape: 25.3081 - val_loss: 0.7268 - val_mae: 0.4039 - val_mse: 0.6174 - val_root_mean_squared_error: 0.7857 - val_smooth_mape: 19.7400 - lr: 2.3289e-06\n",
      "Epoch 57/100\n",
      "507/507 [==============================] - 20s 39ms/step - loss: 0.7671 - mae: 0.4455 - mse: 0.6604 - root_mean_squared_error: 0.8126 - smooth_mape: 25.3943 - val_loss: 0.7181 - val_mae: 0.4094 - val_mse: 0.6137 - val_root_mean_squared_error: 0.7834 - val_smooth_mape: 21.3784 - lr: 6.2374e-05\n",
      "Epoch 58/100\n",
      "507/507 [==============================] - 21s 41ms/step - loss: 0.7690 - mae: 0.4469 - mse: 0.6696 - root_mean_squared_error: 0.8183 - smooth_mape: 25.2775 - val_loss: 0.7278 - val_mae: 0.4093 - val_mse: 0.6335 - val_root_mean_squared_error: 0.7959 - val_smooth_mape: 19.5767 - lr: 7.6924e-05\n",
      "Epoch 59/100\n",
      "507/507 [==============================] - 22s 43ms/step - loss: 0.7632 - mae: 0.4472 - mse: 0.6706 - root_mean_squared_error: 0.8189 - smooth_mape: 25.2525 - val_loss: 0.7001 - val_mae: 0.4038 - val_mse: 0.6087 - val_root_mean_squared_error: 0.7802 - val_smooth_mape: 21.2936 - lr: 8.8765e-05\n",
      "Epoch 60/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.7547 - mae: 0.4472 - mse: 0.6690 - root_mean_squared_error: 0.8179 - smooth_mape: 25.2686\n",
      "Epoch 60: ReduceLROnPlateau reducing learning rate to 4.835262006963604e-06.\n",
      "507/507 [==============================] - 20s 40ms/step - loss: 0.7553 - mae: 0.4474 - mse: 0.6696 - root_mean_squared_error: 0.8183 - smooth_mape: 25.2703 - val_loss: 0.7642 - val_mae: 0.4280 - val_mse: 0.7001 - val_root_mean_squared_error: 0.8367 - val_smooth_mape: 22.1543 - lr: 4.8353e-06\n",
      "Epoch 61/100\n",
      "2028/2028 [==============================] - 18s 9ms/step\n",
      "2028/2028 [==============================] - 16s 8ms/step\n",
      "\n",
      "Epoch 61: Out of range predictions: 0\n",
      "507/507 [==============================] - 59s 116ms/step - loss: 0.7448 - mae: 0.4457 - mse: 0.6658 - root_mean_squared_error: 0.8160 - smooth_mape: 25.0610 - val_loss: 0.6932 - val_mae: 0.4059 - val_mse: 0.6177 - val_root_mean_squared_error: 0.7859 - val_smooth_mape: 20.4987 - lr: 9.9946e-05\n",
      "Epoch 62/100\n",
      "507/507 [==============================] - 19s 38ms/step - loss: 0.7348 - mae: 0.4439 - mse: 0.6628 - root_mean_squared_error: 0.8141 - smooth_mape: 25.0138 - val_loss: 0.6917 - val_mae: 0.4076 - val_mse: 0.6258 - val_root_mean_squared_error: 0.7911 - val_smooth_mape: 21.0874 - lr: 9.8161e-05\n",
      "Epoch 63/100\n",
      "507/507 [==============================] - 19s 38ms/step - loss: 0.7260 - mae: 0.4438 - mse: 0.6602 - root_mean_squared_error: 0.8125 - smooth_mape: 25.0922 - val_loss: 0.6881 - val_mae: 0.4067 - val_mse: 0.6289 - val_root_mean_squared_error: 0.7930 - val_smooth_mape: 20.0478 - lr: 9.1530e-05\n",
      "Epoch 64/100\n",
      "507/507 [==============================] - 20s 40ms/step - loss: 0.7076 - mae: 0.4367 - mse: 0.6451 - root_mean_squared_error: 0.8032 - smooth_mape: 24.4019 - val_loss: 0.6638 - val_mae: 0.4074 - val_mse: 0.5961 - val_root_mean_squared_error: 0.7721 - val_smooth_mape: 20.4913 - lr: 8.0720e-05\n",
      "Epoch 65/100\n",
      "507/507 [==============================] - 18s 36ms/step - loss: 0.7041 - mae: 0.4375 - mse: 0.6469 - root_mean_squared_error: 0.8043 - smooth_mape: 24.5839 - val_loss: 0.6609 - val_mae: 0.4007 - val_mse: 0.6056 - val_root_mean_squared_error: 0.7782 - val_smooth_mape: 19.8633 - lr: 6.6819e-05\n",
      "Epoch 66/100\n",
      "507/507 [==============================] - 19s 37ms/step - loss: 0.6895 - mae: 0.4313 - mse: 0.6349 - root_mean_squared_error: 0.7968 - smooth_mape: 24.1261 - val_loss: 0.6586 - val_mae: 0.4009 - val_mse: 0.6092 - val_root_mean_squared_error: 0.7805 - val_smooth_mape: 21.0485 - lr: 5.1225e-05\n",
      "Epoch 67/100\n",
      "507/507 [==============================] - 19s 38ms/step - loss: 0.6838 - mae: 0.4309 - mse: 0.6313 - root_mean_squared_error: 0.7945 - smooth_mape: 24.2230 - val_loss: 0.6501 - val_mae: 0.3975 - val_mse: 0.6008 - val_root_mean_squared_error: 0.7751 - val_smooth_mape: 20.1558 - lr: 3.5508e-05\n",
      "Epoch 68/100\n",
      "507/507 [==============================] - 19s 38ms/step - loss: 0.6738 - mae: 0.4276 - mse: 0.6227 - root_mean_squared_error: 0.7891 - smooth_mape: 24.0020 - val_loss: 0.6444 - val_mae: 0.3967 - val_mse: 0.5944 - val_root_mean_squared_error: 0.7710 - val_smooth_mape: 20.4472 - lr: 2.1250e-05\n",
      "Epoch 69/100\n",
      "507/507 [==============================] - 21s 41ms/step - loss: 0.6710 - mae: 0.4261 - mse: 0.6204 - root_mean_squared_error: 0.7876 - smooth_mape: 23.7990 - val_loss: 0.6404 - val_mae: 0.3966 - val_mse: 0.5894 - val_root_mean_squared_error: 0.7677 - val_smooth_mape: 20.3464 - lr: 9.8841e-06\n",
      "Epoch 70/100\n",
      "507/507 [==============================] - 21s 41ms/step - loss: 0.6678 - mae: 0.4251 - mse: 0.6176 - root_mean_squared_error: 0.7859 - smooth_mape: 23.8202 - val_loss: 0.6445 - val_mae: 0.3992 - val_mse: 0.5921 - val_root_mean_squared_error: 0.7695 - val_smooth_mape: 19.6060 - lr: 2.5551e-06\n",
      "Epoch 71/100\n",
      "475/507 [===========================>..] - ETA: 0s - loss: 0.6680 - mae: 0.4257 - mse: 0.6182 - root_mean_squared_error: 0.7863 - smooth_mape: 23.9228"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "507/507 [==============================] - 20s 40ms/step - loss: 0.6776 - mae: 0.4345 - mse: 0.6462 - root_mean_squared_error: 0.8039 - smooth_mape: 24.1705 - val_loss: 0.6316 - val_mae: 0.4047 - val_mse: 0.5959 - val_root_mean_squared_error: 0.7720 - val_smooth_mape: 21.0905 - lr: 9.8093e-05\n",
      "Epoch 81/100\n",
      "2028/2028 [==============================] - 16s 8ms/step\n",
      "2028/2028 [==============================] - 18s 9ms/step\n",
      "\n",
      "Epoch 81: Out of range predictions: 0\n",
      "507/507 [==============================] - 58s 114ms/step - loss: 0.6617 - mae: 0.4295 - mse: 0.6308 - root_mean_squared_error: 0.7942 - smooth_mape: 23.8781 - val_loss: 0.6358 - val_mae: 0.4045 - val_mse: 0.6039 - val_root_mean_squared_error: 0.7771 - val_smooth_mape: 19.5664 - lr: 9.9957e-05\n",
      "Epoch 82/100\n",
      "507/507 [==============================] - 20s 39ms/step - loss: 0.6609 - mae: 0.4304 - mse: 0.6342 - root_mean_squared_error: 0.7964 - smooth_mape: 23.8601 - val_loss: 0.6249 - val_mae: 0.3967 - val_mse: 0.5981 - val_root_mean_squared_error: 0.7734 - val_smooth_mape: 19.5850 - lr: 9.6794e-05\n",
      "Epoch 83/100\n",
      "505/507 [============================>.] - ETA: 0s - loss: 0.6528 - mae: 0.4287 - mse: 0.6286 - root_mean_squared_error: 0.7928 - smooth_mape: 23.8442\n",
      "Epoch 83: ReduceLROnPlateau reducing learning rate to 4.446155799087137e-06.\n",
      "507/507 [==============================] - 19s 37ms/step - loss: 0.6541 - mae: 0.4291 - mse: 0.6300 - root_mean_squared_error: 0.7938 - smooth_mape: 23.8694 - val_loss: 0.6230 - val_mae: 0.4001 - val_mse: 0.6031 - val_root_mean_squared_error: 0.7766 - val_smooth_mape: 21.0960 - lr: 4.4462e-06\n",
      "Epoch 84/100\n",
      "507/507 [==============================] - 18s 36ms/step - loss: 0.6433 - mae: 0.4248 - mse: 0.6207 - root_mean_squared_error: 0.7879 - smooth_mape: 23.4925 - val_loss: 0.6229 - val_mae: 0.3979 - val_mse: 0.6052 - val_root_mean_squared_error: 0.7779 - val_smooth_mape: 19.7235 - lr: 7.7135e-05\n",
      "Epoch 85/100\n",
      "507/507 [==============================] - 19s 38ms/step - loss: 0.6355 - mae: 0.4223 - mse: 0.6147 - root_mean_squared_error: 0.7840 - smooth_mape: 23.3794 - val_loss: 0.6273 - val_mae: 0.4044 - val_mse: 0.6084 - val_root_mean_squared_error: 0.7800 - val_smooth_mape: 18.9805 - lr: 6.2617e-05\n",
      "Epoch 86/100\n",
      "507/507 [==============================] - 20s 40ms/step - loss: 0.6296 - mae: 0.4214 - mse: 0.6102 - root_mean_squared_error: 0.7812 - smooth_mape: 23.6014 - val_loss: 0.6144 - val_mae: 0.3965 - val_mse: 0.5984 - val_root_mean_squared_error: 0.7736 - val_smooth_mape: 19.4583 - lr: 4.6829e-05\n",
      "Epoch 87/100\n",
      "507/507 [==============================] - 21s 41ms/step - loss: 0.6258 - mae: 0.4187 - mse: 0.6076 - root_mean_squared_error: 0.7795 - smooth_mape: 23.2483 - val_loss: 0.6128 - val_mae: 0.3962 - val_mse: 0.5974 - val_root_mean_squared_error: 0.7729 - val_smooth_mape: 19.1720 - lr: 3.1360e-05\n",
      "Epoch 88/100\n",
      "507/507 [==============================] - 21s 41ms/step - loss: 0.6178 - mae: 0.4161 - mse: 0.5992 - root_mean_squared_error: 0.7741 - smooth_mape: 23.0963 - val_loss: 0.6083 - val_mae: 0.3958 - val_mse: 0.5927 - val_root_mean_squared_error: 0.7699 - val_smooth_mape: 19.7328 - lr: 1.7767e-05\n",
      "Epoch 89/100\n",
      "507/507 [==============================] - 20s 40ms/step - loss: 0.6124 - mae: 0.4135 - mse: 0.5938 - root_mean_squared_error: 0.7706 - smooth_mape: 22.9061 - val_loss: 0.6062 - val_mae: 0.3971 - val_mse: 0.5881 - val_root_mean_squared_error: 0.7669 - val_smooth_mape: 19.8249 - lr: 7.4173e-06\n",
      "Epoch 90/100\n",
      "507/507 [==============================] - 20s 39ms/step - loss: 0.6135 - mae: 0.4146 - mse: 0.5954 - root_mean_squared_error: 0.7716 - smooth_mape: 22.9713 - val_loss: 0.6091 - val_mae: 0.3964 - val_mse: 0.5932 - val_root_mean_squared_error: 0.7702 - val_smooth_mape: 19.3731 - lr: 1.3523e-06\n",
      "Epoch 91/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.6101 - mae: 0.4127 - mse: 0.5916 - root_mean_squared_error: 0.7692 - smooth_mape: 22.9792\n",
      "Epoch 91: ReduceLROnPlateau reducing learning rate to 1e-07.\n",
      "507/507 [==============================] - 22s 43ms/step - loss: 0.6105 - mae: 0.4128 - mse: 0.5920 - root_mean_squared_error: 0.7694 - smooth_mape: 22.9794 - val_loss: 0.6111 - val_mae: 0.3956 - val_mse: 0.5954 - val_root_mean_squared_error: 0.7716 - val_smooth_mape: 18.9321 - lr: 1.8244e-07\n",
      "Epoch 92/100\n",
      "507/507 [==============================] - 19s 38ms/step - loss: 0.6098 - mae: 0.4129 - mse: 0.5912 - root_mean_squared_error: 0.7689 - smooth_mape: 22.9186 - val_loss: 0.6105 - val_mae: 0.3959 - val_mse: 0.5948 - val_root_mean_squared_error: 0.7712 - val_smooth_mape: 19.1159 - lr: 4.0254e-06\n",
      "Epoch 93/100\n",
      "507/507 [==============================] - 20s 39ms/step - loss: 0.6094 - mae: 0.4129 - mse: 0.5912 - root_mean_squared_error: 0.7689 - smooth_mape: 22.9718 - val_loss: 0.6056 - val_mae: 0.3958 - val_mse: 0.5888 - val_root_mean_squared_error: 0.7673 - val_smooth_mape: 19.7076 - lr: 1.2494e-05\n",
      "Epoch 94/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.6182 - mae: 0.4170 - mse: 0.6016 - root_mean_squared_error: 0.7756 - smooth_mape: 23.1906\n",
      "Epoch 94: ReduceLROnPlateau reducing learning rate to 1.2368702300591396e-06.\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.6187 - mae: 0.4171 - mse: 0.6021 - root_mean_squared_error: 0.7759 - smooth_mape: 23.1873 - val_loss: 0.6096 - val_mae: 0.3954 - val_mse: 0.5967 - val_root_mean_squared_error: 0.7725 - val_smooth_mape: 19.5515 - lr: 1.2369e-06\n",
      "Epoch 95/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.6135 - mae: 0.4156 - mse: 0.5973 - root_mean_squared_error: 0.7729 - smooth_mape: 23.1793\n",
      "Epoch 95: ReduceLROnPlateau reducing learning rate to 7.904486119514332e-06.\n",
      "507/507 [==============================] - 21s 42ms/step - loss: 0.6139 - mae: 0.4157 - mse: 0.5977 - root_mean_squared_error: 0.7731 - smooth_mape: 23.1749 - val_loss: 0.6147 - val_mae: 0.4003 - val_mse: 0.6011 - val_root_mean_squared_error: 0.7753 - val_smooth_mape: 19.1018 - lr: 3.9522e-05\n",
      "Epoch 96/100\n",
      "507/507 [==============================] - 20s 40ms/step - loss: 0.6212 - mae: 0.4190 - mse: 0.6071 - root_mean_squared_error: 0.7792 - smooth_mape: 23.1593 - val_loss: 0.6140 - val_mae: 0.4029 - val_mse: 0.6003 - val_root_mean_squared_error: 0.7748 - val_smooth_mape: 19.5131 - lr: 5.5362e-05\n",
      "Epoch 97/100\n",
      "507/507 [==============================] - ETA: 0s - loss: 0.6205 - mae: 0.4187 - mse: 0.6076 - root_mean_squared_error: 0.7795 - smooth_mape: 23.1693\n",
      "Epoch 97: ReduceLROnPlateau reducing learning rate to 1.4132310752756894e-05.\n",
      "507/507 [==============================] - 22s 43ms/step - loss: 0.6205 - mae: 0.4187 - mse: 0.6076 - root_mean_squared_error: 0.7795 - smooth_mape: 23.1693 - val_loss: 0.6120 - val_mae: 0.3987 - val_mse: 0.6040 - val_root_mean_squared_error: 0.7772 - val_smooth_mape: 19.6054 - lr: 7.0662e-05\n",
      "Epoch 98/100\n",
      "507/507 [==============================] - ETA: 0s - loss: 0.6171 - mae: 0.4189 - mse: 0.6058 - root_mean_squared_error: 0.7783 - smooth_mape: 23.1350\n",
      "Epoch 98: ReduceLROnPlateau reducing learning rate to 4.194115172140301e-06.\n",
      "507/507 [==============================] - 19s 38ms/step - loss: 0.6171 - mae: 0.4189 - mse: 0.6058 - root_mean_squared_error: 0.7783 - smooth_mape: 23.1350 - val_loss: 0.6074 - val_mae: 0.3966 - val_mse: 0.6001 - val_root_mean_squared_error: 0.7747 - val_smooth_mape: 19.3917 - lr: 4.1941e-06\n",
      "Epoch 99/100\n",
      "507/507 [==============================] - 20s 39ms/step - loss: 0.6209 - mae: 0.4207 - mse: 0.6119 - root_mean_squared_error: 0.7822 - smooth_mape: 23.1457 - val_loss: 0.5983 - val_mae: 0.3973 - val_mse: 0.5892 - val_root_mean_squared_error: 0.7676 - val_smooth_mape: 20.8306 - lr: 9.3694e-05\n",
      "Epoch 100/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.6275 - mae: 0.4247 - mse: 0.6214 - root_mean_squared_error: 0.7883 - smooth_mape: 23.6287 - val_loss: 0.6101 - val_mae: 0.4087 - val_mse: 0.5974 - val_root_mean_squared_error: 0.7729 - val_smooth_mape: 20.1332 - lr: 9.9108e-05\n",
      "\n",
      "Training completed successfully!\n"
     ]
    }
   ],
   "source": [
    "# Model creation or loading\n",
    "print(\"\\n2. Model initialization...\")\n",
    "input_shape = (X_train_seq.shape[1], X_train_seq.shape[2])\n",
    "\n",
    "if os.path.exists(model_path):\n",
    "    print(f\"Loading existing model from: {model_path}\")\n",
    "    model = tf.keras.models.load_model(model_path)\n",
    "\n",
    "    # Load existing history if available\n",
    "    if os.path.exists(history_path):\n",
    "        print(f\"Loading existing training history from: {history_path}\")\n",
    "        with open(history_path, 'r') as f:\n",
    "            history_dict = json.load(f)\n",
    "            history = type('History', (), {'history': history_dict})()\n",
    "    else:\n",
    "        history = type('History', (), {'history': {}})()\n",
    "else:\n",
    "    print(\"Creating new model...\")\n",
    "    model = create_uv_index_model(input_shape, folder_name)\n",
    "\n",
    "    print(\"\\n3. Starting training...\")\n",
    "    history = train_hybrid_model(\n",
    "        model=model,\n",
    "        X_train=X_train_seq,\n",
    "        y_train=y_train,\n",
    "        X_test=X_test_seq,\n",
    "        y_test=y_test,\n",
    "        epochs=100,\n",
    "        batch_size=128,\n",
    "        folder_name=folder_name\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f0059ecf-7f4f-496f-bed8-85e2d990ff71",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "4. Generating predictions...\n",
      "2028/2028 [==============================] - 19s 9ms/step\n",
      "\n",
      "5. Model evaluation...\n",
      "\n",
      "Error saving plots: Unknown format code 'd' for object of type 'float'\n",
      "\n",
      "UV Index Prediction Analysis:\n",
      "\n",
      "Raw Metrics:\n",
      "mae: 0.409\n",
      "rmse: 0.773\n",
      "r2: 0.918\n",
      "mean_error: -0.058\n",
      "std_error: 0.771\n",
      "median_error: 0.012\n",
      "p95_abs_error: 1.731\n",
      "within_05: 71.085\n",
      "within_1: 86.023\n",
      "within_15: 93.081\n",
      "within_2: 96.593\n",
      "\n",
      "Rounded Metrics:\n",
      "mae: 0.395\n",
      "rmse: 0.780\n",
      "r2: 0.917\n",
      "within_05: 78.912\n",
      "within_1: 90.167\n",
      "within_15: 95.121\n",
      "within_2: 97.533\n",
      "\n",
      "Analysis by UV Range:\n",
      "\n",
      "Low:\n",
      "  mae: 0.141\n",
      "  count: 41408.000\n",
      "  accuracy_within_05: 90.410\n",
      "  accuracy_within_1: 97.153\n",
      "\n",
      "Moderate:\n",
      "  mae: 0.855\n",
      "  count: 11464.000\n",
      "  accuracy_within_05: 40.448\n",
      "  accuracy_within_1: 67.856\n",
      "\n",
      "High:\n",
      "  mae: 0.847\n",
      "  count: 5534.000\n",
      "  accuracy_within_05: 36.881\n",
      "  accuracy_within_1: 68.395\n",
      "\n",
      "Very High:\n",
      "  mae: 0.927\n",
      "  count: 6225.000\n",
      "  accuracy_within_05: 32.048\n",
      "  accuracy_within_1: 63.711\n",
      "\n",
      "Extreme:\n",
      "  mae: 1.653\n",
      "  count: 235.000\n",
      "  accuracy_within_05: 0.000\n",
      "  accuracy_within_1: 17.447\n",
      "\n",
      "Confusion Matrix:\n",
      "               Low  Moderate    High  Very High  Extreme\n",
      "Low        43039.0    2265.0   129.0       10.0      0.0\n",
      "Moderate    1282.0    7688.0  1215.0      123.0      0.0\n",
      "High          10.0    1133.0  3291.0      439.0      0.0\n",
      "Very High      0.0      92.0  1131.0     3019.0      0.0\n",
      "Extreme        0.0       0.0     0.0        0.0      0.0\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 276520 characters]",
      "text/plain": [
       "<Figure size 2000x1500 with 5 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 33580 characters]",
      "text/plain": [
       "<Figure size 1000x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"\\n4. Generating predictions...\")\n",
    "predictions = model.predict(X_test_seq)\n",
    "predictions = np.clip(predictions, 0, 11)\n",
    "\n",
    "predictions_original = target_scaler.inverse_transform(predictions)\n",
    "y_test_original = target_scaler.inverse_transform(y_test)\n",
    "\n",
    "print(\"\\n5. Model evaluation...\")\n",
    "metrics = evaluate_uv_predictions(y_test_original, predictions_original, folder_name=folder_name)\n",
    "\n",
    "# Save training results only if new training was performed\n",
    "if not os.path.exists(model_path):\n",
    "    training_results = {\n",
    "        'model_params': {\n",
    "            'input_shape': input_shape,\n",
    "            'n_features': len(features),\n",
    "            'sequence_length': X_train_seq.shape[1]\n",
    "        },\n",
    "        'training_params': {\n",
    "            'batch_size': 128,\n",
    "            'total_epochs': len(history.history['loss']),\n",
    "            'best_epoch': np.argmin(history.history['val_loss']) + 1,\n",
    "        },\n",
    "        'performance_metrics': {\n",
    "            'final_loss': float(history.history['val_loss'][-1]),\n",
    "            'final_mae': float(history.history['val_mae'][-1]),\n",
    "            'best_val_loss': float(min(history.history['val_loss'])),\n",
    "            'out_of_range_predictions': int(np.sum((predictions < 0) | (predictions > 11)))\n",
    "        }\n",
    "    }\n",
    "\n",
    "    # Save training history\n",
    "    with open(history_path, 'w') as f:\n",
    "        history_dict = {key: [float(val) for val in values]\n",
    "                        for key, values in history.history.items()}\n",
    "        json.dump(history_dict, f, indent=4)\n",
    "else:\n",
    "    # Load existing training results if available\n",
    "    results_path = f'{folder_name}_training_results.json'\n",
    "    if os.path.exists(results_path):\n",
    "        with open(results_path, 'r') as f:\n",
    "            training_results = json.load(f)\n",
    "    else:\n",
    "        training_results = {}\n",
    "\n",
    "tf.keras.backend.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4365d2bf-daf8-49e1-be13-cce222bbb5bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "6. Predicting missing data...\n",
      "7122/7122 [==============================] - 62s 9ms/step\n",
      "\n",
      "7. Integrating predictions into dataset...\n",
      "Added 227879 predictions to dataset\n",
      "Rows with UV index after integration: 357615\n",
      "Updated dataset saved to: 2024-11-20_11-04_weather_data_uvindex.parquet\n",
      "\n",
      "All files saved with prefix: 2024-11-20_11-04\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n6. Predicting missing data...\")\n",
    "to_predict_predictions = model.predict(X_to_predict_seq)\n",
    "to_predict_predictions = np.clip(to_predict_predictions, 0, 11)\n",
    "\n",
    "print(\"\\n7. Integrating predictions into dataset...\")\n",
    "df_updated = integrate_predictions(df.copy(), to_predict_predictions)\n",
    "\n",
    "output_path = f'../../sources/weather_data_uvindex.parquet'\n",
    "df_updated.to_parquet(output_path)\n",
    "print(f\"Updated dataset saved to: {output_path}\")\n",
    "\n",
    "# Add prediction statistics\n",
    "prediction_stats = {\n",
    "    'n_predictions_added': len(to_predict_predictions),\n",
    "    'mean_predicted_uv': float(to_predict_predictions.mean()),\n",
    "    'min_predicted_uv': float(to_predict_predictions.min()),\n",
    "    'max_predicted_uv': float(to_predict_predictions.max()),\n",
    "}\n",
    "\n",
    "\n",
    "def convert_to_serializable(obj):\n",
    "    \"\"\"Convert numpy types to Python standard types for JSON serialization\"\"\"\n",
    "    if isinstance(obj, (np.int_, np.intc, np.intp, np.int8,\n",
    "                        np.int16, np.int32, np.int64, np.uint8,\n",
    "                        np.uint16, np.uint32, np.uint64)):\n",
    "        return int(obj)\n",
    "    elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)):\n",
    "        return float(obj)\n",
    "    elif isinstance(obj, (np.ndarray,)):\n",
    "        return obj.tolist()\n",
    "    elif isinstance(obj, dict):\n",
    "        return {key: convert_to_serializable(value) for key, value in obj.items()}\n",
    "    elif isinstance(obj, list):\n",
    "        return [convert_to_serializable(item) for item in obj]\n",
    "    return obj\n",
    "\n",
    "\n",
    "if not os.path.exists(model_path):\n",
    "    training_results['prediction_stats'] = prediction_stats\n",
    "\n",
    "    training_results = convert_to_serializable(training_results)\n",
    "    # Save final results\n",
    "    results_path = f'{folder_name}_training_results.json'\n",
    "    with open(results_path, 'w') as f:\n",
    "        json.dump(training_results, f, indent=4)\n",
    "\n",
    "print(f\"\\nAll files saved with prefix: {folder_name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "08fd4208-0afb-4bf1-bdef-b10b4065fe55",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Plot saved as: 2024-11-20_11-04/error_analysis.png\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 138512 characters]",
      "text/plain": [
       "<Figure size 1500x500 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Error statistics:\n",
      "MAE: 0.4087\n",
      "MSE: 0.5974\n",
      "RMSE: 0.7729\n",
      "Mean errors: -0.0576\n",
      "Std errors: 0.7707\n",
      "Predictions within ±0.5: 71.1%\n",
      "Predictions within ±1.0: 86.0%\n",
      "Predictions within ±1.5: 93.1%\n",
      "Predictions within ±2.0: 96.6%\n"
     ]
    }
   ],
   "source": [
    "def plot_error_analysis(y_true, y_pred, folder_name=None):\n",
    "    \"\"\"\n",
    "    Function to visualize prediction error analysis\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    y_true : array-like\n",
    "        Actual values\n",
    "    y_pred : array-like\n",
    "        Predicted values\n",
    "    folder_name : str, optional\n",
    "        Folder to save plots. If None, plots are not saved.\n",
    "    \"\"\"\n",
    "    import os\n",
    "    from datetime import datetime\n",
    "\n",
    "    # Convert to 1D numpy array if necessary\n",
    "    if isinstance(y_true, pd.Series):\n",
    "        y_true = y_true.values\n",
    "    if isinstance(y_pred, pd.Series):\n",
    "        y_pred = y_pred.values\n",
    "\n",
    "    y_true = y_true.ravel()\n",
    "    y_pred = y_pred.ravel()\n",
    "\n",
    "    # Calculate errors\n",
    "    errors = y_pred - y_true\n",
    "\n",
    "    # Create main figure\n",
    "    fig = plt.figure(figsize=(15, 5))\n",
    "\n",
    "    # Plot 1: Error Distribution\n",
    "    plt.subplot(1, 3, 1)\n",
    "    plt.hist(errors, bins=50, alpha=0.7)\n",
    "    plt.title('Prediction Error Distribution')\n",
    "    plt.xlabel('Error')\n",
    "    plt.ylabel('Frequency')\n",
    "\n",
    "    # Plot 2: Actual vs Predicted\n",
    "    plt.subplot(1, 3, 2)\n",
    "    plt.scatter(y_true, y_pred, alpha=0.5)\n",
    "    plt.plot([y_true.min(), y_true.max()], [y_true.min(), y_true.max()], 'r--', lw=2)\n",
    "    plt.title('Actual vs Predicted Values')\n",
    "    plt.xlabel('Actual Values')\n",
    "    plt.ylabel('Predicted Values')\n",
    "\n",
    "    # Plot 3: Errors vs Actual Values\n",
    "    plt.subplot(1, 3, 3)\n",
    "    plt.scatter(y_true, errors, alpha=0.5)\n",
    "    plt.axhline(y=0, color='r', linestyle='--')\n",
    "    plt.title('Errors vs Actual Values')\n",
    "    plt.xlabel('Actual Values')\n",
    "    plt.ylabel('Error')\n",
    "\n",
    "    plt.tight_layout()\n",
    "\n",
    "    # Save plot if folder is specified\n",
    "    if folder_name is not None:\n",
    "        try:\n",
    "            # Create folder if it doesn't exist\n",
    "            os.makedirs(folder_name, exist_ok=True)\n",
    "\n",
    "            # Generate filename with timestamp\n",
    "            filename = os.path.join(folder_name, 'error_analysis.png')\n",
    "\n",
    "            # Save figure\n",
    "            plt.savefig(filename, dpi=300, bbox_inches='tight')\n",
    "            print(f\"\\nPlot saved as: {filename}\")\n",
    "        except Exception as e:\n",
    "            print(f\"\\nError saving plot: {str(e)}\")\n",
    "\n",
    "    plt.show()\n",
    "\n",
    "    # Print error statistics\n",
    "    print(\"\\nError statistics:\")\n",
    "    print(f\"MAE: {np.mean(np.abs(errors)):.4f}\")\n",
    "    print(f\"MSE: {np.mean(errors ** 2):.4f}\")\n",
    "    print(f\"RMSE: {np.sqrt(np.mean(errors ** 2)):.4f}\")\n",
    "    print(f\"Mean errors: {np.mean(errors):.4f}\")\n",
    "    print(f\"Std errors: {np.std(errors):.4f}\")\n",
    "\n",
    "    # Calculate percentage of errors within thresholds\n",
    "    thresholds = [0.5, 1.0, 1.5, 2.0]\n",
    "    for threshold in thresholds:\n",
    "        within_threshold = np.mean(np.abs(errors) <= threshold) * 100\n",
    "        print(f\"Predictions within ±{threshold}: {within_threshold:.1f}%\")\n",
    "\n",
    "\n",
    "plot_error_analysis(y_test, predictions, folder_name=folder_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c57d6b2-00a6-4d31-935e-449a29dafd79",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
