{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8adcbe0819b88578",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-20T00:55:22.066729Z",
     "start_time": "2024-11-20T00:54:13.878615Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Get:1 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB]\n",
      "Hit:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease\n",
      "Hit:3 http://archive.ubuntu.com/ubuntu jammy InRelease                         \n",
      "Get:4 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB]\n",
      "Hit:5 http://archive.ubuntu.com/ubuntu jammy-backports InRelease\n",
      "Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2732 kB]\n",
      "Fetched 2989 kB in 1s (2026 kB/s)  \n",
      "Reading package lists... Done\n",
      "Reading package lists... Done\n",
      "Building dependency tree... Done\n",
      "Reading state information... Done\n",
      "graphviz is already the newest version (2.42.2-6ubuntu0.1).\n",
      "0 upgraded, 0 newly installed, 0 to remove and 121 not upgraded.\n",
      "Requirement already satisfied: tensorflow==2.13.0 in /usr/local/lib/python3.11/dist-packages (2.13.0)\n",
      "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (2.0.0)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (1.6.3)\n",
      "Requirement already satisfied: flatbuffers>=23.1.21 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (23.5.26)\n",
      "Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (0.4.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (0.2.0)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (1.58.0)\n",
      "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (3.9.0)\n",
      "Requirement already satisfied: keras<2.14,>=2.13.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (2.13.1)\n",
      "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (16.0.6)\n",
      "Requirement already satisfied: numpy<=1.24.3,>=1.22 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (1.24.3)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (3.3.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (23.1)\n",
      "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (4.24.3)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (68.2.2)\n",
      "Requirement already satisfied: six>=1.12.0 in /usr/lib/python3/dist-packages (from tensorflow==2.13.0) (1.16.0)\n",
      "Requirement already satisfied: tensorboard<2.14,>=2.13 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (2.13.0)\n",
      "Requirement already satisfied: tensorflow-estimator<2.14,>=2.13.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (2.13.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (2.3.0)\n",
      "Requirement already satisfied: typing-extensions<4.6.0,>=3.6.6 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (4.5.0)\n",
      "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (1.14.1)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (0.37.1)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.11/dist-packages (from astunparse>=1.6.0->tensorflow==2.13.0) (0.41.2)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2.23.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (1.0.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (3.4.4)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2.31.0)\n",
      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (0.7.1)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2.3.7)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (5.3.1)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (0.3.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (4.9)\n",
      "Requirement already satisfied: urllib3>=2.0.5 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2.0.5)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (1.3.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (3.2.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (3.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2023.7.22)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.11/dist-packages (from werkzeug>=1.0.1->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2.1.3)\n",
      "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.11/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (0.5.0)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/lib/python3/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (3.2.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (1.24.3)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (2.2.3)\n",
      "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (1.24.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: keras==2.13.1 in /usr/local/lib/python3.11/dist-packages (2.13.1)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.11/dist-packages (1.5.2)\n",
      "Requirement already satisfied: numpy>=1.19.5 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.24.3)\n",
      "Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.14.1)\n",
      "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.4.2)\n",
      "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (3.5.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: matplotlib in /usr/local/lib/python3.11/dist-packages (3.8.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (4.42.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.4.5)\n",
      "Requirement already satisfied: numpy<2,>=1.21 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.24.3)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (23.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (10.0.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (3.2.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (2.8.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.11/dist-packages (1.4.2)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: pyarrow in /usr/local/lib/python3.11/dist-packages (18.0.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: fastparquet in /usr/local/lib/python3.11/dist-packages (2024.11.0)\n",
      "Requirement already satisfied: pandas>=1.5.0 in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2.2.3)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (from fastparquet) (1.24.3)\n",
      "Requirement already satisfied: cramjam>=2.3 in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2.9.0)\n",
      "Requirement already satisfied: fsspec in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2024.10.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from fastparquet) (23.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2024.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas>=1.5.0->fastparquet) (1.16.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.11/dist-packages (1.14.1)\n",
      "Requirement already satisfied: numpy<2.3,>=1.23.5 in /usr/local/lib/python3.11/dist-packages (from scipy) (1.24.3)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: seaborn in /usr/local/lib/python3.11/dist-packages (0.13.2)\n",
      "Requirement already satisfied: numpy!=1.24.0,>=1.20 in /usr/local/lib/python3.11/dist-packages (from seaborn) (1.24.3)\n",
      "Requirement already satisfied: pandas>=1.2 in /usr/local/lib/python3.11/dist-packages (from seaborn) (2.2.3)\n",
      "Requirement already satisfied: matplotlib!=3.6.1,>=3.4 in /usr/local/lib/python3.11/dist-packages (from seaborn) (3.8.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (4.42.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.4.5)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (23.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (10.0.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (3.2.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.2->seaborn) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.2->seaborn) (2024.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib!=3.6.1,>=3.4->seaborn) (1.16.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (4.67.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: pydot in /usr/local/lib/python3.11/dist-packages (3.0.2)\n",
      "Requirement already satisfied: pyparsing>=3.0.9 in /usr/local/lib/python3.11/dist-packages (from pydot) (3.2.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: tensorflow-io in /usr/local/lib/python3.11/dist-packages (0.37.1)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem==0.37.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow-io) (0.37.1)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.11/dist-packages (0.23.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from tensorflow-addons) (23.1)\n",
      "Requirement already satisfied: typeguard<3.0.0,>=2.7 in /usr/local/lib/python3.11/dist-packages (from tensorflow-addons) (2.13.3)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "from opt_einsum.paths import branch_1\n",
    "!apt-get update\n",
    "!apt-get install graphviz -y\n",
    "\n",
    "!pip install tensorflow==2.13.0\n",
    "!pip install numpy\n",
    "!pip install pandas\n",
    "!pip install keras==2.13.1\n",
    "!pip install scikit-learn\n",
    "!pip install matplotlib\n",
    "!pip install joblib\n",
    "!pip install pyarrow\n",
    "!pip install fastparquet\n",
    "!pip install scipy\n",
    "!pip install seaborn\n",
    "!pip install tqdm\n",
    "!pip install pydot\n",
    "!pip install tensorflow-io\n",
    "!pip install tensorflow-addons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7a813e3cbca057b7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-20T00:55:22.782689Z",
     "start_time": "2024-11-20T00:55:22.089165Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-21 08:23:10.586264: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "/usr/local/lib/python3.11/dist-packages/tensorflow_addons/utils/tfa_eol_msg.py:23: UserWarning: \n",
      "\n",
      "TensorFlow Addons (TFA) has ended development and introduction of new features.\n",
      "TFA has entered a minimal maintenance and release mode until a planned end of life in May 2024.\n",
      "Please modify downstream libraries to take dependencies from other repositories in our TensorFlow community (e.g. Keras, Keras-CV, and Keras-NLP). \n",
      "\n",
      "For more information see: https://github.com/tensorflow/addons/issues/2807 \n",
      "\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Dense, LSTM, MultiHeadAttention, Dropout, BatchNormalization, LayerNormalization, Input, Activation, Lambda, Bidirectional, Add, MaxPooling1D, Conv1D, GlobalAveragePooling1D\n",
    "from tensorflow.keras import regularizers\n",
    "from tensorflow.keras.models import Model\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\n",
    "from tensorflow.keras.optimizers import AdamW\n",
    "import json\n",
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.keras.utils import plot_model\n",
    "import tensorflow_addons as tfa\n",
    "import os\n",
    "import joblib\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import confusion_matrix, mean_absolute_error, mean_squared_error, r2_score\n",
    "\n",
    "folder_name = datetime.now().strftime(\"%Y-%m-%d_%H-%M\")\n",
    "random_state_value = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b3f525e19f78a1da",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_season(date):\n",
    "    month = date.month\n",
    "    day = date.day\n",
    "    if (month == 12 and day >= 21) or (month <= 3 and day < 20):\n",
    "        return 'Winter'\n",
    "    elif (month == 3 and day >= 20) or (month <= 6 and day < 21):\n",
    "        return 'Spring'\n",
    "    elif (month == 6 and day >= 21) or (month <= 9 and day < 23):\n",
    "        return 'Summer'\n",
    "    elif (month == 9 and day >= 23) or (month <= 12 and day < 21):\n",
    "        return 'Autumn'\n",
    "    else:\n",
    "        return 'Unknown'\n",
    "\n",
    "\n",
    "def get_time_period(hour):\n",
    "    if 5 <= hour < 12:\n",
    "        return 'Morning'\n",
    "    elif 12 <= hour < 17:\n",
    "        return 'Afternoon'\n",
    "    elif 17 <= hour < 21:\n",
    "        return 'Evening'\n",
    "    else:\n",
    "        return 'Night'\n",
    "\n",
    "\n",
    "def add_time_features(df):\n",
    "    df['datetime'] = pd.to_datetime(df['datetime'])\n",
    "    df['timestamp'] = df['datetime'].astype(np.int64) // 10 ** 9\n",
    "    df['year'] = df['datetime'].dt.year\n",
    "    df['month'] = df['datetime'].dt.month\n",
    "    df['day'] = df['datetime'].dt.day\n",
    "    df['hour'] = df['datetime'].dt.hour\n",
    "    df['minute'] = df['datetime'].dt.minute\n",
    "    df['hour_sin'] = np.sin(df['hour'] * (2 * np.pi / 24))\n",
    "    df['hour_cos'] = np.cos(df['hour'] * (2 * np.pi / 24))\n",
    "    df['day_of_week'] = df['datetime'].dt.dayofweek\n",
    "    df['day_of_year'] = df['datetime'].dt.dayofyear\n",
    "    df['week_of_year'] = df['datetime'].dt.isocalendar().week.astype(int)\n",
    "    df['quarter'] = df['datetime'].dt.quarter\n",
    "    df['is_month_end'] = df['datetime'].dt.is_month_end.astype(int)\n",
    "    df['is_quarter_end'] = df['datetime'].dt.is_quarter_end.astype(int)\n",
    "    df['is_year_end'] = df['datetime'].dt.is_year_end.astype(int)\n",
    "    df['month_sin'] = np.sin(df['month'] * (2 * np.pi / 12))\n",
    "    df['month_cos'] = np.cos(df['month'] * (2 * np.pi / 12))\n",
    "    df['day_of_year_sin'] = np.sin(df['day_of_year'] * (2 * np.pi / 365.25))\n",
    "    df['day_of_year_cos'] = np.cos(df['day_of_year'] * (2 * np.pi / 365.25))\n",
    "    df['season'] = df['datetime'].apply(get_season)\n",
    "    df['time_period'] = df['hour'].apply(get_time_period)\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_solar_features(df):\n",
    "    # Calculate solar angle\n",
    "    df['solar_angle'] = np.sin(df['day_of_year'] * (2 * np.pi / 365.25)) * np.sin(df['hour'] * (2 * np.pi / 24))\n",
    "\n",
    "    # Interactions between relevant features\n",
    "    df['cloud_temp_interaction'] = df['cloudcover'] * df['temp']\n",
    "    df['visibility_cloud_interaction'] = df['visibility'] * (100 - df['cloudcover'])\n",
    "\n",
    "    # Derived features\n",
    "    df['clear_sky_index'] = (100 - df['cloudcover']) / 100\n",
    "    df['temp_gradient'] = df['temp'] - df['tempmin']\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_solar_specific_features(df):\n",
    "    # Solar angle and day length calculations\n",
    "    df['day_length'] = 12 + 3 * np.sin(2 * np.pi * (df['day_of_year'] - 81) / 365.25)\n",
    "    df['solar_noon'] = 12 - df['hour']\n",
    "    df['solar_elevation'] = np.sin(2 * np.pi * df['day_of_year'] / 365.25) * np.cos(2 * np.pi * df['solar_noon'] / 24)\n",
    "\n",
    "    # Feature interactions\n",
    "    df['cloud_elevation'] = df['cloudcover'] * df['solar_elevation']\n",
    "    df['visibility_elevation'] = df['visibility'] * df['solar_elevation']\n",
    "\n",
    "    # Extended window rolling features\n",
    "    df['cloud_rolling_12h'] = df['cloudcover'].rolling(window=12).mean()\n",
    "    df['temp_rolling_12h'] = df['temp'].rolling(window=12).mean()\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_uv_specific_features(df):\n",
    "    # Solar zenith angle calculation\n",
    "    lat = 41.9  # assuming constant latitude for the dataset - Rome's latitude\n",
    "    df['solar_zenith'] = 90 - np.degrees(\n",
    "        np.arcsin(\n",
    "            np.sin(np.radians(lat)) * np.sin(df['solar_elevation']) +\n",
    "            np.cos(np.radians(lat)) * np.cos(df['solar_elevation']) * np.cos(df['hour'] * 15)\n",
    "        )\n",
    "    )\n",
    "\n",
    "    # UV peak hours indicator (10:00-16:00)\n",
    "    df['is_uv_peak_hours'] = ((df['hour'] >= 10) & (df['hour'] <= 16)).astype(int)\n",
    "\n",
    "    # Atmospheric attenuation factor\n",
    "    df['atmospheric_attenuation'] = (100 - df['cloudcover']) * (df['visibility'] / 100) * (1 - df['humidity'] / 200)\n",
    "\n",
    "    # Seasonal UV factor\n",
    "    df['uv_seasonal_factor'] = np.where(df['season_Summer'], 1.0,\n",
    "                                        np.where(df['season_Spring'], 0.7,\n",
    "                                                 np.where(df['season_Autumn'], 0.5, 0.3)))\n",
    "\n",
    "    # Solar elevation and atmospheric transparency interaction\n",
    "    df['solar_clarity_index'] = df['solar_elevation'] * df['atmospheric_attenuation'] / 100\n",
    "\n",
    "    # UV-specific rolling features\n",
    "    df['clarity_rolling_3h'] = df['atmospheric_attenuation'].rolling(window=3).mean()\n",
    "    df['temp_uv_interaction'] = df['temp'] * df['solar_clarity_index']\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_advanced_features(df):\n",
    "    \"\"\"\n",
    "    Add all advanced features in the correct order\n",
    "    \"\"\"\n",
    "    # 1. First add basic time features\n",
    "    df = add_time_features(df)\n",
    "\n",
    "    # 2. One-hot encoding for categorical features\n",
    "    df = pd.get_dummies(df, columns=['season', 'time_period'])\n",
    "\n",
    "    # 3. Add solar and specific features\n",
    "    df = add_solar_features(df)\n",
    "    df = add_solar_specific_features(df)\n",
    "\n",
    "    # 4. Ensure datetime index\n",
    "    if not isinstance(df.index, pd.DatetimeIndex):\n",
    "        df.index = pd.to_datetime(df.index)\n",
    "\n",
    "    # 5. Add weather variable interactions\n",
    "    df['temp_humidity'] = df['temp'] * df['humidity']\n",
    "    df['temp_cloudcover'] = df['temp'] * df['cloudcover']\n",
    "    df['visibility_cloudcover'] = df['visibility'] * df['cloudcover']\n",
    "\n",
    "    # 6. Add solar radiation derived features\n",
    "    df['clear_sky_factor'] = (100 - df['cloudcover']) / 100\n",
    "    df['day_length'] = np.sin(df['day_of_year_sin']) * 12 + 12\n",
    "\n",
    "    # 7. Add lag features\n",
    "    df['temp_1h_lag'] = df['temp'].shift(1)\n",
    "    df['cloudcover_1h_lag'] = df['cloudcover'].shift(1)\n",
    "    df['humidity_1h_lag'] = df['humidity'].shift(1)\n",
    "\n",
    "    # 8. Add rolling means\n",
    "    df['temp_rolling_mean_6h'] = df['temp'].rolling(window=6).mean()\n",
    "    df['cloudcover_rolling_mean_6h'] = df['cloudcover'].rolling(window=6).mean()\n",
    "    df['temp_humidity_interaction'] = df['temp'] * df['humidity'] / 100\n",
    "\n",
    "    # 9. Add atmospheric stability\n",
    "    df['atmospheric_stability'] = df.groupby(df.index.date)['pressure'].transform(\n",
    "        lambda x: x.std()\n",
    "    ).fillna(0)\n",
    "\n",
    "    # 10. Add extreme conditions indicator\n",
    "    df['extreme_conditions'] = ((df['temp'] > df['temp'].quantile(0.75)) &\n",
    "                                (df['humidity'] < df['humidity'].quantile(0.25))).astype(int)\n",
    "\n",
    "    # 11. Add atmospheric transparency\n",
    "    df['atmospheric_transparency'] = (100 - df['cloudcover']) * (df['visibility'] / 10)\n",
    "\n",
    "    # 12. Add transitional seasons indicator\n",
    "    df['is_transition_season'] = ((df['season_Spring'] | df['season_Autumn'])).astype(int)\n",
    "\n",
    "    # 13. Add solar cloud effect\n",
    "    if 'solar_elevation' in df.columns:\n",
    "        df['solar_cloud_effect'] = df['solar_elevation'] * (100 - df['cloudcover']) / 100\n",
    "\n",
    "    # 14. Finally add UV specific features\n",
    "    df = add_uv_specific_features(df)\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def prepare_advanced_data(df):\n",
    "    \"\"\"\n",
    "    Prepares data for UV index prediction model with advanced feature engineering\n",
    "    and optimized preprocessing.\n",
    "\n",
    "    Args:\n",
    "        df: DataFrame with meteorological data\n",
    "\n",
    "    Returns:\n",
    "        tuple: (X_train_scaled, X_test_scaled, y_train, y_test, scaler, final_features, X_to_predict_scaled)\n",
    "    \"\"\"\n",
    "    # Apply feature engineering functions\n",
    "    df = add_advanced_features(df)\n",
    "\n",
    "    # Optimized feature selection for UV index\n",
    "    selected_features = {\n",
    "        # Primary meteorological features\n",
    "        'atmospheric': [\n",
    "            'temp', 'humidity', 'cloudcover', 'visibility',\n",
    "            'clear_sky_index', 'atmospheric_transparency'\n",
    "        ],\n",
    "\n",
    "        # Essential temporal features\n",
    "        'temporal': [\n",
    "            'hour_sin', 'hour_cos',\n",
    "            'day_of_year_sin', 'day_of_year_cos'\n",
    "        ],\n",
    "\n",
    "        # Solar features\n",
    "        'solar': [\n",
    "            'solar_angle', 'solar_elevation',\n",
    "            'day_length', 'solar_noon',\n",
    "            'solar_cloud_effect'\n",
    "        ],\n",
    "\n",
    "        # Key interactions\n",
    "        'interactions': [\n",
    "            'cloud_temp_interaction',\n",
    "            'visibility_cloud_interaction',\n",
    "            'temp_humidity_interaction',\n",
    "            'solar_clarity_index'\n",
    "        ],\n",
    "\n",
    "        # Rolling features\n",
    "        'rolling': [\n",
    "            'cloud_rolling_12h',\n",
    "            'temp_rolling_mean_6h'\n",
    "        ]\n",
    "    }\n",
    "\n",
    "    # Flatten feature list\n",
    "    base_features = [item for sublist in selected_features.values() for item in sublist]\n",
    "\n",
    "    # Add categorical features (one-hot encoded)\n",
    "    categorical_columns = [col for col in df.columns if col.startswith(('season_', 'time_period_'))]\n",
    "    final_features = base_features + categorical_columns\n",
    "\n",
    "    # Temporal preprocessing\n",
    "    df = df.sort_values('datetime')\n",
    "    df.set_index('datetime', inplace=True)\n",
    "\n",
    "    # Advanced interpolation for missing values\n",
    "    for column in final_features:\n",
    "        if column in df.columns:\n",
    "            if df[column].isnull().any():\n",
    "                if column in selected_features['rolling']:\n",
    "                    df[column] = df[column].ffill().bfill()\n",
    "                else:\n",
    "                    df[column] = df[column].interpolate(method='time', limit_direction='both')\n",
    "\n",
    "    # Temporal data split\n",
    "    data_after_2010 = df[df.index.year >= 2010].copy()\n",
    "    data_before_2010 = df[df.index.year < 2010].copy()\n",
    "\n",
    "    print(f\"\\nTemporal distribution of data:\")\n",
    "    print(f\"Records after 2010: {len(data_after_2010):,}\")\n",
    "    print(f\"Records before 2010: {len(data_before_2010):,}\")\n",
    "\n",
    "    # Feature and target preparation\n",
    "    X = data_after_2010[final_features]\n",
    "    y = data_after_2010['uvindex']\n",
    "    X_to_predict = data_before_2010[final_features]\n",
    "\n",
    "    # Data validation\n",
    "    if X.isnull().any().any() or y.isnull().any():\n",
    "        print(\"\\nWarning: Found missing values after preprocessing\")\n",
    "        print(\"Features with missing values:\", X.columns[X.isnull().any()].tolist())\n",
    "        X = X.fillna(X.mean())\n",
    "        y = y.fillna(y.mean())\n",
    "\n",
    "    # Stratified data split\n",
    "    X_train, X_test, y_train, y_test = train_test_split(\n",
    "        X, y,\n",
    "        test_size=0.5,\n",
    "        random_state=random_state_value,\n",
    "        stratify=pd.qcut(y, q=5, duplicates='drop', labels=False)\n",
    "    )\n",
    "\n",
    "    # Robust feature scaling\n",
    "    feature_scaler = RobustScaler()\n",
    "    X_train_scaled = feature_scaler.fit_transform(X_train)\n",
    "    X_test_scaled = feature_scaler.transform(X_test)\n",
    "    X_to_predict_scaled = feature_scaler.transform(X_to_predict)\n",
    "\n",
    "    target_scaler = RobustScaler()\n",
    "    y_train_scaled = target_scaler.fit_transform(y_train.values.reshape(-1, 1)).ravel()\n",
    "    y_test_scaled = target_scaler.transform(y_test.values.reshape(-1, 1)).ravel()\n",
    "\n",
    "    # Final validation\n",
    "    assert not np.isnan(X_train_scaled).any(), \"Found NaN in X_train_scaled\"\n",
    "    assert not np.isnan(X_test_scaled).any(), \"Found NaN in X_test_scaled\"\n",
    "    assert not np.isnan(X_to_predict_scaled).any(), \"Found NaN in X_to_predict_scaled\"\n",
    "\n",
    "    # Print feature information\n",
    "    print(\"\\nNumber of features used:\", len(final_features))\n",
    "    print(\"\\nFeature categories:\")\n",
    "    for category, features in selected_features.items():\n",
    "        print(f\"{category}: {len(features)} features\")\n",
    "    print(f\"Categorical: {len(categorical_columns)} features\")\n",
    "\n",
    "    return (X_train_scaled, X_test_scaled, y_train_scaled, y_test_scaled,\n",
    "            feature_scaler, target_scaler, final_features, X_to_predict_scaled)\n",
    "\n",
    "\n",
    "def create_sequence_data(X, sequence_length=24):\n",
    "    \"\"\"\n",
    "    Converts data into sequences for LSTM input\n",
    "    sequence_length represents how many previous hours to consider\n",
    "    \"\"\"\n",
    "    sequences = []\n",
    "    for i in range(len(X) - sequence_length + 1):\n",
    "        sequences.append(X[i:i + sequence_length])\n",
    "    return np.array(sequences)\n",
    "\n",
    "\n",
    "def prepare_hybrid_data(df):\n",
    "    # Use existing data preparation\n",
    "    X_train_scaled, X_test_scaled, y_train, y_test, feature_scaler, target_scaler, features, X_to_predict_scaled = prepare_advanced_data(df)\n",
    "\n",
    "    # Convert data to sequences\n",
    "    sequence_length = 24  # 24 hours of historical data\n",
    "\n",
    "    X_train_seq = create_sequence_data(X_train_scaled, sequence_length)\n",
    "    X_test_seq = create_sequence_data(X_test_scaled, sequence_length)\n",
    "\n",
    "    # Adjust y by removing the first (sequence_length-1) elements\n",
    "    y_train = y_train[sequence_length - 1:]\n",
    "    y_test = y_test[sequence_length - 1:]\n",
    "\n",
    "    X_to_predict_seq = create_sequence_data(X_to_predict_scaled, sequence_length)\n",
    "\n",
    "    return X_train_seq, X_test_seq, y_train, y_test, feature_scaler, target_scaler, features, X_to_predict_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9dff3259-b376-4cfc-89d8-ab2ea18aaa5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_residual_lstm_layer(x, units, dropout_rate, l2_reg=0.01,\n",
    "                               survival_probability=0.8, return_sequences=True):\n",
    "    \"\"\"LSTM layer with stochastic depth\"\"\"\n",
    "    residual = x\n",
    "\n",
    "    # Main path\n",
    "    x = Bidirectional(LSTM(units, return_sequences=return_sequences,\n",
    "                           kernel_regularizer=regularizers.l2(l2_reg)))(x)\n",
    "    x = LayerNormalization()(x)\n",
    "    x = Dropout(dropout_rate)(x)\n",
    "\n",
    "    # Adjust residual dimension if needed\n",
    "    if return_sequences:\n",
    "        # For Bidirectional LSTM, the output dimension is 2 * units\n",
    "        target_dim = 2 * units\n",
    "        if int(residual.shape[-1]) != target_dim:\n",
    "            # Use Dense layer instead of Conv1D for better dimension matching\n",
    "            residual = Dense(target_dim)(residual)\n",
    "\n",
    "    # Apply stochastic depth only if dimensions match\n",
    "    if x.shape[-1] == residual.shape[-1]:\n",
    "        x = tfa.layers.StochasticDepth(survival_probability)([x, residual])\n",
    "    else:\n",
    "        print(f\"Warning: Dimension mismatch - x: {x.shape}, residual: {residual.shape}\")\n",
    "        # Skip residual connection if dimensions don't match\n",
    "        pass\n",
    "\n",
    "    return x\n",
    "\n",
    "\n",
    "def attention_block(x, units, num_heads=8, survival_probability=0.8):\n",
    "    \"\"\"\n",
    "    Attention block with stochastic depth.\n",
    "    \"\"\"\n",
    "    original_x = x\n",
    "\n",
    "    # Compute self-attention\n",
    "    attention = MultiHeadAttention(num_heads=num_heads, key_dim=units)(x, x)\n",
    "\n",
    "    # Ensure dimensions match before applying stochastic depth\n",
    "    if attention.shape[-1] != original_x.shape[-1]:\n",
    "        original_x = Dense(attention.shape[-1])(original_x)\n",
    "\n",
    "    # Apply stochastic depth to the attention path\n",
    "    x = tfa.layers.StochasticDepth(survival_probability)([attention, original_x])\n",
    "    x = LayerNormalization()(x)\n",
    "\n",
    "    # Store the input to the FFN\n",
    "    ffn_input = x\n",
    "\n",
    "    # FFN block\n",
    "    x = Dense(units * 4, activation='swish')(x)\n",
    "    x = Dense(ffn_input.shape[-1])(x)  # Match the input dimension\n",
    "\n",
    "    # Apply stochastic depth to the FFN\n",
    "    x = tfa.layers.StochasticDepth(survival_probability)([x, ffn_input])\n",
    "    x = LayerNormalization()(x)\n",
    "\n",
    "    return x\n",
    "\n",
    "\n",
    "def create_uv_index_model(input_shape, folder_name, l2_lambda=0.005, max_output=11):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    # Further adjusted hyperparameters\n",
    "    survival_probs = [0.98, 0.95, 0.92]  # Even higher survival probabilities\n",
    "    attention_survival_probs = [0.95, 0.92, 0.9]\n",
    "\n",
    "    # First LSTM block\n",
    "    x = create_residual_lstm_layer(\n",
    "        inputs, 64, dropout_rate=0.2,  # Further reduced dropout\n",
    "        l2_reg=l2_lambda,\n",
    "        survival_probability=survival_probs[0],\n",
    "        return_sequences=True\n",
    "    )\n",
    "    x = attention_block(x, 128, num_heads=2,  # Reduced heads\n",
    "                        survival_probability=attention_survival_probs[0])\n",
    "\n",
    "    # Second LSTM block\n",
    "    x = create_residual_lstm_layer(\n",
    "        x, 32, dropout_rate=0.15,\n",
    "        l2_reg=l2_lambda,\n",
    "        survival_probability=survival_probs[1],\n",
    "        return_sequences=True\n",
    "    )\n",
    "    x = attention_block(x, 64, num_heads=2,\n",
    "                        survival_probability=attention_survival_probs[1])\n",
    "\n",
    "    # Third LSTM block\n",
    "    x = create_residual_lstm_layer(\n",
    "        x, 16, dropout_rate=0.1,\n",
    "        l2_reg=l2_lambda,\n",
    "        survival_probability=survival_probs[2],\n",
    "        return_sequences=True\n",
    "    )\n",
    "    x = attention_block(x, 32, num_heads=2,\n",
    "                        survival_probability=attention_survival_probs[2])\n",
    "\n",
    "    # Global attention with reduced complexity\n",
    "    x_input = x\n",
    "    x = MultiHeadAttention(num_heads=2, key_dim=32)(x, x)\n",
    "\n",
    "    if x.shape[-1] != x_input.shape[-1]:\n",
    "        x_input = Dense(x.shape[-1])(x_input)\n",
    "\n",
    "    x = tfa.layers.StochasticDepth(survival_probability=0.95)([x, x_input])\n",
    "    x = LayerNormalization()(x)\n",
    "\n",
    "    # Simplified dense layers\n",
    "    x = GlobalAveragePooling1D()(x)\n",
    "\n",
    "    # Gradual dimension reduction\n",
    "    x = Dense(32, activation='swish', kernel_regularizer=regularizers.l2(l2_lambda / 2), kernel_constraint=tf.keras.constraints.MaxNorm(3))(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dropout(0.05)(x)  # Minimal dropout\n",
    "\n",
    "    x = Dense(16, activation='swish',\n",
    "              kernel_regularizer=regularizers.l2(l2_lambda / 2))(x)\n",
    "    x = BatchNormalization()(x)\n",
    "\n",
    "    # Modified output layer\n",
    "    x = Dense(8, activation='swish')(x)\n",
    "    outputs = Dense(1, activation='sigmoid')(x)  # Sigmoid activation\n",
    "    outputs = Lambda(lambda x: x * max_output)(outputs)  # Scale to [0, 11] range\n",
    "\n",
    "    model = Model(inputs=inputs, outputs=outputs, name=\"UvModel\")\n",
    "\n",
    "    # More stable learning rate schedule\n",
    "    initial_learning_rate = 0.0001  # Further reduced\n",
    "    warmup_steps = 1000\n",
    "    decay_steps = 5000\n",
    "\n",
    "    # Corretto learning rate schedule\n",
    "    class CustomLRSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
    "        def __init__(self, initial_lr=0.0001, warmup_steps=1000, decay_steps=5000):\n",
    "            super().__init__()\n",
    "            self.initial_lr = initial_lr\n",
    "            self.warmup_steps = warmup_steps\n",
    "            self.decay_steps = decay_steps\n",
    "\n",
    "        def __call__(self, step):\n",
    "            # Convert to float32\n",
    "            step_f = tf.cast(step, tf.float32)\n",
    "            warmup_steps_f = tf.cast(self.warmup_steps, tf.float32)\n",
    "            decay_steps_f = tf.cast(self.decay_steps, tf.float32)\n",
    "\n",
    "            # Warmup phase\n",
    "            warmup_progress = step_f / warmup_steps_f\n",
    "            warmup_lr = self.initial_lr * warmup_progress\n",
    "\n",
    "            # Decay phase\n",
    "            decay_progress = (step_f - warmup_steps_f) / decay_steps_f\n",
    "            decay_factor = 0.5 * (1.0 + tf.cos(tf.constant(np.pi) * decay_progress))\n",
    "            decay_lr = self.initial_lr * decay_factor\n",
    "\n",
    "            # Combine phases\n",
    "            lr = tf.where(step_f < warmup_steps_f, warmup_lr, decay_lr)\n",
    "            return lr\n",
    "\n",
    "        def get_config(self):\n",
    "            return {\n",
    "                \"initial_lr\": self.initial_lr,\n",
    "                \"warmup_steps\": self.warmup_steps,\n",
    "                \"decay_steps\": self.decay_steps\n",
    "            }\n",
    "\n",
    "    # Utilizzo dello schedule corretto\n",
    "    lr_schedule = CustomLRSchedule(\n",
    "        initial_lr=initial_learning_rate,\n",
    "        warmup_steps=warmup_steps,\n",
    "        decay_steps=decay_steps\n",
    "    )\n",
    "\n",
    "    optimizer = AdamW(\n",
    "        learning_rate=lr_schedule,\n",
    "        weight_decay=0.0005,\n",
    "        beta_1=0.9,\n",
    "        beta_2=0.999,\n",
    "        epsilon=1e-7\n",
    "    )\n",
    "\n",
    "    # Improved loss function\n",
    "    def smooth_uv_loss(y_true, y_pred):\n",
    "        # Basic MSE with smoothing\n",
    "        mse = tf.square(y_true - y_pred)\n",
    "\n",
    "        # Smooth L1 component for better stability\n",
    "        abs_diff = tf.abs(y_true - y_pred)\n",
    "        smooth_l1 = tf.where(abs_diff < 1.0,\n",
    "                             0.5 * tf.square(abs_diff),\n",
    "                             abs_diff - 0.5)\n",
    "\n",
    "        # Combined loss with dynamic weighting\n",
    "        combined_loss = 0.7 * mse + 0.3 * smooth_l1\n",
    "\n",
    "        # Gentle weighting for high UV values\n",
    "        high_uv_weight = tf.where(y_true >= 8.0, 1.2, 1.0)\n",
    "\n",
    "        # Smooth peak hours weight\n",
    "        time_of_day = tf.cast(tf.math.floormod(tf.range(tf.shape(y_true)[0]), 24),\n",
    "                              tf.float32)\n",
    "        peak_weight = 1.0 + 0.2 * tf.math.sigmoid((time_of_day - 10.0) * 0.5) * \\\n",
    "                      tf.math.sigmoid((16.0 - time_of_day) * 0.5)\n",
    "\n",
    "        total_weight = high_uv_weight * peak_weight\n",
    "\n",
    "        return tf.reduce_mean(combined_loss * total_weight)\n",
    "\n",
    "    # Improved MAPE metric\n",
    "    def smooth_mape(y_true, y_pred):\n",
    "        epsilon = 1e-7\n",
    "        diff = tf.abs(y_true - y_pred)\n",
    "        scale = tf.maximum(tf.abs(y_true) + epsilon, 0.5)  # Minimum scale of 0.5\n",
    "        return tf.reduce_mean(diff / scale) * 100\n",
    "\n",
    "    model.compile(\n",
    "        optimizer=optimizer,\n",
    "        loss=smooth_uv_loss,\n",
    "        metrics=[\n",
    "            'mae',\n",
    "            'mse',\n",
    "            tf.keras.metrics.RootMeanSquaredError(),\n",
    "            smooth_mape\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    model.summary()\n",
    "\n",
    "    plot_model(model,\n",
    "               to_file=f'{folder_name}_model_architecture.png',\n",
    "               show_shapes=True,\n",
    "               show_layer_names=True,\n",
    "               dpi=150,\n",
    "               show_layer_activations=True)\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "def evaluate_uv_predictions(y_true, y_pred, folder_name=None):\n",
    "    \"\"\"\n",
    "    Comprehensive evaluation of UV index predictions with detailed analysis and visualizations.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    y_true : array-like\n",
    "        Actual UV index values\n",
    "    y_pred : array-like\n",
    "        Predicted UV index values\n",
    "    folder_name : str, optional\n",
    "        Folder to save analysis plots\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    dict\n",
    "        Dictionary containing all calculated metrics\n",
    "    \"\"\"\n",
    "\n",
    "    # Initialize plot paths\n",
    "    main_plot_path = None\n",
    "    conf_matrix_path = None\n",
    "\n",
    "    # Data preprocessing\n",
    "    y_true = np.array(y_true).ravel()\n",
    "    y_pred = np.array(y_pred).ravel()\n",
    "\n",
    "    # Rounding and clipping predictions\n",
    "    y_pred_rounded = np.round(y_pred * 2) / 2  # Round to nearest 0.5\n",
    "    y_pred_clipped = np.clip(y_pred_rounded, 0, 11)\n",
    "\n",
    "    # Calculate errors\n",
    "    errors = y_pred - y_true\n",
    "    errors_rounded = y_pred_clipped - y_true\n",
    "\n",
    "    # Function to determine UV risk level\n",
    "    def get_uv_risk_level(values):\n",
    "        levels = np.full_like(values, 'Low', dtype=object)\n",
    "        levels[(values > 2) & (values <= 5)] = 'Moderate'\n",
    "        levels[(values > 5) & (values <= 7)] = 'High'\n",
    "        levels[(values > 7) & (values <= 10)] = 'Very High'\n",
    "        levels[values > 10] = 'Extreme'\n",
    "        return levels\n",
    "\n",
    "    # Calculate basic metrics\n",
    "    metrics = {\n",
    "        'raw': {\n",
    "            'mae': mean_absolute_error(y_true, y_pred),\n",
    "            'rmse': np.sqrt(mean_squared_error(y_true, y_pred)),\n",
    "            'r2': r2_score(y_true, y_pred),\n",
    "            'mean_error': np.mean(errors),\n",
    "            'std_error': np.std(errors),\n",
    "            'median_error': np.median(errors),\n",
    "            'p95_abs_error': np.percentile(np.abs(errors), 95)\n",
    "        },\n",
    "        'rounded': {\n",
    "            'mae': mean_absolute_error(y_true, y_pred_clipped),\n",
    "            'rmse': np.sqrt(mean_squared_error(y_true, y_pred_clipped)),\n",
    "            'r2': r2_score(y_true, y_pred_clipped)\n",
    "        }\n",
    "    }\n",
    "\n",
    "    # Calculate accuracies for different margins\n",
    "    for data_type, errors_data in [('raw', errors), ('rounded', errors_rounded)]:\n",
    "        metrics[data_type].update({\n",
    "            'within_05': np.mean(np.abs(errors_data) <= 0.5) * 100,\n",
    "            'within_1': np.mean(np.abs(errors_data) <= 1.0) * 100,\n",
    "            'within_15': np.mean(np.abs(errors_data) <= 1.5) * 100,\n",
    "            'within_2': np.mean(np.abs(errors_data) <= 2.0) * 100\n",
    "        })\n",
    "\n",
    "    # Analysis by UV risk level\n",
    "    y_true_risk = get_uv_risk_level(y_true)\n",
    "    y_pred_risk = get_uv_risk_level(y_pred_clipped)\n",
    "\n",
    "    # Calculate confusion matrix with handling for missing classes\n",
    "    risk_levels = ['Low', 'Moderate', 'High', 'Very High', 'Extreme']\n",
    "\n",
    "    # Get unique labels present in the data\n",
    "    present_labels = np.unique(np.concatenate([y_true_risk, y_pred_risk]))\n",
    "\n",
    "    # Calculate confusion matrix for present labels\n",
    "    cm = confusion_matrix(y_true_risk, y_pred_risk, labels=present_labels)\n",
    "\n",
    "    # Create full confusion matrix with zeros\n",
    "    full_cm = np.zeros((len(risk_levels), len(risk_levels)))\n",
    "\n",
    "    # Map present labels to their positions in the full matrix\n",
    "    label_positions = {label: i for i, label in enumerate(risk_levels)}\n",
    "    for i, true_label in enumerate(present_labels):\n",
    "        for j, pred_label in enumerate(present_labels):\n",
    "            full_cm[label_positions[true_label], label_positions[pred_label]] = cm[i, j]\n",
    "\n",
    "    # Create DataFrame with all risk levels\n",
    "    cm_df = pd.DataFrame(full_cm, columns=risk_levels, index=risk_levels)\n",
    "\n",
    "    # Analysis by UV range\n",
    "    uv_ranges = [\n",
    "        (0, 2, 'Low'),\n",
    "        (2, 5, 'Moderate'),\n",
    "        (5, 7, 'High'),\n",
    "        (7, 10, 'Very High'),\n",
    "        (10, 11, 'Extreme')\n",
    "    ]\n",
    "\n",
    "    range_analysis = {}\n",
    "    for low, high, label in uv_ranges:\n",
    "        mask = (y_true >= low) & (y_true < high)\n",
    "        if mask.any():\n",
    "            range_analysis[label] = {\n",
    "                'mae': mean_absolute_error(y_true[mask], y_pred[mask]),\n",
    "                'count': np.sum(mask),\n",
    "                'accuracy_within_05': np.mean(np.abs(errors[mask]) <= 0.5) * 100,\n",
    "                'accuracy_within_1': np.mean(np.abs(errors[mask]) <= 1.0) * 100\n",
    "            }\n",
    "\n",
    "    # Visualizations\n",
    "    if folder_name is not None:\n",
    "        try:\n",
    "            # Main figure with 4 subplots\n",
    "            fig = plt.figure(figsize=(20, 15))\n",
    "\n",
    "            # 1. Error distribution\n",
    "            plt.subplot(2, 2, 1)\n",
    "            plt.hist(errors, bins=50, alpha=0.7)\n",
    "            plt.title('Prediction Error Distribution')\n",
    "            plt.xlabel('Error')\n",
    "            plt.ylabel('Frequency')\n",
    "\n",
    "            # 2. Actual vs Predicted scatter plot\n",
    "            plt.subplot(2, 2, 2)\n",
    "            plt.scatter(y_true, y_pred, alpha=0.5)\n",
    "            plt.plot([0, 11], [0, 11], 'r--', lw=2)\n",
    "            plt.title('Actual vs Predicted Values')\n",
    "            plt.xlabel('Actual Values')\n",
    "            plt.ylabel('Predicted Values')\n",
    "\n",
    "            # 3. Errors vs Actual Values\n",
    "            plt.subplot(2, 2, 3)\n",
    "            plt.scatter(y_true, errors, alpha=0.5)\n",
    "            plt.axhline(y=0, color='r', linestyle='--')\n",
    "            plt.title('Errors vs Actual Values')\n",
    "            plt.xlabel('Actual Values')\n",
    "            plt.ylabel('Error')\n",
    "\n",
    "            # 4. Accuracy and MAE by range\n",
    "            ax = plt.subplot(2, 2, 4)\n",
    "            x_labels = [f\"{label}\\n({low}-{high})\" for low, high, label in uv_ranges]\n",
    "            accuracies = [range_analysis[label]['accuracy_within_05']\n",
    "                          for _, _, label in uv_ranges if label in range_analysis]\n",
    "            mae_values = [range_analysis[label]['mae']\n",
    "                          for _, _, label in uv_ranges if label in range_analysis]\n",
    "\n",
    "            bars = plt.bar(x_labels, accuracies, alpha=0.6)\n",
    "            plt.ylabel('Accuracy within ±0.5 (%)')\n",
    "            plt.title('Accuracy and MAE by UV Range')\n",
    "\n",
    "            # Add MAE as line\n",
    "            ax2 = ax.twinx()\n",
    "            ax2.plot(x_labels, mae_values, 'r-o', label='MAE')\n",
    "            ax2.set_ylabel('MAE', color='red')\n",
    "\n",
    "            plt.tight_layout()\n",
    "\n",
    "            # Save main figure\n",
    "            main_plot_path = f'{folder_name}_uv_analysis.png'\n",
    "            plt.savefig(main_plot_path, dpi=300, bbox_inches='tight')\n",
    "\n",
    "            # Confusion matrix as separate plot\n",
    "            plt.figure(figsize=(10, 8))\n",
    "            sns.heatmap(cm_df, annot=True, fmt='d', cmap='Blues')\n",
    "            plt.title('Confusion Matrix for UV Risk Levels')\n",
    "\n",
    "            conf_matrix_path = f'{folder_name}_confusion_matrix.png'\n",
    "            plt.savefig(conf_matrix_path, dpi=300, bbox_inches='tight')\n",
    "\n",
    "            plt.close('all')\n",
    "\n",
    "        except Exception as e:\n",
    "            print(f\"\\nError saving plots: {str(e)}\")\n",
    "            main_plot_path = None\n",
    "            conf_matrix_path = None\n",
    "\n",
    "    # Print detailed report\n",
    "    print(\"\\nUV Index Prediction Analysis:\")\n",
    "    print(\"\\nRaw Metrics:\")\n",
    "    for key, value in metrics['raw'].items():\n",
    "        print(f\"{key}: {value:.3f}\")\n",
    "\n",
    "    print(\"\\nRounded Metrics:\")\n",
    "    for key, value in metrics['rounded'].items():\n",
    "        print(f\"{key}: {value:.3f}\")\n",
    "\n",
    "    print(\"\\nAnalysis by UV Range:\")\n",
    "    for label, stats in range_analysis.items():\n",
    "        print(f\"\\n{label}:\")\n",
    "        for key, value in stats.items():\n",
    "            print(f\"  {key}: {value:.3f}\")\n",
    "\n",
    "    print(\"\\nConfusion Matrix:\")\n",
    "    print(cm_df)\n",
    "\n",
    "    # Add range analysis and confusion matrix to metrics dictionary\n",
    "    metrics.update({\n",
    "        'range_analysis': range_analysis,\n",
    "        'confusion_matrix': cm_df.to_dict(),\n",
    "        'plot_paths': {\n",
    "            'main_analysis': main_plot_path,\n",
    "            'confusion_matrix': conf_matrix_path\n",
    "        }\n",
    "    })\n",
    "\n",
    "    return metrics\n",
    "\n",
    "\n",
    "def plot_training_history(history, folder_name=None):\n",
    "    \"\"\"\n",
    "    Visualize and save the loss and metrics plots during training\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    history : tensorflow.keras.callbacks.History\n",
    "        The history object returned by model training\n",
    "    folder_name : str\n",
    "        Folder where to save the plot\n",
    "    \"\"\"\n",
    "\n",
    "    try:\n",
    "        # Create the figure\n",
    "        plt.figure(figsize=(12, 4))\n",
    "\n",
    "        # Loss Plot\n",
    "        plt.subplot(1, 2, 1)\n",
    "        plt.plot(history.history['loss'], label='Training Loss')\n",
    "        plt.plot(history.history['val_loss'], label='Validation Loss')\n",
    "        plt.title('Model Loss')\n",
    "        plt.xlabel('Epoch')\n",
    "        plt.ylabel('Loss')\n",
    "        plt.legend()\n",
    "        plt.grid(True)\n",
    "\n",
    "        # MAE Plot\n",
    "        plt.subplot(1, 2, 2)\n",
    "        plt.plot(history.history['mae'], label='Training MAE')\n",
    "        plt.plot(history.history['val_mae'], label='Validation MAE')\n",
    "        plt.title('Model MAE')\n",
    "        plt.xlabel('Epoch')\n",
    "        plt.ylabel('MAE')\n",
    "        plt.legend()\n",
    "        plt.grid(True)\n",
    "\n",
    "        plt.tight_layout()\n",
    "\n",
    "        if folder_name is not None:\n",
    "            os.makedirs(folder_name, exist_ok=True)\n",
    "            # Generate filename with timestamp\n",
    "            filename = os.path.join(folder_name, 'training_history.png')\n",
    "\n",
    "            # Save the figure\n",
    "            plt.savefig(filename, dpi=300, bbox_inches='tight')\n",
    "            print(f\"\\nTraining history plot saved as: {filename}\")\n",
    "\n",
    "        # Also save numerical data in CSV format\n",
    "        history_df = pd.DataFrame({\n",
    "            'epoch': range(1, len(history.history['loss']) + 1),\n",
    "            'training_loss': history.history['loss'],\n",
    "            'validation_loss': history.history['val_loss'],\n",
    "            'training_mae': history.history['mae'],\n",
    "            'validation_mae': history.history['val_mae']\n",
    "        })\n",
    "\n",
    "        if folder_name is not None:\n",
    "            csv_filename = os.path.join(folder_name, 'training_history.csv')\n",
    "            history_df.to_csv(csv_filename, index=False)\n",
    "            print(f\"Training history data saved as: {csv_filename}\")\n",
    "\n",
    "        # Calculate and save final statistics\n",
    "        final_stats = {\n",
    "            'final_training_loss': history.history['loss'][-1],\n",
    "            'final_validation_loss': history.history['val_loss'][-1],\n",
    "            'final_training_mae': history.history['mae'][-1],\n",
    "            'final_validation_mae': history.history['val_mae'][-1],\n",
    "            'best_validation_loss': min(history.history['val_loss']),\n",
    "            'best_validation_mae': min(history.history['val_mae']),\n",
    "            'epochs': len(history.history['loss']),\n",
    "        }\n",
    "\n",
    "        if folder_name is not None:\n",
    "            # Save statistics in JSON format\n",
    "            stats_filename = os.path.join(folder_name, 'training_stats.json')\n",
    "            with open(stats_filename, 'w') as f:\n",
    "                json.dump(final_stats, f, indent=4)\n",
    "            print(f\"Final statistics saved as: {stats_filename}\")\n",
    "\n",
    "        # Print main statistics\n",
    "        print(\"\\nFinal training statistics:\")\n",
    "        print(f\"Final Loss (train/val): {final_stats['final_training_loss']:.4f}/{final_stats['final_validation_loss']:.4f}\")\n",
    "        print(f\"Final MAE (train/val): {final_stats['final_training_mae']:.4f}/{final_stats['final_validation_mae']:.4f}\")\n",
    "        print(f\"Best validation loss: {final_stats['best_validation_loss']:.4f}\")\n",
    "        print(f\"Best validation MAE: {final_stats['best_validation_mae']:.4f}\")\n",
    "\n",
    "        plt.show()\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"\\nError during plot creation or saving: {str(e)}\")\n",
    "\n",
    "\n",
    "def train_hybrid_model(model, X_train, y_train, X_test, y_test, epochs=100, batch_size=32, folder_name='uv_index'):\n",
    "    \"\"\"\n",
    "    Advanced training function for the hybrid UV index model with detailed monitoring\n",
    "    and training management.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    model : keras.Model\n",
    "        The compiled hybrid model\n",
    "    X_train : numpy.ndarray\n",
    "        Training data\n",
    "    y_train : numpy.ndarray\n",
    "        Training targets\n",
    "    X_test : numpy.ndarray\n",
    "        Validation data\n",
    "    y_test : numpy.ndarray\n",
    "        Validation targets\n",
    "    epochs : int, optional\n",
    "        Maximum number of training epochs\n",
    "    batch_size : int, optional\n",
    "        Batch size\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    history : keras.callbacks.History\n",
    "        Training history with all metrics\n",
    "    \"\"\"\n",
    "\n",
    "    # Advanced callbacks for training\n",
    "    callbacks = [\n",
    "        # Advanced Early Stopping\n",
    "        EarlyStopping(\n",
    "            monitor='mae',\n",
    "            patience=15,\n",
    "            restore_best_weights=True,\n",
    "            mode='min',\n",
    "            verbose=1,\n",
    "            min_delta=1e-6\n",
    "        ),\n",
    "        ReduceLROnPlateau(\n",
    "            monitor='mae',\n",
    "            factor=0.05,\n",
    "            patience=3,\n",
    "            verbose=1,\n",
    "            mode='min',\n",
    "            min_delta=1e-6,\n",
    "            cooldown=2,\n",
    "            min_lr=1e-7\n",
    "        ),\n",
    "        ReduceLROnPlateau(\n",
    "            monitor='val_loss',\n",
    "            factor=0.2,\n",
    "            patience=2,\n",
    "            verbose=1,\n",
    "            mode='min',\n",
    "            min_delta=1e-6,\n",
    "            cooldown=1,\n",
    "            min_lr=1e-7\n",
    "        ),\n",
    "        tf.keras.callbacks.ModelCheckpoint(\n",
    "            filepath=f'{folder_name}_best_uv_model.h5',\n",
    "            monitor='mae',\n",
    "            save_best_only=True,\n",
    "            mode='min'\n",
    "        ),\n",
    "        tf.keras.callbacks.TensorBoard(\n",
    "            log_dir=f'./{folder_name}_logs',\n",
    "            histogram_freq=1,\n",
    "            write_graph=True,\n",
    "            update_freq='epoch'\n",
    "        ),\n",
    "        tf.keras.callbacks.LambdaCallback(\n",
    "            on_epoch_end=lambda epoch, logs: print(\n",
    "                f\"\\nEpoch {epoch + 1}: Out of range predictions: \"\n",
    "                f\"{np.sum((model.predict(X_test) < 0) | (model.predict(X_test) > 11))}\"\n",
    "            ) if epoch % 20 == 0 else None\n",
    "        )\n",
    "    ]\n",
    "\n",
    "    try:\n",
    "        history = model.fit(\n",
    "            X_train, y_train,\n",
    "            validation_data=(X_test, y_test),\n",
    "            epochs=epochs,\n",
    "            batch_size=batch_size,\n",
    "            callbacks=callbacks,\n",
    "            verbose=1,\n",
    "            shuffle=False,\n",
    "            validation_freq=1,\n",
    "        )\n",
    "\n",
    "        # Post-training analysis\n",
    "        print(\"\\nTraining completed successfully!\")\n",
    "\n",
    "        return history\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"\\nError during training: {str(e)}\")\n",
    "        raise\n",
    "\n",
    "    finally:\n",
    "        # Memory cleanup\n",
    "        tf.keras.backend.clear_session()\n",
    "\n",
    "\n",
    "def integrate_predictions(df, predictions, sequence_length=24):\n",
    "    \"\"\"\n",
    "    Integrate UV index predictions into the original dataset for pre-2010 data.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    df : pandas.DataFrame\n",
    "        Original dataset\n",
    "    predictions : numpy.ndarray\n",
    "        Array of UV index predictions\n",
    "    sequence_length : int\n",
    "        Sequence length used for predictions\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    pandas.DataFrame\n",
    "        Updated dataset with UV index predictions\n",
    "    \"\"\"\n",
    "    # Convert datetime to datetime format if not already\n",
    "    df['datetime'] = pd.to_datetime(df['datetime'])\n",
    "\n",
    "    # Identify pre-2010 rows\n",
    "    mask_pre_2010 = df['datetime'].dt.year < 2010\n",
    "\n",
    "    # Create temporary DataFrame with predictions\n",
    "    dates_pre_2010 = df[mask_pre_2010]['datetime'].iloc[sequence_length - 1:]\n",
    "    predictions_df = pd.DataFrame({\n",
    "        'datetime': dates_pre_2010,\n",
    "        'uvindex_predicted': predictions.flatten()\n",
    "    })\n",
    "\n",
    "    # Merge with original dataset\n",
    "    df = df.merge(predictions_df, on='datetime', how='left')\n",
    "\n",
    "    # Update uvindex column where missing\n",
    "    df['uvindex'] = df['uvindex'].fillna(df['uvindex_predicted'])\n",
    "\n",
    "    # Remove temporary column\n",
    "    df = df.drop('uvindex_predicted', axis=1)\n",
    "\n",
    "    print(f\"Added {len(predictions)} predictions to dataset\")\n",
    "    print(f\"Rows with UV index after integration: {df['uvindex'].notna().sum()}\")\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "initial_id",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing UV index model training...\n",
      "\n",
      "1. Preparing data...\n",
      "\n",
      "Temporal distribution of data:\n",
      "Records after 2010: 129,777\n",
      "Records before 2010: 227,902\n",
      "\n",
      "Warning: Found missing values after preprocessing\n",
      "Features with missing values: []\n",
      "\n",
      "Number of features used: 30\n",
      "\n",
      "Feature categories:\n",
      "atmospheric: 6 features\n",
      "temporal: 4 features\n",
      "solar: 5 features\n",
      "interactions: 4 features\n",
      "rolling: 2 features\n",
      "Categorical: 9 features\n",
      "Training data shape: (64865, 24, 30)\n",
      "Test data shape: (64866, 24, 30)\n",
      "Saving scaler to: 2024-11-21_08-23_feature_scaler.joblib\n",
      "Saving scaler to: 2024-11-21_08-23_target_scaler.joblib\n",
      "Saving features to: 2024-11-21_08-23_features.json\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_parquet('../../sources/weather_data.parquet')\n",
    "\n",
    "print(\"Initializing UV index model training...\")\n",
    "\n",
    "# Data preparation\n",
    "print(\"\\n1. Preparing data...\")\n",
    "X_train_seq, X_test_seq, y_train, y_test, feature_scaler, target_scaler, features, X_to_predict_seq = prepare_hybrid_data(df)\n",
    "\n",
    "print(f\"Training data shape: {X_train_seq.shape}\")\n",
    "print(f\"Test data shape: {X_test_seq.shape}\")\n",
    "\n",
    "# Save or load scaler and features\n",
    "feature_scaler_path = f'{folder_name}_feature_scaler.joblib'\n",
    "target_scaler_path = f'{folder_name}_target_scaler.joblib'\n",
    "features_path = f'{folder_name}_features.json'\n",
    "model_path = f'{folder_name}_best_model.h5'\n",
    "history_path = f'{folder_name}_training_history.json'\n",
    "\n",
    "if os.path.exists(feature_scaler_path):\n",
    "    print(f\"Loading existing scaler from: {feature_scaler_path}\")\n",
    "    scaler = joblib.load(feature_scaler_path)\n",
    "else:\n",
    "    print(f\"Saving scaler to: {feature_scaler_path}\")\n",
    "    joblib.dump(feature_scaler, feature_scaler_path)\n",
    "\n",
    "if os.path.exists(target_scaler_path):\n",
    "    print(f\"Loading existing scaler from: {target_scaler_path}\")\n",
    "    scaler = joblib.load(target_scaler_path)\n",
    "else:\n",
    "    print(f\"Saving scaler to: {target_scaler_path}\")\n",
    "    joblib.dump(target_scaler, target_scaler_path)\n",
    "\n",
    "if os.path.exists(features_path):\n",
    "    print(f\"Loading existing features from: {features_path}\")\n",
    "    with open(features_path, 'r') as f:\n",
    "        features = json.load(f)\n",
    "else:\n",
    "    print(f\"Saving features to: {features_path}\")\n",
    "    with open(features_path, 'w') as f:\n",
    "        json.dump(features, f)\n",
    "\n",
    "# Data quality verification\n",
    "if np.isnan(X_train_seq).any() or np.isnan(y_train).any():\n",
    "    raise ValueError(\"Found NaN values in training data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "83771453-71db-4bb2-833d-7b81c022863d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "2. Model initialization...\n",
      "Creating new model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-21 08:26:35.683631: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 43404 MB memory:  -> device: 0, name: NVIDIA L40, pci bus id: 0000:25:00.0, compute capability: 8.9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"UvModel\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                Output Shape                 Param #   Connected to                  \n",
      "==================================================================================================\n",
      " input_1 (InputLayer)        [(None, 24, 30)]             0         []                            \n",
      "                                                                                                  \n",
      " bidirectional (Bidirection  (None, 24, 128)              48640     ['input_1[0][0]']             \n",
      " al)                                                                                              \n",
      "                                                                                                  \n",
      " layer_normalization (Layer  (None, 24, 128)              256       ['bidirectional[0][0]']       \n",
      " Normalization)                                                                                   \n",
      "                                                                                                  \n",
      " dropout (Dropout)           (None, 24, 128)              0         ['layer_normalization[0][0]'] \n",
      "                                                                                                  \n",
      " dense (Dense)               (None, 24, 128)              3968      ['input_1[0][0]']             \n",
      "                                                                                                  \n",
      " stochastic_depth (Stochast  (None, 24, 128)              0         ['dropout[0][0]',             \n",
      " icDepth)                                                            'dense[0][0]']               \n",
      "                                                                                                  \n",
      " multi_head_attention (Mult  (None, 24, 128)              131968    ['stochastic_depth[0][0]',    \n",
      " iHeadAttention)                                                     'stochastic_depth[0][0]']    \n",
      "                                                                                                  \n",
      " stochastic_depth_1 (Stocha  (None, 24, 128)              0         ['multi_head_attention[0][0]',\n",
      " sticDepth)                                                          'stochastic_depth[0][0]']    \n",
      "                                                                                                  \n",
      " layer_normalization_1 (Lay  (None, 24, 128)              256       ['stochastic_depth_1[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dense_1 (Dense)             (None, 24, 512)              66048     ['layer_normalization_1[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_2 (Dense)             (None, 24, 128)              65664     ['dense_1[0][0]']             \n",
      "                                                                                                  \n",
      " stochastic_depth_2 (Stocha  (None, 24, 128)              0         ['dense_2[0][0]',             \n",
      " sticDepth)                                                          'layer_normalization_1[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " layer_normalization_2 (Lay  (None, 24, 128)              256       ['stochastic_depth_2[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " bidirectional_1 (Bidirecti  (None, 24, 64)               41216     ['layer_normalization_2[0][0]'\n",
      " onal)                                                              ]                             \n",
      "                                                                                                  \n",
      " layer_normalization_3 (Lay  (None, 24, 64)               128       ['bidirectional_1[0][0]']     \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dropout_1 (Dropout)         (None, 24, 64)               0         ['layer_normalization_3[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_3 (Dense)             (None, 24, 64)               8256      ['layer_normalization_2[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " stochastic_depth_3 (Stocha  (None, 24, 64)               0         ['dropout_1[0][0]',           \n",
      " sticDepth)                                                          'dense_3[0][0]']             \n",
      "                                                                                                  \n",
      " multi_head_attention_1 (Mu  (None, 24, 64)               33216     ['stochastic_depth_3[0][0]',  \n",
      " ltiHeadAttention)                                                   'stochastic_depth_3[0][0]']  \n",
      "                                                                                                  \n",
      " stochastic_depth_4 (Stocha  (None, 24, 64)               0         ['multi_head_attention_1[0][0]\n",
      " sticDepth)                                                         ',                            \n",
      "                                                                     'stochastic_depth_3[0][0]']  \n",
      "                                                                                                  \n",
      " layer_normalization_4 (Lay  (None, 24, 64)               128       ['stochastic_depth_4[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dense_4 (Dense)             (None, 24, 256)              16640     ['layer_normalization_4[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_5 (Dense)             (None, 24, 64)               16448     ['dense_4[0][0]']             \n",
      "                                                                                                  \n",
      " stochastic_depth_5 (Stocha  (None, 24, 64)               0         ['dense_5[0][0]',             \n",
      " sticDepth)                                                          'layer_normalization_4[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " layer_normalization_5 (Lay  (None, 24, 64)               128       ['stochastic_depth_5[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " bidirectional_2 (Bidirecti  (None, 24, 32)               10368     ['layer_normalization_5[0][0]'\n",
      " onal)                                                              ]                             \n",
      "                                                                                                  \n",
      " layer_normalization_6 (Lay  (None, 24, 32)               64        ['bidirectional_2[0][0]']     \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dropout_2 (Dropout)         (None, 24, 32)               0         ['layer_normalization_6[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_6 (Dense)             (None, 24, 32)               2080      ['layer_normalization_5[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " stochastic_depth_6 (Stocha  (None, 24, 32)               0         ['dropout_2[0][0]',           \n",
      " sticDepth)                                                          'dense_6[0][0]']             \n",
      "                                                                                                  \n",
      " multi_head_attention_2 (Mu  (None, 24, 32)               8416      ['stochastic_depth_6[0][0]',  \n",
      " ltiHeadAttention)                                                   'stochastic_depth_6[0][0]']  \n",
      "                                                                                                  \n",
      " stochastic_depth_7 (Stocha  (None, 24, 32)               0         ['multi_head_attention_2[0][0]\n",
      " sticDepth)                                                         ',                            \n",
      "                                                                     'stochastic_depth_6[0][0]']  \n",
      "                                                                                                  \n",
      " layer_normalization_7 (Lay  (None, 24, 32)               64        ['stochastic_depth_7[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dense_7 (Dense)             (None, 24, 128)              4224      ['layer_normalization_7[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_8 (Dense)             (None, 24, 32)               4128      ['dense_7[0][0]']             \n",
      "                                                                                                  \n",
      " stochastic_depth_8 (Stocha  (None, 24, 32)               0         ['dense_8[0][0]',             \n",
      " sticDepth)                                                          'layer_normalization_7[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " layer_normalization_8 (Lay  (None, 24, 32)               64        ['stochastic_depth_8[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " multi_head_attention_3 (Mu  (None, 24, 32)               8416      ['layer_normalization_8[0][0]'\n",
      " ltiHeadAttention)                                                  , 'layer_normalization_8[0][0]\n",
      "                                                                    ']                            \n",
      "                                                                                                  \n",
      " stochastic_depth_9 (Stocha  (None, 24, 32)               0         ['multi_head_attention_3[0][0]\n",
      " sticDepth)                                                         ',                            \n",
      "                                                                     'layer_normalization_8[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " layer_normalization_9 (Lay  (None, 24, 32)               64        ['stochastic_depth_9[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " global_average_pooling1d (  (None, 32)                   0         ['layer_normalization_9[0][0]'\n",
      " GlobalAveragePooling1D)                                            ]                             \n",
      "                                                                                                  \n",
      " dense_9 (Dense)             (None, 32)                   1056      ['global_average_pooling1d[0][\n",
      "                                                                    0]']                          \n",
      "                                                                                                  \n",
      " batch_normalization (Batch  (None, 32)                   128       ['dense_9[0][0]']             \n",
      " Normalization)                                                                                   \n",
      "                                                                                                  \n",
      " dropout_3 (Dropout)         (None, 32)                   0         ['batch_normalization[0][0]'] \n",
      "                                                                                                  \n",
      " dense_10 (Dense)            (None, 16)                   528       ['dropout_3[0][0]']           \n",
      "                                                                                                  \n",
      " batch_normalization_1 (Bat  (None, 16)                   64        ['dense_10[0][0]']            \n",
      " chNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dense_11 (Dense)            (None, 8)                    136       ['batch_normalization_1[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_12 (Dense)            (None, 1)                    9         ['dense_11[0][0]']            \n",
      "                                                                                                  \n",
      " lambda (Lambda)             (None, 1)                    0         ['dense_12[0][0]']            \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 473025 (1.80 MB)\n",
      "Trainable params: 472929 (1.80 MB)\n",
      "Non-trainable params: 96 (384.00 Byte)\n",
      "__________________________________________________________________________________________________\n",
      "\n",
      "3. Starting training...\n",
      "Epoch 1/100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-21 08:26:51.620818: I tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:606] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n",
      "2024-11-21 08:26:51.695976: I tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:432] Loaded cuDNN version 8905\n",
      "2024-11-21 08:26:51.911310: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0xd713390 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2024-11-21 08:26:51.911349: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): NVIDIA L40, Compute Capability 8.9\n",
      "2024-11-21 08:26:51.921786: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:255] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n",
      "2024-11-21 08:26:52.001781: I tensorflow/tsl/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory\n",
      "2024-11-21 08:26:52.063791: I ./tensorflow/compiler/jit/device_compiler.h:186] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "507/507 [==============================] - ETA: 0s - loss: 4.4444 - mae: 1.3032 - mse: 2.1820 - root_mean_squared_error: 1.4772 - smooth_mape: 226.3000"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.11/dist-packages/keras/src/engine/training.py:3000: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n",
      "  saving_api.save_model(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2028/2028 [==============================] - 25s 11ms/step\n",
      "2028/2028 [==============================] - 19s 9ms/step\n",
      "\n",
      "Epoch 1: Out of range predictions: 0\n",
      "507/507 [==============================] - 95s 151ms/step - loss: 4.4444 - mae: 1.3032 - mse: 2.1820 - root_mean_squared_error: 1.4772 - smooth_mape: 226.3000 - val_loss: 3.3901 - val_mae: 0.7726 - val_mse: 1.0065 - val_root_mean_squared_error: 1.0033 - val_smooth_mape: 96.0347 - lr: 5.0600e-05\n",
      "Epoch 2/100\n",
      "507/507 [==============================] - 24s 48ms/step - loss: 3.4513 - mae: 0.9422 - mse: 1.2050 - root_mean_squared_error: 1.0977 - smooth_mape: 144.6453 - val_loss: 3.0292 - val_mae: 0.6905 - val_mse: 0.9136 - val_root_mean_squared_error: 0.9558 - val_smooth_mape: 72.3569 - lr: 9.9998e-05\n",
      "Epoch 3/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 2.8616 - mae: 0.7835 - mse: 0.9255 - root_mean_squared_error: 0.9620 - smooth_mape: 105.5174 - val_loss: 2.6105 - val_mae: 0.6461 - val_mse: 0.8668 - val_root_mean_squared_error: 0.9310 - val_smooth_mape: 62.3054 - lr: 9.7355e-05\n",
      "Epoch 4/100\n",
      "507/507 [==============================] - 27s 52ms/step - loss: 2.2615 - mae: 0.6304 - mse: 0.6460 - root_mean_squared_error: 0.8038 - smooth_mape: 84.4009 - val_loss: 1.9317 - val_mae: 0.4135 - val_mse: 0.4540 - val_root_mean_squared_error: 0.6738 - val_smooth_mape: 35.5852 - lr: 8.9946e-05\n",
      "Epoch 5/100\n",
      "507/507 [==============================] - 26s 52ms/step - loss: 1.6904 - mae: 0.4345 - mse: 0.3313 - root_mean_squared_error: 0.5756 - smooth_mape: 59.1463 - val_loss: 1.4432 - val_mae: 0.2685 - val_mse: 0.1950 - val_root_mean_squared_error: 0.4416 - val_smooth_mape: 27.2014 - lr: 7.8518e-05\n",
      "Epoch 6/100\n",
      "507/507 [==============================] - 27s 54ms/step - loss: 1.3637 - mae: 0.3450 - mse: 0.2236 - root_mean_squared_error: 0.4729 - smooth_mape: 45.5589 - val_loss: 1.2321 - val_mae: 0.2588 - val_mse: 0.1828 - val_root_mean_squared_error: 0.4276 - val_smooth_mape: 25.4509 - lr: 6.4221e-05\n",
      "Epoch 7/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 1.1519 - mae: 0.2973 - mse: 0.1789 - root_mean_squared_error: 0.4229 - smooth_mape: 38.0580 - val_loss: 1.0643 - val_mae: 0.2375 - val_mse: 0.1577 - val_root_mean_squared_error: 0.3971 - val_smooth_mape: 23.5643 - lr: 4.8492e-05\n",
      "Epoch 8/100\n",
      "507/507 [==============================] - 28s 55ms/step - loss: 1.0083 - mae: 0.2665 - mse: 0.1546 - root_mean_squared_error: 0.3932 - smooth_mape: 32.9333 - val_loss: 0.9257 - val_mae: 0.2038 - val_mse: 0.1143 - val_root_mean_squared_error: 0.3380 - val_smooth_mape: 21.4354 - lr: 3.2915e-05\n",
      "Epoch 9/100\n",
      "507/507 [==============================] - 28s 55ms/step - loss: 0.9148 - mae: 0.2470 - mse: 0.1407 - root_mean_squared_error: 0.3751 - smooth_mape: 29.7929 - val_loss: 0.8520 - val_mae: 0.1890 - val_mse: 0.1027 - val_root_mean_squared_error: 0.3204 - val_smooth_mape: 20.0954 - lr: 1.9058e-05\n",
      "Epoch 10/100\n",
      "507/507 [==============================] - 27s 54ms/step - loss: 0.8584 - mae: 0.2366 - mse: 0.1317 - root_mean_squared_error: 0.3628 - smooth_mape: 28.3349 - val_loss: 0.8144 - val_mae: 0.1853 - val_mse: 0.0991 - val_root_mean_squared_error: 0.3148 - val_smooth_mape: 19.7012 - lr: 8.3134e-06\n",
      "Epoch 11/100\n",
      "507/507 [==============================] - 28s 55ms/step - loss: 0.8331 - mae: 0.2331 - mse: 0.1292 - root_mean_squared_error: 0.3594 - smooth_mape: 27.8952 - val_loss: 0.7991 - val_mae: 0.1829 - val_mse: 0.0966 - val_root_mean_squared_error: 0.3108 - val_smooth_mape: 19.7143 - lr: 1.7639e-06\n",
      "Epoch 12/100\n",
      "507/507 [==============================] - 28s 55ms/step - loss: 0.8266 - mae: 0.2326 - mse: 0.1289 - root_mean_squared_error: 0.3591 - smooth_mape: 27.6276 - val_loss: 0.8002 - val_mae: 0.1851 - val_mse: 0.0996 - val_root_mean_squared_error: 0.3155 - val_smooth_mape: 19.6762 - lr: 6.7976e-08\n",
      "Epoch 13/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.8256 - mae: 0.2332 - mse: 0.1297 - root_mean_squared_error: 0.3601 - smooth_mape: 27.8768 - val_loss: 0.7938 - val_mae: 0.1827 - val_mse: 0.0977 - val_root_mean_squared_error: 0.3126 - val_smooth_mape: 19.5952 - lr: 3.3964e-06\n",
      "Epoch 14/100\n",
      "507/507 [==============================] - 27s 52ms/step - loss: 0.8108 - mae: 0.2311 - mse: 0.1283 - root_mean_squared_error: 0.3582 - smooth_mape: 27.5029 - val_loss: 0.7689 - val_mae: 0.1841 - val_mse: 0.0983 - val_root_mean_squared_error: 0.3135 - val_smooth_mape: 19.2959 - lr: 1.1414e-05\n",
      "Epoch 15/100\n",
      "507/507 [==============================] - 26s 52ms/step - loss: 0.7666 - mae: 0.2260 - mse: 0.1253 - root_mean_squared_error: 0.3539 - smooth_mape: 26.7909 - val_loss: 0.7071 - val_mae: 0.1791 - val_mse: 0.0942 - val_root_mean_squared_error: 0.3069 - val_smooth_mape: 18.7677 - lr: 2.3315e-05\n",
      "Epoch 16/100\n",
      "507/507 [==============================] - 26s 52ms/step - loss: 0.6896 - mae: 0.2200 - mse: 0.1214 - root_mean_squared_error: 0.3484 - smooth_mape: 25.8531 - val_loss: 0.6141 - val_mae: 0.1740 - val_mse: 0.0874 - val_root_mean_squared_error: 0.2956 - val_smooth_mape: 18.6023 - lr: 3.7901e-05\n",
      "Epoch 17/100\n",
      "507/507 [==============================] - 25s 50ms/step - loss: 0.5905 - mae: 0.2116 - mse: 0.1160 - root_mean_squared_error: 0.3406 - smooth_mape: 24.5564 - val_loss: 0.5156 - val_mae: 0.1681 - val_mse: 0.0874 - val_root_mean_squared_error: 0.2956 - val_smooth_mape: 17.4665 - lr: 5.3704e-05\n",
      "Epoch 18/100\n",
      "507/507 [==============================] - 27s 54ms/step - loss: 0.4901 - mae: 0.2037 - mse: 0.1116 - root_mean_squared_error: 0.3341 - smooth_mape: 23.3802 - val_loss: 0.4216 - val_mae: 0.1617 - val_mse: 0.0852 - val_root_mean_squared_error: 0.2919 - val_smooth_mape: 17.2917 - lr: 6.9134e-05\n",
      "Epoch 19/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.3984 - mae: 0.1930 - mse: 0.1038 - root_mean_squared_error: 0.3221 - smooth_mape: 21.8713 - val_loss: 0.3391 - val_mae: 0.1637 - val_mse: 0.0790 - val_root_mean_squared_error: 0.2810 - val_smooth_mape: 17.4952 - lr: 8.2639e-05\n",
      "Epoch 20/100\n",
      "507/507 [==============================] - 27s 54ms/step - loss: 0.3280 - mae: 0.1882 - mse: 0.1019 - root_mean_squared_error: 0.3192 - smooth_mape: 21.1186 - val_loss: 0.2774 - val_mae: 0.1532 - val_mse: 0.0766 - val_root_mean_squared_error: 0.2767 - val_smooth_mape: 16.5548 - lr: 9.2860e-05\n",
      "Epoch 21/100\n",
      "2028/2028 [==============================] - 22s 11ms/step\n",
      "2028/2028 [==============================] - 23s 12ms/step\n",
      "\n",
      "Epoch 21: Out of range predictions: 0\n",
      "507/507 [==============================] - 75s 148ms/step - loss: 0.2717 - mae: 0.1800 - mse: 0.0959 - root_mean_squared_error: 0.3097 - smooth_mape: 19.9770 - val_loss: 0.2327 - val_mae: 0.1514 - val_mse: 0.0756 - val_root_mean_squared_error: 0.2750 - val_smooth_mape: 16.9079 - lr: 9.8768e-05\n",
      "Epoch 22/100\n",
      "507/507 [==============================] - 27s 52ms/step - loss: 0.2290 - mae: 0.1732 - mse: 0.0907 - root_mean_squared_error: 0.3011 - smooth_mape: 19.0873 - val_loss: 0.1969 - val_mae: 0.1482 - val_mse: 0.0722 - val_root_mean_squared_error: 0.2687 - val_smooth_mape: 16.2890 - lr: 9.9769e-05\n",
      "Epoch 23/100\n",
      "507/507 [==============================] - 26s 50ms/step - loss: 0.1994 - mae: 0.1705 - mse: 0.0889 - root_mean_squared_error: 0.2982 - smooth_mape: 18.7545 - val_loss: 0.1750 - val_mae: 0.1452 - val_mse: 0.0739 - val_root_mean_squared_error: 0.2719 - val_smooth_mape: 15.6235 - lr: 9.5762e-05\n",
      "Epoch 24/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.1768 - mae: 0.1661 - mse: 0.0861 - root_mean_squared_error: 0.2934 - smooth_mape: 18.1383 - val_loss: 0.1559 - val_mae: 0.1482 - val_mse: 0.0716 - val_root_mean_squared_error: 0.2676 - val_smooth_mape: 16.7181 - lr: 8.7150e-05\n",
      "Epoch 25/100\n",
      "507/507 [==============================] - 26s 52ms/step - loss: 0.1601 - mae: 0.1629 - mse: 0.0835 - root_mean_squared_error: 0.2890 - smooth_mape: 17.7203 - val_loss: 0.1425 - val_mae: 0.1434 - val_mse: 0.0702 - val_root_mean_squared_error: 0.2649 - val_smooth_mape: 15.5010 - lr: 7.4800e-05\n",
      "Epoch 26/100\n",
      "507/507 [==============================] - 27s 54ms/step - loss: 0.1472 - mae: 0.1586 - mse: 0.0806 - root_mean_squared_error: 0.2839 - smooth_mape: 17.2387 - val_loss: 0.1347 - val_mae: 0.1454 - val_mse: 0.0710 - val_root_mean_squared_error: 0.2665 - val_smooth_mape: 16.1275 - lr: 5.9955e-05\n",
      "Epoch 27/100\n",
      "507/507 [==============================] - 27s 54ms/step - loss: 0.1399 - mae: 0.1584 - mse: 0.0803 - root_mean_squared_error: 0.2834 - smooth_mape: 17.2506 - val_loss: 0.1270 - val_mae: 0.1401 - val_mse: 0.0687 - val_root_mean_squared_error: 0.2621 - val_smooth_mape: 15.1573 - lr: 4.4108e-05\n",
      "Epoch 28/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.1344 - mae: 0.1563 - mse: 0.0792 - root_mean_squared_error: 0.2815 - smooth_mape: 16.9289 - val_loss: 0.1229 - val_mae: 0.1394 - val_mse: 0.0682 - val_root_mean_squared_error: 0.2611 - val_smooth_mape: 15.1774 - lr: 2.8853e-05\n",
      "Epoch 29/100\n",
      "507/507 [==============================] - 26s 52ms/step - loss: 0.1293 - mae: 0.1536 - mse: 0.0767 - root_mean_squared_error: 0.2770 - smooth_mape: 16.6836 - val_loss: 0.1206 - val_mae: 0.1383 - val_mse: 0.0679 - val_root_mean_squared_error: 0.2606 - val_smooth_mape: 14.8600 - lr: 1.5727e-05\n",
      "Epoch 30/100\n",
      "507/507 [==============================] - 27s 54ms/step - loss: 0.1275 - mae: 0.1526 - mse: 0.0763 - root_mean_squared_error: 0.2763 - smooth_mape: 16.5544 - val_loss: 0.1198 - val_mae: 0.1375 - val_mse: 0.0683 - val_root_mean_squared_error: 0.2613 - val_smooth_mape: 14.7848 - lr: 6.0491e-06\n",
      "Epoch 31/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.1259 - mae: 0.1517 - mse: 0.0753 - root_mean_squared_error: 0.2744 - smooth_mape: 16.4806 - val_loss: 0.1192 - val_mae: 0.1370 - val_mse: 0.0678 - val_root_mean_squared_error: 0.2605 - val_smooth_mape: 14.5789 - lr: 7.9394e-07\n",
      "Epoch 32/100\n",
      "507/507 [==============================] - 25s 50ms/step - loss: 0.1263 - mae: 0.1522 - mse: 0.0759 - root_mean_squared_error: 0.2754 - smooth_mape: 16.5490 - val_loss: 0.1192 - val_mae: 0.1368 - val_mse: 0.0679 - val_root_mean_squared_error: 0.2606 - val_smooth_mape: 14.5403 - lr: 4.9000e-07\n",
      "Epoch 33/100\n",
      "507/507 [==============================] - 26s 52ms/step - loss: 0.1258 - mae: 0.1518 - mse: 0.0754 - root_mean_squared_error: 0.2745 - smooth_mape: 16.4660 - val_loss: 0.1189 - val_mae: 0.1376 - val_mse: 0.0678 - val_root_mean_squared_error: 0.2605 - val_smooth_mape: 14.7214 - lr: 5.1679e-06\n",
      "Epoch 34/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.1255 - mae: 0.1520 - mse: 0.0756 - root_mean_squared_error: 0.2749 - smooth_mape: 16.4794\n",
      "Epoch 34: ReduceLROnPlateau reducing learning rate to 7.178531632234808e-07.\n",
      "507/507 [==============================] - 27s 54ms/step - loss: 0.1255 - mae: 0.1520 - mse: 0.0756 - root_mean_squared_error: 0.2749 - smooth_mape: 16.4811 - val_loss: 0.1179 - val_mae: 0.1389 - val_mse: 0.0676 - val_root_mean_squared_error: 0.2601 - val_smooth_mape: 14.8385 - lr: 7.1785e-07\n",
      "Epoch 35/100\n",
      "507/507 [==============================] - 27s 52ms/step - loss: 0.1245 - mae: 0.1527 - mse: 0.0760 - root_mean_squared_error: 0.2756 - smooth_mape: 16.5704 - val_loss: 0.1169 - val_mae: 0.1410 - val_mse: 0.0685 - val_root_mean_squared_error: 0.2618 - val_smooth_mape: 15.6455 - lr: 2.7133e-05\n",
      "Epoch 36/100\n",
      "507/507 [==============================] - 27s 54ms/step - loss: 0.1227 - mae: 0.1525 - mse: 0.0766 - root_mean_squared_error: 0.2767 - smooth_mape: 16.5028 - val_loss: 0.1139 - val_mae: 0.1381 - val_mse: 0.0683 - val_root_mean_squared_error: 0.2614 - val_smooth_mape: 14.6552 - lr: 4.2209e-05\n",
      "Epoch 37/100\n",
      "507/507 [==============================] - 27s 54ms/step - loss: 0.1198 - mae: 0.1535 - mse: 0.0769 - root_mean_squared_error: 0.2773 - smooth_mape: 16.6359 - val_loss: 0.1107 - val_mae: 0.1409 - val_mse: 0.0686 - val_root_mean_squared_error: 0.2619 - val_smooth_mape: 14.7102 - lr: 5.8070e-05\n",
      "Epoch 38/100\n",
      "507/507 [==============================] - ETA: 0s - loss: 0.1160 - mae: 0.1532 - mse: 0.0769 - root_mean_squared_error: 0.2772 - smooth_mape: 16.5710\n",
      "Epoch 38: ReduceLROnPlateau reducing learning rate to 3.6559198633767668e-06.\n",
      "507/507 [==============================] - 26s 51ms/step - loss: 0.1160 - mae: 0.1532 - mse: 0.0769 - root_mean_squared_error: 0.2772 - smooth_mape: 16.5710 - val_loss: 0.1057 - val_mae: 0.1380 - val_mse: 0.0675 - val_root_mean_squared_error: 0.2599 - val_smooth_mape: 14.8251 - lr: 3.6559e-06\n",
      "Epoch 39/100\n",
      "507/507 [==============================] - 28s 55ms/step - loss: 0.1113 - mae: 0.1525 - mse: 0.0762 - root_mean_squared_error: 0.2761 - smooth_mape: 16.4623 - val_loss: 0.1040 - val_mae: 0.1391 - val_mse: 0.0703 - val_root_mean_squared_error: 0.2652 - val_smooth_mape: 14.3343 - lr: 8.5841e-05\n",
      "Epoch 40/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.1080 - mae: 0.1531 - mse: 0.0770 - root_mean_squared_error: 0.2775 - smooth_mape: 16.5384 - val_loss: 0.0995 - val_mae: 0.1393 - val_mse: 0.0697 - val_root_mean_squared_error: 0.2640 - val_smooth_mape: 15.2621 - lr: 9.4957e-05\n",
      "Epoch 41/100\n",
      "2028/2028 [==============================] - 22s 11ms/step\n",
      "2028/2028 [==============================] - 23s 11ms/step\n",
      "\n",
      "Epoch 41: Out of range predictions: 0\n",
      "507/507 [==============================] - 73s 144ms/step - loss: 0.1038 - mae: 0.1523 - mse: 0.0766 - root_mean_squared_error: 0.2767 - smooth_mape: 16.4194 - val_loss: 0.0982 - val_mae: 0.1425 - val_mse: 0.0723 - val_root_mean_squared_error: 0.2689 - val_smooth_mape: 14.8658 - lr: 9.9549e-05\n",
      "Epoch 42/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.1026 - mae: 0.1539 - mse: 0.0783 - root_mean_squared_error: 0.2798 - smooth_mape: 16.6916\n",
      "Epoch 42: ReduceLROnPlateau reducing learning rate to 4.957754936185666e-06.\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.1026 - mae: 0.1539 - mse: 0.0783 - root_mean_squared_error: 0.2798 - smooth_mape: 16.6892 - val_loss: 0.0915 - val_mae: 0.1368 - val_mse: 0.0677 - val_root_mean_squared_error: 0.2602 - val_smooth_mape: 14.4180 - lr: 4.9578e-06\n",
      "Epoch 43/100\n",
      "507/507 [==============================] - 28s 55ms/step - loss: 0.0964 - mae: 0.1500 - mse: 0.0749 - root_mean_squared_error: 0.2736 - smooth_mape: 16.2060 - val_loss: 0.0881 - val_mae: 0.1362 - val_mse: 0.0671 - val_root_mean_squared_error: 0.2591 - val_smooth_mape: 14.6337 - lr: 9.3815e-05\n",
      "Epoch 44/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.0926 - mae: 0.1483 - mse: 0.0735 - root_mean_squared_error: 0.2711 - smooth_mape: 15.9664 - val_loss: 0.0856 - val_mae: 0.1355 - val_mse: 0.0668 - val_root_mean_squared_error: 0.2585 - val_smooth_mape: 14.4813 - lr: 8.4067e-05\n",
      "Epoch 45/100\n",
      "507/507 [==============================] - 26s 52ms/step - loss: 0.0904 - mae: 0.1477 - mse: 0.0732 - root_mean_squared_error: 0.2706 - smooth_mape: 15.9141 - val_loss: 0.0839 - val_mae: 0.1354 - val_mse: 0.0669 - val_root_mean_squared_error: 0.2587 - val_smooth_mape: 14.4705 - lr: 7.0890e-05\n",
      "Epoch 46/100\n",
      "507/507 [==============================] - 27s 54ms/step - loss: 0.0876 - mae: 0.1461 - mse: 0.0718 - root_mean_squared_error: 0.2680 - smooth_mape: 15.7518 - val_loss: 0.0825 - val_mae: 0.1373 - val_mse: 0.0668 - val_root_mean_squared_error: 0.2585 - val_smooth_mape: 14.4117 - lr: 5.5612e-05\n",
      "Epoch 47/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.0853 - mae: 0.1446 - mse: 0.0706 - root_mean_squared_error: 0.2658 - smooth_mape: 15.5602 - val_loss: 0.0805 - val_mae: 0.1370 - val_mse: 0.0658 - val_root_mean_squared_error: 0.2566 - val_smooth_mape: 14.8418 - lr: 3.9768e-05\n",
      "Epoch 48/100\n",
      "507/507 [==============================] - 28s 55ms/step - loss: 0.0851 - mae: 0.1449 - mse: 0.0713 - root_mean_squared_error: 0.2671 - smooth_mape: 15.6229 - val_loss: 0.0801 - val_mae: 0.1363 - val_mse: 0.0660 - val_root_mean_squared_error: 0.2570 - val_smooth_mape: 14.7760 - lr: 2.4955e-05\n",
      "Epoch 49/100\n",
      "507/507 [==============================] - 26s 51ms/step - loss: 0.0833 - mae: 0.1432 - mse: 0.0699 - root_mean_squared_error: 0.2643 - smooth_mape: 15.4598 - val_loss: 0.0791 - val_mae: 0.1348 - val_mse: 0.0654 - val_root_mean_squared_error: 0.2557 - val_smooth_mape: 14.3369 - lr: 1.2661e-05\n",
      "Epoch 50/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.0819 - mae: 0.1421 - mse: 0.0687 - root_mean_squared_error: 0.2620 - smooth_mape: 15.3369 - val_loss: 0.0790 - val_mae: 0.1342 - val_mse: 0.0655 - val_root_mean_squared_error: 0.2558 - val_smooth_mape: 14.2211 - lr: 4.1248e-06\n",
      "Epoch 51/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.0818 - mae: 0.1421 - mse: 0.0686 - root_mean_squared_error: 0.2619 - smooth_mape: 15.3209 - val_loss: 0.0791 - val_mae: 0.1332 - val_mse: 0.0656 - val_root_mean_squared_error: 0.2562 - val_smooth_mape: 14.1215 - lr: 2.0452e-07\n",
      "Epoch 52/100\n",
      "507/507 [==============================] - ETA: 0s - loss: 0.0820 - mae: 0.1420 - mse: 0.0689 - root_mean_squared_error: 0.2625 - smooth_mape: 15.3575\n",
      "Epoch 52: ReduceLROnPlateau reducing learning rate to 2.589756149973255e-07.\n",
      "507/507 [==============================] - 28s 55ms/step - loss: 0.0820 - mae: 0.1420 - mse: 0.0689 - root_mean_squared_error: 0.2625 - smooth_mape: 15.3575 - val_loss: 0.0791 - val_mae: 0.1334 - val_mse: 0.0656 - val_root_mean_squared_error: 0.2561 - val_smooth_mape: 14.1793 - lr: 1.2949e-06\n",
      "Epoch 53/100\n",
      "507/507 [==============================] - 28s 55ms/step - loss: 0.0818 - mae: 0.1417 - mse: 0.0687 - root_mean_squared_error: 0.2621 - smooth_mape: 15.2994 - val_loss: 0.0790 - val_mae: 0.1349 - val_mse: 0.0655 - val_root_mean_squared_error: 0.2560 - val_smooth_mape: 14.1680 - lr: 7.2861e-06\n",
      "Epoch 54/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.0818 - mae: 0.1421 - mse: 0.0689 - root_mean_squared_error: 0.2625 - smooth_mape: 15.3289 - val_loss: 0.0786 - val_mae: 0.1360 - val_mse: 0.0654 - val_root_mean_squared_error: 0.2558 - val_smooth_mape: 14.6983 - lr: 1.7575e-05\n",
      "Epoch 55/100\n",
      "507/507 [==============================] - 26s 51ms/step - loss: 0.0826 - mae: 0.1438 - mse: 0.0701 - root_mean_squared_error: 0.2648 - smooth_mape: 15.5292 - val_loss: 0.0784 - val_mae: 0.1367 - val_mse: 0.0656 - val_root_mean_squared_error: 0.2562 - val_smooth_mape: 14.7914 - lr: 3.1127e-05\n",
      "Epoch 56/100\n",
      "507/507 [==============================] - ETA: 0s - loss: 0.0820 - mae: 0.1433 - mse: 0.0700 - root_mean_squared_error: 0.2647 - smooth_mape: 15.4452\n",
      "Epoch 56: ReduceLROnPlateau reducing learning rate to 2.3289143427973617e-06.\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.0820 - mae: 0.1433 - mse: 0.0700 - root_mean_squared_error: 0.2647 - smooth_mape: 15.4452 - val_loss: 0.0777 - val_mae: 0.1360 - val_mse: 0.0656 - val_root_mean_squared_error: 0.2561 - val_smooth_mape: 14.4766 - lr: 2.3289e-06\n",
      "Epoch 57/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.0823 - mae: 0.1451 - mse: 0.0711 - root_mean_squared_error: 0.2667 - smooth_mape: 15.6261 - val_loss: 0.0785 - val_mae: 0.1375 - val_mse: 0.0674 - val_root_mean_squared_error: 0.2596 - val_smooth_mape: 15.0211 - lr: 6.2374e-05\n",
      "Epoch 58/100\n",
      "507/507 [==============================] - 26s 52ms/step - loss: 0.0818 - mae: 0.1454 - mse: 0.0715 - root_mean_squared_error: 0.2675 - smooth_mape: 15.6269 - val_loss: 0.0772 - val_mae: 0.1366 - val_mse: 0.0669 - val_root_mean_squared_error: 0.2586 - val_smooth_mape: 14.1091 - lr: 7.6924e-05\n",
      "Epoch 59/100\n",
      "507/507 [==============================] - 27s 54ms/step - loss: 0.0817 - mae: 0.1461 - mse: 0.0723 - root_mean_squared_error: 0.2689 - smooth_mape: 15.7262 - val_loss: 0.0761 - val_mae: 0.1370 - val_mse: 0.0663 - val_root_mean_squared_error: 0.2575 - val_smooth_mape: 14.8032 - lr: 8.8765e-05\n",
      "Epoch 60/100\n",
      "507/507 [==============================] - ETA: 0s - loss: 0.0805 - mae: 0.1461 - mse: 0.0719 - root_mean_squared_error: 0.2681 - smooth_mape: 15.7620\n",
      "Epoch 60: ReduceLROnPlateau reducing learning rate to 4.835262006963604e-06.\n",
      "507/507 [==============================] - 26s 52ms/step - loss: 0.0805 - mae: 0.1461 - mse: 0.0719 - root_mean_squared_error: 0.2681 - smooth_mape: 15.7620 - val_loss: 0.0759 - val_mae: 0.1358 - val_mse: 0.0674 - val_root_mean_squared_error: 0.2597 - val_smooth_mape: 14.4084 - lr: 4.8353e-06\n",
      "Epoch 61/100\n",
      "2028/2028 [==============================] - 22s 11ms/step\n",
      "2028/2028 [==============================] - 23s 11ms/step\n",
      "\n",
      "Epoch 61: Out of range predictions: 0\n",
      "507/507 [==============================] - 75s 147ms/step - loss: 0.0787 - mae: 0.1447 - mse: 0.0711 - root_mean_squared_error: 0.2666 - smooth_mape: 15.5565 - val_loss: 0.0737 - val_mae: 0.1352 - val_mse: 0.0661 - val_root_mean_squared_error: 0.2571 - val_smooth_mape: 14.3196 - lr: 9.9946e-05\n",
      "Epoch 62/100\n",
      "507/507 [==============================] - 27s 54ms/step - loss: 0.0781 - mae: 0.1451 - mse: 0.0715 - root_mean_squared_error: 0.2674 - smooth_mape: 15.5903 - val_loss: 0.0726 - val_mae: 0.1344 - val_mse: 0.0658 - val_root_mean_squared_error: 0.2565 - val_smooth_mape: 14.3699 - lr: 9.8161e-05\n",
      "Epoch 63/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.0771 - mae: 0.1446 - mse: 0.0713 - root_mean_squared_error: 0.2670 - smooth_mape: 15.5551 - val_loss: 0.0721 - val_mae: 0.1350 - val_mse: 0.0661 - val_root_mean_squared_error: 0.2571 - val_smooth_mape: 14.3152 - lr: 9.1530e-05\n",
      "Epoch 64/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.0757 - mae: 0.1441 - mse: 0.0705 - root_mean_squared_error: 0.2655 - smooth_mape: 15.5067\n",
      "Epoch 64: ReduceLROnPlateau reducing learning rate to 4.035990059492178e-06.\n",
      "507/507 [==============================] - 27s 54ms/step - loss: 0.0757 - mae: 0.1441 - mse: 0.0705 - root_mean_squared_error: 0.2655 - smooth_mape: 15.5081 - val_loss: 0.0716 - val_mae: 0.1347 - val_mse: 0.0663 - val_root_mean_squared_error: 0.2574 - val_smooth_mape: 14.7350 - lr: 4.0360e-06\n",
      "Epoch 65/100\n",
      "507/507 [==============================] - 26s 50ms/step - loss: 0.0745 - mae: 0.1431 - mse: 0.0698 - root_mean_squared_error: 0.2642 - smooth_mape: 15.3922 - val_loss: 0.0710 - val_mae: 0.1360 - val_mse: 0.0661 - val_root_mean_squared_error: 0.2571 - val_smooth_mape: 14.1923 - lr: 6.6819e-05\n",
      "Epoch 66/100\n",
      "507/507 [==============================] - 27s 54ms/step - loss: 0.0729 - mae: 0.1413 - mse: 0.0686 - root_mean_squared_error: 0.2619 - smooth_mape: 15.1677 - val_loss: 0.0697 - val_mae: 0.1349 - val_mse: 0.0652 - val_root_mean_squared_error: 0.2553 - val_smooth_mape: 14.2538 - lr: 5.1225e-05\n",
      "Epoch 67/100\n",
      "507/507 [==============================] - 28s 55ms/step - loss: 0.0722 - mae: 0.1408 - mse: 0.0682 - root_mean_squared_error: 0.2612 - smooth_mape: 15.0982 - val_loss: 0.0691 - val_mae: 0.1346 - val_mse: 0.0650 - val_root_mean_squared_error: 0.2549 - val_smooth_mape: 14.2971 - lr: 3.5508e-05\n",
      "Epoch 68/100\n",
      "507/507 [==============================] - 25s 48ms/step - loss: 0.0707 - mae: 0.1389 - mse: 0.0669 - root_mean_squared_error: 0.2587 - smooth_mape: 14.9200 - val_loss: 0.0686 - val_mae: 0.1339 - val_mse: 0.0646 - val_root_mean_squared_error: 0.2542 - val_smooth_mape: 14.3511 - lr: 2.1250e-05\n",
      "Epoch 69/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.0705 - mae: 0.1388 - mse: 0.0669 - root_mean_squared_error: 0.2586 - smooth_mape: 14.9215 - val_loss: 0.0684 - val_mae: 0.1325 - val_mse: 0.0645 - val_root_mean_squared_error: 0.2540 - val_smooth_mape: 14.0225 - lr: 9.8841e-06\n",
      "Epoch 70/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.0698 - mae: 0.1379 - mse: 0.0662 - root_mean_squared_error: 0.2573 - smooth_mape: 14.8352 - val_loss: 0.0684 - val_mae: 0.1320 - val_mse: 0.0646 - val_root_mean_squared_error: 0.2542 - val_smooth_mape: 13.9647 - lr: 2.5551e-06\n",
      "Epoch 71/100\n",
      "507/507 [==============================] - 25s 49ms/step - loss: 0.0695 - mae: 0.1374 - mse: 0.0658 - root_mean_squared_error: 0.2565 - smooth_mape: 14.7516 - val_loss: 0.0685 - val_mae: 0.1317 - val_mse: 0.0648 - val_root_mean_squared_error: 0.2545 - val_smooth_mape: 13.8579 - lr: 1.5795e-10\n",
      "Epoch 72/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.0696 - mae: 0.1376 - mse: 0.0660 - root_mean_squared_error: 0.2569 - smooth_mape: 14.7707\n",
      "Epoch 72: ReduceLROnPlateau reducing learning rate to 4.95273479828029e-07.\n",
      "507/507 [==============================] - 25s 50ms/step - loss: 0.0696 - mae: 0.1376 - mse: 0.0660 - root_mean_squared_error: 0.2569 - smooth_mape: 14.7708 - val_loss: 0.0684 - val_mae: 0.1318 - val_mse: 0.0646 - val_root_mean_squared_error: 0.2542 - val_smooth_mape: 13.9681 - lr: 2.4764e-06\n",
      "Epoch 73/100\n",
      "507/507 [==============================] - 27s 54ms/step - loss: 0.0694 - mae: 0.1373 - mse: 0.0658 - root_mean_squared_error: 0.2565 - smooth_mape: 14.7647 - val_loss: 0.0683 - val_mae: 0.1327 - val_mse: 0.0646 - val_root_mean_squared_error: 0.2541 - val_smooth_mape: 13.9221 - lr: 9.7346e-06\n",
      "Epoch 74/100\n",
      "507/507 [==============================] - 24s 47ms/step - loss: 0.0698 - mae: 0.1380 - mse: 0.0663 - root_mean_squared_error: 0.2575 - smooth_mape: 14.8505 - val_loss: 0.0683 - val_mae: 0.1332 - val_mse: 0.0647 - val_root_mean_squared_error: 0.2543 - val_smooth_mape: 14.4362 - lr: 2.1044e-05\n",
      "Epoch 75/100\n",
      "507/507 [==============================] - 25s 50ms/step - loss: 0.0697 - mae: 0.1381 - mse: 0.0664 - root_mean_squared_error: 0.2577 - smooth_mape: 14.8281 - val_loss: 0.0680 - val_mae: 0.1335 - val_mse: 0.0646 - val_root_mean_squared_error: 0.2541 - val_smooth_mape: 14.2706 - lr: 3.5268e-05\n",
      "Epoch 76/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.0713 - mae: 0.1407 - mse: 0.0685 - root_mean_squared_error: 0.2616 - smooth_mape: 15.0952\n",
      "Epoch 76: ReduceLROnPlateau reducing learning rate to 2.548691554693505e-06.\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.0713 - mae: 0.1407 - mse: 0.0685 - root_mean_squared_error: 0.2616 - smooth_mape: 15.0982 - val_loss: 0.0680 - val_mae: 0.1339 - val_mse: 0.0648 - val_root_mean_squared_error: 0.2546 - val_smooth_mape: 14.0539 - lr: 2.5487e-06\n",
      "Epoch 77/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.0706 - mae: 0.1401 - mse: 0.0679 - root_mean_squared_error: 0.2606 - smooth_mape: 15.0332\n",
      "Epoch 77: ReduceLROnPlateau reducing learning rate to 1.3316345575731248e-05.\n",
      "507/507 [==============================] - 25s 49ms/step - loss: 0.0706 - mae: 0.1401 - mse: 0.0679 - root_mean_squared_error: 0.2606 - smooth_mape: 15.0357 - val_loss: 0.0682 - val_mae: 0.1346 - val_mse: 0.0653 - val_root_mean_squared_error: 0.2556 - val_smooth_mape: 14.3903 - lr: 6.6582e-05\n",
      "Epoch 78/100\n",
      "507/507 [==============================] - 25s 50ms/step - loss: 0.0707 - mae: 0.1408 - mse: 0.0683 - root_mean_squared_error: 0.2614 - smooth_mape: 15.1099 - val_loss: 0.0680 - val_mae: 0.1327 - val_mse: 0.0656 - val_root_mean_squared_error: 0.2561 - val_smooth_mape: 14.0229 - lr: 8.0521e-05\n",
      "Epoch 79/100\n",
      "507/507 [==============================] - 24s 47ms/step - loss: 0.0710 - mae: 0.1417 - mse: 0.0691 - root_mean_squared_error: 0.2629 - smooth_mape: 15.2021 - val_loss: 0.0677 - val_mae: 0.1354 - val_mse: 0.0656 - val_root_mean_squared_error: 0.2561 - val_smooth_mape: 14.2230 - lr: 9.1389e-05\n",
      "Epoch 80/100\n",
      "507/507 [==============================] - ETA: 0s - loss: 0.0726 - mae: 0.1444 - mse: 0.0712 - root_mean_squared_error: 0.2668 - smooth_mape: 15.5279\n",
      "Epoch 80: ReduceLROnPlateau reducing learning rate to 4.904638990410604e-06.\n",
      "507/507 [==============================] - 27s 54ms/step - loss: 0.0726 - mae: 0.1444 - mse: 0.0712 - root_mean_squared_error: 0.2668 - smooth_mape: 15.5279 - val_loss: 0.0682 - val_mae: 0.1343 - val_mse: 0.0661 - val_root_mean_squared_error: 0.2570 - val_smooth_mape: 14.2839 - lr: 4.9046e-06\n",
      "Epoch 81/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.0713 - mae: 0.1438 - mse: 0.0699 - root_mean_squared_error: 0.2645 - smooth_mape: 15.4713\n",
      "Epoch 81: ReduceLROnPlateau reducing learning rate to 1.9991402223240587e-05.\n",
      "2028/2028 [==============================] - 23s 11ms/step\n",
      "2028/2028 [==============================] - 23s 11ms/step\n",
      "\n",
      "Epoch 81: Out of range predictions: 0\n",
      "507/507 [==============================] - 75s 148ms/step - loss: 0.0713 - mae: 0.1438 - mse: 0.0700 - root_mean_squared_error: 0.2645 - smooth_mape: 15.4736 - val_loss: 0.0690 - val_mae: 0.1379 - val_mse: 0.0677 - val_root_mean_squared_error: 0.2602 - val_smooth_mape: 15.5488 - lr: 9.9957e-05\n",
      "Epoch 82/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.0696 - mae: 0.1419 - mse: 0.0686 - root_mean_squared_error: 0.2619 - smooth_mape: 15.2732 - val_loss: 0.0667 - val_mae: 0.1334 - val_mse: 0.0656 - val_root_mean_squared_error: 0.2562 - val_smooth_mape: 14.2230 - lr: 9.6794e-05\n",
      "Epoch 83/100\n",
      "507/507 [==============================] - 26s 51ms/step - loss: 0.0688 - mae: 0.1409 - mse: 0.0682 - root_mean_squared_error: 0.2612 - smooth_mape: 15.1230 - val_loss: 0.0659 - val_mae: 0.1343 - val_mse: 0.0652 - val_root_mean_squared_error: 0.2553 - val_smooth_mape: 14.2547 - lr: 8.8923e-05\n",
      "Epoch 84/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.0674 - mae: 0.1393 - mse: 0.0670 - root_mean_squared_error: 0.2589 - smooth_mape: 14.9698\n",
      "Epoch 84: ReduceLROnPlateau reducing learning rate to 3.8567635783692825e-06.\n",
      "507/507 [==============================] - 25s 50ms/step - loss: 0.0674 - mae: 0.1393 - mse: 0.0670 - root_mean_squared_error: 0.2589 - smooth_mape: 14.9716 - val_loss: 0.0655 - val_mae: 0.1338 - val_mse: 0.0650 - val_root_mean_squared_error: 0.2550 - val_smooth_mape: 14.2066 - lr: 3.8568e-06\n",
      "Epoch 85/100\n",
      "507/507 [==============================] - 26s 52ms/step - loss: 0.0673 - mae: 0.1395 - mse: 0.0672 - root_mean_squared_error: 0.2593 - smooth_mape: 14.9972 - val_loss: 0.0680 - val_mae: 0.1333 - val_mse: 0.0682 - val_root_mean_squared_error: 0.2611 - val_smooth_mape: 13.8129 - lr: 6.2617e-05\n",
      "Epoch 86/100\n",
      "507/507 [==============================] - 25s 49ms/step - loss: 0.0669 - mae: 0.1389 - mse: 0.0670 - root_mean_squared_error: 0.2588 - smooth_mape: 14.9195 - val_loss: 0.0647 - val_mae: 0.1343 - val_mse: 0.0645 - val_root_mean_squared_error: 0.2539 - val_smooth_mape: 14.2430 - lr: 4.6829e-05\n",
      "Epoch 87/100\n",
      "507/507 [==============================] - 23s 46ms/step - loss: 0.0656 - mae: 0.1372 - mse: 0.0657 - root_mean_squared_error: 0.2563 - smooth_mape: 14.7366 - val_loss: 0.0643 - val_mae: 0.1317 - val_mse: 0.0644 - val_root_mean_squared_error: 0.2537 - val_smooth_mape: 14.1979 - lr: 3.1360e-05\n",
      "Epoch 88/100\n",
      "507/507 [==============================] - 22s 44ms/step - loss: 0.0649 - mae: 0.1365 - mse: 0.0651 - root_mean_squared_error: 0.2552 - smooth_mape: 14.6326 - val_loss: 0.0638 - val_mae: 0.1331 - val_mse: 0.0639 - val_root_mean_squared_error: 0.2528 - val_smooth_mape: 14.3781 - lr: 1.7767e-05\n",
      "Epoch 89/100\n",
      "507/507 [==============================] - 23s 45ms/step - loss: 0.0647 - mae: 0.1361 - mse: 0.0649 - root_mean_squared_error: 0.2548 - smooth_mape: 14.6184 - val_loss: 0.0637 - val_mae: 0.1317 - val_mse: 0.0638 - val_root_mean_squared_error: 0.2526 - val_smooth_mape: 13.9675 - lr: 7.4173e-06\n",
      "Epoch 90/100\n",
      "507/507 [==============================] - 26s 51ms/step - loss: 0.0642 - mae: 0.1356 - mse: 0.0644 - root_mean_squared_error: 0.2538 - smooth_mape: 14.5656 - val_loss: 0.0638 - val_mae: 0.1309 - val_mse: 0.0640 - val_root_mean_squared_error: 0.2530 - val_smooth_mape: 13.8811 - lr: 1.3523e-06\n",
      "Epoch 91/100\n",
      "507/507 [==============================] - ETA: 0s - loss: 0.0636 - mae: 0.1346 - mse: 0.0637 - root_mean_squared_error: 0.2524 - smooth_mape: 14.4922\n",
      "Epoch 91: ReduceLROnPlateau reducing learning rate to 1e-07.\n",
      "507/507 [==============================] - 24s 47ms/step - loss: 0.0636 - mae: 0.1346 - mse: 0.0637 - root_mean_squared_error: 0.2524 - smooth_mape: 14.4922 - val_loss: 0.0638 - val_mae: 0.1311 - val_mse: 0.0640 - val_root_mean_squared_error: 0.2530 - val_smooth_mape: 13.8034 - lr: 1.8244e-07\n",
      "Epoch 92/100\n",
      "507/507 [==============================] - 25s 49ms/step - loss: 0.0634 - mae: 0.1342 - mse: 0.0635 - root_mean_squared_error: 0.2520 - smooth_mape: 14.4498 - val_loss: 0.0637 - val_mae: 0.1312 - val_mse: 0.0639 - val_root_mean_squared_error: 0.2527 - val_smooth_mape: 13.8449 - lr: 4.0254e-06\n",
      "Epoch 93/100\n",
      "507/507 [==============================] - 26s 51ms/step - loss: 0.0636 - mae: 0.1346 - mse: 0.0638 - root_mean_squared_error: 0.2526 - smooth_mape: 14.4781 - val_loss: 0.0636 - val_mae: 0.1318 - val_mse: 0.0638 - val_root_mean_squared_error: 0.2526 - val_smooth_mape: 14.0036 - lr: 1.2494e-05\n",
      "Epoch 94/100\n",
      "507/507 [==============================] - 26s 51ms/step - loss: 0.0639 - mae: 0.1350 - mse: 0.0642 - root_mean_squared_error: 0.2534 - smooth_mape: 14.4772 - val_loss: 0.0635 - val_mae: 0.1324 - val_mse: 0.0638 - val_root_mean_squared_error: 0.2526 - val_smooth_mape: 14.1743 - lr: 2.4737e-05\n",
      "Epoch 95/100\n",
      "507/507 [==============================] - ETA: 0s - loss: 0.0648 - mae: 0.1366 - mse: 0.0653 - root_mean_squared_error: 0.2556 - smooth_mape: 14.6348\n",
      "Epoch 95: ReduceLROnPlateau reducing learning rate to 1.976121529878583e-06.\n",
      "507/507 [==============================] - 25s 50ms/step - loss: 0.0648 - mae: 0.1366 - mse: 0.0653 - root_mean_squared_error: 0.2556 - smooth_mape: 14.6348 - val_loss: 0.0638 - val_mae: 0.1331 - val_mse: 0.0643 - val_root_mean_squared_error: 0.2535 - val_smooth_mape: 14.4393 - lr: 1.9761e-06\n",
      "Epoch 96/100\n",
      "507/507 [==============================] - ETA: 0s - loss: 0.0652 - mae: 0.1374 - mse: 0.0659 - root_mean_squared_error: 0.2567 - smooth_mape: 14.7371\n",
      "Epoch 96: ReduceLROnPlateau reducing learning rate to 1.1072350753238425e-05.\n",
      "507/507 [==============================] - 25s 50ms/step - loss: 0.0652 - mae: 0.1374 - mse: 0.0659 - root_mean_squared_error: 0.2567 - smooth_mape: 14.7371 - val_loss: 0.0644 - val_mae: 0.1353 - val_mse: 0.0650 - val_root_mean_squared_error: 0.2550 - val_smooth_mape: 14.2866 - lr: 5.5362e-05\n",
      "Epoch 97/100\n",
      "507/507 [==============================] - 27s 53ms/step - loss: 0.0660 - mae: 0.1386 - mse: 0.0668 - root_mean_squared_error: 0.2585 - smooth_mape: 14.8711 - val_loss: 0.0640 - val_mae: 0.1324 - val_mse: 0.0647 - val_root_mean_squared_error: 0.2544 - val_smooth_mape: 14.0692 - lr: 7.0662e-05\n",
      "Epoch 98/100\n",
      "507/507 [==============================] - ETA: 0s - loss: 0.0653 - mae: 0.1380 - mse: 0.0662 - root_mean_squared_error: 0.2574 - smooth_mape: 14.7726\n",
      "Epoch 98: ReduceLROnPlateau reducing learning rate to 1.6776460688561202e-05.\n",
      "507/507 [==============================] - 26s 51ms/step - loss: 0.0653 - mae: 0.1380 - mse: 0.0662 - root_mean_squared_error: 0.2574 - smooth_mape: 14.7726 - val_loss: 0.0652 - val_mae: 0.1359 - val_mse: 0.0662 - val_root_mean_squared_error: 0.2572 - val_smooth_mape: 14.0106 - lr: 8.3882e-05\n",
      "Epoch 99/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.0660 - mae: 0.1395 - mse: 0.0672 - root_mean_squared_error: 0.2593 - smooth_mape: 14.9081\n",
      "Epoch 99: ReduceLROnPlateau reducing learning rate to 4.684684972744436e-06.\n",
      "507/507 [==============================] - 24s 47ms/step - loss: 0.0660 - mae: 0.1395 - mse: 0.0672 - root_mean_squared_error: 0.2593 - smooth_mape: 14.9098 - val_loss: 0.0642 - val_mae: 0.1338 - val_mse: 0.0653 - val_root_mean_squared_error: 0.2555 - val_smooth_mape: 14.2021 - lr: 4.6847e-06\n",
      "Epoch 100/100\n",
      "506/507 [============================>.] - ETA: 0s - loss: 0.0649 - mae: 0.1379 - mse: 0.0662 - root_mean_squared_error: 0.2574 - smooth_mape: 14.7397\n",
      "Epoch 100: ReduceLROnPlateau reducing learning rate to 1.9821693422272803e-05.\n",
      "507/507 [==============================] - 24s 46ms/step - loss: 0.0649 - mae: 0.1379 - mse: 0.0662 - root_mean_squared_error: 0.2574 - smooth_mape: 14.7407 - val_loss: 0.0652 - val_mae: 0.1356 - val_mse: 0.0667 - val_root_mean_squared_error: 0.2582 - val_smooth_mape: 13.8309 - lr: 9.9108e-05\n",
      "\n",
      "Training completed successfully!\n"
     ]
    }
   ],
   "source": [
    "# Model creation or loading\n",
    "print(\"\\n2. Model initialization...\")\n",
    "input_shape = (X_train_seq.shape[1], X_train_seq.shape[2])\n",
    "\n",
    "MAX_UVINDEX = 11\n",
    "\n",
    "max_val_scaled = target_scaler.transform([[MAX_UVINDEX]])[0][0]\n",
    "\n",
    "if os.path.exists(model_path):\n",
    "    print(f\"Loading existing model from: {model_path}\")\n",
    "    model = tf.keras.models.load_model(model_path)\n",
    "\n",
    "    # Load existing history if available\n",
    "    if os.path.exists(history_path):\n",
    "        print(f\"Loading existing training history from: {history_path}\")\n",
    "        with open(history_path, 'r') as f:\n",
    "            history_dict = json.load(f)\n",
    "            history = type('History', (), {'history': history_dict})()\n",
    "    else:\n",
    "        history = type('History', (), {'history': {}})()\n",
    "else:\n",
    "    print(\"Creating new model...\")\n",
    "    model = create_uv_index_model(input_shape=input_shape, folder_name=folder_name, max_output=max_val_scaled)\n",
    "\n",
    "    print(\"\\n3. Starting training...\")\n",
    "    history = train_hybrid_model(\n",
    "        model=model,\n",
    "        X_train=X_train_seq,\n",
    "        y_train=y_train,\n",
    "        X_test=X_test_seq,\n",
    "        y_test=y_test,\n",
    "        epochs=100,\n",
    "        batch_size=128,\n",
    "        folder_name=folder_name\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f0059ecf-7f4f-496f-bed8-85e2d990ff71",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "4. Generating predictions...\n",
      "2028/2028 [==============================] - 18s 9ms/step\n",
      "\n",
      "5. Model evaluation...\n",
      "\n",
      "Error saving plots: Unknown format code 'd' for object of type 'float'\n",
      "\n",
      "UV Index Prediction Analysis:\n",
      "\n",
      "Raw Metrics:\n",
      "mae: 0.407\n",
      "rmse: 0.775\n",
      "r2: 0.918\n",
      "mean_error: -0.076\n",
      "std_error: 0.771\n",
      "median_error: 0.012\n",
      "p95_abs_error: 1.745\n",
      "within_05: 71.379\n",
      "within_1: 86.040\n",
      "within_15: 92.984\n",
      "within_2: 96.562\n",
      "\n",
      "Rounded Metrics:\n",
      "mae: 0.393\n",
      "rmse: 0.782\n",
      "r2: 0.916\n",
      "within_05: 78.975\n",
      "within_1: 90.160\n",
      "within_15: 95.037\n",
      "within_2: 97.478\n",
      "\n",
      "Analysis by UV Range:\n",
      "\n",
      "Low:\n",
      "  mae: 0.133\n",
      "  count: 41407.000\n",
      "  accuracy_within_05: 90.828\n",
      "  accuracy_within_1: 97.341\n",
      "\n",
      "Moderate:\n",
      "  mae: 0.874\n",
      "  count: 11467.000\n",
      "  accuracy_within_05: 36.418\n",
      "  accuracy_within_1: 66.713\n",
      "\n",
      "High:\n",
      "  mae: 0.871\n",
      "  count: 5415.000\n",
      "  accuracy_within_05: 37.876\n",
      "  accuracy_within_1: 65.614\n",
      "\n",
      "Very High:\n",
      "  mae: 0.905\n",
      "  count: 6343.000\n",
      "  accuracy_within_05: 38.862\n",
      "  accuracy_within_1: 66.404\n",
      "\n",
      "Extreme:\n",
      "  mae: 1.649\n",
      "  count: 234.000\n",
      "  accuracy_within_05: 0.000\n",
      "  accuracy_within_1: 38.462\n",
      "\n",
      "Confusion Matrix:\n",
      "               Low  Moderate    High  Very High  Extreme\n",
      "Low        43040.0    2283.0    97.0       17.0      0.0\n",
      "Moderate    1336.0    7625.0  1181.0      107.0      0.0\n",
      "High          10.0    1155.0  3149.0      576.0      0.0\n",
      "Very High      0.0     114.0  1110.0     3066.0      0.0\n",
      "Extreme        0.0       0.0     0.0        0.0      0.0\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 276404 characters]",
      "text/plain": [
       "<Figure size 2000x1500 with 5 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 33588 characters]",
      "text/plain": [
       "<Figure size 1000x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"\\n4. Generating predictions...\")\n",
    "predictions = model.predict(X_test_seq)\n",
    "predictions = np.clip(predictions, 0, max_val_scaled)\n",
    "\n",
    "predictions_original = target_scaler.inverse_transform(predictions.reshape(-1, 1))\n",
    "y_test_original = target_scaler.inverse_transform(y_test.reshape(-1, 1))\n",
    "\n",
    "print(\"\\n5. Model evaluation...\")\n",
    "metrics = evaluate_uv_predictions(y_test_original, predictions_original, folder_name=folder_name)\n",
    "\n",
    "# Save training results only if new training was performed\n",
    "if not os.path.exists(model_path):\n",
    "    training_results = {\n",
    "        'model_params': {\n",
    "            'input_shape': input_shape,\n",
    "            'n_features': len(features),\n",
    "            'sequence_length': X_train_seq.shape[1]\n",
    "        },\n",
    "        'training_params': {\n",
    "            'batch_size': 128,\n",
    "            'total_epochs': len(history.history['loss']),\n",
    "            'best_epoch': np.argmin(history.history['val_loss']) + 1,\n",
    "        },\n",
    "        'performance_metrics': {\n",
    "            'final_loss': float(history.history['val_loss'][-1]),\n",
    "            'final_mae': float(history.history['val_mae'][-1]),\n",
    "            'best_val_loss': float(min(history.history['val_loss'])),\n",
    "            'out_of_range_predictions': int(np.sum((predictions < 0) | (predictions > 11)))\n",
    "        }\n",
    "    }\n",
    "\n",
    "    # Save training history\n",
    "    with open(history_path, 'w') as f:\n",
    "        history_dict = {key: [float(val) for val in values]\n",
    "                        for key, values in history.history.items()}\n",
    "        json.dump(history_dict, f, indent=4)\n",
    "else:\n",
    "    # Load existing training results if available\n",
    "    results_path = f'{folder_name}_training_results.json'\n",
    "    if os.path.exists(results_path):\n",
    "        with open(results_path, 'r') as f:\n",
    "            training_results = json.load(f)\n",
    "    else:\n",
    "        training_results = {}\n",
    "\n",
    "tf.keras.backend.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4365d2bf-daf8-49e1-be13-cce222bbb5bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "6. Predicting missing data...\n",
      "7122/7122 [==============================] - 74s 10ms/step\n",
      "\n",
      "7. Integrating predictions into dataset...\n",
      "Added 227879 predictions to dataset\n",
      "Rows with UV index after integration: 357615\n",
      "Updated dataset saved to: ../../sources/weather_data_uvindex.parquet\n",
      "\n",
      "All files saved with prefix: 2024-11-21_08-23\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n6. Predicting missing data...\")\n",
    "to_predict_predictions = model.predict(X_to_predict_seq)\n",
    "to_predict_predictions = np.clip(to_predict_predictions, 0, max_val_scaled)\n",
    "\n",
    "to_predict_predictions_original = target_scaler.inverse_transform(to_predict_predictions.reshape(-1, 1))\n",
    "\n",
    "print(\"\\n7. Integrating predictions into dataset...\")\n",
    "df_updated = integrate_predictions(df.copy(), to_predict_predictions_original)\n",
    "\n",
    "output_path = f'../../sources/weather_data_uvindex.parquet'\n",
    "df_updated.to_parquet(output_path)\n",
    "print(f\"Updated dataset saved to: {output_path}\")\n",
    "\n",
    "# Add prediction statistics\n",
    "prediction_stats = {\n",
    "    'n_predictions_added': len(to_predict_predictions),\n",
    "    'mean_predicted_uv': float(to_predict_predictions.mean()),\n",
    "    'min_predicted_uv': float(to_predict_predictions.min()),\n",
    "    'max_predicted_uv': float(to_predict_predictions.max()),\n",
    "}\n",
    "\n",
    "\n",
    "def convert_to_serializable(obj):\n",
    "    \"\"\"Convert numpy types to Python standard types for JSON serialization\"\"\"\n",
    "    if isinstance(obj, (np.int_, np.intc, np.intp, np.int8,\n",
    "                        np.int16, np.int32, np.int64, np.uint8,\n",
    "                        np.uint16, np.uint32, np.uint64)):\n",
    "        return int(obj)\n",
    "    elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)):\n",
    "        return float(obj)\n",
    "    elif isinstance(obj, (np.ndarray,)):\n",
    "        return obj.tolist()\n",
    "    elif isinstance(obj, dict):\n",
    "        return {key: convert_to_serializable(value) for key, value in obj.items()}\n",
    "    elif isinstance(obj, list):\n",
    "        return [convert_to_serializable(item) for item in obj]\n",
    "    return obj\n",
    "\n",
    "\n",
    "if not os.path.exists(model_path):\n",
    "    training_results['prediction_stats'] = prediction_stats\n",
    "\n",
    "    training_results = convert_to_serializable(training_results)\n",
    "    # Save final results\n",
    "    results_path = f'{folder_name}_training_results.json'\n",
    "    with open(results_path, 'w') as f:\n",
    "        json.dump(training_results, f, indent=4)\n",
    "\n",
    "print(f\"\\nAll files saved with prefix: {folder_name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "08fd4208-0afb-4bf1-bdef-b10b4065fe55",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Plot saved as: 2024-11-21_08-23/error_analysis.png\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 137692 characters]",
      "text/plain": [
       "<Figure size 1500x500 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Error statistics:\n",
      "MAE: 0.1356\n",
      "MSE: 0.0667\n",
      "RMSE: 0.2582\n",
      "Mean errors: -0.0252\n",
      "Std errors: 0.2569\n",
      "Predictions within ±0.5: 93.0%\n",
      "Predictions within ±1.0: 99.0%\n",
      "Predictions within ±1.5: 99.9%\n",
      "Predictions within ±2.0: 100.0%\n"
     ]
    }
   ],
   "source": [
    "def plot_error_analysis(y_true, y_pred, folder_name=None):\n",
    "    \"\"\"\n",
    "    Function to visualize prediction error analysis\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    y_true : array-like\n",
    "        Actual values\n",
    "    y_pred : array-like\n",
    "        Predicted values\n",
    "    folder_name : str, optional\n",
    "        Folder to save plots. If None, plots are not saved.\n",
    "    \"\"\"\n",
    "\n",
    "    # Convert to 1D numpy array if necessary\n",
    "    if isinstance(y_true, pd.Series):\n",
    "        y_true = y_true.values\n",
    "    if isinstance(y_pred, pd.Series):\n",
    "        y_pred = y_pred.values\n",
    "\n",
    "    y_true = y_true.ravel()\n",
    "    y_pred = y_pred.ravel()\n",
    "\n",
    "    # Calculate errors\n",
    "    errors = y_pred - y_true\n",
    "\n",
    "    # Create main figure\n",
    "    fig = plt.figure(figsize=(15, 5))\n",
    "\n",
    "    # Plot 1: Error Distribution\n",
    "    plt.subplot(1, 3, 1)\n",
    "    plt.hist(errors, bins=50, alpha=0.7)\n",
    "    plt.title('Prediction Error Distribution')\n",
    "    plt.xlabel('Error')\n",
    "    plt.ylabel('Frequency')\n",
    "\n",
    "    # Plot 2: Actual vs Predicted\n",
    "    plt.subplot(1, 3, 2)\n",
    "    plt.scatter(y_true, y_pred, alpha=0.5)\n",
    "    plt.plot([y_true.min(), y_true.max()], [y_true.min(), y_true.max()], 'r--', lw=2)\n",
    "    plt.title('Actual vs Predicted Values')\n",
    "    plt.xlabel('Actual Values')\n",
    "    plt.ylabel('Predicted Values')\n",
    "\n",
    "    # Plot 3: Errors vs Actual Values\n",
    "    plt.subplot(1, 3, 3)\n",
    "    plt.scatter(y_true, errors, alpha=0.5)\n",
    "    plt.axhline(y=0, color='r', linestyle='--')\n",
    "    plt.title('Errors vs Actual Values')\n",
    "    plt.xlabel('Actual Values')\n",
    "    plt.ylabel('Error')\n",
    "\n",
    "    plt.tight_layout()\n",
    "\n",
    "    # Save plot if folder is specified\n",
    "    if folder_name is not None:\n",
    "        try:\n",
    "            # Create folder if it doesn't exist\n",
    "            os.makedirs(folder_name, exist_ok=True)\n",
    "\n",
    "            # Generate filename with timestamp\n",
    "            filename = os.path.join(folder_name, 'error_analysis.png')\n",
    "\n",
    "            # Save figure\n",
    "            plt.savefig(filename, dpi=300, bbox_inches='tight')\n",
    "            print(f\"\\nPlot saved as: {filename}\")\n",
    "        except Exception as e:\n",
    "            print(f\"\\nError saving plot: {str(e)}\")\n",
    "\n",
    "    plt.show()\n",
    "\n",
    "    # Print error statistics\n",
    "    print(\"\\nError statistics:\")\n",
    "    print(f\"MAE: {np.mean(np.abs(errors)):.4f}\")\n",
    "    print(f\"MSE: {np.mean(errors ** 2):.4f}\")\n",
    "    print(f\"RMSE: {np.sqrt(np.mean(errors ** 2)):.4f}\")\n",
    "    print(f\"Mean errors: {np.mean(errors):.4f}\")\n",
    "    print(f\"Std errors: {np.std(errors):.4f}\")\n",
    "\n",
    "    # Calculate percentage of errors within thresholds\n",
    "    thresholds = [0.5, 1.0, 1.5, 2.0]\n",
    "    for threshold in thresholds:\n",
    "        within_threshold = np.mean(np.abs(errors) <= threshold) * 100\n",
    "        print(f\"Predictions within ±{threshold}: {within_threshold:.1f}%\")\n",
    "\n",
    "\n",
    "plot_error_analysis(y_test, predictions, folder_name=folder_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c57d6b2-00a6-4d31-935e-449a29dafd79",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
