{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8adcbe0819b88578",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease\n",
      "Hit:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease               \n",
      "Hit:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease\n",
      "Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease             \n",
      "Hit:5 http://security.ubuntu.com/ubuntu jammy-security InRelease\n",
      "Reading package lists... Done\n",
      "Reading package lists... Done\n",
      "Building dependency tree... Done\n",
      "Reading state information... Done\n",
      "graphviz is already the newest version (2.42.2-6ubuntu0.1).\n",
      "0 upgraded, 0 newly installed, 0 to remove and 121 not upgraded.\n",
      "Requirement already satisfied: tensorflow in /usr/local/lib/python3.11/dist-packages (2.14.0)\n",
      "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.0.0)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.6.3)\n",
      "Requirement already satisfied: flatbuffers>=23.5.26 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (23.5.26)\n",
      "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.5.4)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.2.0)\n",
      "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (3.9.0)\n",
      "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (16.0.6)\n",
      "Requirement already satisfied: ml-dtypes==0.2.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.2.0)\n",
      "Requirement already satisfied: numpy>=1.23.5 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.26.0)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (3.3.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from tensorflow) (23.1)\n",
      "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (4.24.3)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.11/dist-packages (from tensorflow) (68.2.2)\n",
      "Requirement already satisfied: six>=1.12.0 in /usr/lib/python3/dist-packages (from tensorflow) (1.16.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.3.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (4.8.0)\n",
      "Requirement already satisfied: wrapt<1.15,>=1.11.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.14.1)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.37.1)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.58.0)\n",
      "Requirement already satisfied: tensorboard<2.15,>=2.14 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.14.0)\n",
      "Requirement already satisfied: tensorflow-estimator<2.15,>=2.14.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.14.0)\n",
      "Requirement already satisfied: keras<2.15,>=2.14.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.14.0)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.11/dist-packages (from astunparse>=1.6.0->tensorflow) (0.41.2)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (2.23.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (1.0.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (3.4.4)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (2.31.0)\n",
      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (0.7.1)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (2.3.7)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (5.3.1)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (0.3.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (4.9)\n",
      "Requirement already satisfied: urllib3>=2.0.5 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (2.0.5)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard<2.15,>=2.14->tensorflow) (1.3.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.15,>=2.14->tensorflow) (3.2.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.15,>=2.14->tensorflow) (3.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.15,>=2.14->tensorflow) (2023.7.22)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.11/dist-packages (from werkzeug>=1.0.1->tensorboard<2.15,>=2.14->tensorflow) (2.1.3)\n",
      "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.11/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (0.5.0)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/lib/python3/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard<2.15,>=2.14->tensorflow) (3.2.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (1.26.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (2.2.3)\n",
      "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (1.26.0)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: keras in /usr/local/lib/python3.11/dist-packages (2.14.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.11/dist-packages (1.5.2)\n",
      "Requirement already satisfied: numpy>=1.19.5 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.26.0)\n",
      "Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.14.1)\n",
      "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.4.2)\n",
      "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (3.5.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: matplotlib in /usr/local/lib/python3.11/dist-packages (3.8.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (4.42.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.4.5)\n",
      "Requirement already satisfied: numpy<2,>=1.21 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.26.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (23.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (10.0.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (3.2.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (2.8.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.11/dist-packages (1.4.2)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: pyarrow in /usr/local/lib/python3.11/dist-packages (18.1.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: fastparquet in /usr/local/lib/python3.11/dist-packages (2024.11.0)\n",
      "Requirement already satisfied: pandas>=1.5.0 in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2.2.3)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (from fastparquet) (1.26.0)\n",
      "Requirement already satisfied: cramjam>=2.3 in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2.9.0)\n",
      "Requirement already satisfied: fsspec in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2024.10.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from fastparquet) (23.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2024.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas>=1.5.0->fastparquet) (1.16.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.11/dist-packages (1.14.1)\n",
      "Requirement already satisfied: numpy<2.3,>=1.23.5 in /usr/local/lib/python3.11/dist-packages (from scipy) (1.26.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: seaborn in /usr/local/lib/python3.11/dist-packages (0.13.2)\n",
      "Requirement already satisfied: numpy!=1.24.0,>=1.20 in /usr/local/lib/python3.11/dist-packages (from seaborn) (1.26.0)\n",
      "Requirement already satisfied: pandas>=1.2 in /usr/local/lib/python3.11/dist-packages (from seaborn) (2.2.3)\n",
      "Requirement already satisfied: matplotlib!=3.6.1,>=3.4 in /usr/local/lib/python3.11/dist-packages (from seaborn) (3.8.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (4.42.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.4.5)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (23.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (10.0.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (3.2.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.2->seaborn) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.2->seaborn) (2024.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib!=3.6.1,>=3.4->seaborn) (1.16.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (4.67.1)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: pydot in /usr/local/lib/python3.11/dist-packages (3.0.2)\n",
      "Requirement already satisfied: pyparsing>=3.0.9 in /usr/local/lib/python3.11/dist-packages (from pydot) (3.2.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: tensorflow-io in /usr/local/lib/python3.11/dist-packages (0.37.1)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem==0.37.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow-io) (0.37.1)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.11/dist-packages (0.23.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from tensorflow-addons) (23.1)\n",
      "Requirement already satisfied: typeguard<3.0.0,>=2.7 in /usr/local/lib/python3.11/dist-packages (from tensorflow-addons) (2.13.3)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "from opt_einsum.paths import branch_1\n",
    "!apt-get update\n",
    "!apt-get install graphviz -y\n",
    "\n",
    "!pip install tensorflow\n",
    "!pip install numpy\n",
    "!pip install pandas\n",
    "\n",
    "!pip install keras\n",
    "!pip install scikit-learn\n",
    "!pip install matplotlib\n",
    "!pip install joblib\n",
    "!pip install pyarrow\n",
    "!pip install fastparquet\n",
    "!pip install scipy\n",
    "!pip install seaborn\n",
    "!pip install tqdm\n",
    "!pip install pydot\n",
    "!pip install tensorflow-io\n",
    "!pip install tensorflow-addons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e6fe6bb613168a8a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-26 05:41:43.497052: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-11-26 05:41:43.497104: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-11-26 05:41:43.497156: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-11-26 05:41:43.506575: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "/usr/local/lib/python3.11/dist-packages/tensorflow_addons/utils/tfa_eol_msg.py:23: UserWarning: \n",
      "\n",
      "TensorFlow Addons (TFA) has ended development and introduction of new features.\n",
      "TFA has entered a minimal maintenance and release mode until a planned end of life in May 2024.\n",
      "Please modify downstream libraries to take dependencies from other repositories in our TensorFlow community (e.g. Keras, Keras-CV, and Keras-NLP). \n",
      "\n",
      "For more information see: https://github.com/tensorflow/addons/issues/2807 \n",
      "\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Dense, LSTM, MultiHeadAttention, Dropout, BatchNormalization, LayerNormalization, Input, Activation, Lambda, Bidirectional, Add, MaxPooling1D, SpatialDropout1D, GlobalAveragePooling1D, \\\n",
    "    GlobalMaxPooling1D, Concatenate, ThresholdedReLU, Average\n",
    "from tensorflow.keras import regularizers\n",
    "from tensorflow.keras.models import Model\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\n",
    "from tensorflow.keras.optimizers import AdamW\n",
    "import json\n",
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.keras.utils import plot_model\n",
    "import tensorflow_addons as tfa\n",
    "import os\n",
    "import joblib\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score, confusion_matrix, classification_report, roc_auc_score\n",
    "from tensorflow.keras.metrics import AUC\n",
    "from scipy import stats\n",
    "\n",
    "folder_name = datetime.now().strftime(\"%Y-%m-%d_%H-%M\")\n",
    "\n",
    "random_state_value = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3da8b15c7eb9833f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_season(date):\n",
    "    month = date.month\n",
    "    day = date.day\n",
    "    if (month == 12 and day >= 21) or (month <= 3 and day < 20):\n",
    "        return 'Winter'\n",
    "    elif (month == 3 and day >= 20) or (month <= 6 and day < 21):\n",
    "        return 'Spring'\n",
    "    elif (month == 6 and day >= 21) or (month <= 9 and day < 23):\n",
    "        return 'Summer'\n",
    "    elif (month == 9 and day >= 23) or (month <= 12 and day < 21):\n",
    "        return 'Autumn'\n",
    "    else:\n",
    "        return 'Unknown'\n",
    "\n",
    "\n",
    "def get_time_period(hour):\n",
    "    if 5 <= hour < 12:\n",
    "        return 'Morning'\n",
    "    elif 12 <= hour < 17:\n",
    "        return 'Afternoon'\n",
    "    elif 17 <= hour < 21:\n",
    "        return 'Evening'\n",
    "    else:\n",
    "        return 'Night'\n",
    "\n",
    "\n",
    "def add_time_features(df):\n",
    "    \"\"\"\n",
    "    Add time-based features to the DataFrame.\n",
    "    Works with both 'datetime' as column or index.\n",
    "    \"\"\"\n",
    "    # Se datetime è l'indice, lo usiamo direttamente\n",
    "    if isinstance(df.index, pd.DatetimeIndex):\n",
    "        datetime_col = df.index\n",
    "    else:\n",
    "        # Se datetime è una colonna, la convertiamo\n",
    "        if 'datetime' in df.columns:\n",
    "            datetime_col = pd.to_datetime(df['datetime'])\n",
    "        else:\n",
    "            raise ValueError(\"No datetime column or index found in DataFrame\")\n",
    "\n",
    "    # Creazione delle feature temporali\n",
    "    df['timestamp'] = datetime_col.astype(np.int64) // 10 ** 9\n",
    "    df['year'] = datetime_col.year\n",
    "    df['month'] = datetime_col.month\n",
    "    df['day'] = datetime_col.day\n",
    "    df['hour'] = datetime_col.hour\n",
    "    df['minute'] = datetime_col.minute\n",
    "    df['hour_sin'] = np.sin(datetime_col.hour * (2 * np.pi / 24))\n",
    "    df['hour_cos'] = np.cos(datetime_col.hour * (2 * np.pi / 24))\n",
    "    df['day_of_week'] = datetime_col.dayofweek\n",
    "    df['day_of_year'] = datetime_col.dayofyear\n",
    "    df['week_of_year'] = datetime_col.isocalendar().week.astype(int)\n",
    "    df['quarter'] = datetime_col.quarter\n",
    "    df['is_month_end'] = datetime_col.is_month_end.astype(int)\n",
    "    df['is_quarter_end'] = datetime_col.is_quarter_end.astype(int)\n",
    "    df['is_year_end'] = datetime_col.is_year_end.astype(int)\n",
    "    df['month_sin'] = np.sin(datetime_col.month * (2 * np.pi / 12))\n",
    "    df['month_cos'] = np.cos(datetime_col.month * (2 * np.pi / 12))\n",
    "    df['day_of_year_sin'] = np.sin(datetime_col.dayofyear * (2 * np.pi / 365.25))\n",
    "    df['day_of_year_cos'] = np.cos(datetime_col.dayofyear * (2 * np.pi / 365.25))\n",
    "    df['season'] = datetime_col.map(get_season)\n",
    "    df['time_period'] = datetime_col.hour.map(get_time_period)\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_solar_features(df):\n",
    "    # Solar angle calculation\n",
    "    df['solar_angle'] = np.sin(df['day_of_year'] * (2 * np.pi / 365.25)) * np.sin(df['hour'] * (2 * np.pi / 24))\n",
    "\n",
    "    # Interactions between relevant features\n",
    "    df['cloud_temp_interaction'] = df['cloudcover'] * df['temp']\n",
    "    df['visibility_cloud_interaction'] = df['visibility'] * (100 - df['cloudcover'])\n",
    "\n",
    "    # Derived features\n",
    "    df['clear_sky_index'] = (100 - df['cloudcover']) / 100\n",
    "    df['temp_gradient'] = df['temp'] - df['tempmin']\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_solar_specific_features(df):\n",
    "    \"\"\"\n",
    "    Aggiunge feature specifiche per la predizione della radiazione solare\n",
    "    combinando caratteristiche astronomiche e meteorologiche\n",
    "    \"\"\"\n",
    "    # Caratteristiche astronomiche\n",
    "    df['day_length'] = 12 + 3 * np.sin(2 * np.pi * (df['day_of_year'] - 81) / 365.25)\n",
    "    df['solar_noon'] = 12 - df['hour']\n",
    "    df['solar_elevation'] = np.sin(2 * np.pi * df['day_of_year'] / 365.25) * np.cos(2 * np.pi * df['solar_noon'] / 24)\n",
    "\n",
    "    # Angolo solare teorico\n",
    "    df['solar_angle'] = np.sin(df['hour_sin']) * np.sin(df['day_of_year_sin'])\n",
    "\n",
    "    # Interazioni con condizioni atmosferiche\n",
    "    df['cloud_elevation'] = df['cloudcover'] * df['solar_elevation']\n",
    "    df['visibility_elevation'] = df['visibility'] * df['solar_elevation']\n",
    "    df['uv_cloud_interaction'] = df['uvindex'] * (100 - df['cloudcover']) / 100\n",
    "\n",
    "    # Indici di chiarezza e trasmissione\n",
    "    df['clearness_index'] = (100 - df['cloudcover']) * df['visibility'] / 10000\n",
    "    df['atmospheric_attenuation'] = (df['pressure'] / 1013.25) * (1 - (df['humidity'] / 100) * 0.6)\n",
    "\n",
    "    # Radiazione teorica e attenuazione\n",
    "    df['theoretical_radiation'] = df['solar_angle'].clip(0, 1) * 1000\n",
    "    df['expected_radiation'] = df['theoretical_radiation'] * df['clearness_index']\n",
    "\n",
    "    # Rolling features\n",
    "    df['cloud_rolling_12h'] = df['cloudcover'].rolling(window=12).mean()\n",
    "    df['temp_rolling_12h'] = df['temp'].rolling(window=12).mean()\n",
    "    df['uv_rolling_12h'] = df['uvindex'].rolling(window=12).mean()\n",
    "\n",
    "    # Interazioni temperatura-radiazione\n",
    "    df['temp_radiation_potential'] = df['temp'] * df['solar_elevation']\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_radiation_energy_features(df):\n",
    "    \"\"\"Adds specific features based on solarenergy and uvindex\"\"\"\n",
    "\n",
    "    # Assicuriamoci che l'indice sia di tipo datetime\n",
    "    if not isinstance(df.index, pd.DatetimeIndex):\n",
    "        df.index = pd.to_datetime(df['datetime'])\n",
    "\n",
    "    # Solar energy to UV ratio (independent from solarradiation)\n",
    "    df['energy_uv_ratio'] = df['solarenergy'] / (df['uvindex'] + 1e-6)\n",
    "\n",
    "    # Time aggregations\n",
    "    # Moving averages\n",
    "    windows = [3, 6, 12, 24]  # hours\n",
    "    for w in windows:\n",
    "        df[f'energy_rolling_mean_{w}h'] = df['solarenergy'].rolling(window=w).mean()\n",
    "        df[f'uv_rolling_mean_{w}h'] = df['uvindex'].rolling(window=w).mean()\n",
    "\n",
    "    # Daily aggregations utilizzando datetime\n",
    "    df['energy_daily_sum'] = df.groupby(df.index.date)['solarenergy'].transform('sum')\n",
    "    df['uv_daily_max'] = df.groupby(df.index.date)['uvindex'].transform('max')\n",
    "\n",
    "    # Changes\n",
    "    df['energy_change'] = df['solarenergy'].diff()\n",
    "    df['uv_change'] = df['uvindex'].diff()\n",
    "\n",
    "    # Lag features\n",
    "    lags = [1, 2, 3, 6, 12, 24]  # hours\n",
    "    for lag in lags:\n",
    "        df[f'energy_lag_{lag}h'] = df['solarenergy'].shift(lag)\n",
    "        df[f'uv_lag_{lag}h'] = df['uvindex'].shift(lag)\n",
    "\n",
    "    # Peak indicators\n",
    "    df['is_energy_peak'] = (df['solarenergy'] > df['energy_rolling_mean_6h'] * 1.2).astype(int)\n",
    "    df['is_uv_peak'] = (df['uvindex'] > df['uv_rolling_mean_6h'] * 1.2).astype(int)\n",
    "\n",
    "    # Aggiungiamo alcune metriche di volatilità\n",
    "    df['energy_volatility'] = df['energy_change'].rolling(window=24).std()\n",
    "    df['uv_volatility'] = df['uv_change'].rolling(window=24).std()\n",
    "\n",
    "    # Indice di intensità solare composito\n",
    "    df['solar_intensity_index'] = (df['solarenergy'] * df['uvindex']) / (df['cloudcover'] + 1e-6)\n",
    "\n",
    "    # Interazioni\n",
    "    df['uv_cloud_interaction'] = df['uvindex'] * (100 - df['cloudcover']) / 100\n",
    "    df['energy_temp_interaction'] = df['solarenergy'] * df['temp']\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_advanced_features(df):\n",
    "    \"\"\"\n",
    "    Add all advanced features to the DataFrame\n",
    "    Assumes df has a DatetimeIndex\n",
    "    \"\"\"\n",
    "    # Verifichiamo che abbiamo un DatetimeIndex\n",
    "    if not isinstance(df.index, pd.DatetimeIndex):\n",
    "        raise ValueError(\"DataFrame must have a DatetimeIndex\")\n",
    "\n",
    "    # Existing features\n",
    "    df = add_time_features(df)\n",
    "    df = add_solar_features(df)\n",
    "    df = add_solar_specific_features(df)\n",
    "    df = add_radiation_energy_features(df)\n",
    "\n",
    "    # Weather variable interactions\n",
    "    df['temp_humidity'] = df['temp'] * df['humidity']\n",
    "    df['temp_cloudcover'] = df['temp'] * df['cloudcover']\n",
    "    df['visibility_cloudcover'] = df['visibility'] * df['cloudcover']\n",
    "\n",
    "    # Derived features\n",
    "    df['clear_sky_factor'] = (100 - df['cloudcover']) / 100\n",
    "    df['temp_humidity_interaction'] = df['temp'] * df['humidity'] / 100\n",
    "    df['atmospheric_transparency'] = (100 - df['cloudcover']) * (df['visibility'] / 10)\n",
    "\n",
    "    # Rolling means\n",
    "    df['temp_rolling_mean_6h'] = df['temp'].rolling(window=6).mean()\n",
    "    df['cloudcover_rolling_mean_6h'] = df['cloudcover'].rolling(window=6).mean()\n",
    "\n",
    "    # Lag features\n",
    "    df['temp_1h_lag'] = df['temp'].shift(1)\n",
    "    df['cloudcover_1h_lag'] = df['cloudcover'].shift(1)\n",
    "    df['humidity_1h_lag'] = df['humidity'].shift(1)\n",
    "\n",
    "    # Extreme conditions indicator\n",
    "    df['extreme_conditions'] = ((df['temp'] > df['temp'].quantile(0.75)) &\n",
    "                                (df['humidity'] < df['humidity'].quantile(0.25))).astype(int)\n",
    "\n",
    "    # One-hot encoding for categorical features\n",
    "    df = pd.get_dummies(df, columns=['season', 'time_period'])\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def prepare_advanced_data(df):\n",
    "    \"\"\"\n",
    "    Prepare data for advanced modeling with proper datetime handling\n",
    "    \"\"\"\n",
    "    # Assicuriamoci che abbiamo una copia del DataFrame\n",
    "    df = df.copy()\n",
    "\n",
    "    # Verifichiamo se datetime è già l'indice\n",
    "    if not isinstance(df.index, pd.DatetimeIndex):\n",
    "        if 'datetime' in df.columns:\n",
    "            df['datetime'] = pd.to_datetime(df['datetime'])\n",
    "            df.set_index('datetime', inplace=True)\n",
    "        else:\n",
    "            raise ValueError(\"No datetime column or index found in DataFrame\")\n",
    "\n",
    "    # Ordiniamo il DataFrame per datetime\n",
    "    df = df.sort_index()\n",
    "\n",
    "    # Apply feature engineering functions\n",
    "    df = add_advanced_features(df)\n",
    "\n",
    "    #all_columns = list(df.columns)\n",
    "    #print(all_columns)\n",
    "\n",
    "    features = {\n",
    "        # Primary Features (strong direct correlation)\n",
    "        'primary_features': [\n",
    "            'uvindex',  # Direct radiation indicator\n",
    "            'cloudcover',  # Cloud coverage\n",
    "            'visibility',  # Atmospheric transparency\n",
    "            'temp',  # Temperature\n",
    "            'pressure',  # Atmospheric pressure\n",
    "            'humidity',  # Humidity\n",
    "        ],\n",
    "\n",
    "        # Astronomical and Temporal Features\n",
    "        'astronomical_features': [\n",
    "            'solar_elevation',  # Solar elevation\n",
    "            'solar_angle',  # Solar angle\n",
    "            'day_length',  # Day length\n",
    "            'hour_sin',  # Daily cycle\n",
    "            'hour_cos',\n",
    "            'day_of_year_sin',  # Annual cycle\n",
    "            'day_of_year_cos',\n",
    "            'month_sin',  # Monthly cycle\n",
    "            'month_cos',\n",
    "        ],\n",
    "\n",
    "        # Key Indices and Interactions\n",
    "        'key_interactions': [\n",
    "            'clear_sky_index',  # Clear sky index\n",
    "            'atmospheric_attenuation',  # Atmospheric attenuation\n",
    "            'theoretical_radiation',  # Theoretical radiation\n",
    "            'expected_radiation',  # Expected radiation\n",
    "            'cloud_elevation',  # Cloud-elevation interaction\n",
    "            'visibility_elevation',  # Visibility-elevation interaction\n",
    "            'uv_cloud_interaction',  # UV-cloud interaction\n",
    "            'temp_radiation_potential',  # Temperature-radiation potential\n",
    "        ],\n",
    "\n",
    "        # Rolling Features (temporal trends)\n",
    "        'rolling_features': [\n",
    "            'cloud_rolling_12h',  # Cloud coverage moving average\n",
    "            'temp_rolling_12h',  # Temperature moving average\n",
    "            'uv_rolling_12h',  # UV moving average\n",
    "            'cloudcover_rolling_mean_6h',\n",
    "            'temp_rolling_mean_6h',\n",
    "        ],\n",
    "\n",
    "        # Lag Features (most recent)\n",
    "        'lag_features': [\n",
    "            'temp_1h_lag',  # 1-hour temperature lag\n",
    "            'cloudcover_1h_lag',  # 1-hour cloud coverage lag\n",
    "            'humidity_1h_lag',  # 1-hour humidity lag\n",
    "            'uv_lag_1h',  # 1-hour UV lag\n",
    "        ],\n",
    "\n",
    "        # Categorical Features\n",
    "        'categorical_features': [\n",
    "            'season_Spring',  # Seasons\n",
    "            'season_Summer',\n",
    "            'season_Autumn',\n",
    "            'season_Winter',\n",
    "            'time_period_Morning',  # Time periods\n",
    "            'time_period_Afternoon',\n",
    "            'time_period_Evening',\n",
    "            'time_period_Night',\n",
    "        ]\n",
    "    }\n",
    "\n",
    "    final_features = [feature for group in features.values() for feature in group]\n",
    "\n",
    "    # Handle missing values\n",
    "    target_variables = ['solarradiation', 'solarenergy', 'uvindex']\n",
    "    for column in final_features + target_variables:\n",
    "        if column in df.columns:\n",
    "            df[column] = df[column].interpolate(method='time')\n",
    "\n",
    "    df.fillna(0, inplace=True)\n",
    "\n",
    "    # Temporal split\n",
    "    data_after_2010 = df[df['year'] >= 2010].copy()\n",
    "    data_before_2010 = df[df['year'] < 2010].copy()\n",
    "\n",
    "    X = data_after_2010[final_features]\n",
    "    y = data_after_2010['solarradiation']\n",
    "    X_to_predict = data_before_2010[final_features]\n",
    "\n",
    "    # Train-test split\n",
    "    X_train, X_test, y_train, y_test = train_test_split(\n",
    "        X, y, test_size=0.2, random_state=random_state_value, shuffle=False\n",
    "    )\n",
    "\n",
    "    # Scaling\n",
    "    scaler_X = RobustScaler()\n",
    "    X_train_scaled = scaler_X.fit_transform(X_train)\n",
    "    X_test_scaled = scaler_X.transform(X_test)\n",
    "    X_to_predict_scaled = scaler_X.transform(X_to_predict)\n",
    "\n",
    "    scaler_y = RobustScaler()\n",
    "    y_train_scaled = scaler_y.fit_transform(y_train.values.reshape(-1, 1))\n",
    "    y_test_scaled = scaler_y.transform(y_test.values.reshape(-1, 1))\n",
    "\n",
    "    # Print info about selected features\n",
    "    print(\"\\nSelected features:\")\n",
    "    print(f\"Number of features: {len(final_features)}\")\n",
    "    print(\"Features list:\", final_features)\n",
    "\n",
    "    return X_train_scaled, X_test_scaled, y_train_scaled, y_test_scaled, scaler_X, scaler_y, final_features, X_to_predict_scaled\n",
    "\n",
    "\n",
    "def create_sequence_data(X, sequence_length=24):\n",
    "    \"\"\"\n",
    "    Converts data into sequences for LSTM input\n",
    "    sequence_length represents how many previous hours to consider\n",
    "    \"\"\"\n",
    "    sequences = []\n",
    "    for i in range(len(X) - sequence_length + 1):\n",
    "        sequences.append(X[i:i + sequence_length])\n",
    "    return np.array(sequences)\n",
    "\n",
    "\n",
    "def prepare_hybrid_data(df):\n",
    "    X_train_scaled, X_test_scaled, y_train_scaled, y_test_scaled, scaler_X, scaler_y, features, X_to_predict_scaled = prepare_advanced_data(df)\n",
    "\n",
    "    # Convert data into sequences\n",
    "    sequence_length = 24  # 24 hours of historical data\n",
    "\n",
    "    X_train_seq = create_sequence_data(X_train_scaled, sequence_length)\n",
    "    X_test_seq = create_sequence_data(X_test_scaled, sequence_length)\n",
    "\n",
    "    # Adjust y by removing the first (sequence_length-1) elements\n",
    "    y_train = y_train_scaled[sequence_length - 1:]\n",
    "    y_test = y_test_scaled[sequence_length - 1:]\n",
    "\n",
    "    X_to_predict_seq = create_sequence_data(X_to_predict_scaled, sequence_length)\n",
    "\n",
    "    return X_train_seq, X_test_seq, y_train, y_test, scaler_X, scaler_y, features, X_to_predict_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "570b18f2caa3e0db",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_residual_lstm_layer(x, units, dropout_rate, l2_reg=0.01, return_sequences=True, survival_probability=0.8):\n",
    "    \"\"\"\n",
    "    Creates a bidirectional LSTM layer with residual connections and regularization.\n",
    "\n",
    "    Parameters:\n",
    "        x: Input tensor\n",
    "        units: Number of LSTM units\n",
    "        dropout_rate: Dropout rate for regularization\n",
    "        l2_reg: L2 regularization factor\n",
    "        return_sequences: Whether to return sequences or just the last output\n",
    "        survival_probability: Probability of layer survival for stochastic depth\n",
    "    \"\"\"\n",
    "    residual = x\n",
    "    x = Bidirectional(LSTM(units, return_sequences=return_sequences, kernel_regularizer=regularizers.l2(l2_reg)))(x)\n",
    "    x = LayerNormalization()(x)\n",
    "    x = Dropout(dropout_rate)(x)\n",
    "\n",
    "    if return_sequences:\n",
    "        if int(residual.shape[-1]) != 2 * units:\n",
    "            residual = Dense(2 * units, activation='linear')(residual)\n",
    "        x = tfa.layers.StochasticDepth(survival_probability)([x, residual])\n",
    "    return x\n",
    "\n",
    "\n",
    "def attention_block(x, units, num_heads=8, survival_probability=0.8):\n",
    "    \"\"\"\n",
    "    Creates a multi-head attention block with residual connections.\n",
    "\n",
    "    Parameters:\n",
    "        x: Input tensor\n",
    "        units: Dimension of the key space\n",
    "        num_heads: Number of attention heads\n",
    "        survival_probability: Probability of layer survival for stochastic depth\n",
    "    \"\"\"\n",
    "    attention = MultiHeadAttention(num_heads=num_heads, key_dim=units)(x, x)\n",
    "    x = tfa.layers.StochasticDepth(survival_probability)([x, attention])\n",
    "    x = LayerNormalization()(x)\n",
    "    return x\n",
    "\n",
    "\n",
    "def asymmetric_loss(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    Loss function che penalizza maggiormente la sottostima dei valori alti\n",
    "    \"\"\"\n",
    "    diff = y_true - y_pred\n",
    "    abs_diff = tf.abs(diff)\n",
    "\n",
    "    # Calcola il peso basato sul valore reale\n",
    "    value_weight = tf.exp(y_true / tf.reduce_max(y_true)) - 1\n",
    "\n",
    "    # Penalizza maggiormente la sottostima (quando y_pred < y_true)\n",
    "    underestimation_penalty = tf.where(diff > 0, 2.0, 1.0)\n",
    "\n",
    "    # Combina i pesi\n",
    "    total_weight = value_weight * underestimation_penalty\n",
    "\n",
    "    # Calcola la loss pesata\n",
    "    weighted_loss = total_weight * abs_diff\n",
    "\n",
    "    return tf.reduce_mean(weighted_loss)\n",
    "\n",
    "def add_peak_features(x):\n",
    "    \"\"\"\n",
    "    Aggiunge feature specifiche per identificare potenziali picchi\n",
    "    \"\"\"\n",
    "    # Moving average delle ultime n osservazioni\n",
    "    ma = tf.keras.layers.Conv1D(1, kernel_size=5, padding='same')(x)\n",
    "\n",
    "    # Differenza dal moving average (identifica anomalie)\n",
    "    diff_ma = Lambda(lambda x: x[0] - x[1])([x, ma])\n",
    "\n",
    "    # Rate of change\n",
    "    roc = Lambda(lambda x: x[:, 1:] - x[:, :-1])(x)\n",
    "    roc = tf.pad(roc, [[0, 0], [1, 0], [0, 0]])\n",
    "\n",
    "    # Concatena tutte le feature\n",
    "    enhanced_x = Concatenate()([x, diff_ma, roc])\n",
    "\n",
    "    return enhanced_x\n",
    "\n",
    "def create_regression_branch(shared_features, l2_lambda=0.005, name_suffix=''):\n",
    "    \"\"\"\n",
    "    Branch di regressione migliorato per valori alti\n",
    "    \"\"\"\n",
    "    # Branch principale\n",
    "    main_branch = shared_features\n",
    "    dense_units = [512, 256, 128, 64]  # Unità aumentate\n",
    "\n",
    "    for units in dense_units:\n",
    "        main_branch = Dense(\n",
    "            units,\n",
    "            kernel_regularizer=regularizers.l2(l2_lambda)\n",
    "        )(main_branch)\n",
    "        main_branch = BatchNormalization()(main_branch)\n",
    "        main_branch = Activation('swish')(main_branch)\n",
    "\n",
    "    # Branch specializzato per valori alti\n",
    "    high_values_branch = shared_features\n",
    "    for units in [256, 128, 64]:\n",
    "        high_values_branch = Dense(\n",
    "            units,\n",
    "            kernel_regularizer=regularizers.l2(l2_lambda),\n",
    "            activation='relu'  # Usa ReLU per preservare valori alti\n",
    "        )(high_values_branch)\n",
    "\n",
    "    # Gate per decidere quanto pesare il branch dei valori alti\n",
    "    gate = Dense(1, activation='sigmoid')(shared_features)\n",
    "\n",
    "    # Combina i branch\n",
    "    main_output = Dense(1)(main_branch)\n",
    "    high_values_output = Dense(1)(high_values_branch)\n",
    "\n",
    "    # Output finale pesato\n",
    "    final_output = Lambda(lambda x: x[0] * (1 - x[2]) + x[1] * x[2])(\n",
    "        [main_output, high_values_output, gate]\n",
    "    )\n",
    "\n",
    "    return final_output\n",
    "\n",
    "def create_peak_specialized_ensemble(shared_features, l2_lambda=0.005):\n",
    "    \"\"\"\n",
    "    Ensemble di modelli specializzati per diverse fasce di valori\n",
    "    \"\"\"\n",
    "    # Modello generale\n",
    "    general_model = create_regression_branch(shared_features, name_suffix='general')\n",
    "\n",
    "    # Modello specializzato per valori alti\n",
    "    high_values_features = Dense(256, activation='relu')(shared_features)\n",
    "    high_values_model = create_regression_branch(high_values_features, name_suffix='high')\n",
    "\n",
    "    # Modello specializzato per picchi estremi\n",
    "    peak_features = Dense(512, activation='relu')(shared_features)\n",
    "    peak_model = create_regression_branch(peak_features, name_suffix='peak')\n",
    "\n",
    "    # Gate network per pesare i modelli\n",
    "    gate_features = Concatenate()([shared_features,\n",
    "                                   Dense(32)(shared_features),\n",
    "                                   Dense(32)(high_values_features),\n",
    "                                   Dense(32)(peak_features)])\n",
    "\n",
    "    gates = Dense(3, activation='softmax')(gate_features)\n",
    "\n",
    "    # Combina le predizioni\n",
    "    final_output = Lambda(lambda x: (x[0] * x[3][:, 0:1] +\n",
    "                                     x[1] * x[3][:, 1:2] +\n",
    "                                     x[2] * x[3][:, 2:3]))([general_model,\n",
    "                                                            high_values_model,\n",
    "                                                            peak_model,\n",
    "                                                            gates])\n",
    "\n",
    "    return final_output\n",
    "\n",
    "def create_solarradiation_model(input_shape, folder_name, l2_lambda=0.005, min_output=0, max_output=1):\n",
    "    \"\"\"\n",
    "    Creates a hybrid model with enhanced peak prediction capabilities\n",
    "    \"\"\"\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    # Backbone comune\n",
    "    survival_probs = [0.9, 0.8, 0.7, 0.6]\n",
    "    attention_survival_probs = [0.85, 0.75, 0.65, 0.55]\n",
    "    lstm_units = [256, 128, 64, 32]\n",
    "    dropout_rates = [0.4, 0.3, 0.2, 0.2]\n",
    "    attention_heads = [32, 24, 16, 8]\n",
    "\n",
    "    x = inputs\n",
    "    lstm_blocks = 4\n",
    "    for i in range(lstm_blocks):\n",
    "        x = create_residual_lstm_layer(\n",
    "            x,\n",
    "            units=lstm_units[i],\n",
    "            dropout_rate=dropout_rates[i],\n",
    "            l2_reg=l2_lambda,\n",
    "            return_sequences=True,\n",
    "            survival_probability=survival_probs[i]\n",
    "        )\n",
    "        x = attention_block(\n",
    "            x,\n",
    "            units=lstm_units[i],\n",
    "            num_heads=attention_heads[i],\n",
    "            survival_probability=attention_survival_probs[i]\n",
    "        )\n",
    "        if i < lstm_blocks - 1:\n",
    "            x = MaxPooling1D()(x)\n",
    "\n",
    "    # Final shared LSTM layer\n",
    "    shared_features = create_residual_lstm_layer(\n",
    "        x,\n",
    "        units=32,\n",
    "        dropout_rate=0.1,\n",
    "        l2_reg=l2_lambda,\n",
    "        return_sequences=False,\n",
    "        survival_probability=0.6\n",
    "    )\n",
    "\n",
    "    # Enhance features for peak detection\n",
    "    enhanced_features = add_peak_features(x)\n",
    "    enhanced_shared_features = create_residual_lstm_layer(\n",
    "        enhanced_features,\n",
    "        units=64,  # Increased units for enhanced features\n",
    "        dropout_rate=0.1,\n",
    "        l2_reg=l2_lambda,\n",
    "        return_sequences=False,\n",
    "        survival_probability=0.6\n",
    "    )\n",
    "\n",
    "    # Classification branch\n",
    "    classification_x = Dense(64, kernel_regularizer=regularizers.l2(l2_lambda))(shared_features)\n",
    "    classification_x = BatchNormalization()(classification_x)\n",
    "    classification_x = Activation('swish')(classification_x)\n",
    "    classification_x = Dropout(0.2)(classification_x)\n",
    "    classification_x = Dense(32, kernel_regularizer=regularizers.l2(l2_lambda))(classification_x)\n",
    "    classification_x = BatchNormalization()(classification_x)\n",
    "    classification_x = Activation('swish')(classification_x)\n",
    "    classification_output = Dense(1, activation='sigmoid', name='classification_output')(classification_x)\n",
    "\n",
    "    # Combined features for regression\n",
    "    regression_features = Concatenate()([shared_features, enhanced_shared_features])\n",
    "\n",
    "    # Create specialized ensemble for regression\n",
    "    regression_output = create_peak_specialized_ensemble(regression_features, l2_lambda)\n",
    "\n",
    "    # Clip regression values\n",
    "    regression_output = Lambda(\n",
    "        lambda x: tf.clip_by_value(x, min_output, max_output),\n",
    "        name='regression_output'\n",
    "    )(regression_output)\n",
    "\n",
    "    # Combine outputs using threshold activation\n",
    "    thresholded_classification = ThresholdedReLU(theta=0.5)(classification_output)\n",
    "    normalized_classification = Lambda(lambda x: tf.cast(x > 0, tf.float32))(thresholded_classification)\n",
    "    final_output = Lambda(\n",
    "        lambda inputs: inputs[0] * inputs[1],\n",
    "        name='final_output'\n",
    "    )([regression_output, normalized_classification])\n",
    "\n",
    "    # Create model\n",
    "    model = Model(\n",
    "        inputs=inputs,\n",
    "        outputs=[\n",
    "            classification_output,\n",
    "            regression_output,\n",
    "            final_output\n",
    "        ],\n",
    "        name=\"SolarRadiationModel\"\n",
    "    )\n",
    "\n",
    "    # Custom loss functions\n",
    "    def hybrid_focal_loss(y_true, y_pred):\n",
    "        mse = tf.square(y_true - y_pred)\n",
    "        error_ratio = tf.abs(y_true - y_pred) / (tf.abs(y_true) + 1.0)\n",
    "        focal_weight = tf.pow(error_ratio, 2)\n",
    "        weighted_mse = focal_weight * mse\n",
    "        mae = tf.abs(y_true - y_pred)\n",
    "        return tf.reduce_mean(0.7 * weighted_mse + 0.3 * mae)\n",
    "\n",
    "    def masked_regression_loss(y_true, y_pred):\n",
    "        mask = tf.cast(tf.not_equal(y_true, 0), tf.float32)\n",
    "        return asymmetric_loss(y_true * mask, y_pred * mask)\n",
    "\n",
    "    # Metrics\n",
    "    def rmse(y_true, y_pred):\n",
    "        return tf.sqrt(tf.reduce_mean(tf.square(y_true - y_pred)))\n",
    "\n",
    "    def custom_mape(y_true, y_pred):\n",
    "        epsilon = 1e-7\n",
    "        diff = tf.abs((y_true - y_pred) / (y_true + epsilon))\n",
    "        diff = tf.clip_by_value(diff, 0, 1)\n",
    "        return tf.reduce_mean(diff) * 100\n",
    "\n",
    "    # Optimizer with reduced initial learning rate\n",
    "    optimizer = AdamW(\n",
    "        learning_rate=0.0002,  # Reduced from 0.0003\n",
    "        beta_1=0.9,\n",
    "        beta_2=0.999,\n",
    "        epsilon=1e-7,\n",
    "        weight_decay=0.001,\n",
    "        amsgrad=True\n",
    "    )\n",
    "\n",
    "    # Compile model\n",
    "    model.compile(\n",
    "        optimizer=optimizer,\n",
    "        loss={\n",
    "            'classification_output': 'binary_crossentropy',\n",
    "            'regression_output': masked_regression_loss,\n",
    "            'final_output': hybrid_focal_loss\n",
    "        },\n",
    "        loss_weights={\n",
    "            'classification_output': 0.2,\n",
    "            'regression_output': 0.5,\n",
    "            'final_output': 0.3\n",
    "        },\n",
    "        metrics={\n",
    "            'classification_output': ['accuracy', AUC()],\n",
    "            'regression_output': ['mse', 'mae', rmse, custom_mape],\n",
    "            'final_output': ['mse', 'mae', rmse, custom_mape]\n",
    "        }\n",
    "    )\n",
    "\n",
    "    model.summary()\n",
    "\n",
    "    # Save model architecture visualization\n",
    "    plot_model(\n",
    "        model,\n",
    "        to_file=f'{folder_name}_model_architecture.png',\n",
    "        show_shapes=True,\n",
    "        show_layer_names=True,\n",
    "        dpi=150,\n",
    "        show_layer_activations=True\n",
    "    )\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "def evaluate_solarradiation_predictions(y_true, y_pred, hour=None, folder_name=None):\n",
    "    \"\"\"\n",
    "    Comprehensive evaluation of solar radiation predictions with detailed analysis and visualizations.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    y_true : array-like\n",
    "        Actual solar radiation values (W/m²)\n",
    "    y_pred : array-like\n",
    "        Predicted solar radiation values (W/m²)\n",
    "    hour : array-like, optional\n",
    "        Array of hours corresponding to predictions, for temporal analysis\n",
    "    folder_name : str, optional\n",
    "        Directory to save analysis plots\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    dict\n",
    "        Dictionary containing all calculated metrics\n",
    "    \"\"\"\n",
    "\n",
    "    # Data preparation\n",
    "    y_true = np.array(y_true).ravel()\n",
    "    y_pred = np.array(y_pred).ravel()\n",
    "    errors = y_pred - y_true\n",
    "\n",
    "    # Basic metrics calculation\n",
    "    mae_raw = mean_absolute_error(y_true, y_pred)\n",
    "    rmse_raw = np.sqrt(mean_squared_error(y_true, y_pred))\n",
    "    r2_raw = r2_score(y_true, y_pred)\n",
    "\n",
    "    # Corrected MAPE calculation\n",
    "    mask = y_true > 10  # Consider only values above 10 W/m²\n",
    "    if np.any(mask):\n",
    "        mape = np.mean(np.abs((y_true[mask] - y_pred[mask]) / y_true[mask])) * 100\n",
    "    else:\n",
    "        mape = np.nan\n",
    "\n",
    "    # Corrected error margin accuracy\n",
    "    within_5_percent = np.mean(np.abs(errors) <= 5) * 100  # Within 5 W/m²\n",
    "    within_10_percent = np.mean(np.abs(errors) <= 10) * 100  # Within 10 W/m²\n",
    "    within_20_percent = np.mean(np.abs(errors) <= 20) * 100  # Within 20 W/m²\n",
    "\n",
    "    # Radiation level classification\n",
    "    def get_radiation_level(value):\n",
    "        if value <= 200:\n",
    "            return 'Very Low'\n",
    "        elif value <= 400:\n",
    "            return 'Low'\n",
    "        elif value <= 600:\n",
    "            return 'Moderate'\n",
    "        elif value <= 800:\n",
    "            return 'High'\n",
    "        elif value <= 1000:\n",
    "            return 'Very High'\n",
    "        else:\n",
    "            return 'Extreme'\n",
    "\n",
    "    # Calculate radiation levels\n",
    "    y_true_levels = [get_radiation_level(v) for v in y_true]\n",
    "    y_pred_levels = [get_radiation_level(v) for v in y_pred]\n",
    "    level_accuracy = np.mean([t == p for t, p in zip(y_true_levels, y_pred_levels)])\n",
    "\n",
    "    # Print main metrics\n",
    "    print(\"\\nSolar Radiation Prediction Metrics:\")\n",
    "    print(\"\\nAbsolute Metrics:\")\n",
    "    print(f\"MAE: {mae_raw:.2f} W/m²\")\n",
    "    print(f\"RMSE: {rmse_raw:.2f} W/m²\")\n",
    "    print(f\"R² Score: {r2_raw:.3f}\")\n",
    "    print(f\"MAPE: {mape:.2f}%\" if not np.isnan(mape) else \"MAPE: N/A (insufficient data)\")\n",
    "\n",
    "    print(\"\\nAccuracy Metrics:\")\n",
    "    print(f\"Within ±5 W/m²: {within_5_percent:.1f}%\")\n",
    "    print(f\"Within ±10 W/m²: {within_10_percent:.1f}%\")\n",
    "    print(f\"Within ±20 W/m²: {within_20_percent:.1f}%\")\n",
    "\n",
    "    print(\"\\nLevel Accuracy:\")\n",
    "    print(f\"Level Accuracy: {level_accuracy * 100:.1f}%\")\n",
    "\n",
    "    # Confusion matrix for radiation levels\n",
    "    cm = confusion_matrix(y_true_levels, y_pred_levels)\n",
    "    print(\"\\nConfusion Matrix for Radiation Levels:\")\n",
    "    cm_df = pd.DataFrame(\n",
    "        cm,\n",
    "        columns=['Very Low', 'Low', 'Moderate', 'High', 'Very High', 'Extreme'],\n",
    "        index=['Very Low', 'Low', 'Moderate', 'High', 'Very High', 'Extreme']\n",
    "    )\n",
    "    print(cm_df)\n",
    "\n",
    "    # Time period analysis\n",
    "    if hour is not None:\n",
    "        day_periods = {\n",
    "            'Morning (5-11)': (5, 11),\n",
    "            'Noon (11-13)': (11, 13),\n",
    "            'Afternoon (13-17)': (13, 17),\n",
    "            'Evening (17-21)': (17, 21),\n",
    "            'Night (21-5)': (21, 5)\n",
    "        }\n",
    "\n",
    "        print(\"\\nAnalysis by Time Period:\")\n",
    "        for period, (start, end) in day_periods.items():\n",
    "            if start < end:\n",
    "                mask = (hour >= start) & (hour < end)\n",
    "            else:\n",
    "                mask = (hour >= start) | (hour < end)\n",
    "\n",
    "            if np.any(mask):\n",
    "                period_mae = mean_absolute_error(y_true[mask], y_pred[mask])\n",
    "\n",
    "                # Corrected period MAPE calculation\n",
    "                period_mask = mask & (y_true > 10)\n",
    "                if np.any(period_mask):\n",
    "                    period_mape = np.mean(np.abs((y_true[period_mask] - y_pred[period_mask]) / y_true[period_mask])) * 100\n",
    "                    print(f\"\\n{period}:\")\n",
    "                    print(f\"MAE: {period_mae:.2f} W/m²\")\n",
    "                    print(f\"MAPE: {period_mape:.2f}%\")\n",
    "                else:\n",
    "                    print(f\"\\n{period}:\")\n",
    "                    print(f\"MAE: {period_mae:.2f} W/m²\")\n",
    "                    print(\"MAPE: N/A (insufficient data)\")\n",
    "\n",
    "    # Visualizations\n",
    "    if folder_name is not None:\n",
    "        try:\n",
    "            # Figure 1: Main analysis plots\n",
    "            plt.figure(figsize=(20, 15))\n",
    "\n",
    "            # Plot 1: Scatter plot of actual vs predicted values\n",
    "            plt.subplot(3, 2, 1)\n",
    "            plt.scatter(y_true, y_pred, alpha=0.5)\n",
    "            plt.plot([y_true.min(), y_true.max()], [y_true.min(), y_true.max()], 'r--', lw=2)\n",
    "            plt.xlabel('Actual Radiation (W/m²)')\n",
    "            plt.ylabel('Predicted Radiation (W/m²)')\n",
    "            plt.title('Actual vs Predicted Values')\n",
    "            plt.grid(True)\n",
    "\n",
    "            # Plot 2: Absolute error distribution\n",
    "            plt.subplot(3, 2, 2)\n",
    "            plt.hist(errors, bins=50, alpha=0.7)\n",
    "            plt.xlabel('Prediction Error (W/m²)')\n",
    "            plt.ylabel('Frequency')\n",
    "            plt.title('Error Distribution')\n",
    "            plt.grid(True)\n",
    "\n",
    "            # Plot 3: Percentage error distribution (only for values > 10 W/m²)\n",
    "            plt.subplot(3, 2, 3)\n",
    "            mask = y_true > 10\n",
    "            if np.any(mask):\n",
    "                percentage_errors = ((y_pred[mask] - y_true[mask]) / y_true[mask]) * 100\n",
    "                plt.hist(np.clip(percentage_errors, -100, 100), bins=50, alpha=0.7)\n",
    "                plt.xlabel('Percentage Error (%)')\n",
    "                plt.ylabel('Frequency')\n",
    "                plt.title('Percentage Error Distribution (for values > 10 W/m²)')\n",
    "                plt.grid(True)\n",
    "\n",
    "            # Plot 4: Errors vs actual values\n",
    "            plt.subplot(3, 2, 4)\n",
    "            plt.scatter(y_true, errors, alpha=0.5)\n",
    "            plt.axhline(y=0, color='r', linestyle='--')\n",
    "            plt.xlabel('Actual Radiation (W/m²)')\n",
    "            plt.ylabel('Error (W/m²)')\n",
    "            plt.title('Errors vs Actual Values')\n",
    "            plt.grid(True)\n",
    "\n",
    "            # Plot 5: Error boxplot by radiation level\n",
    "            plt.subplot(3, 2, 5)\n",
    "            sns.boxplot(x=[get_radiation_level(v) for v in y_true], y=errors)\n",
    "            plt.xticks(rotation=45)\n",
    "            plt.xlabel('Radiation Level')\n",
    "            plt.ylabel('Error (W/m²)')\n",
    "            plt.title('Error Distribution by Level')\n",
    "\n",
    "            # Plot 6: Confusion matrix heatmap\n",
    "            plt.subplot(3, 2, 6)\n",
    "            sns.heatmap(cm_df, annot=True, fmt='d', cmap='Blues')\n",
    "            plt.title('Confusion Matrix')\n",
    "            plt.xticks(rotation=45)\n",
    "            plt.yticks(rotation=45)\n",
    "\n",
    "            plt.tight_layout()\n",
    "            filename = f'{folder_name}_radiation_analysis.png'\n",
    "            plt.savefig(filename, dpi=300, bbox_inches='tight')\n",
    "            print(f\"\\nPlot saved as: {filename}\")\n",
    "            plt.close()\n",
    "\n",
    "        except Exception as e:\n",
    "            print(f\"\\nError saving plots: {str(e)}\")\n",
    "\n",
    "    # Additional error statistics\n",
    "    print(\"\\nError Statistics:\")\n",
    "    print(f\"Mean error: {np.mean(errors):.3f}\")\n",
    "    print(f\"Error standard deviation: {np.std(errors):.3f}\")\n",
    "    print(f\"Median error: {np.median(errors):.3f}\")\n",
    "    print(f\"95th percentile absolute error: {np.percentile(np.abs(errors), 95):.3f}\")\n",
    "\n",
    "    # Return structured metrics\n",
    "    metrics = {\n",
    "        'absolute': {\n",
    "            'mae': mae_raw,\n",
    "            'rmse': rmse_raw,\n",
    "            'r2': r2_raw,\n",
    "            'mape': float(mape) if not np.isnan(mape) else None\n",
    "        },\n",
    "        'accuracy': {\n",
    "            'within_5_wm2': float(within_5_percent),\n",
    "            'within_10_wm2': float(within_10_percent),\n",
    "            'within_20_wm2': float(within_20_percent)\n",
    "        },\n",
    "        'categorical': {\n",
    "            'level_accuracy': float(level_accuracy)\n",
    "        },\n",
    "        'error_stats': {\n",
    "            'mean': float(np.mean(errors)),\n",
    "            'std': float(np.std(errors)),\n",
    "            'median': float(np.median(errors)),\n",
    "            'p95_abs': float(np.percentile(np.abs(errors), 95))\n",
    "        }\n",
    "    }\n",
    "\n",
    "    return metrics\n",
    "\n",
    "\n",
    "def plot_training_history(history, folder_name=None):\n",
    "    \"\"\"\n",
    "    Visualize and save training history for the hybrid model\n",
    "    \"\"\"\n",
    "    plt.figure(figsize=(15, 10))\n",
    "\n",
    "    # Loss plots\n",
    "    plt.subplot(2, 2, 1)\n",
    "    plt.plot(history.history['classification_output_loss'], label='Class Loss')\n",
    "    plt.plot(history.history['regression_output_loss'], label='Reg Loss')\n",
    "    plt.plot(history.history['final_output_loss'], label='Final Loss')\n",
    "    plt.plot(history.history['val_classification_output_loss'], label='Val Class Loss')\n",
    "    plt.plot(history.history['val_regression_output_loss'], label='Val Reg Loss')\n",
    "    plt.plot(history.history['val_final_output_loss'], label='Val Final Loss')\n",
    "    plt.title('Model Losses')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.legend()\n",
    "    plt.grid(True)\n",
    "\n",
    "    # Classification metrics\n",
    "    plt.subplot(2, 2, 2)\n",
    "    plt.plot(history.history['classification_output_accuracy'], label='Class Acc')\n",
    "    plt.plot(history.history['val_classification_output_accuracy'], label='Val Class Acc')\n",
    "    plt.plot(history.history['classification_output_auc'], label='Class AUC')\n",
    "    plt.plot(history.history['val_classification_output_auc'], label='Val Class AUC')\n",
    "    plt.title('Classification Metrics')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.ylabel('Metric Value')\n",
    "    plt.legend()\n",
    "    plt.grid(True)\n",
    "\n",
    "    # Regression metrics\n",
    "    plt.subplot(2, 2, 3)\n",
    "    plt.plot(history.history['regression_output_mae'], label='Reg MAE')\n",
    "    plt.plot(history.history['val_regression_output_mae'], label='Val Reg MAE')\n",
    "    plt.title('Regression MAE')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.ylabel('MAE')\n",
    "    plt.legend()\n",
    "    plt.grid(True)\n",
    "\n",
    "    # Final output metrics\n",
    "    plt.subplot(2, 2, 4)\n",
    "    plt.plot(history.history['final_output_mae'], label='Final MAE')\n",
    "    plt.plot(history.history['val_final_output_mae'], label='Val Final MAE')\n",
    "    plt.title('Final Output MAE')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.ylabel('MAE')\n",
    "    plt.legend()\n",
    "    plt.grid(True)\n",
    "\n",
    "    plt.tight_layout()\n",
    "\n",
    "    if folder_name is not None:\n",
    "        filename = f'{folder_name}_training_history.png'\n",
    "        plt.savefig(filename, dpi=300, bbox_inches='tight')\n",
    "        print(f\"\\nTraining history plot saved as: {filename}\")\n",
    "\n",
    "        # Save history to JSON\n",
    "        history_dict = history.history\n",
    "        json_filename = f'{folder_name}_training_history.json'\n",
    "        with open(json_filename, 'w') as f:\n",
    "            json.dump(history_dict, f)\n",
    "        print(f\"Training history saved as: {json_filename}\")\n",
    "\n",
    "    plt.show()\n",
    "\n",
    "def calculate_metrics(y_true, y_class, y_reg, y_final, min_output, max_output):\n",
    "        \"\"\"\n",
    "        Helper function to calculate and print metrics for all outputs\n",
    "    \n",
    "        Parameters:\n",
    "        - y_true: true values\n",
    "        - y_class: classification predictions\n",
    "        - y_reg: regression predictions\n",
    "        - y_final: final combined predictions\n",
    "        \"\"\"\n",
    "        from sklearn.metrics import roc_auc_score, classification_report, confusion_matrix\n",
    "    \n",
    "        y_true = np.array(y_true).flatten()\n",
    "        y_class = np.array(y_class).flatten()\n",
    "        y_reg = np.array(y_reg).flatten()\n",
    "        y_final = np.array(y_final).flatten()\n",
    "    \n",
    "        # Classification metrics\n",
    "        print(\"\\nClassification Metrics:\")\n",
    "        y_true_binary = (y_true > 0).astype(int)\n",
    "        y_pred_binary = (y_class > 0.5).astype(int)\n",
    "        \n",
    "        accuracy = np.mean((y_class > 0.5) == (y_true > 0)) * 100\n",
    "        auc_roc = roc_auc_score(y_true > 0, y_class)\n",
    "        print(f\"Accuracy: {accuracy:.2f}%\")\n",
    "        print(f\"AUC-ROC: {auc_roc:.4f}\")\n",
    "        \n",
    "        print(\"\\nConfusion Matrix:\")\n",
    "        print(confusion_matrix(y_true_binary, y_pred_binary))\n",
    "        \n",
    "        print(\"\\nClassification Report:\")\n",
    "        print(classification_report(y_true_binary, y_pred_binary, \n",
    "                                  target_names=['Zero', 'Non-Zero'],\n",
    "                                  digits=4))\n",
    "    \n",
    "        # Regression metrics (non-zero values)\n",
    "        print(\"\\nRegression Metrics (non-zero values):\")\n",
    "        mask_nonzero = y_true > 0\n",
    "        if np.any(mask_nonzero):  # verifichiamo che ci siano valori non-zero\n",
    "            y_true_nonzero = y_true[mask_nonzero]\n",
    "            y_reg_nonzero = y_reg[mask_nonzero]\n",
    "    \n",
    "            out_of_range = np.sum((y_reg_nonzero < min_output) | (y_reg_nonzero > max_output))\n",
    "            diff = np.abs((y_true_nonzero - y_reg_nonzero) / (y_true_nonzero + 1e-7))\n",
    "            diff = np.clip(diff, 0, 1)\n",
    "            mape = np.mean(diff) * 100\n",
    "            within_10_percent = np.mean(diff <= 0.10) * 100\n",
    "            mae = np.mean(np.abs(y_true_nonzero - y_reg_nonzero))\n",
    "            rmse = np.sqrt(np.mean(np.square(y_true_nonzero - y_reg_nonzero)))\n",
    "    \n",
    "            print(f\"Out of range: {out_of_range} predictions\")\n",
    "            print(f\"MAPE: {mape:.2f}%\")\n",
    "            print(f\"Within ±10%: {within_10_percent:.2f}%\")\n",
    "            print(f\"MAE: {mae:.2f}\")\n",
    "            print(f\"RMSE: {rmse:.2f}\")\n",
    "        else:\n",
    "            print(\"No non-zero values in this batch\")\n",
    "    \n",
    "        # Final combined output metrics\n",
    "        print(\"\\nFinal Combined Output Metrics:\")\n",
    "        out_of_range = np.sum((y_final < min_output) | (y_final > max_output))\n",
    "        diff = np.abs((y_true - y_final) / (y_true + 1e-7))\n",
    "        diff = np.clip(diff, 0, 1)\n",
    "        mape = np.mean(diff) * 100\n",
    "        within_10_percent = np.mean(diff <= 0.10) * 100\n",
    "        mae = np.mean(np.abs(y_true - y_final))\n",
    "        rmse = np.sqrt(np.mean(np.square(y_true - y_final)))\n",
    "    \n",
    "        print(f\"Out of range: {out_of_range} predictions\")\n",
    "        print(f\"MAPE: {mape:.2f}%\")\n",
    "        print(f\"Within ±10%: {within_10_percent:.2f}%\")\n",
    "        print(f\"MAE: {mae:.2f}\")\n",
    "        print(f\"RMSE: {rmse:.2f}\")\n",
    "\n",
    "def train_hybrid_model(model, X_train, y_train, X_test, y_test, epochs=100, batch_size=32, folder_name='solarradiation', min_output=0, max_output=1):\n",
    "    \"\"\"\n",
    "    Advanced training function for the hybrid solar radiation model\n",
    "    \"\"\" \n",
    "    # Prepare binary targets for classification\n",
    "    y_train_binary = (y_train > 0).astype(float)\n",
    "    y_test_binary = (y_test > 0).astype(float)\n",
    "\n",
    "    # Training targets dictionary - usando i nomi esatti degli output del modello\n",
    "    train_targets = {\n",
    "        'classification_output': y_train_binary,\n",
    "        'regression_output': y_train,  # Questo nome corrisponde a quello nel modello\n",
    "        'final_output': y_train\n",
    "    }\n",
    "\n",
    "    # Validation targets dictionary\n",
    "    test_targets = {\n",
    "        'classification_output': y_test_binary,\n",
    "        'regression_output': y_test,   # Questo nome corrisponde a quello nel modello\n",
    "        'final_output': y_test\n",
    "    }\n",
    "\n",
    "    callbacks = [\n",
    "        EarlyStopping(\n",
    "            monitor='val_final_output_loss',\n",
    "            patience=15,\n",
    "            restore_best_weights=True,\n",
    "            mode='min',\n",
    "            verbose=1,\n",
    "            min_delta=1e-4\n",
    "        ),\n",
    "        ReduceLROnPlateau(\n",
    "            monitor='val_final_output_loss',\n",
    "            factor=0.5,\n",
    "            patience=7,\n",
    "            verbose=1,\n",
    "            mode='min',\n",
    "            min_delta=1e-4,\n",
    "            cooldown=2,\n",
    "            min_lr=1e-7\n",
    "        ),\n",
    "        tf.keras.callbacks.ModelCheckpoint(\n",
    "            filepath=f'{folder_name}_best_model.h5',\n",
    "            monitor='val_final_output_loss',\n",
    "            save_best_only=True,\n",
    "            mode='min',\n",
    "            save_weights_only=False\n",
    "        ),\n",
    "        tf.keras.callbacks.TensorBoard(\n",
    "            log_dir=f'./{folder_name}_logs',\n",
    "            histogram_freq=1,\n",
    "            write_graph=True,\n",
    "            update_freq='epoch'\n",
    "        ),\n",
    "        tf.keras.callbacks.LambdaCallback(\n",
    "            on_epoch_end=lambda epoch, logs: (\n",
    "                print(f\"\\nEpoch {epoch + 1} Detailed Metrics:\") and\n",
    "                calculate_metrics(y_test, *model.predict(X_test, verbose=0), min_output, max_output)\n",
    "            ) if epoch % 10 == 0 else None\n",
    "        )\n",
    "    ]\n",
    "\n",
    "    try:\n",
    "        history = model.fit(\n",
    "            X_train,\n",
    "            train_targets,\n",
    "            validation_data=(X_test, test_targets),\n",
    "            epochs=epochs,\n",
    "            batch_size=batch_size,\n",
    "            callbacks=callbacks,\n",
    "            verbose=1,\n",
    "            shuffle=False\n",
    "        )\n",
    "\n",
    "        print(\"\\nTraining completed successfully!\")\n",
    "\n",
    "        # Final evaluation\n",
    "        predictions = model.predict(X_test, verbose=0)\n",
    "        calculate_metrics(y_test, *predictions, min_output, max_output)\n",
    "\n",
    "        return history\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"\\nError during training: {str(e)}\")\n",
    "        print(\"\\nModel output names:\", [output.name for output in model.outputs])\n",
    "        print(\"Training targets keys:\", train_targets.keys())\n",
    "        raise\n",
    "\n",
    "    finally:\n",
    "        tf.keras.backend.clear_session()\n",
    "\n",
    "\n",
    "def integrate_predictions(df, predictions, sequence_length=24):\n",
    "    \"\"\"\n",
    "    Integrates solar radiation predictions into the original dataset for pre-2010 data.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    df : pandas.DataFrame\n",
    "        Original dataset\n",
    "    predictions : tuple\n",
    "        Tuple containing (classification_pred, regression_pred, final_pred)\n",
    "        - classification_pred: probability of non-zero values\n",
    "        - regression_pred: predicted values (used for non-zero cases)\n",
    "        - final_pred: final combined predictions\n",
    "    sequence_length : int\n",
    "        Sequence length used for predictions\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    pandas.DataFrame\n",
    "        Updated dataset with solar radiation predictions and additional prediction details\n",
    "    \"\"\"\n",
    "    # Convert datetime to datetime format if not already\n",
    "    df['datetime'] = pd.to_datetime(df['datetime'])\n",
    "\n",
    "    # Identify pre-2010 rows\n",
    "    mask_pre_2010 = df['datetime'].dt.year < 2010\n",
    "\n",
    "    # Unpack predictions\n",
    "    classification_pred, regression_pred, final_pred = predictions\n",
    "\n",
    "    # Create temporary DataFrame with all predictions\n",
    "    dates_pre_2010 = df[mask_pre_2010]['datetime'].iloc[sequence_length - 1:]\n",
    "    predictions_df = pd.DataFrame({\n",
    "        'datetime': dates_pre_2010,\n",
    "        'solarradiation_predicted': final_pred.flatten(),\n",
    "        'solarradiation_classification': classification_pred.flatten(),\n",
    "        'solarradiation_regression': regression_pred.flatten()\n",
    "    })\n",
    "\n",
    "    # Merge with original dataset\n",
    "    df = df.merge(predictions_df, on='datetime', how='left')\n",
    "\n",
    "    # Update solar radiation column where missing\n",
    "    df['solarradiation'] = df['solarradiation'].fillna(df['solarradiation_predicted'])\n",
    "\n",
    "    # Print detailed statistics\n",
    "    print(\"\\nPrediction Integration Statistics:\")\n",
    "    print(f\"Added {len(final_pred)} predictions to dataset\")\n",
    "    print(f\"Rows with solar radiation after integration: {df['solarradiation'].notna().sum()}\")\n",
    "\n",
    "    # Analyze prediction components for the filled values\n",
    "    mask_filled = df['solarradiation'] == df['solarradiation_predicted']\n",
    "    if mask_filled.any():\n",
    "        filled_data = df[mask_filled]\n",
    "\n",
    "        print(\"\\nFilled Values Analysis:\")\n",
    "        print(f\"Zero predictions (classification < 0.5): {(filled_data['solarradiation_classification'] < 0.5).sum()}\")\n",
    "        print(f\"Non-zero predictions (classification >= 0.5): {(filled_data['solarradiation_classification'] >= 0.5).sum()}\")\n",
    "\n",
    "        # Distribution of predicted values\n",
    "        non_zero_pred = filled_data[filled_data['solarradiation_predicted'] > 0]\n",
    "        if len(non_zero_pred) > 0:\n",
    "            print(f\"\\nNon-zero predictions statistics:\")\n",
    "            print(f\"Mean: {non_zero_pred['solarradiation_predicted'].mean():.2f}\")\n",
    "            print(f\"Median: {non_zero_pred['solarradiation_predicted'].median():.2f}\")\n",
    "            print(f\"Std: {non_zero_pred['solarradiation_predicted'].std():.2f}\")\n",
    "\n",
    "    # Optionally, you can keep or remove the intermediate prediction columns\n",
    "    columns_to_drop = ['solarradiation_predicted', 'solarradiation_classification',\n",
    "                       'solarradiation_regression']\n",
    "    df = df.drop(columns_to_drop, axis=1)\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b3b0c2e65ddf484",
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyze_distribution(data, solar_column='solarenergy', name = 'Solar Energy'):\n",
    "    \"\"\"\n",
    "    Analizza dettagliatamente la distribuzione della variabile solarenergy.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    data : pandas.DataFrame\n",
    "        DataFrame contenente la colonna solarenergy\n",
    "    solar_column : str, default='solarenergy'\n",
    "        Nome della colonna da analizzare\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    dict\n",
    "        Dizionario contenente le statistiche principali\n",
    "    \"\"\"\n",
    "\n",
    "    # Creiamo una figura con più subplot\n",
    "    fig = plt.figure(figsize=(20, 12))\n",
    "\n",
    "    # 1. Statistiche di base\n",
    "    stats_dict = {\n",
    "        'count': len(data[solar_column]),\n",
    "        'missing': data[solar_column].isnull().sum(),\n",
    "        'zeros': (data[solar_column] == 0).sum(),\n",
    "        'mean': data[solar_column].mean(),\n",
    "        'median': data[solar_column].median(),\n",
    "        'std': data[solar_column].std(),\n",
    "        'min': data[solar_column].min(),\n",
    "        'max': data[solar_column].max(),\n",
    "        'skewness': stats.skew(data[solar_column].dropna()),\n",
    "        'kurtosis': stats.kurtosis(data[solar_column].dropna())\n",
    "    }\n",
    "\n",
    "    # Calcolo dei percentili\n",
    "    percentiles = [1, 5, 10, 25, 50, 75, 90, 95, 99]\n",
    "    for p in percentiles:\n",
    "        stats_dict[f'percentile_{p}'] = np.percentile(data[solar_column].dropna(), p)\n",
    "\n",
    "    # 2. Visualizzazioni\n",
    "\n",
    "    # 2.1 Distribuzione\n",
    "    plt.subplot(2, 2, 1)\n",
    "    sns.histplot(data=data, x=solar_column, kde=True)\n",
    "    plt.title(f'Distribuzione di {name}')\n",
    "    plt.xlabel(f'{name}')\n",
    "    plt.ylabel('Frequenza')\n",
    "\n",
    "    # 2.2 Box Plot\n",
    "    plt.subplot(2, 2, 2)\n",
    "    sns.boxplot(y=data[solar_column])\n",
    "    plt.title(f'Box Plot di {name}')\n",
    "\n",
    "    # 2.3 QQ Plot\n",
    "    plt.subplot(2, 2, 3)\n",
    "    stats.probplot(data[solar_column].dropna(), dist=\"norm\", plot=plt)\n",
    "    plt.title(f'Q-Q Plot di {name}')\n",
    "\n",
    "    # 2.4 Distribuzione Log-trasformata\n",
    "    plt.subplot(2, 2, 4)\n",
    "    sns.histplot(data=np.log1p(data[solar_column]), kde=True)\n",
    "    plt.title(f'Distribuzione Log-trasformata di {name}')\n",
    "    plt.xlabel(f'Log({name} + 1)')\n",
    "    plt.ylabel('Frequenza')\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "    # 3. Analisi temporale se disponibile\n",
    "    if 'timestamp' in data.columns or 'datetime' in data.columns:\n",
    "        time_col = 'timestamp' if 'timestamp' in data.columns else 'datetime'\n",
    "        if isinstance(data[time_col].iloc[0], (int, float)):\n",
    "            data['temp_datetime'] = pd.to_datetime(data[time_col], unit='s')\n",
    "        else:\n",
    "            data['temp_datetime'] = pd.to_datetime(data[time_col])\n",
    "\n",
    "        # Plot temporale\n",
    "        plt.figure(figsize=(15, 6))\n",
    "        plt.plot(data['temp_datetime'], data[solar_column])\n",
    "        plt.title(f'Serie Temporale di {name}')\n",
    "        plt.xlabel('Data')\n",
    "        plt.ylabel(f'{name}')\n",
    "        plt.xticks(rotation=45)\n",
    "        plt.tight_layout()\n",
    "        plt.show()\n",
    "\n",
    "        # Analisi stagionale\n",
    "        data['month'] = data['temp_datetime'].dt.month\n",
    "        seasonal_stats = data.groupby('month')[solar_column].agg(['mean', 'std', 'median'])\n",
    "\n",
    "        plt.figure(figsize=(12, 6))\n",
    "        seasonal_stats['mean'].plot(kind='bar')\n",
    "        plt.title(f'Media Mensile di {name}')\n",
    "        plt.xlabel('Mese')\n",
    "        plt.ylabel(f'{name} Media')\n",
    "        plt.tight_layout()\n",
    "        plt.show()\n",
    "\n",
    "    # 4. Stampa delle statistiche principali\n",
    "    print(f\"\\nStatistiche principali di {name}:\")\n",
    "    print(\"-\" * 50)\n",
    "    for key, value in stats_dict.items():\n",
    "        print(f\"{key:15}: {value:,.4f}\")\n",
    "\n",
    "    # 5. Suggerimenti per la normalizzazione\n",
    "    print(\"\\nSuggerimenti per la normalizzazione:\")\n",
    "    print(\"-\" * 50)\n",
    "\n",
    "    skewness = abs(stats_dict['skewness'])\n",
    "    if skewness > 1:\n",
    "        print(\"- La distribuzione è fortemente asimmetrica (skewness > 1)\")\n",
    "        print(\"- Considerare una trasformazione logaritmica: np.log1p(x)\")\n",
    "\n",
    "    range_ratio = stats_dict['max'] / stats_dict['std']\n",
    "    if range_ratio > 10:\n",
    "        print(\"- La variabile ha una scala molto ampia\")\n",
    "        print(\"- Considerare RobustScaler o StandardScaler per la normalizzazione\")\n",
    "\n",
    "    zero_ratio = stats_dict['zeros'] / stats_dict['count']\n",
    "    if zero_ratio > 0.1:\n",
    "        print(f\"- Alta presenza di zeri ({zero_ratio:.2%})\")\n",
    "        print(\"- Considerare un modello in due parti: classificazione degli zeri + regressione sui valori non-zero\")\n",
    "\n",
    "    return stats_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1b1ee91d1573ec66",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing solar radiation model training...\n",
      "\n",
      "1. Preparing data...\n",
      "\n",
      "Selected features:\n",
      "Number of features: 40\n",
      "Features list: ['uvindex', 'cloudcover', 'visibility', 'temp', 'pressure', 'humidity', 'solar_elevation', 'solar_angle', 'day_length', 'hour_sin', 'hour_cos', 'day_of_year_sin', 'day_of_year_cos', 'month_sin', 'month_cos', 'clear_sky_index', 'atmospheric_attenuation', 'theoretical_radiation', 'expected_radiation', 'cloud_elevation', 'visibility_elevation', 'uv_cloud_interaction', 'temp_radiation_potential', 'cloud_rolling_12h', 'temp_rolling_12h', 'uv_rolling_12h', 'cloudcover_rolling_mean_6h', 'temp_rolling_mean_6h', 'temp_1h_lag', 'cloudcover_1h_lag', 'humidity_1h_lag', 'uv_lag_1h', 'season_Spring', 'season_Summer', 'season_Autumn', 'season_Winter', 'time_period_Morning', 'time_period_Afternoon', 'time_period_Evening', 'time_period_Night']\n",
      "Training data shape: (103798, 24, 40)\n",
      "Test data shape: (25933, 24, 40)\n",
      "Saving scaler X to: 2024-11-26_05-41_scale_X.joblib\n",
      "Saving scaler X to: 2024-11-26_05-41_scale_y.joblib\n",
      "Saving features to: 2024-11-26_05-41_features.json\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_parquet('../../sources/weather_data_uvindex.parquet')\n",
    "\n",
    "print(\"Initializing solar radiation model training...\")\n",
    "\n",
    "# Data preparation\n",
    "print(\"\\n1. Preparing data...\")\n",
    "X_train_seq, X_test_seq, y_train, y_test, scaler_X, scaler_y, features, X_to_predict_seq = prepare_hybrid_data(df)\n",
    "\n",
    "print(f\"Training data shape: {X_train_seq.shape}\")\n",
    "print(f\"Test data shape: {X_test_seq.shape}\")\n",
    "\n",
    "# Save or load scaler and features\n",
    "scaler_X_path = f'{folder_name}_scale_X.joblib'\n",
    "scaler_y_path = f'{folder_name}_scale_y.joblib'\n",
    "features_path = f'{folder_name}_features.json'\n",
    "model_path = f'{folder_name}_best_model.h5'\n",
    "history_path = f'{folder_name}_training_history.json'\n",
    "\n",
    "if os.path.exists(scaler_X_path):\n",
    "    print(f\"Loading existing scaler X from: {scaler_X_path}\")\n",
    "    scaler = joblib.load(scaler_X_path)\n",
    "else:\n",
    "    print(f\"Saving scaler X to: {scaler_X_path}\")\n",
    "    joblib.dump(scaler_X, scaler_X_path)\n",
    "\n",
    "if os.path.exists(scaler_y_path):\n",
    "    print(f\"Loading existing scaler X from: {scaler_y_path}\")\n",
    "    scaler = joblib.load(scaler_y_path)\n",
    "else:\n",
    "    print(f\"Saving scaler X to: {scaler_y_path}\")\n",
    "    joblib.dump(scaler_y, scaler_y_path)\n",
    "\n",
    "if os.path.exists(features_path):\n",
    "    print(f\"Loading existing features from: {features_path}\")\n",
    "    with open(features_path, 'r') as f:\n",
    "        features = json.load(f)\n",
    "else:\n",
    "    print(f\"Saving features to: {features_path}\")\n",
    "    with open(features_path, 'w') as f:\n",
    "        json.dump(features, f)\n",
    "\n",
    "# Data quality verification\n",
    "if np.isnan(X_train_seq).any() or np.isnan(y_train).any():\n",
    "    raise ValueError(\"Found NaN values in training data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "76deb4deb84dc4c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "2. Creating model...\n",
      "\n",
      "Max dataset solar radiation : 1113.0 - Scaled Version : 3.2535460992907805\n",
      "Max dataset solar radiation increased by 15% : 1279.9499999999998 - Scaled Version : 3.7415780141843973\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-26 05:41:50.507143: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1886] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 43404 MB memory:  -> device: 0, name: NVIDIA L40, pci bus id: 0000:01:00.0, compute capability: 8.9\n",
      "2024-11-26 05:41:51.386109: I tensorflow/tsl/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"SolarRadiationModel\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                Output Shape                 Param #   Connected to                  \n",
      "==================================================================================================\n",
      " input_1 (InputLayer)        [(None, 24, 40)]             0         []                            \n",
      "                                                                                                  \n",
      " bidirectional (Bidirection  (None, 24, 512)              608256    ['input_1[0][0]']             \n",
      " al)                                                                                              \n",
      "                                                                                                  \n",
      " layer_normalization (Layer  (None, 24, 512)              1024      ['bidirectional[0][0]']       \n",
      " Normalization)                                                                                   \n",
      "                                                                                                  \n",
      " dropout (Dropout)           (None, 24, 512)              0         ['layer_normalization[0][0]'] \n",
      "                                                                                                  \n",
      " dense (Dense)               (None, 24, 512)              20992     ['input_1[0][0]']             \n",
      "                                                                                                  \n",
      " stochastic_depth (Stochast  (None, 24, 512)              0         ['dropout[0][0]',             \n",
      " icDepth)                                                            'dense[0][0]']               \n",
      "                                                                                                  \n",
      " multi_head_attention (Mult  (None, 24, 512)              1680230   ['stochastic_depth[0][0]',    \n",
      " iHeadAttention)                                          4          'stochastic_depth[0][0]']    \n",
      "                                                                                                  \n",
      " stochastic_depth_1 (Stocha  (None, 24, 512)              0         ['stochastic_depth[0][0]',    \n",
      " sticDepth)                                                          'multi_head_attention[0][0]']\n",
      "                                                                                                  \n",
      " layer_normalization_1 (Lay  (None, 24, 512)              1024      ['stochastic_depth_1[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " max_pooling1d (MaxPooling1  (None, 12, 512)              0         ['layer_normalization_1[0][0]'\n",
      " D)                                                                 ]                             \n",
      "                                                                                                  \n",
      " bidirectional_1 (Bidirecti  (None, 12, 256)              656384    ['max_pooling1d[0][0]']       \n",
      " onal)                                                                                            \n",
      "                                                                                                  \n",
      " layer_normalization_2 (Lay  (None, 12, 256)              512       ['bidirectional_1[0][0]']     \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dropout_1 (Dropout)         (None, 12, 256)              0         ['layer_normalization_2[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_1 (Dense)             (None, 12, 256)              131328    ['max_pooling1d[0][0]']       \n",
      "                                                                                                  \n",
      " stochastic_depth_2 (Stocha  (None, 12, 256)              0         ['dropout_1[0][0]',           \n",
      " sticDepth)                                                          'dense_1[0][0]']             \n",
      "                                                                                                  \n",
      " multi_head_attention_1 (Mu  (None, 12, 256)              3155200   ['stochastic_depth_2[0][0]',  \n",
      " ltiHeadAttention)                                                   'stochastic_depth_2[0][0]']  \n",
      "                                                                                                  \n",
      " stochastic_depth_3 (Stocha  (None, 12, 256)              0         ['stochastic_depth_2[0][0]',  \n",
      " sticDepth)                                                          'multi_head_attention_1[0][0]\n",
      "                                                                    ']                            \n",
      "                                                                                                  \n",
      " layer_normalization_3 (Lay  (None, 12, 256)              512       ['stochastic_depth_3[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " max_pooling1d_1 (MaxPoolin  (None, 6, 256)               0         ['layer_normalization_3[0][0]'\n",
      " g1D)                                                               ]                             \n",
      "                                                                                                  \n",
      " bidirectional_2 (Bidirecti  (None, 6, 128)               164352    ['max_pooling1d_1[0][0]']     \n",
      " onal)                                                                                            \n",
      "                                                                                                  \n",
      " layer_normalization_4 (Lay  (None, 6, 128)               256       ['bidirectional_2[0][0]']     \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dropout_2 (Dropout)         (None, 6, 128)               0         ['layer_normalization_4[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_2 (Dense)             (None, 6, 128)               32896     ['max_pooling1d_1[0][0]']     \n",
      "                                                                                                  \n",
      " stochastic_depth_4 (Stocha  (None, 6, 128)               0         ['dropout_2[0][0]',           \n",
      " sticDepth)                                                          'dense_2[0][0]']             \n",
      "                                                                                                  \n",
      " multi_head_attention_2 (Mu  (None, 6, 128)               527488    ['stochastic_depth_4[0][0]',  \n",
      " ltiHeadAttention)                                                   'stochastic_depth_4[0][0]']  \n",
      "                                                                                                  \n",
      " stochastic_depth_5 (Stocha  (None, 6, 128)               0         ['stochastic_depth_4[0][0]',  \n",
      " sticDepth)                                                          'multi_head_attention_2[0][0]\n",
      "                                                                    ']                            \n",
      "                                                                                                  \n",
      " layer_normalization_5 (Lay  (None, 6, 128)               256       ['stochastic_depth_5[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " max_pooling1d_2 (MaxPoolin  (None, 3, 128)               0         ['layer_normalization_5[0][0]'\n",
      " g1D)                                                               ]                             \n",
      "                                                                                                  \n",
      " bidirectional_3 (Bidirecti  (None, 3, 64)                41216     ['max_pooling1d_2[0][0]']     \n",
      " onal)                                                                                            \n",
      "                                                                                                  \n",
      " layer_normalization_6 (Lay  (None, 3, 64)                128       ['bidirectional_3[0][0]']     \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dropout_3 (Dropout)         (None, 3, 64)                0         ['layer_normalization_6[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_3 (Dense)             (None, 3, 64)                8256      ['max_pooling1d_2[0][0]']     \n",
      "                                                                                                  \n",
      " stochastic_depth_6 (Stocha  (None, 3, 64)                0         ['dropout_3[0][0]',           \n",
      " sticDepth)                                                          'dense_3[0][0]']             \n",
      "                                                                                                  \n",
      " multi_head_attention_3 (Mu  (None, 3, 64)                66368     ['stochastic_depth_6[0][0]',  \n",
      " ltiHeadAttention)                                                   'stochastic_depth_6[0][0]']  \n",
      "                                                                                                  \n",
      " stochastic_depth_7 (Stocha  (None, 3, 64)                0         ['stochastic_depth_6[0][0]',  \n",
      " sticDepth)                                                          'multi_head_attention_3[0][0]\n",
      "                                                                    ']                            \n",
      "                                                                                                  \n",
      " layer_normalization_7 (Lay  (None, 3, 64)                128       ['stochastic_depth_7[0][0]']  \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " conv1d (Conv1D)             (None, 3, 1)                 321       ['layer_normalization_7[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " lambda_1 (Lambda)           (None, 2, 64)                0         ['layer_normalization_7[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " lambda (Lambda)             (None, 3, 64)                0         ['layer_normalization_7[0][0]'\n",
      "                                                                    , 'conv1d[0][0]']             \n",
      "                                                                                                  \n",
      " tf.compat.v1.pad (TFOpLamb  (None, 3, 64)                0         ['lambda_1[0][0]']            \n",
      " da)                                                                                              \n",
      "                                                                                                  \n",
      " concatenate (Concatenate)   (None, 3, 192)               0         ['layer_normalization_7[0][0]'\n",
      "                                                                    , 'lambda[0][0]',             \n",
      "                                                                     'tf.compat.v1.pad[0][0]']    \n",
      "                                                                                                  \n",
      " bidirectional_4 (Bidirecti  (None, 64)                   24832     ['layer_normalization_7[0][0]'\n",
      " onal)                                                              ]                             \n",
      "                                                                                                  \n",
      " bidirectional_5 (Bidirecti  (None, 128)                  131584    ['concatenate[0][0]']         \n",
      " onal)                                                                                            \n",
      "                                                                                                  \n",
      " layer_normalization_8 (Lay  (None, 64)                   128       ['bidirectional_4[0][0]']     \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " layer_normalization_9 (Lay  (None, 128)                  256       ['bidirectional_5[0][0]']     \n",
      " erNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dropout_4 (Dropout)         (None, 64)                   0         ['layer_normalization_8[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dropout_5 (Dropout)         (None, 128)                  0         ['layer_normalization_9[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " concatenate_1 (Concatenate  (None, 192)                  0         ['dropout_4[0][0]',           \n",
      " )                                                                   'dropout_5[0][0]']           \n",
      "                                                                                                  \n",
      " dense_16 (Dense)            (None, 256)                  49408     ['concatenate_1[0][0]']       \n",
      "                                                                                                  \n",
      " dense_27 (Dense)            (None, 512)                  98816     ['concatenate_1[0][0]']       \n",
      "                                                                                                  \n",
      " dense_6 (Dense)             (None, 512)                  98816     ['concatenate_1[0][0]']       \n",
      "                                                                                                  \n",
      " dense_17 (Dense)            (None, 512)                  131584    ['dense_16[0][0]']            \n",
      "                                                                                                  \n",
      " dense_28 (Dense)            (None, 512)                  262656    ['dense_27[0][0]']            \n",
      "                                                                                                  \n",
      " batch_normalization_2 (Bat  (None, 512)                  2048      ['dense_6[0][0]']             \n",
      " chNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " batch_normalization_6 (Bat  (None, 512)                  2048      ['dense_17[0][0]']            \n",
      " chNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " batch_normalization_10 (Ba  (None, 512)                  2048      ['dense_28[0][0]']            \n",
      " tchNormalization)                                                                                \n",
      "                                                                                                  \n",
      " activation_2 (Activation)   (None, 512)                  0         ['batch_normalization_2[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " activation_6 (Activation)   (None, 512)                  0         ['batch_normalization_6[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " activation_10 (Activation)  (None, 512)                  0         ['batch_normalization_10[0][0]\n",
      "                                                                    ']                            \n",
      "                                                                                                  \n",
      " dense_7 (Dense)             (None, 256)                  131328    ['activation_2[0][0]']        \n",
      "                                                                                                  \n",
      " dense_18 (Dense)            (None, 256)                  131328    ['activation_6[0][0]']        \n",
      "                                                                                                  \n",
      " dense_29 (Dense)            (None, 256)                  131328    ['activation_10[0][0]']       \n",
      "                                                                                                  \n",
      " batch_normalization_3 (Bat  (None, 256)                  1024      ['dense_7[0][0]']             \n",
      " chNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " batch_normalization_7 (Bat  (None, 256)                  1024      ['dense_18[0][0]']            \n",
      " chNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " batch_normalization_11 (Ba  (None, 256)                  1024      ['dense_29[0][0]']            \n",
      " tchNormalization)                                                                                \n",
      "                                                                                                  \n",
      " activation_3 (Activation)   (None, 256)                  0         ['batch_normalization_3[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " activation_7 (Activation)   (None, 256)                  0         ['batch_normalization_7[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " activation_11 (Activation)  (None, 256)                  0         ['batch_normalization_11[0][0]\n",
      "                                                                    ']                            \n",
      "                                                                                                  \n",
      " dense_4 (Dense)             (None, 64)                   4160      ['dropout_4[0][0]']           \n",
      "                                                                                                  \n",
      " dense_8 (Dense)             (None, 128)                  32896     ['activation_3[0][0]']        \n",
      "                                                                                                  \n",
      " dense_19 (Dense)            (None, 128)                  32896     ['activation_7[0][0]']        \n",
      "                                                                                                  \n",
      " dense_30 (Dense)            (None, 128)                  32896     ['activation_11[0][0]']       \n",
      "                                                                                                  \n",
      " batch_normalization (Batch  (None, 64)                   256       ['dense_4[0][0]']             \n",
      " Normalization)                                                                                   \n",
      "                                                                                                  \n",
      " batch_normalization_4 (Bat  (None, 128)                  512       ['dense_8[0][0]']             \n",
      " chNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " batch_normalization_8 (Bat  (None, 128)                  512       ['dense_19[0][0]']            \n",
      " chNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " batch_normalization_12 (Ba  (None, 128)                  512       ['dense_30[0][0]']            \n",
      " tchNormalization)                                                                                \n",
      "                                                                                                  \n",
      " activation (Activation)     (None, 64)                   0         ['batch_normalization[0][0]'] \n",
      "                                                                                                  \n",
      " activation_4 (Activation)   (None, 128)                  0         ['batch_normalization_4[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " activation_8 (Activation)   (None, 128)                  0         ['batch_normalization_8[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " activation_12 (Activation)  (None, 128)                  0         ['batch_normalization_12[0][0]\n",
      "                                                                    ']                            \n",
      "                                                                                                  \n",
      " dropout_6 (Dropout)         (None, 64)                   0         ['activation[0][0]']          \n",
      "                                                                                                  \n",
      " dense_9 (Dense)             (None, 64)                   8256      ['activation_4[0][0]']        \n",
      "                                                                                                  \n",
      " dense_10 (Dense)            (None, 256)                  49408     ['concatenate_1[0][0]']       \n",
      "                                                                                                  \n",
      " dense_20 (Dense)            (None, 64)                   8256      ['activation_8[0][0]']        \n",
      "                                                                                                  \n",
      " dense_21 (Dense)            (None, 256)                  65792     ['dense_16[0][0]']            \n",
      "                                                                                                  \n",
      " dense_31 (Dense)            (None, 64)                   8256      ['activation_12[0][0]']       \n",
      "                                                                                                  \n",
      " dense_32 (Dense)            (None, 256)                  131328    ['dense_27[0][0]']            \n",
      "                                                                                                  \n",
      " dense_5 (Dense)             (None, 32)                   2080      ['dropout_6[0][0]']           \n",
      "                                                                                                  \n",
      " batch_normalization_5 (Bat  (None, 64)                   256       ['dense_9[0][0]']             \n",
      " chNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dense_11 (Dense)            (None, 128)                  32896     ['dense_10[0][0]']            \n",
      "                                                                                                  \n",
      " batch_normalization_9 (Bat  (None, 64)                   256       ['dense_20[0][0]']            \n",
      " chNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dense_22 (Dense)            (None, 128)                  32896     ['dense_21[0][0]']            \n",
      "                                                                                                  \n",
      " batch_normalization_13 (Ba  (None, 64)                   256       ['dense_31[0][0]']            \n",
      " tchNormalization)                                                                                \n",
      "                                                                                                  \n",
      " dense_33 (Dense)            (None, 128)                  32896     ['dense_32[0][0]']            \n",
      "                                                                                                  \n",
      " batch_normalization_1 (Bat  (None, 32)                   128       ['dense_5[0][0]']             \n",
      " chNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " activation_5 (Activation)   (None, 64)                   0         ['batch_normalization_5[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_12 (Dense)            (None, 64)                   8256      ['dense_11[0][0]']            \n",
      "                                                                                                  \n",
      " activation_9 (Activation)   (None, 64)                   0         ['batch_normalization_9[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_23 (Dense)            (None, 64)                   8256      ['dense_22[0][0]']            \n",
      "                                                                                                  \n",
      " activation_13 (Activation)  (None, 64)                   0         ['batch_normalization_13[0][0]\n",
      "                                                                    ']                            \n",
      "                                                                                                  \n",
      " dense_34 (Dense)            (None, 64)                   8256      ['dense_33[0][0]']            \n",
      "                                                                                                  \n",
      " dense_38 (Dense)            (None, 32)                   6176      ['concatenate_1[0][0]']       \n",
      "                                                                                                  \n",
      " dense_39 (Dense)            (None, 32)                   8224      ['dense_16[0][0]']            \n",
      "                                                                                                  \n",
      " dense_40 (Dense)            (None, 32)                   16416     ['dense_27[0][0]']            \n",
      "                                                                                                  \n",
      " activation_1 (Activation)   (None, 32)                   0         ['batch_normalization_1[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " dense_14 (Dense)            (None, 1)                    65        ['activation_5[0][0]']        \n",
      "                                                                                                  \n",
      " dense_15 (Dense)            (None, 1)                    65        ['dense_12[0][0]']            \n",
      "                                                                                                  \n",
      " dense_13 (Dense)            (None, 1)                    193       ['concatenate_1[0][0]']       \n",
      "                                                                                                  \n",
      " dense_25 (Dense)            (None, 1)                    65        ['activation_9[0][0]']        \n",
      "                                                                                                  \n",
      " dense_26 (Dense)            (None, 1)                    65        ['dense_23[0][0]']            \n",
      "                                                                                                  \n",
      " dense_24 (Dense)            (None, 1)                    257       ['dense_16[0][0]']            \n",
      "                                                                                                  \n",
      " dense_36 (Dense)            (None, 1)                    65        ['activation_13[0][0]']       \n",
      "                                                                                                  \n",
      " dense_37 (Dense)            (None, 1)                    65        ['dense_34[0][0]']            \n",
      "                                                                                                  \n",
      " dense_35 (Dense)            (None, 1)                    513       ['dense_27[0][0]']            \n",
      "                                                                                                  \n",
      " concatenate_2 (Concatenate  (None, 288)                  0         ['concatenate_1[0][0]',       \n",
      " )                                                                   'dense_38[0][0]',            \n",
      "                                                                     'dense_39[0][0]',            \n",
      "                                                                     'dense_40[0][0]']            \n",
      "                                                                                                  \n",
      " classification_output (Den  (None, 1)                    33        ['activation_1[0][0]']        \n",
      " se)                                                                                              \n",
      "                                                                                                  \n",
      " lambda_2 (Lambda)           (None, 1)                    0         ['dense_14[0][0]',            \n",
      "                                                                     'dense_15[0][0]',            \n",
      "                                                                     'dense_13[0][0]']            \n",
      "                                                                                                  \n",
      " lambda_3 (Lambda)           (None, 1)                    0         ['dense_25[0][0]',            \n",
      "                                                                     'dense_26[0][0]',            \n",
      "                                                                     'dense_24[0][0]']            \n",
      "                                                                                                  \n",
      " lambda_4 (Lambda)           (None, 1)                    0         ['dense_36[0][0]',            \n",
      "                                                                     'dense_37[0][0]',            \n",
      "                                                                     'dense_35[0][0]']            \n",
      "                                                                                                  \n",
      " dense_41 (Dense)            (None, 3)                    867       ['concatenate_2[0][0]']       \n",
      "                                                                                                  \n",
      " lambda_5 (Lambda)           (None, 1)                    0         ['lambda_2[0][0]',            \n",
      "                                                                     'lambda_3[0][0]',            \n",
      "                                                                     'lambda_4[0][0]',            \n",
      "                                                                     'dense_41[0][0]']            \n",
      "                                                                                                  \n",
      " thresholded_re_lu (Thresho  (None, 1)                    0         ['classification_output[0][0]'\n",
      " ldedReLU)                                                          ]                             \n",
      "                                                                                                  \n",
      " regression_output (Lambda)  (None, 1)                    0         ['lambda_5[0][0]']            \n",
      "                                                                                                  \n",
      " lambda_6 (Lambda)           (None, 1)                    0         ['thresholded_re_lu[0][0]']   \n",
      "                                                                                                  \n",
      " final_output (Lambda)       (None, 1)                    0         ['regression_output[0][0]',   \n",
      "                                                                     'lambda_6[0][0]']            \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 23955918 (91.38 MB)\n",
      "Trainable params: 23949966 (91.36 MB)\n",
      "Non-trainable params: 5952 (23.25 KB)\n",
      "__________________________________________________________________________________________________\n",
      "\n",
      "Class distribution in training set:\n",
      "Zeros: 52022 (50.12%)\n",
      "Non-zeros: 51776 (49.88%)\n",
      "\n",
      "Class distribution in test set:\n",
      "Zeros: 13007 (50.16%)\n",
      "Non-zeros: 12926 (49.84%)\n",
      "\n",
      "Model output names: ['classification_output', 'regression_output', 'final_output']\n",
      "\n",
      "4. Starting training...\n",
      "Epoch 1/100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-11-26 05:42:25.841427: I tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:442] Loaded cuDNN version 8905\n",
      "2024-11-26 05:42:26.758143: I tensorflow/tsl/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory\n",
      "2024-11-26 05:42:28.319667: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x74802ce90ad0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2024-11-26 05:42:28.319705: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): NVIDIA L40, Compute Capability 8.9\n",
      "2024-11-26 05:42:28.325479: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:269] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n",
      "2024-11-26 05:42:28.469866: I ./tensorflow/compiler/jit/device_compiler.h:186] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "541/541 [==============================] - ETA: 0s - loss: 14.9229 - classification_output_loss: 0.2997 - regression_output_loss: 0.2514 - final_output_loss: 0.1790 - classification_output_accuracy: 0.8674 - classification_output_auc: 0.9483 - regression_output_mse: 0.3870 - regression_output_mae: 0.4665 - regression_output_rmse: 0.5816 - regression_output_custom_mape: 68.0181 - final_output_mse: 0.2366 - final_output_mae: 0.2930 - final_output_rmse: 0.4493 - final_output_custom_mape: 76.9850"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.11/dist-packages/keras/src/engine/training.py:3079: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n",
      "  saving_api.save_model(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 1 Detailed Metrics:\n",
      "541/541 [==============================] - 106s 111ms/step - loss: 14.9229 - classification_output_loss: 0.2997 - regression_output_loss: 0.2514 - final_output_loss: 0.1790 - classification_output_accuracy: 0.8674 - classification_output_auc: 0.9483 - regression_output_mse: 0.3870 - regression_output_mae: 0.4665 - regression_output_rmse: 0.5816 - regression_output_custom_mape: 68.0181 - final_output_mse: 0.2366 - final_output_mae: 0.2930 - final_output_rmse: 0.4493 - final_output_custom_mape: 76.9850 - val_loss: 4.8619 - val_classification_output_loss: 0.2702 - val_regression_output_loss: 0.1998 - val_final_output_loss: 0.1203 - val_classification_output_accuracy: 0.8850 - val_classification_output_auc: 0.9648 - val_regression_output_mse: 2.6679 - val_regression_output_mae: 1.1926 - val_regression_output_rmse: 1.4470 - val_regression_output_custom_mape: 76.3626 - val_final_output_mse: 0.1619 - val_final_output_mae: 0.2603 - val_final_output_rmse: 0.3866 - val_final_output_custom_mape: 77.3426 - lr: 2.0000e-04\n",
      "Epoch 2/100\n",
      "541/541 [==============================] - 53s 97ms/step - loss: 2.5852 - classification_output_loss: 0.1672 - regression_output_loss: 0.1392 - final_output_loss: 0.0880 - classification_output_accuracy: 0.9350 - classification_output_auc: 0.9830 - regression_output_mse: 5.1786 - regression_output_mae: 1.6349 - regression_output_rmse: 2.2598 - regression_output_custom_mape: 71.9229 - final_output_mse: 0.1133 - final_output_mae: 0.2027 - final_output_rmse: 0.3209 - final_output_custom_mape: 72.9816 - val_loss: 1.4136 - val_classification_output_loss: 0.2138 - val_regression_output_loss: 0.1684 - val_final_output_loss: 0.1213 - val_classification_output_accuracy: 0.9236 - val_classification_output_auc: 0.9842 - val_regression_output_mse: 1.8603 - val_regression_output_mae: 1.0648 - val_regression_output_rmse: 1.3566 - val_regression_output_custom_mape: 75.8244 - val_final_output_mse: 0.1512 - val_final_output_mae: 0.2497 - val_final_output_rmse: 0.3643 - val_final_output_custom_mape: 75.8084 - lr: 2.0000e-04\n",
      "Epoch 3/100\n",
      "541/541 [==============================] - 53s 99ms/step - loss: 0.9558 - classification_output_loss: 0.1719 - regression_output_loss: 0.1366 - final_output_loss: 0.0895 - classification_output_accuracy: 0.9322 - classification_output_auc: 0.9815 - regression_output_mse: 5.6874 - regression_output_mae: 1.7310 - regression_output_rmse: 2.3695 - regression_output_custom_mape: 71.9037 - final_output_mse: 0.1126 - final_output_mae: 0.2038 - final_output_rmse: 0.3184 - final_output_custom_mape: 72.9206 - val_loss: 0.9508 - val_classification_output_loss: 0.4633 - val_regression_output_loss: 0.3961 - val_final_output_loss: 0.3536 - val_classification_output_accuracy: 0.8129 - val_classification_output_auc: 0.9060 - val_regression_output_mse: 1.6205 - val_regression_output_mae: 0.8607 - val_regression_output_rmse: 1.2559 - val_regression_output_custom_mape: 66.2296 - val_final_output_mse: 0.4082 - val_final_output_mae: 0.4125 - val_final_output_rmse: 0.6197 - val_final_output_custom_mape: 78.8389 - lr: 2.0000e-04\n",
      "Epoch 4/100\n",
      "541/541 [==============================] - 54s 100ms/step - loss: 0.5850 - classification_output_loss: 0.1792 - regression_output_loss: 0.2007 - final_output_loss: 0.1027 - classification_output_accuracy: 0.9293 - classification_output_auc: 0.9798 - regression_output_mse: 2.2127 - regression_output_mae: 0.9460 - regression_output_rmse: 1.2398 - regression_output_custom_mape: 73.1762 - final_output_mse: 0.1435 - final_output_mae: 0.2295 - final_output_rmse: 0.3588 - final_output_custom_mape: 74.5266 - val_loss: 0.4332 - val_classification_output_loss: 0.1561 - val_regression_output_loss: 0.1114 - val_final_output_loss: 0.1238 - val_classification_output_accuracy: 0.9345 - val_classification_output_auc: 0.9856 - val_regression_output_mse: 4.7324 - val_regression_output_mae: 1.5163 - val_regression_output_rmse: 2.1370 - val_regression_output_custom_mape: 76.3284 - val_final_output_mse: 0.1509 - val_final_output_mae: 0.2353 - val_final_output_rmse: 0.3733 - val_final_output_custom_mape: 76.9648 - lr: 2.0000e-04\n",
      "Epoch 5/100\n",
      "541/541 [==============================] - 58s 106ms/step - loss: 0.3949 - classification_output_loss: 0.1662 - regression_output_loss: 0.1463 - final_output_loss: 0.0885 - classification_output_accuracy: 0.9320 - classification_output_auc: 0.9831 - regression_output_mse: 2.7376 - regression_output_mae: 0.9965 - regression_output_rmse: 1.2987 - regression_output_custom_mape: 71.5179 - final_output_mse: 0.1125 - final_output_mae: 0.2059 - final_output_rmse: 0.3178 - final_output_custom_mape: 73.1178 - val_loss: 0.3969 - val_classification_output_loss: 0.2723 - val_regression_output_loss: 0.2098 - val_final_output_loss: 0.0787 - val_classification_output_accuracy: 0.8948 - val_classification_output_auc: 0.9729 - val_regression_output_mse: 0.1621 - val_regression_output_mae: 0.3438 - val_regression_output_rmse: 0.3980 - val_regression_output_custom_mape: 72.4170 - val_final_output_mse: 0.1050 - val_final_output_mae: 0.1904 - val_final_output_rmse: 0.3098 - val_final_output_custom_mape: 74.6832 - lr: 2.0000e-04\n",
      "Epoch 6/100\n",
      "541/541 [==============================] - 56s 103ms/step - loss: 0.2983 - classification_output_loss: 0.1354 - regression_output_loss: 0.1470 - final_output_loss: 0.0704 - classification_output_accuracy: 0.9464 - classification_output_auc: 0.9883 - regression_output_mse: 0.2598 - regression_output_mae: 0.4423 - regression_output_rmse: 0.5024 - regression_output_custom_mape: 71.7011 - final_output_mse: 0.0909 - final_output_mae: 0.1841 - final_output_rmse: 0.2871 - final_output_custom_mape: 71.9341 - val_loss: 0.2936 - val_classification_output_loss: 0.1702 - val_regression_output_loss: 0.1509 - val_final_output_loss: 0.1213 - val_classification_output_accuracy: 0.9348 - val_classification_output_auc: 0.9879 - val_regression_output_mse: 0.2456 - val_regression_output_mae: 0.4147 - val_regression_output_rmse: 0.4831 - val_regression_output_custom_mape: 74.9922 - val_final_output_mse: 0.1488 - val_final_output_mae: 0.2237 - val_final_output_rmse: 0.3632 - val_final_output_custom_mape: 74.9167 - lr: 2.0000e-04\n",
      "Epoch 7/100\n",
      "541/541 [==============================] - 56s 103ms/step - loss: 0.2505 - classification_output_loss: 0.1442 - regression_output_loss: 0.1384 - final_output_loss: 0.0775 - classification_output_accuracy: 0.9443 - classification_output_auc: 0.9866 - regression_output_mse: 1.4617 - regression_output_mae: 0.7991 - regression_output_rmse: 1.0508 - regression_output_custom_mape: 71.1954 - final_output_mse: 0.0988 - final_output_mae: 0.1902 - final_output_rmse: 0.2984 - final_output_custom_mape: 72.2324 - val_loss: 0.3214 - val_classification_output_loss: 0.3465 - val_regression_output_loss: 0.1456 - val_final_output_loss: 0.2147 - val_classification_output_accuracy: 0.8648 - val_classification_output_auc: 0.9798 - val_regression_output_mse: 0.8371 - val_regression_output_mae: 0.7323 - val_regression_output_rmse: 0.9107 - val_regression_output_custom_mape: 73.4417 - val_final_output_mse: 0.2713 - val_final_output_mae: 0.3340 - val_final_output_rmse: 0.4815 - val_final_output_custom_mape: 76.5578 - lr: 2.0000e-04\n",
      "Epoch 8/100\n",
      "541/541 [==============================] - 56s 104ms/step - loss: 0.2237 - classification_output_loss: 0.1397 - regression_output_loss: 0.1364 - final_output_loss: 0.0796 - classification_output_accuracy: 0.9446 - classification_output_auc: 0.9878 - regression_output_mse: 5.1974 - regression_output_mae: 1.6560 - regression_output_rmse: 2.2692 - regression_output_custom_mape: 71.2507 - final_output_mse: 0.1033 - final_output_mae: 0.1957 - final_output_rmse: 0.3040 - final_output_custom_mape: 72.5075 - val_loss: 0.2685 - val_classification_output_loss: 0.3914 - val_regression_output_loss: 0.1217 - val_final_output_loss: 0.1222 - val_classification_output_accuracy: 0.8613 - val_classification_output_auc: 0.9606 - val_regression_output_mse: 3.7820 - val_regression_output_mae: 1.2866 - val_regression_output_rmse: 1.8121 - val_regression_output_custom_mape: 68.2919 - val_final_output_mse: 0.1599 - val_final_output_mae: 0.2535 - val_final_output_rmse: 0.3599 - val_final_output_custom_mape: 71.1078 - lr: 2.0000e-04\n",
      "Epoch 9/100\n",
      "541/541 [==============================] - 54s 100ms/step - loss: 0.1754 - classification_output_loss: 0.1188 - regression_output_loss: 0.0985 - final_output_loss: 0.0632 - classification_output_accuracy: 0.9523 - classification_output_auc: 0.9911 - regression_output_mse: 5.4764 - regression_output_mae: 1.6783 - regression_output_rmse: 2.3144 - regression_output_custom_mape: 69.8070 - final_output_mse: 0.0766 - final_output_mae: 0.1685 - final_output_rmse: 0.2609 - final_output_custom_mape: 70.8609 - val_loss: 0.1706 - val_classification_output_loss: 0.1096 - val_regression_output_loss: 0.1077 - val_final_output_loss: 0.0658 - val_classification_output_accuracy: 0.9538 - val_classification_output_auc: 0.9930 - val_regression_output_mse: 5.1203 - val_regression_output_mae: 1.6260 - val_regression_output_rmse: 2.2470 - val_regression_output_custom_mape: 70.2256 - val_final_output_mse: 0.0762 - val_final_output_mae: 0.1588 - val_final_output_rmse: 0.2636 - val_final_output_custom_mape: 72.1157 - lr: 2.0000e-04\n",
      "Epoch 10/100\n",
      "541/541 [==============================] - 53s 98ms/step - loss: 0.1727 - classification_output_loss: 0.1290 - regression_output_loss: 0.1097 - final_output_loss: 0.0692 - classification_output_accuracy: 0.9479 - classification_output_auc: 0.9894 - regression_output_mse: 5.5291 - regression_output_mae: 1.6998 - regression_output_rmse: 2.3152 - regression_output_custom_mape: 70.2356 - final_output_mse: 0.0878 - final_output_mae: 0.1771 - final_output_rmse: 0.2731 - final_output_custom_mape: 71.4182 - val_loss: 0.1912 - val_classification_output_loss: 0.1751 - val_regression_output_loss: 0.0882 - val_final_output_loss: 0.0951 - val_classification_output_accuracy: 0.9463 - val_classification_output_auc: 0.9833 - val_regression_output_mse: 6.5049 - val_regression_output_mae: 1.9127 - val_regression_output_rmse: 2.5318 - val_regression_output_custom_mape: 75.4162 - val_final_output_mse: 0.1245 - val_final_output_mae: 0.2154 - val_final_output_rmse: 0.3377 - val_final_output_custom_mape: 75.3060 - lr: 2.0000e-04\n",
      "Epoch 11/100\n",
      "541/541 [==============================] - ETA: 0s - loss: 0.1454 - classification_output_loss: 0.0976 - regression_output_loss: 0.0864 - final_output_loss: 0.0573 - classification_output_accuracy: 0.9615 - classification_output_auc: 0.9941 - regression_output_mse: 5.8449 - regression_output_mae: 1.7543 - regression_output_rmse: 2.4040 - regression_output_custom_mape: 69.6693 - final_output_mse: 0.0682 - final_output_mae: 0.1573 - final_output_rmse: 0.2424 - final_output_custom_mape: 70.2452\n",
      "Epoch 11 Detailed Metrics:\n",
      "541/541 [==============================] - 54s 101ms/step - loss: 0.1454 - classification_output_loss: 0.0976 - regression_output_loss: 0.0864 - final_output_loss: 0.0573 - classification_output_accuracy: 0.9615 - classification_output_auc: 0.9941 - regression_output_mse: 5.8449 - regression_output_mae: 1.7543 - regression_output_rmse: 2.4040 - regression_output_custom_mape: 69.6693 - final_output_mse: 0.0682 - final_output_mae: 0.1573 - final_output_rmse: 0.2424 - final_output_custom_mape: 70.2452 - val_loss: 0.2093 - val_classification_output_loss: 0.2277 - val_regression_output_loss: 0.1437 - val_final_output_loss: 0.1177 - val_classification_output_accuracy: 0.9283 - val_classification_output_auc: 0.9779 - val_regression_output_mse: 6.0570 - val_regression_output_mae: 1.8178 - val_regression_output_rmse: 2.4544 - val_regression_output_custom_mape: 72.7221 - val_final_output_mse: 0.1364 - val_final_output_mae: 0.2321 - val_final_output_rmse: 0.3485 - val_final_output_custom_mape: 73.0335 - lr: 2.0000e-04\n",
      "Epoch 12/100\n",
      "541/541 [==============================] - 55s 101ms/step - loss: 0.1464 - classification_output_loss: 0.1072 - regression_output_loss: 0.1053 - final_output_loss: 0.0601 - classification_output_accuracy: 0.9561 - classification_output_auc: 0.9927 - regression_output_mse: 6.1092 - regression_output_mae: 1.8112 - regression_output_rmse: 2.4620 - regression_output_custom_mape: 69.8113 - final_output_mse: 0.0740 - final_output_mae: 0.1632 - final_output_rmse: 0.2523 - final_output_custom_mape: 70.3597 - val_loss: 0.1542 - val_classification_output_loss: 0.1503 - val_regression_output_loss: 0.0985 - val_final_output_loss: 0.0841 - val_classification_output_accuracy: 0.9489 - val_classification_output_auc: 0.9879 - val_regression_output_mse: 5.9678 - val_regression_output_mae: 1.7956 - val_regression_output_rmse: 2.4203 - val_regression_output_custom_mape: 72.0743 - val_final_output_mse: 0.1016 - val_final_output_mae: 0.1921 - val_final_output_rmse: 0.2866 - val_final_output_custom_mape: 72.4481 - lr: 2.0000e-04\n",
      "Epoch 13/100\n",
      "541/541 [==============================] - 56s 103ms/step - loss: 0.1351 - classification_output_loss: 0.0971 - regression_output_loss: 0.0999 - final_output_loss: 0.0592 - classification_output_accuracy: 0.9601 - classification_output_auc: 0.9939 - regression_output_mse: 6.1488 - regression_output_mae: 1.8152 - regression_output_rmse: 2.4696 - regression_output_custom_mape: 69.9159 - final_output_mse: 0.0739 - final_output_mae: 0.1643 - final_output_rmse: 0.2517 - final_output_custom_mape: 70.7436 - val_loss: 0.2396 - val_classification_output_loss: 0.3289 - val_regression_output_loss: 0.2191 - val_final_output_loss: 0.0616 - val_classification_output_accuracy: 0.8966 - val_classification_output_auc: 0.9705 - val_regression_output_mse: 0.7583 - val_regression_output_mae: 0.6739 - val_regression_output_rmse: 0.8690 - val_regression_output_custom_mape: 73.5880 - val_final_output_mse: 0.0818 - val_final_output_mae: 0.1722 - val_final_output_rmse: 0.2787 - val_final_output_custom_mape: 73.8794 - lr: 2.0000e-04\n",
      "Epoch 14/100\n",
      "541/541 [==============================] - 56s 104ms/step - loss: 0.1342 - classification_output_loss: 0.1155 - regression_output_loss: 0.0989 - final_output_loss: 0.0567 - classification_output_accuracy: 0.9515 - classification_output_auc: 0.9914 - regression_output_mse: 6.1207 - regression_output_mae: 1.8088 - regression_output_rmse: 2.4631 - regression_output_custom_mape: 69.6780 - final_output_mse: 0.0681 - final_output_mae: 0.1613 - final_output_rmse: 0.2447 - final_output_custom_mape: 70.4782 - val_loss: 0.1483 - val_classification_output_loss: 0.1125 - val_regression_output_loss: 0.1075 - val_final_output_loss: 0.0949 - val_classification_output_accuracy: 0.9545 - val_classification_output_auc: 0.9921 - val_regression_output_mse: 3.9937 - val_regression_output_mae: 1.5182 - val_regression_output_rmse: 1.9883 - val_regression_output_custom_mape: 74.0972 - val_final_output_mse: 0.1191 - val_final_output_mae: 0.2071 - val_final_output_rmse: 0.3226 - val_final_output_custom_mape: 73.9669 - lr: 2.0000e-04\n",
      "Epoch 15/100\n",
      "541/541 [==============================] - 57s 105ms/step - loss: 0.1104 - classification_output_loss: 0.0837 - regression_output_loss: 0.0778 - final_output_loss: 0.0495 - classification_output_accuracy: 0.9664 - classification_output_auc: 0.9953 - regression_output_mse: 6.4510 - regression_output_mae: 1.8607 - regression_output_rmse: 2.5321 - regression_output_custom_mape: 68.8861 - final_output_mse: 0.0558 - final_output_mae: 0.1437 - final_output_rmse: 0.2201 - final_output_custom_mape: 69.3373 - val_loss: 0.1422 - val_classification_output_loss: 0.1152 - val_regression_output_loss: 0.1280 - val_final_output_loss: 0.0574 - val_classification_output_accuracy: 0.9537 - val_classification_output_auc: 0.9933 - val_regression_output_mse: 5.8958 - val_regression_output_mae: 1.7291 - val_regression_output_rmse: 2.4158 - val_regression_output_custom_mape: 68.3024 - val_final_output_mse: 0.0661 - val_final_output_mae: 0.1556 - val_final_output_rmse: 0.2529 - val_final_output_custom_mape: 69.4068 - lr: 2.0000e-04\n",
      "Epoch 16/100\n",
      "541/541 [==============================] - 56s 103ms/step - loss: 0.1238 - classification_output_loss: 0.1192 - regression_output_loss: 0.0907 - final_output_loss: 0.0547 - classification_output_accuracy: 0.9500 - classification_output_auc: 0.9910 - regression_output_mse: 6.1427 - regression_output_mae: 1.7993 - regression_output_rmse: 2.4683 - regression_output_custom_mape: 68.9479 - final_output_mse: 0.0637 - final_output_mae: 0.1549 - final_output_rmse: 0.2359 - final_output_custom_mape: 69.7157 - val_loss: 0.1268 - val_classification_output_loss: 0.1301 - val_regression_output_loss: 0.0858 - val_final_output_loss: 0.0656 - val_classification_output_accuracy: 0.9430 - val_classification_output_auc: 0.9895 - val_regression_output_mse: 6.4156 - val_regression_output_mae: 1.8944 - val_regression_output_rmse: 2.5145 - val_regression_output_custom_mape: 71.3515 - val_final_output_mse: 0.0826 - val_final_output_mae: 0.1624 - val_final_output_rmse: 0.2670 - val_final_output_custom_mape: 71.2686 - lr: 2.0000e-04\n",
      "Epoch 17/100\n",
      "541/541 [==============================] - 58s 108ms/step - loss: 0.0942 - classification_output_loss: 0.0838 - regression_output_loss: 0.0617 - final_output_loss: 0.0405 - classification_output_accuracy: 0.9650 - classification_output_auc: 0.9955 - regression_output_mse: 6.3983 - regression_output_mae: 1.8268 - regression_output_rmse: 2.5217 - regression_output_custom_mape: 67.0560 - final_output_mse: 0.0399 - final_output_mae: 0.1244 - final_output_rmse: 0.1906 - final_output_custom_mape: 67.8066 - val_loss: 0.0979 - val_classification_output_loss: 0.1114 - val_regression_output_loss: 0.0596 - val_final_output_loss: 0.0473 - val_classification_output_accuracy: 0.9581 - val_classification_output_auc: 0.9919 - val_regression_output_mse: 5.9015 - val_regression_output_mae: 1.7669 - val_regression_output_rmse: 2.4102 - val_regression_output_custom_mape: 67.6186 - val_final_output_mse: 0.0482 - val_final_output_mae: 0.1296 - val_final_output_rmse: 0.2055 - val_final_output_custom_mape: 67.8666 - lr: 2.0000e-04\n",
      "Epoch 18/100\n",
      "541/541 [==============================] - 55s 102ms/step - loss: 0.0781 - classification_output_loss: 0.0719 - regression_output_loss: 0.0469 - final_output_loss: 0.0356 - classification_output_accuracy: 0.9714 - classification_output_auc: 0.9967 - regression_output_mse: 6.5024 - regression_output_mae: 1.8395 - regression_output_rmse: 2.5405 - regression_output_custom_mape: 65.9929 - final_output_mse: 0.0320 - final_output_mae: 0.1099 - final_output_rmse: 0.1657 - final_output_custom_mape: 66.5092 - val_loss: 0.1399 - val_classification_output_loss: 0.2071 - val_regression_output_loss: 0.1092 - val_final_output_loss: 0.0458 - val_classification_output_accuracy: 0.9400 - val_classification_output_auc: 0.9852 - val_regression_output_mse: 6.7711 - val_regression_output_mae: 1.9289 - val_regression_output_rmse: 2.5859 - val_regression_output_custom_mape: 69.2916 - val_final_output_mse: 0.0527 - val_final_output_mae: 0.1335 - val_final_output_rmse: 0.2186 - val_final_output_custom_mape: 69.7183 - lr: 2.0000e-04\n",
      "Epoch 19/100\n",
      "541/541 [==============================] - 54s 100ms/step - loss: 0.1056 - classification_output_loss: 0.1087 - regression_output_loss: 0.0770 - final_output_loss: 0.0488 - classification_output_accuracy: 0.9540 - classification_output_auc: 0.9925 - regression_output_mse: 6.1650 - regression_output_mae: 1.7928 - regression_output_rmse: 2.4682 - regression_output_custom_mape: 68.5078 - final_output_mse: 0.0533 - final_output_mae: 0.1445 - final_output_rmse: 0.2165 - final_output_custom_mape: 69.2497 - val_loss: 0.1351 - val_classification_output_loss: 0.2118 - val_regression_output_loss: 0.0784 - val_final_output_loss: 0.0744 - val_classification_output_accuracy: 0.9309 - val_classification_output_auc: 0.9809 - val_regression_output_mse: 6.2044 - val_regression_output_mae: 1.8057 - val_regression_output_rmse: 2.4610 - val_regression_output_custom_mape: 71.1880 - val_final_output_mse: 0.0989 - val_final_output_mae: 0.1831 - val_final_output_rmse: 0.2888 - val_final_output_custom_mape: 71.7069 - lr: 2.0000e-04\n",
      "Epoch 20/100\n",
      "541/541 [==============================] - 52s 97ms/step - loss: 0.0826 - classification_output_loss: 0.0899 - regression_output_loss: 0.0503 - final_output_loss: 0.0377 - classification_output_accuracy: 0.9641 - classification_output_auc: 0.9946 - regression_output_mse: 6.4599 - regression_output_mae: 1.8346 - regression_output_rmse: 2.5319 - regression_output_custom_mape: 66.5618 - final_output_mse: 0.0347 - final_output_mae: 0.1170 - final_output_rmse: 0.1767 - final_output_custom_mape: 67.1488 - val_loss: 0.1313 - val_classification_output_loss: 0.1914 - val_regression_output_loss: 0.0979 - val_final_output_loss: 0.0590 - val_classification_output_accuracy: 0.9393 - val_classification_output_auc: 0.9835 - val_regression_output_mse: 7.0486 - val_regression_output_mae: 2.0071 - val_regression_output_rmse: 2.6313 - val_regression_output_custom_mape: 72.4229 - val_final_output_mse: 0.0695 - val_final_output_mae: 0.1603 - val_final_output_rmse: 0.2545 - val_final_output_custom_mape: 72.8319 - lr: 2.0000e-04\n",
      "Epoch 21/100\n",
      "541/541 [==============================] - ETA: 0s - loss: 0.0978 - classification_output_loss: 0.0855 - regression_output_loss: 0.0795 - final_output_loss: 0.0458 - classification_output_accuracy: 0.9647 - classification_output_auc: 0.9953 - regression_output_mse: 6.5122 - regression_output_mae: 1.8662 - regression_output_rmse: 2.5424 - regression_output_custom_mape: 68.3928 - final_output_mse: 0.0492 - final_output_mae: 0.1379 - final_output_rmse: 0.2084 - final_output_custom_mape: 68.8971\n",
      "Epoch 21 Detailed Metrics:\n",
      "541/541 [==============================] - 54s 101ms/step - loss: 0.0978 - classification_output_loss: 0.0855 - regression_output_loss: 0.0795 - final_output_loss: 0.0458 - classification_output_accuracy: 0.9647 - classification_output_auc: 0.9953 - regression_output_mse: 6.5122 - regression_output_mae: 1.8662 - regression_output_rmse: 2.5424 - regression_output_custom_mape: 68.3928 - final_output_mse: 0.0492 - final_output_mae: 0.1379 - final_output_rmse: 0.2084 - final_output_custom_mape: 68.8971 - val_loss: 0.1567 - val_classification_output_loss: 0.1925 - val_regression_output_loss: 0.1182 - val_final_output_loss: 0.1052 - val_classification_output_accuracy: 0.9335 - val_classification_output_auc: 0.9888 - val_regression_output_mse: 6.3370 - val_regression_output_mae: 1.8864 - val_regression_output_rmse: 2.4981 - val_regression_output_custom_mape: 76.4414 - val_final_output_mse: 0.1555 - val_final_output_mae: 0.2558 - val_final_output_rmse: 0.3667 - val_final_output_custom_mape: 76.3954 - lr: 2.0000e-04\n",
      "Epoch 22/100\n",
      "541/541 [==============================] - 53s 99ms/step - loss: 0.0895 - classification_output_loss: 0.0784 - regression_output_loss: 0.0675 - final_output_loss: 0.0448 - classification_output_accuracy: 0.9675 - classification_output_auc: 0.9960 - regression_output_mse: 6.3547 - regression_output_mae: 1.8204 - regression_output_rmse: 2.5032 - regression_output_custom_mape: 67.3088 - final_output_mse: 0.0469 - final_output_mae: 0.1335 - final_output_rmse: 0.2007 - final_output_custom_mape: 68.4431 - val_loss: 0.1160 - val_classification_output_loss: 0.1100 - val_regression_output_loss: 0.1048 - val_final_output_loss: 0.0471 - val_classification_output_accuracy: 0.9584 - val_classification_output_auc: 0.9936 - val_regression_output_mse: 6.0390 - val_regression_output_mae: 1.7293 - val_regression_output_rmse: 2.4455 - val_regression_output_custom_mape: 66.8834 - val_final_output_mse: 0.0490 - val_final_output_mae: 0.1419 - val_final_output_rmse: 0.2174 - val_final_output_custom_mape: 69.0459 - lr: 2.0000e-04\n",
      "Epoch 23/100\n",
      "541/541 [==============================] - 53s 98ms/step - loss: 0.0919 - classification_output_loss: 0.0994 - regression_output_loss: 0.0648 - final_output_loss: 0.0429 - classification_output_accuracy: 0.9578 - classification_output_auc: 0.9935 - regression_output_mse: 6.2936 - regression_output_mae: 1.8038 - regression_output_rmse: 2.4971 - regression_output_custom_mape: 67.2289 - final_output_mse: 0.0428 - final_output_mae: 0.1306 - final_output_rmse: 0.1954 - final_output_custom_mape: 68.1846 - val_loss: 0.1030 - val_classification_output_loss: 0.1001 - val_regression_output_loss: 0.0829 - val_final_output_loss: 0.0521 - val_classification_output_accuracy: 0.9620 - val_classification_output_auc: 0.9930 - val_regression_output_mse: 4.9840 - val_regression_output_mae: 1.6330 - val_regression_output_rmse: 2.2237 - val_regression_output_custom_mape: 67.5453 - val_final_output_mse: 0.0686 - val_final_output_mae: 0.1495 - val_final_output_rmse: 0.2307 - val_final_output_custom_mape: 68.2594 - lr: 2.0000e-04\n",
      "Epoch 24/100\n",
      "541/541 [==============================] - 55s 101ms/step - loss: 0.0805 - classification_output_loss: 0.0740 - regression_output_loss: 0.0572 - final_output_loss: 0.0406 - classification_output_accuracy: 0.9696 - classification_output_auc: 0.9964 - regression_output_mse: 6.5178 - regression_output_mae: 1.8487 - regression_output_rmse: 2.5431 - regression_output_custom_mape: 66.7225 - final_output_mse: 0.0413 - final_output_mae: 0.1227 - final_output_rmse: 0.1848 - final_output_custom_mape: 67.3500 - val_loss: 0.1503 - val_classification_output_loss: 0.1089 - val_regression_output_loss: 0.1781 - val_final_output_loss: 0.0504 - val_classification_output_accuracy: 0.9567 - val_classification_output_auc: 0.9940 - val_regression_output_mse: 7.3781 - val_regression_output_mae: 2.0860 - val_regression_output_rmse: 2.7043 - val_regression_output_custom_mape: 71.4390 - val_final_output_mse: 0.0652 - val_final_output_mae: 0.1535 - val_final_output_rmse: 0.2412 - val_final_output_custom_mape: 71.1141 - lr: 2.0000e-04\n",
      "Epoch 25/100\n",
      "541/541 [==============================] - ETA: 0s - loss: 0.0978 - classification_output_loss: 0.0906 - regression_output_loss: 0.0818 - final_output_loss: 0.0454 - classification_output_accuracy: 0.9611 - classification_output_auc: 0.9948 - regression_output_mse: 6.4449 - regression_output_mae: 1.8471 - regression_output_rmse: 2.5298 - regression_output_custom_mape: 68.0068 - final_output_mse: 0.0497 - final_output_mae: 0.1380 - final_output_rmse: 0.2085 - final_output_custom_mape: 68.7826\n",
      "Epoch 25: ReduceLROnPlateau reducing learning rate to 9.999999747378752e-05.\n",
      "541/541 [==============================] - 55s 101ms/step - loss: 0.0978 - classification_output_loss: 0.0906 - regression_output_loss: 0.0818 - final_output_loss: 0.0454 - classification_output_accuracy: 0.9611 - classification_output_auc: 0.9948 - regression_output_mse: 6.4449 - regression_output_mae: 1.8471 - regression_output_rmse: 2.5298 - regression_output_custom_mape: 68.0068 - final_output_mse: 0.0497 - final_output_mae: 0.1380 - final_output_rmse: 0.2085 - final_output_custom_mape: 68.7826 - val_loss: 0.0920 - val_classification_output_loss: 0.1283 - val_regression_output_loss: 0.0552 - val_final_output_loss: 0.0492 - val_classification_output_accuracy: 0.9526 - val_classification_output_auc: 0.9923 - val_regression_output_mse: 6.4349 - val_regression_output_mae: 1.8355 - val_regression_output_rmse: 2.5185 - val_regression_output_custom_mape: 69.3148 - val_final_output_mse: 0.0507 - val_final_output_mae: 0.1446 - val_final_output_rmse: 0.2100 - val_final_output_custom_mape: 69.2727 - lr: 2.0000e-04\n",
      "Epoch 26/100\n",
      "541/541 [==============================] - 56s 103ms/step - loss: 0.0650 - classification_output_loss: 0.0610 - regression_output_loss: 0.0419 - final_output_loss: 0.0313 - classification_output_accuracy: 0.9754 - classification_output_auc: 0.9975 - regression_output_mse: 6.6869 - regression_output_mae: 1.8694 - regression_output_rmse: 2.5765 - regression_output_custom_mape: 65.2689 - final_output_mse: 0.0247 - final_output_mae: 0.0999 - final_output_rmse: 0.1486 - final_output_custom_mape: 65.4804 - val_loss: 0.0663 - val_classification_output_loss: 0.0825 - val_regression_output_loss: 0.0378 - val_final_output_loss: 0.0327 - val_classification_output_accuracy: 0.9638 - val_classification_output_auc: 0.9952 - val_regression_output_mse: 6.7468 - val_regression_output_mae: 1.8843 - val_regression_output_rmse: 2.5794 - val_regression_output_custom_mape: 64.9190 - val_final_output_mse: 0.0274 - val_final_output_mae: 0.1009 - val_final_output_rmse: 0.1557 - val_final_output_custom_mape: 64.8372 - lr: 1.0000e-04\n",
      "Epoch 27/100\n",
      "541/541 [==============================] - 55s 102ms/step - loss: 0.0520 - classification_output_loss: 0.0582 - regression_output_loss: 0.0251 - final_output_loss: 0.0263 - classification_output_accuracy: 0.9761 - classification_output_auc: 0.9978 - regression_output_mse: 6.7296 - regression_output_mae: 1.8683 - regression_output_rmse: 2.5843 - regression_output_custom_mape: 64.1190 - final_output_mse: 0.0169 - final_output_mae: 0.0854 - final_output_rmse: 0.1247 - final_output_custom_mape: 64.2327 - val_loss: 0.0596 - val_classification_output_loss: 0.0822 - val_regression_output_loss: 0.0315 - val_final_output_loss: 0.0279 - val_classification_output_accuracy: 0.9638 - val_classification_output_auc: 0.9952 - val_regression_output_mse: 6.7299 - val_regression_output_mae: 1.8644 - val_regression_output_rmse: 2.5772 - val_regression_output_custom_mape: 64.0526 - val_final_output_mse: 0.0205 - val_final_output_mae: 0.0887 - val_final_output_rmse: 0.1379 - val_final_output_custom_mape: 64.1891 - lr: 1.0000e-04\n",
      "Epoch 28/100\n",
      "541/541 [==============================] - 54s 101ms/step - loss: 0.0545 - classification_output_loss: 0.0621 - regression_output_loss: 0.0298 - final_output_loss: 0.0277 - classification_output_accuracy: 0.9743 - classification_output_auc: 0.9975 - regression_output_mse: 6.5792 - regression_output_mae: 1.8345 - regression_output_rmse: 2.5551 - regression_output_custom_mape: 63.7945 - final_output_mse: 0.0191 - final_output_mae: 0.0891 - final_output_rmse: 0.1301 - final_output_custom_mape: 64.3093 - val_loss: 0.0707 - val_classification_output_loss: 0.0879 - val_regression_output_loss: 0.0470 - val_final_output_loss: 0.0366 - val_classification_output_accuracy: 0.9646 - val_classification_output_auc: 0.9949 - val_regression_output_mse: 6.6182 - val_regression_output_mae: 1.8513 - val_regression_output_rmse: 2.5551 - val_regression_output_custom_mape: 65.1533 - val_final_output_mse: 0.0342 - val_final_output_mae: 0.1136 - val_final_output_rmse: 0.1726 - val_final_output_custom_mape: 65.5781 - lr: 1.0000e-04\n",
      "Epoch 29/100\n",
      "541/541 [==============================] - 53s 98ms/step - loss: 0.0504 - classification_output_loss: 0.0586 - regression_output_loss: 0.0257 - final_output_loss: 0.0264 - classification_output_accuracy: 0.9756 - classification_output_auc: 0.9978 - regression_output_mse: 6.7302 - regression_output_mae: 1.8661 - regression_output_rmse: 2.5837 - regression_output_custom_mape: 63.8101 - final_output_mse: 0.0171 - final_output_mae: 0.0858 - final_output_rmse: 0.1252 - final_output_custom_mape: 64.0432 - val_loss: 0.0755 - val_classification_output_loss: 0.0734 - val_regression_output_loss: 0.0692 - val_final_output_loss: 0.0298 - val_classification_output_accuracy: 0.9703 - val_classification_output_auc: 0.9959 - val_regression_output_mse: 6.5720 - val_regression_output_mae: 1.8334 - val_regression_output_rmse: 2.5485 - val_regression_output_custom_mape: 65.7695 - val_final_output_mse: 0.0219 - val_final_output_mae: 0.0957 - val_final_output_rmse: 0.1415 - val_final_output_custom_mape: 65.9068 - lr: 1.0000e-04\n",
      "Epoch 30/100\n",
      "541/541 [==============================] - 55s 102ms/step - loss: 0.0618 - classification_output_loss: 0.0646 - regression_output_loss: 0.0430 - final_output_loss: 0.0325 - classification_output_accuracy: 0.9726 - classification_output_auc: 0.9974 - regression_output_mse: 6.6295 - regression_output_mae: 1.8588 - regression_output_rmse: 2.5643 - regression_output_custom_mape: 65.4595 - final_output_mse: 0.0267 - final_output_mae: 0.1036 - final_output_rmse: 0.1522 - final_output_custom_mape: 65.8207 - val_loss: 0.0563 - val_classification_output_loss: 0.0878 - val_regression_output_loss: 0.0261 - val_final_output_loss: 0.0269 - val_classification_output_accuracy: 0.9639 - val_classification_output_auc: 0.9949 - val_regression_output_mse: 6.6181 - val_regression_output_mae: 1.8267 - val_regression_output_rmse: 2.5541 - val_regression_output_custom_mape: 63.1844 - val_final_output_mse: 0.0184 - val_final_output_mae: 0.0849 - val_final_output_rmse: 0.1292 - val_final_output_custom_mape: 63.5460 - lr: 1.0000e-04\n",
      "Epoch 31/100\n",
      "541/541 [==============================] - ETA: 0s - loss: 0.0539 - classification_output_loss: 0.0608 - regression_output_loss: 0.0321 - final_output_loss: 0.0282 - classification_output_accuracy: 0.9746 - classification_output_auc: 0.9975 - regression_output_mse: 6.6719 - regression_output_mae: 1.8572 - regression_output_rmse: 2.5729 - regression_output_custom_mape: 64.1531 - final_output_mse: 0.0197 - final_output_mae: 0.0908 - final_output_rmse: 0.1326 - final_output_custom_mape: 64.4701\n",
      "Epoch 31 Detailed Metrics:\n",
      "541/541 [==============================] - 54s 99ms/step - loss: 0.0539 - classification_output_loss: 0.0608 - regression_output_loss: 0.0321 - final_output_loss: 0.0282 - classification_output_accuracy: 0.9746 - classification_output_auc: 0.9975 - regression_output_mse: 6.6719 - regression_output_mae: 1.8572 - regression_output_rmse: 2.5729 - regression_output_custom_mape: 64.1531 - final_output_mse: 0.0197 - final_output_mae: 0.0908 - final_output_rmse: 0.1326 - final_output_custom_mape: 64.4701 - val_loss: 0.0803 - val_classification_output_loss: 0.1105 - val_regression_output_loss: 0.0560 - val_final_output_loss: 0.0436 - val_classification_output_accuracy: 0.9555 - val_classification_output_auc: 0.9951 - val_regression_output_mse: 6.3193 - val_regression_output_mae: 1.7972 - val_regression_output_rmse: 2.4978 - val_regression_output_custom_mape: 67.5192 - val_final_output_mse: 0.0457 - val_final_output_mae: 0.1347 - val_final_output_rmse: 0.2015 - val_final_output_custom_mape: 67.7615 - lr: 1.0000e-04\n",
      "Epoch 32/100\n",
      "541/541 [==============================] - 55s 102ms/step - loss: 0.0588 - classification_output_loss: 0.0668 - regression_output_loss: 0.0386 - final_output_loss: 0.0303 - classification_output_accuracy: 0.9719 - classification_output_auc: 0.9971 - regression_output_mse: 6.6258 - regression_output_mae: 1.8496 - regression_output_rmse: 2.5637 - regression_output_custom_mape: 64.4690 - final_output_mse: 0.0235 - final_output_mae: 0.0968 - final_output_rmse: 0.1423 - final_output_custom_mape: 64.9170 - val_loss: 0.0554 - val_classification_output_loss: 0.0832 - val_regression_output_loss: 0.0290 - val_final_output_loss: 0.0247 - val_classification_output_accuracy: 0.9658 - val_classification_output_auc: 0.9956 - val_regression_output_mse: 6.7171 - val_regression_output_mae: 1.8475 - val_regression_output_rmse: 2.5747 - val_regression_output_custom_mape: 63.5893 - val_final_output_mse: 0.0151 - val_final_output_mae: 0.0793 - val_final_output_rmse: 0.1190 - val_final_output_custom_mape: 63.5201 - lr: 1.0000e-04\n",
      "Epoch 33/100\n",
      "541/541 [==============================] - 53s 97ms/step - loss: 0.0562 - classification_output_loss: 0.0654 - regression_output_loss: 0.0351 - final_output_loss: 0.0293 - classification_output_accuracy: 0.9728 - classification_output_auc: 0.9973 - regression_output_mse: 6.6587 - regression_output_mae: 1.8556 - regression_output_rmse: 2.5699 - regression_output_custom_mape: 64.4570 - final_output_mse: 0.0214 - final_output_mae: 0.0943 - final_output_rmse: 0.1371 - final_output_custom_mape: 64.8190 - val_loss: 0.0688 - val_classification_output_loss: 0.0870 - val_regression_output_loss: 0.0540 - val_final_output_loss: 0.0265 - val_classification_output_accuracy: 0.9638 - val_classification_output_auc: 0.9946 - val_regression_output_mse: 6.6598 - val_regression_output_mae: 1.8470 - val_regression_output_rmse: 2.5624 - val_regression_output_custom_mape: 63.3716 - val_final_output_mse: 0.0182 - val_final_output_mae: 0.0846 - val_final_output_rmse: 0.1293 - val_final_output_custom_mape: 63.4470 - lr: 1.0000e-04\n",
      "Epoch 34/100\n",
      "541/541 [==============================] - 52s 97ms/step - loss: 0.0557 - classification_output_loss: 0.0589 - regression_output_loss: 0.0381 - final_output_loss: 0.0284 - classification_output_accuracy: 0.9753 - classification_output_auc: 0.9977 - regression_output_mse: 6.7001 - regression_output_mae: 1.8630 - regression_output_rmse: 2.5779 - regression_output_custom_mape: 63.9678 - final_output_mse: 0.0203 - final_output_mae: 0.0921 - final_output_rmse: 0.1348 - final_output_custom_mape: 64.2577 - val_loss: 0.0620 - val_classification_output_loss: 0.0928 - val_regression_output_loss: 0.0371 - val_final_output_loss: 0.0294 - val_classification_output_accuracy: 0.9603 - val_classification_output_auc: 0.9940 - val_regression_output_mse: 6.7330 - val_regression_output_mae: 1.8643 - val_regression_output_rmse: 2.5765 - val_regression_output_custom_mape: 64.8509 - val_final_output_mse: 0.0217 - val_final_output_mae: 0.0925 - val_final_output_rmse: 0.1422 - val_final_output_custom_mape: 65.1600 - lr: 1.0000e-04\n",
      "Epoch 35/100\n",
      "541/541 [==============================] - 52s 97ms/step - loss: 0.0557 - classification_output_loss: 0.0614 - regression_output_loss: 0.0367 - final_output_loss: 0.0297 - classification_output_accuracy: 0.9745 - classification_output_auc: 0.9976 - regression_output_mse: 6.6579 - regression_output_mae: 1.8582 - regression_output_rmse: 2.5696 - regression_output_custom_mape: 64.6971 - final_output_mse: 0.0221 - final_output_mae: 0.0956 - final_output_rmse: 0.1398 - final_output_custom_mape: 65.0271 - val_loss: 0.0594 - val_classification_output_loss: 0.0939 - val_regression_output_loss: 0.0300 - val_final_output_loss: 0.0317 - val_classification_output_accuracy: 0.9612 - val_classification_output_auc: 0.9941 - val_regression_output_mse: 6.8011 - val_regression_output_mae: 1.8931 - val_regression_output_rmse: 2.5893 - val_regression_output_custom_mape: 65.7507 - val_final_output_mse: 0.0254 - val_final_output_mae: 0.1008 - val_final_output_rmse: 0.1520 - val_final_output_custom_mape: 65.6841 - lr: 1.0000e-04\n",
      "Epoch 36/100\n",
      "541/541 [==============================] - 57s 106ms/step - loss: 0.0461 - classification_output_loss: 0.0654 - regression_output_loss: 0.0198 - final_output_loss: 0.0250 - classification_output_accuracy: 0.9728 - classification_output_auc: 0.9971 - regression_output_mse: 6.6669 - regression_output_mae: 1.8474 - regression_output_rmse: 2.5716 - regression_output_custom_mape: 63.3631 - final_output_mse: 0.0152 - final_output_mae: 0.0813 - final_output_rmse: 0.1173 - final_output_custom_mape: 63.6640 - val_loss: 0.0649 - val_classification_output_loss: 0.1065 - val_regression_output_loss: 0.0418 - val_final_output_loss: 0.0242 - val_classification_output_accuracy: 0.9594 - val_classification_output_auc: 0.9931 - val_regression_output_mse: 6.5540 - val_regression_output_mae: 1.8143 - val_regression_output_rmse: 2.5381 - val_regression_output_custom_mape: 62.4616 - val_final_output_mse: 0.0149 - val_final_output_mae: 0.0784 - val_final_output_rmse: 0.1183 - val_final_output_custom_mape: 62.9478 - lr: 1.0000e-04\n",
      "Epoch 37/100\n",
      "541/541 [==============================] - 58s 107ms/step - loss: 0.0585 - classification_output_loss: 0.0619 - regression_output_loss: 0.0427 - final_output_loss: 0.0304 - classification_output_accuracy: 0.9744 - classification_output_auc: 0.9975 - regression_output_mse: 6.6088 - regression_output_mae: 1.8483 - regression_output_rmse: 2.5604 - regression_output_custom_mape: 64.4680 - final_output_mse: 0.0237 - final_output_mae: 0.0974 - final_output_rmse: 0.1426 - final_output_custom_mape: 64.9015 - val_loss: 0.0691 - val_classification_output_loss: 0.1106 - val_regression_output_loss: 0.0482 - val_final_output_loss: 0.0240 - val_classification_output_accuracy: 0.9576 - val_classification_output_auc: 0.9934 - val_regression_output_mse: 6.5947 - val_regression_output_mae: 1.8196 - val_regression_output_rmse: 2.5455 - val_regression_output_custom_mape: 62.2371 - val_final_output_mse: 0.0145 - val_final_output_mae: 0.0779 - val_final_output_rmse: 0.1159 - val_final_output_custom_mape: 62.1254 - lr: 1.0000e-04\n",
      "Epoch 38/100\n",
      "541/541 [==============================] - 55s 103ms/step - loss: 0.0531 - classification_output_loss: 0.0608 - regression_output_loss: 0.0336 - final_output_loss: 0.0279 - classification_output_accuracy: 0.9748 - classification_output_auc: 0.9977 - regression_output_mse: 6.5831 - regression_output_mae: 1.8327 - regression_output_rmse: 2.5548 - regression_output_custom_mape: 63.5282 - final_output_mse: 0.0194 - final_output_mae: 0.0900 - final_output_rmse: 0.1307 - final_output_custom_mape: 64.1170 - val_loss: 0.0563 - val_classification_output_loss: 0.0809 - val_regression_output_loss: 0.0317 - val_final_output_loss: 0.0292 - val_classification_output_accuracy: 0.9680 - val_classification_output_auc: 0.9955 - val_regression_output_mse: 6.5566 - val_regression_output_mae: 1.8266 - val_regression_output_rmse: 2.5428 - val_regression_output_custom_mape: 63.8044 - val_final_output_mse: 0.0222 - val_final_output_mae: 0.0928 - val_final_output_rmse: 0.1405 - val_final_output_custom_mape: 64.0602 - lr: 1.0000e-04\n",
      "Epoch 39/100\n",
      "541/541 [==============================] - 54s 100ms/step - loss: 0.0515 - classification_output_loss: 0.0622 - regression_output_loss: 0.0313 - final_output_loss: 0.0272 - classification_output_accuracy: 0.9739 - classification_output_auc: 0.9975 - regression_output_mse: 6.6419 - regression_output_mae: 1.8481 - regression_output_rmse: 2.5667 - regression_output_custom_mape: 63.7719 - final_output_mse: 0.0181 - final_output_mae: 0.0884 - final_output_rmse: 0.1283 - final_output_custom_mape: 64.1751 - val_loss: 0.0704 - val_classification_output_loss: 0.1092 - val_regression_output_loss: 0.0453 - val_final_output_loss: 0.0350 - val_classification_output_accuracy: 0.9609 - val_classification_output_auc: 0.9938 - val_regression_output_mse: 6.1988 - val_regression_output_mae: 1.7405 - val_regression_output_rmse: 2.4718 - val_regression_output_custom_mape: 63.3963 - val_final_output_mse: 0.0315 - val_final_output_mae: 0.1112 - val_final_output_rmse: 0.1671 - val_final_output_custom_mape: 65.2804 - lr: 1.0000e-04\n",
      "Epoch 40/100\n",
      "541/541 [==============================] - 54s 101ms/step - loss: 0.0517 - classification_output_loss: 0.0625 - regression_output_loss: 0.0314 - final_output_loss: 0.0282 - classification_output_accuracy: 0.9742 - classification_output_auc: 0.9975 - regression_output_mse: 6.6585 - regression_output_mae: 1.8533 - regression_output_rmse: 2.5700 - regression_output_custom_mape: 64.0952 - final_output_mse: 0.0199 - final_output_mae: 0.0913 - final_output_rmse: 0.1320 - final_output_custom_mape: 64.4904 - val_loss: 0.0712 - val_classification_output_loss: 0.0779 - val_regression_output_loss: 0.0654 - val_final_output_loss: 0.0267 - val_classification_output_accuracy: 0.9666 - val_classification_output_auc: 0.9957 - val_regression_output_mse: 6.7348 - val_regression_output_mae: 1.8600 - val_regression_output_rmse: 2.5770 - val_regression_output_custom_mape: 63.5495 - val_final_output_mse: 0.0176 - val_final_output_mae: 0.0874 - val_final_output_rmse: 0.1274 - val_final_output_custom_mape: 63.5438 - lr: 1.0000e-04\n",
      "Epoch 41/100\n",
      "541/541 [==============================] - ETA: 0s - loss: 0.0537 - classification_output_loss: 0.0611 - regression_output_loss: 0.0359 - final_output_loss: 0.0285 - classification_output_accuracy: 0.9743 - classification_output_auc: 0.9976 - regression_output_mse: 6.7033 - regression_output_mae: 1.8642 - regression_output_rmse: 2.5791 - regression_output_custom_mape: 64.1775 - final_output_mse: 0.0202 - final_output_mae: 0.0923 - final_output_rmse: 0.1340 - final_output_custom_mape: 64.4763\n",
      "Epoch 41 Detailed Metrics:\n",
      "541/541 [==============================] - 56s 103ms/step - loss: 0.0537 - classification_output_loss: 0.0611 - regression_output_loss: 0.0359 - final_output_loss: 0.0285 - classification_output_accuracy: 0.9743 - classification_output_auc: 0.9976 - regression_output_mse: 6.7033 - regression_output_mae: 1.8642 - regression_output_rmse: 2.5791 - regression_output_custom_mape: 64.1775 - final_output_mse: 0.0202 - final_output_mae: 0.0923 - final_output_rmse: 0.1340 - final_output_custom_mape: 64.4763 - val_loss: 0.0600 - val_classification_output_loss: 0.0930 - val_regression_output_loss: 0.0397 - val_final_output_loss: 0.0232 - val_classification_output_accuracy: 0.9621 - val_classification_output_auc: 0.9955 - val_regression_output_mse: 6.4476 - val_regression_output_mae: 1.7785 - val_regression_output_rmse: 2.5232 - val_regression_output_custom_mape: 62.2171 - val_final_output_mse: 0.0129 - val_final_output_mae: 0.0759 - val_final_output_rmse: 0.1099 - val_final_output_custom_mape: 62.3951 - lr: 1.0000e-04\n",
      "Epoch 42/100\n",
      "541/541 [==============================] - 54s 99ms/step - loss: 0.0544 - classification_output_loss: 0.0687 - regression_output_loss: 0.0347 - final_output_loss: 0.0285 - classification_output_accuracy: 0.9719 - classification_output_auc: 0.9968 - regression_output_mse: 6.6631 - regression_output_mae: 1.8573 - regression_output_rmse: 2.5712 - regression_output_custom_mape: 64.0519 - final_output_mse: 0.0203 - final_output_mae: 0.0917 - final_output_rmse: 0.1332 - final_output_custom_mape: 64.3852 - val_loss: 0.0628 - val_classification_output_loss: 0.1047 - val_regression_output_loss: 0.0382 - val_final_output_loss: 0.0251 - val_classification_output_accuracy: 0.9576 - val_classification_output_auc: 0.9943 - val_regression_output_mse: 6.5854 - val_regression_output_mae: 1.8202 - val_regression_output_rmse: 2.5484 - val_regression_output_custom_mape: 62.9115 - val_final_output_mse: 0.0159 - val_final_output_mae: 0.0809 - val_final_output_rmse: 0.1218 - val_final_output_custom_mape: 62.8272 - lr: 1.0000e-04\n",
      "Epoch 43/100\n",
      "541/541 [==============================] - 53s 99ms/step - loss: 0.0545 - classification_output_loss: 0.0614 - regression_output_loss: 0.0368 - final_output_loss: 0.0291 - classification_output_accuracy: 0.9748 - classification_output_auc: 0.9975 - regression_output_mse: 6.7100 - regression_output_mae: 1.8688 - regression_output_rmse: 2.5810 - regression_output_custom_mape: 64.6285 - final_output_mse: 0.0208 - final_output_mae: 0.0943 - final_output_rmse: 0.1371 - final_output_custom_mape: 64.8706 - val_loss: 0.0639 - val_classification_output_loss: 0.1024 - val_regression_output_loss: 0.0432 - val_final_output_loss: 0.0238 - val_classification_output_accuracy: 0.9622 - val_classification_output_auc: 0.9938 - val_regression_output_mse: 6.9339 - val_regression_output_mae: 1.9011 - val_regression_output_rmse: 2.6127 - val_regression_output_custom_mape: 62.8598 - val_final_output_mse: 0.0143 - val_final_output_mae: 0.0768 - val_final_output_rmse: 0.1158 - val_final_output_custom_mape: 62.6952 - lr: 1.0000e-04\n",
      "Epoch 44/100\n",
      "541/541 [==============================] - 57s 106ms/step - loss: 0.0503 - classification_output_loss: 0.0579 - regression_output_loss: 0.0320 - final_output_loss: 0.0272 - classification_output_accuracy: 0.9755 - classification_output_auc: 0.9979 - regression_output_mse: 6.7157 - regression_output_mae: 1.8642 - regression_output_rmse: 2.5809 - regression_output_custom_mape: 63.8468 - final_output_mse: 0.0185 - final_output_mae: 0.0884 - final_output_rmse: 0.1284 - final_output_custom_mape: 64.1197 - val_loss: 0.0535 - val_classification_output_loss: 0.0933 - val_regression_output_loss: 0.0278 - val_final_output_loss: 0.0222 - val_classification_output_accuracy: 0.9624 - val_classification_output_auc: 0.9953 - val_regression_output_mse: 6.5462 - val_regression_output_mae: 1.7951 - val_regression_output_rmse: 2.5414 - val_regression_output_custom_mape: 61.7450 - val_final_output_mse: 0.0123 - val_final_output_mae: 0.0720 - val_final_output_rmse: 0.1075 - val_final_output_custom_mape: 61.9004 - lr: 1.0000e-04\n",
      "Epoch 45/100\n",
      "541/541 [==============================] - 55s 102ms/step - loss: 0.0461 - classification_output_loss: 0.0569 - regression_output_loss: 0.0258 - final_output_loss: 0.0260 - classification_output_accuracy: 0.9755 - classification_output_auc: 0.9980 - regression_output_mse: 6.7333 - regression_output_mae: 1.8649 - regression_output_rmse: 2.5850 - regression_output_custom_mape: 63.6520 - final_output_mse: 0.0163 - final_output_mae: 0.0845 - final_output_rmse: 0.1212 - final_output_custom_mape: 63.8765 - val_loss: 0.0649 - val_classification_output_loss: 0.1188 - val_regression_output_loss: 0.0354 - val_final_output_loss: 0.0316 - val_classification_output_accuracy: 0.9471 - val_classification_output_auc: 0.9966 - val_regression_output_mse: 6.0672 - val_regression_output_mae: 1.7044 - val_regression_output_rmse: 2.4462 - val_regression_output_custom_mape: 62.7387 - val_final_output_mse: 0.0251 - val_final_output_mae: 0.1018 - val_final_output_rmse: 0.1497 - val_final_output_custom_mape: 63.6189 - lr: 1.0000e-04\n",
      "Epoch 46/100\n",
      "541/541 [==============================] - 53s 99ms/step - loss: 0.0510 - classification_output_loss: 0.0628 - regression_output_loss: 0.0325 - final_output_loss: 0.0277 - classification_output_accuracy: 0.9741 - classification_output_auc: 0.9974 - regression_output_mse: 6.6063 - regression_output_mae: 1.8383 - regression_output_rmse: 2.5597 - regression_output_custom_mape: 63.7547 - final_output_mse: 0.0190 - final_output_mae: 0.0901 - final_output_rmse: 0.1300 - final_output_custom_mape: 64.2525 - val_loss: 0.0523 - val_classification_output_loss: 0.0814 - val_regression_output_loss: 0.0291 - val_final_output_loss: 0.0254 - val_classification_output_accuracy: 0.9658 - val_classification_output_auc: 0.9951 - val_regression_output_mse: 6.7808 - val_regression_output_mae: 1.8736 - val_regression_output_rmse: 2.5860 - val_regression_output_custom_mape: 65.4947 - val_final_output_mse: 0.0155 - val_final_output_mae: 0.0831 - val_final_output_rmse: 0.1224 - val_final_output_custom_mape: 65.5416 - lr: 1.0000e-04\n",
      "Epoch 47/100\n",
      "541/541 [==============================] - 51s 94ms/step - loss: 0.0509 - classification_output_loss: 0.0589 - regression_output_loss: 0.0343 - final_output_loss: 0.0272 - classification_output_accuracy: 0.9755 - classification_output_auc: 0.9977 - regression_output_mse: 6.7076 - regression_output_mae: 1.8620 - regression_output_rmse: 2.5796 - regression_output_custom_mape: 63.6718 - final_output_mse: 0.0182 - final_output_mae: 0.0885 - final_output_rmse: 0.1278 - final_output_custom_mape: 63.9957 - val_loss: 0.0655 - val_classification_output_loss: 0.1041 - val_regression_output_loss: 0.0332 - val_final_output_loss: 0.0474 - val_classification_output_accuracy: 0.9581 - val_classification_output_auc: 0.9955 - val_regression_output_mse: 5.7525 - val_regression_output_mae: 1.6878 - val_regression_output_rmse: 2.3943 - val_regression_output_custom_mape: 62.4874 - val_final_output_mse: 0.0233 - val_final_output_mae: 0.0817 - val_final_output_rmse: 0.1449 - val_final_output_custom_mape: 62.4108 - lr: 1.0000e-04\n",
      "Epoch 48/100\n",
      "541/541 [==============================] - 53s 99ms/step - loss: 0.0480 - classification_output_loss: 0.0625 - regression_output_loss: 0.0275 - final_output_loss: 0.0264 - classification_output_accuracy: 0.9739 - classification_output_auc: 0.9975 - regression_output_mse: 6.6081 - regression_output_mae: 1.8378 - regression_output_rmse: 2.5612 - regression_output_custom_mape: 63.5768 - final_output_mse: 0.0170 - final_output_mae: 0.0853 - final_output_rmse: 0.1224 - final_output_custom_mape: 63.9522 - val_loss: 0.0500 - val_classification_output_loss: 0.0769 - val_regression_output_loss: 0.0292 - val_final_output_loss: 0.0217 - val_classification_output_accuracy: 0.9676 - val_classification_output_auc: 0.9960 - val_regression_output_mse: 6.5384 - val_regression_output_mae: 1.7942 - val_regression_output_rmse: 2.5409 - val_regression_output_custom_mape: 61.9763 - val_final_output_mse: 0.0115 - val_final_output_mae: 0.0710 - val_final_output_rmse: 0.1040 - val_final_output_custom_mape: 62.2404 - lr: 1.0000e-04\n",
      "Epoch 49/100\n",
      "541/541 [==============================] - 51s 95ms/step - loss: 0.0474 - classification_output_loss: 0.0564 - regression_output_loss: 0.0296 - final_output_loss: 0.0263 - classification_output_accuracy: 0.9760 - classification_output_auc: 0.9980 - regression_output_mse: 6.7164 - regression_output_mae: 1.8615 - regression_output_rmse: 2.5813 - regression_output_custom_mape: 63.5453 - final_output_mse: 0.0171 - final_output_mae: 0.0857 - final_output_rmse: 0.1235 - final_output_custom_mape: 63.7933 - val_loss: 0.0498 - val_classification_output_loss: 0.0727 - val_regression_output_loss: 0.0283 - val_final_output_loss: 0.0260 - val_classification_output_accuracy: 0.9693 - val_classification_output_auc: 0.9961 - val_regression_output_mse: 6.5693 - val_regression_output_mae: 1.8158 - val_regression_output_rmse: 2.5473 - val_regression_output_custom_mape: 64.0865 - val_final_output_mse: 0.0165 - val_final_output_mae: 0.0844 - val_final_output_rmse: 0.1243 - val_final_output_custom_mape: 64.5164 - lr: 1.0000e-04\n",
      "Epoch 50/100\n",
      "541/541 [==============================] - 49s 91ms/step - loss: 0.0395 - classification_output_loss: 0.0549 - regression_output_loss: 0.0169 - final_output_loss: 0.0237 - classification_output_accuracy: 0.9766 - classification_output_auc: 0.9981 - regression_output_mse: 6.7595 - regression_output_mae: 1.8651 - regression_output_rmse: 2.5900 - regression_output_custom_mape: 63.1470 - final_output_mse: 0.0132 - final_output_mae: 0.0776 - final_output_rmse: 0.1109 - final_output_custom_mape: 63.3583 - val_loss: 0.0451 - val_classification_output_loss: 0.0768 - val_regression_output_loss: 0.0201 - val_final_output_loss: 0.0230 - val_classification_output_accuracy: 0.9683 - val_classification_output_auc: 0.9956 - val_regression_output_mse: 6.4914 - val_regression_output_mae: 1.8122 - val_regression_output_rmse: 2.5338 - val_regression_output_custom_mape: 62.2374 - val_final_output_mse: 0.0132 - val_final_output_mae: 0.0753 - val_final_output_rmse: 0.1099 - val_final_output_custom_mape: 62.2972 - lr: 1.0000e-04\n",
      "Epoch 51/100\n",
      "541/541 [==============================] - ETA: 0s - loss: 0.0401 - classification_output_loss: 0.0564 - regression_output_loss: 0.0186 - final_output_loss: 0.0232 - classification_output_accuracy: 0.9757 - classification_output_auc: 0.9979 - regression_output_mse: 6.6958 - regression_output_mae: 1.8472 - regression_output_rmse: 2.5775 - regression_output_custom_mape: 62.6248 - final_output_mse: 0.0125 - final_output_mae: 0.0761 - final_output_rmse: 0.1077 - final_output_custom_mape: 62.9354\n",
      "Epoch 51 Detailed Metrics:\n",
      "541/541 [==============================] - 55s 102ms/step - loss: 0.0401 - classification_output_loss: 0.0564 - regression_output_loss: 0.0186 - final_output_loss: 0.0232 - classification_output_accuracy: 0.9757 - classification_output_auc: 0.9979 - regression_output_mse: 6.6958 - regression_output_mae: 1.8472 - regression_output_rmse: 2.5775 - regression_output_custom_mape: 62.6248 - final_output_mse: 0.0125 - final_output_mae: 0.0761 - final_output_rmse: 0.1077 - final_output_custom_mape: 62.9354 - val_loss: 0.0727 - val_classification_output_loss: 0.0944 - val_regression_output_loss: 0.0562 - val_final_output_loss: 0.0432 - val_classification_output_accuracy: 0.9653 - val_classification_output_auc: 0.9951 - val_regression_output_mse: 4.2029 - val_regression_output_mae: 1.5086 - val_regression_output_rmse: 2.0452 - val_regression_output_custom_mape: 67.4825 - val_final_output_mse: 0.0431 - val_final_output_mae: 0.1288 - val_final_output_rmse: 0.1947 - val_final_output_custom_mape: 67.4020 - lr: 1.0000e-04\n",
      "Epoch 52/100\n",
      "541/541 [==============================] - 55s 101ms/step - loss: 0.0518 - classification_output_loss: 0.0650 - regression_output_loss: 0.0344 - final_output_loss: 0.0286 - classification_output_accuracy: 0.9733 - classification_output_auc: 0.9972 - regression_output_mse: 6.6534 - regression_output_mae: 1.8507 - regression_output_rmse: 2.5694 - regression_output_custom_mape: 64.0731 - final_output_mse: 0.0204 - final_output_mae: 0.0922 - final_output_rmse: 0.1333 - final_output_custom_mape: 64.4775 - val_loss: 0.0449 - val_classification_output_loss: 0.0869 - val_regression_output_loss: 0.0155 - val_final_output_loss: 0.0226 - val_classification_output_accuracy: 0.9648 - val_classification_output_auc: 0.9948 - val_regression_output_mse: 6.4651 - val_regression_output_mae: 1.8048 - val_regression_output_rmse: 2.5262 - val_regression_output_custom_mape: 62.5471 - val_final_output_mse: 0.0125 - val_final_output_mae: 0.0736 - val_final_output_rmse: 0.1087 - val_final_output_custom_mape: 62.6503 - lr: 1.0000e-04\n",
      "Epoch 53/100\n",
      "541/541 [==============================] - 53s 99ms/step - loss: 0.0411 - classification_output_loss: 0.0548 - regression_output_loss: 0.0207 - final_output_loss: 0.0237 - classification_output_accuracy: 0.9775 - classification_output_auc: 0.9980 - regression_output_mse: 6.7604 - regression_output_mae: 1.8647 - regression_output_rmse: 2.5895 - regression_output_custom_mape: 62.7709 - final_output_mse: 0.0133 - final_output_mae: 0.0777 - final_output_rmse: 0.1099 - final_output_custom_mape: 62.9869 - val_loss: 0.0771 - val_classification_output_loss: 0.0905 - val_regression_output_loss: 0.0766 - val_final_output_loss: 0.0281 - val_classification_output_accuracy: 0.9636 - val_classification_output_auc: 0.9944 - val_regression_output_mse: 6.3203 - val_regression_output_mae: 1.8295 - val_regression_output_rmse: 2.5019 - val_regression_output_custom_mape: 64.9538 - val_final_output_mse: 0.0197 - val_final_output_mae: 0.0919 - val_final_output_rmse: 0.1340 - val_final_output_custom_mape: 64.7748 - lr: 1.0000e-04\n",
      "Epoch 54/100\n",
      "541/541 [==============================] - 52s 96ms/step - loss: 0.0483 - classification_output_loss: 0.0616 - regression_output_loss: 0.0307 - final_output_loss: 0.0264 - classification_output_accuracy: 0.9738 - classification_output_auc: 0.9975 - regression_output_mse: 6.6227 - regression_output_mae: 1.8377 - regression_output_rmse: 2.5629 - regression_output_custom_mape: 63.1048 - final_output_mse: 0.0169 - final_output_mae: 0.0860 - final_output_rmse: 0.1239 - final_output_custom_mape: 63.6682 - val_loss: 0.0536 - val_classification_output_loss: 0.0971 - val_regression_output_loss: 0.0289 - val_final_output_loss: 0.0225 - val_classification_output_accuracy: 0.9632 - val_classification_output_auc: 0.9942 - val_regression_output_mse: 6.6649 - val_regression_output_mae: 1.8270 - val_regression_output_rmse: 2.5626 - val_regression_output_custom_mape: 62.0142 - val_final_output_mse: 0.0125 - val_final_output_mae: 0.0733 - val_final_output_rmse: 0.1081 - val_final_output_custom_mape: 62.4347 - lr: 1.0000e-04\n",
      "Epoch 55/100\n",
      "541/541 [==============================] - ETA: 0s - loss: 0.0466 - classification_output_loss: 0.0573 - regression_output_loss: 0.0293 - final_output_loss: 0.0257 - classification_output_accuracy: 0.9759 - classification_output_auc: 0.9979 - regression_output_mse: 6.7103 - regression_output_mae: 1.8587 - regression_output_rmse: 2.5801 - regression_output_custom_mape: 63.0620 - final_output_mse: 0.0164 - final_output_mae: 0.0839 - final_output_rmse: 0.1207 - final_output_custom_mape: 63.4037\n",
      "Epoch 55: ReduceLROnPlateau reducing learning rate to 4.999999873689376e-05.\n",
      "541/541 [==============================] - 54s 99ms/step - loss: 0.0466 - classification_output_loss: 0.0573 - regression_output_loss: 0.0293 - final_output_loss: 0.0257 - classification_output_accuracy: 0.9759 - classification_output_auc: 0.9979 - regression_output_mse: 6.7103 - regression_output_mae: 1.8587 - regression_output_rmse: 2.5801 - regression_output_custom_mape: 63.0620 - final_output_mse: 0.0164 - final_output_mae: 0.0839 - final_output_rmse: 0.1207 - final_output_custom_mape: 63.4037 - val_loss: 0.0547 - val_classification_output_loss: 0.1083 - val_regression_output_loss: 0.0260 - val_final_output_loss: 0.0251 - val_classification_output_accuracy: 0.9587 - val_classification_output_auc: 0.9929 - val_regression_output_mse: 6.7626 - val_regression_output_mae: 1.8696 - val_regression_output_rmse: 2.5790 - val_regression_output_custom_mape: 64.1653 - val_final_output_mse: 0.0161 - val_final_output_mae: 0.0809 - val_final_output_rmse: 0.1229 - val_final_output_custom_mape: 64.2964 - lr: 1.0000e-04\n",
      "Epoch 56/100\n",
      "541/541 [==============================] - 57s 106ms/step - loss: 0.0357 - classification_output_loss: 0.0520 - regression_output_loss: 0.0131 - final_output_loss: 0.0217 - classification_output_accuracy: 0.9775 - classification_output_auc: 0.9983 - regression_output_mse: 6.7730 - regression_output_mae: 1.8622 - regression_output_rmse: 2.5912 - regression_output_custom_mape: 62.1226 - final_output_mse: 0.0110 - final_output_mae: 0.0715 - final_output_rmse: 0.1009 - final_output_custom_mape: 62.2862 - val_loss: 0.0406 - val_classification_output_loss: 0.0722 - val_regression_output_loss: 0.0164 - val_final_output_loss: 0.0202 - val_classification_output_accuracy: 0.9707 - val_classification_output_auc: 0.9960 - val_regression_output_mse: 6.5865 - val_regression_output_mae: 1.8079 - val_regression_output_rmse: 2.5506 - val_regression_output_custom_mape: 62.1180 - val_final_output_mse: 0.0098 - val_final_output_mae: 0.0662 - val_final_output_rmse: 0.0959 - val_final_output_custom_mape: 62.3044 - lr: 5.0000e-05\n",
      "Epoch 57/100\n",
      "541/541 [==============================] - 56s 103ms/step - loss: 0.0331 - classification_output_loss: 0.0515 - regression_output_loss: 0.0100 - final_output_loss: 0.0206 - classification_output_accuracy: 0.9785 - classification_output_auc: 0.9983 - regression_output_mse: 6.7417 - regression_output_mae: 1.8522 - regression_output_rmse: 2.5856 - regression_output_custom_mape: 61.5541 - final_output_mse: 0.0094 - final_output_mae: 0.0674 - final_output_rmse: 0.0938 - final_output_custom_mape: 61.7115 - val_loss: 0.0406 - val_classification_output_loss: 0.0763 - val_regression_output_loss: 0.0160 - val_final_output_loss: 0.0196 - val_classification_output_accuracy: 0.9690 - val_classification_output_auc: 0.9960 - val_regression_output_mse: 6.5545 - val_regression_output_mae: 1.7924 - val_regression_output_rmse: 2.5441 - val_regression_output_custom_mape: 61.5195 - val_final_output_mse: 0.0089 - val_final_output_mae: 0.0644 - val_final_output_rmse: 0.0920 - val_final_output_custom_mape: 61.8914 - lr: 5.0000e-05\n",
      "Epoch 58/100\n",
      "541/541 [==============================] - 54s 100ms/step - loss: 0.0325 - classification_output_loss: 0.0506 - regression_output_loss: 0.0099 - final_output_loss: 0.0203 - classification_output_accuracy: 0.9789 - classification_output_auc: 0.9983 - regression_output_mse: 6.7571 - regression_output_mae: 1.8542 - regression_output_rmse: 2.5887 - regression_output_custom_mape: 61.4581 - final_output_mse: 0.0094 - final_output_mae: 0.0670 - final_output_rmse: 0.0932 - final_output_custom_mape: 61.6506 - val_loss: 0.0405 - val_classification_output_loss: 0.0714 - val_regression_output_loss: 0.0185 - val_final_output_loss: 0.0197 - val_classification_output_accuracy: 0.9710 - val_classification_output_auc: 0.9961 - val_regression_output_mse: 6.6865 - val_regression_output_mae: 1.8298 - val_regression_output_rmse: 2.5699 - val_regression_output_custom_mape: 62.0339 - val_final_output_mse: 0.0091 - val_final_output_mae: 0.0648 - val_final_output_rmse: 0.0928 - val_final_output_custom_mape: 62.0572 - lr: 5.0000e-05\n",
      "Epoch 59/100\n",
      "541/541 [==============================] - 54s 99ms/step - loss: 0.0322 - classification_output_loss: 0.0514 - regression_output_loss: 0.0097 - final_output_loss: 0.0201 - classification_output_accuracy: 0.9784 - classification_output_auc: 0.9983 - regression_output_mse: 6.7708 - regression_output_mae: 1.8569 - regression_output_rmse: 2.5913 - regression_output_custom_mape: 61.4620 - final_output_mse: 0.0090 - final_output_mae: 0.0665 - final_output_rmse: 0.0918 - final_output_custom_mape: 61.6222 - val_loss: 0.0479 - val_classification_output_loss: 0.0727 - val_regression_output_loss: 0.0320 - val_final_output_loss: 0.0218 - val_classification_output_accuracy: 0.9702 - val_classification_output_auc: 0.9959 - val_regression_output_mse: 6.6692 - val_regression_output_mae: 1.8295 - val_regression_output_rmse: 2.5661 - val_regression_output_custom_mape: 63.2262 - val_final_output_mse: 0.0109 - val_final_output_mae: 0.0716 - val_final_output_rmse: 0.1014 - val_final_output_custom_mape: 63.3540 - lr: 5.0000e-05\n",
      "Epoch 60/100\n",
      "541/541 [==============================] - 54s 100ms/step - loss: 0.0336 - classification_output_loss: 0.0512 - regression_output_loss: 0.0125 - final_output_loss: 0.0210 - classification_output_accuracy: 0.9784 - classification_output_auc: 0.9983 - regression_output_mse: 6.7842 - regression_output_mae: 1.8627 - regression_output_rmse: 2.5937 - regression_output_custom_mape: 61.7020 - final_output_mse: 0.0100 - final_output_mae: 0.0691 - final_output_rmse: 0.0961 - final_output_custom_mape: 61.8323 - val_loss: 0.0496 - val_classification_output_loss: 0.0797 - val_regression_output_loss: 0.0308 - val_final_output_loss: 0.0249 - val_classification_output_accuracy: 0.9691 - val_classification_output_auc: 0.9953 - val_regression_output_mse: 6.8019 - val_regression_output_mae: 1.8752 - val_regression_output_rmse: 2.5909 - val_regression_output_custom_mape: 66.9446 - val_final_output_mse: 0.0148 - val_final_output_mae: 0.0816 - val_final_output_rmse: 0.1192 - val_final_output_custom_mape: 67.0260 - lr: 5.0000e-05\n",
      "Epoch 61/100\n",
      "541/541 [==============================] - ETA: 0s - loss: 0.0320 - classification_output_loss: 0.0501 - regression_output_loss: 0.0103 - final_output_loss: 0.0206 - classification_output_accuracy: 0.9793 - classification_output_auc: 0.9984 - regression_output_mse: 6.7789 - regression_output_mae: 1.8593 - regression_output_rmse: 2.5925 - regression_output_custom_mape: 61.5656 - final_output_mse: 0.0095 - final_output_mae: 0.0677 - final_output_rmse: 0.0939 - final_output_custom_mape: 61.7795\n",
      "Epoch 61 Detailed Metrics:\n",
      "541/541 [==============================] - 55s 101ms/step - loss: 0.0320 - classification_output_loss: 0.0501 - regression_output_loss: 0.0103 - final_output_loss: 0.0206 - classification_output_accuracy: 0.9793 - classification_output_auc: 0.9984 - regression_output_mse: 6.7789 - regression_output_mae: 1.8593 - regression_output_rmse: 2.5925 - regression_output_custom_mape: 61.5656 - final_output_mse: 0.0095 - final_output_mae: 0.0677 - final_output_rmse: 0.0939 - final_output_custom_mape: 61.7795 - val_loss: 0.0505 - val_classification_output_loss: 0.0742 - val_regression_output_loss: 0.0356 - val_final_output_loss: 0.0244 - val_classification_output_accuracy: 0.9709 - val_classification_output_auc: 0.9958 - val_regression_output_mse: 6.6354 - val_regression_output_mae: 1.8309 - val_regression_output_rmse: 2.5603 - val_regression_output_custom_mape: 65.7531 - val_final_output_mse: 0.0137 - val_final_output_mae: 0.0801 - val_final_output_rmse: 0.1142 - val_final_output_custom_mape: 65.9202 - lr: 5.0000e-05\n",
      "Epoch 62/100\n",
      "541/541 [==============================] - 53s 98ms/step - loss: 0.0332 - classification_output_loss: 0.0518 - regression_output_loss: 0.0119 - final_output_loss: 0.0209 - classification_output_accuracy: 0.9784 - classification_output_auc: 0.9983 - regression_output_mse: 6.7770 - regression_output_mae: 1.8601 - regression_output_rmse: 2.5926 - regression_output_custom_mape: 61.7351 - final_output_mse: 0.0097 - final_output_mae: 0.0689 - final_output_rmse: 0.0954 - final_output_custom_mape: 61.8624 - val_loss: 0.0481 - val_classification_output_loss: 0.0746 - val_regression_output_loss: 0.0310 - val_final_output_loss: 0.0238 - val_classification_output_accuracy: 0.9690 - val_classification_output_auc: 0.9959 - val_regression_output_mse: 6.7120 - val_regression_output_mae: 1.8451 - val_regression_output_rmse: 2.5737 - val_regression_output_custom_mape: 65.7528 - val_final_output_mse: 0.0131 - val_final_output_mae: 0.0782 - val_final_output_rmse: 0.1123 - val_final_output_custom_mape: 65.8633 - lr: 5.0000e-05\n",
      "Epoch 63/100\n",
      "541/541 [==============================] - 56s 104ms/step - loss: 0.0304 - classification_output_loss: 0.0504 - regression_output_loss: 0.0079 - final_output_loss: 0.0198 - classification_output_accuracy: 0.9785 - classification_output_auc: 0.9984 - regression_output_mse: 6.7759 - regression_output_mae: 1.8572 - regression_output_rmse: 2.5922 - regression_output_custom_mape: 61.3674 - final_output_mse: 0.0087 - final_output_mae: 0.0655 - final_output_rmse: 0.0902 - final_output_custom_mape: 61.5143 - val_loss: 0.0364 - val_classification_output_loss: 0.0714 - val_regression_output_loss: 0.0121 - val_final_output_loss: 0.0193 - val_classification_output_accuracy: 0.9703 - val_classification_output_auc: 0.9962 - val_regression_output_mse: 6.6819 - val_regression_output_mae: 1.8267 - val_regression_output_rmse: 2.5687 - val_regression_output_custom_mape: 62.2443 - val_final_output_mse: 0.0086 - val_final_output_mae: 0.0635 - val_final_output_rmse: 0.0902 - val_final_output_custom_mape: 62.3224 - lr: 5.0000e-05\n",
      "Epoch 64/100\n",
      "541/541 [==============================] - 56s 103ms/step - loss: 0.0309 - classification_output_loss: 0.0507 - regression_output_loss: 0.0090 - final_output_loss: 0.0198 - classification_output_accuracy: 0.9783 - classification_output_auc: 0.9984 - regression_output_mse: 6.7723 - regression_output_mae: 1.8552 - regression_output_rmse: 2.5911 - regression_output_custom_mape: 61.1519 - final_output_mse: 0.0086 - final_output_mae: 0.0653 - final_output_rmse: 0.0898 - final_output_custom_mape: 61.3432 - val_loss: 0.0393 - val_classification_output_loss: 0.0725 - val_regression_output_loss: 0.0162 - val_final_output_loss: 0.0217 - val_classification_output_accuracy: 0.9709 - val_classification_output_auc: 0.9960 - val_regression_output_mse: 6.7377 - val_regression_output_mae: 1.8477 - val_regression_output_rmse: 2.5799 - val_regression_output_custom_mape: 64.4454 - val_final_output_mse: 0.0108 - val_final_output_mae: 0.0712 - val_final_output_rmse: 0.1021 - val_final_output_custom_mape: 64.4477 - lr: 5.0000e-05\n",
      "Epoch 65/100\n",
      "541/541 [==============================] - 54s 100ms/step - loss: 0.0312 - classification_output_loss: 0.0508 - regression_output_loss: 0.0096 - final_output_loss: 0.0202 - classification_output_accuracy: 0.9785 - classification_output_auc: 0.9983 - regression_output_mse: 6.7588 - regression_output_mae: 1.8526 - regression_output_rmse: 2.5891 - regression_output_custom_mape: 61.3250 - final_output_mse: 0.0091 - final_output_mae: 0.0665 - final_output_rmse: 0.0922 - final_output_custom_mape: 61.5369 - val_loss: 0.0398 - val_classification_output_loss: 0.0764 - val_regression_output_loss: 0.0174 - val_final_output_loss: 0.0194 - val_classification_output_accuracy: 0.9673 - val_classification_output_auc: 0.9959 - val_regression_output_mse: 6.6505 - val_regression_output_mae: 1.8149 - val_regression_output_rmse: 2.5623 - val_regression_output_custom_mape: 61.7415 - val_final_output_mse: 0.0087 - val_final_output_mae: 0.0638 - val_final_output_rmse: 0.0908 - val_final_output_custom_mape: 61.9581 - lr: 5.0000e-05\n",
      "Epoch 66/100\n",
      "541/541 [==============================] - 54s 100ms/step - loss: 0.0352 - classification_output_loss: 0.0543 - regression_output_loss: 0.0094 - final_output_loss: 0.0300 - classification_output_accuracy: 0.9775 - classification_output_auc: 0.9981 - regression_output_mse: 6.6501 - regression_output_mae: 1.8334 - regression_output_rmse: 2.5658 - regression_output_custom_mape: 61.4441 - final_output_mse: 0.0123 - final_output_mae: 0.0678 - final_output_rmse: 0.0945 - final_output_custom_mape: 61.6992 - val_loss: 0.0378 - val_classification_output_loss: 0.0773 - val_regression_output_loss: 0.0096 - val_final_output_loss: 0.0197 - val_classification_output_accuracy: 0.9685 - val_classification_output_auc: 0.9959 - val_regression_output_mse: 6.5211 - val_regression_output_mae: 1.7843 - val_regression_output_rmse: 2.5370 - val_regression_output_custom_mape: 61.5626 - val_final_output_mse: 0.0090 - val_final_output_mae: 0.0649 - val_final_output_rmse: 0.0924 - val_final_output_custom_mape: 62.0307 - lr: 5.0000e-05\n",
      "Epoch 67/100\n",
      "541/541 [==============================] - 55s 102ms/step - loss: 0.0315 - classification_output_loss: 0.0499 - regression_output_loss: 0.0088 - final_output_loss: 0.0201 - classification_output_accuracy: 0.9785 - classification_output_auc: 0.9985 - regression_output_mse: 6.7018 - regression_output_mae: 1.8423 - regression_output_rmse: 2.5784 - regression_output_custom_mape: 61.3784 - final_output_mse: 0.0090 - final_output_mae: 0.0662 - final_output_rmse: 0.0914 - final_output_custom_mape: 61.5973 - val_loss: 0.0422 - val_classification_output_loss: 0.0714 - val_regression_output_loss: 0.0222 - val_final_output_loss: 0.0203 - val_classification_output_accuracy: 0.9717 - val_classification_output_auc: 0.9961 - val_regression_output_mse: 6.6187 - val_regression_output_mae: 1.8151 - val_regression_output_rmse: 2.5569 - val_regression_output_custom_mape: 62.3132 - val_final_output_mse: 0.0093 - val_final_output_mae: 0.0670 - val_final_output_rmse: 0.0939 - val_final_output_custom_mape: 62.4799 - lr: 5.0000e-05\n",
      "Epoch 68/100\n",
      "541/541 [==============================] - 56s 104ms/step - loss: 0.0337 - classification_output_loss: 0.0514 - regression_output_loss: 0.0130 - final_output_loss: 0.0211 - classification_output_accuracy: 0.9783 - classification_output_auc: 0.9983 - regression_output_mse: 6.7455 - regression_output_mae: 1.8556 - regression_output_rmse: 2.5868 - regression_output_custom_mape: 61.6488 - final_output_mse: 0.0101 - final_output_mae: 0.0693 - final_output_rmse: 0.0959 - final_output_custom_mape: 61.7737 - val_loss: 0.0471 - val_classification_output_loss: 0.0707 - val_regression_output_loss: 0.0315 - val_final_output_loss: 0.0225 - val_classification_output_accuracy: 0.9718 - val_classification_output_auc: 0.9964 - val_regression_output_mse: 6.6541 - val_regression_output_mae: 1.8264 - val_regression_output_rmse: 2.5656 - val_regression_output_custom_mape: 63.9875 - val_final_output_mse: 0.0115 - val_final_output_mae: 0.0740 - val_final_output_rmse: 0.1040 - val_final_output_custom_mape: 64.1717 - lr: 5.0000e-05\n",
      "Epoch 69/100\n",
      "462/541 [========================>.....] - ETA: 7s - loss: 0.0330 - classification_output_loss: 0.0495 - regression_output_loss: 0.0129 - final_output_loss: 0.0208 - classification_output_accuracy: 0.9793 - classification_output_auc: 0.9984 - regression_output_mse: 6.8607 - regression_output_mae: 1.8819 - regression_output_rmse: 2.6091 - regression_output_custom_mape: 61.7354 - final_output_mse: 0.0096 - final_output_mae: 0.0686 - final_output_rmse: 0.0946 - final_output_custom_mape: 61.9095"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "541/541 [==============================] - 56s 104ms/step - loss: 0.0270 - classification_output_loss: 0.0498 - regression_output_loss: 0.0037 - final_output_loss: 0.0185 - classification_output_accuracy: 0.9783 - classification_output_auc: 0.9984 - regression_output_mse: 6.8101 - regression_output_mae: 1.8609 - regression_output_rmse: 2.5989 - regression_output_custom_mape: 60.8799 - final_output_mse: 0.0074 - final_output_mae: 0.0611 - final_output_rmse: 0.0833 - final_output_custom_mape: 60.9907 - val_loss: 0.0326 - val_classification_output_loss: 0.0688 - val_regression_output_loss: 0.0070 - val_final_output_loss: 0.0193 - val_classification_output_accuracy: 0.9723 - val_classification_output_auc: 0.9961 - val_regression_output_mse: 6.7669 - val_regression_output_mae: 1.8484 - val_regression_output_rmse: 2.5858 - val_regression_output_custom_mape: 62.3941 - val_final_output_mse: 0.0084 - val_final_output_mae: 0.0634 - val_final_output_rmse: 0.0894 - val_final_output_custom_mape: 62.3456 - lr: 2.5000e-05\n",
      "Epoch 74/100\n",
      "541/541 [==============================] - 54s 100ms/step - loss: 0.0266 - classification_output_loss: 0.0496 - regression_output_loss: 0.0034 - final_output_loss: 0.0184 - classification_output_accuracy: 0.9787 - classification_output_auc: 0.9984 - regression_output_mse: 6.8160 - regression_output_mae: 1.8619 - regression_output_rmse: 2.6000 - regression_output_custom_mape: 60.7902 - final_output_mse: 0.0072 - final_output_mae: 0.0608 - final_output_rmse: 0.0824 - final_output_custom_mape: 60.9197 - val_loss: 0.0329 - val_classification_output_loss: 0.0714 - val_regression_output_loss: 0.0070 - val_final_output_loss: 0.0189 - val_classification_output_accuracy: 0.9700 - val_classification_output_auc: 0.9961 - val_regression_output_mse: 6.7283 - val_regression_output_mae: 1.8351 - val_regression_output_rmse: 2.5782 - val_regression_output_custom_mape: 61.8653 - val_final_output_mse: 0.0081 - val_final_output_mae: 0.0621 - val_final_output_rmse: 0.0877 - val_final_output_custom_mape: 61.8723 - lr: 2.5000e-05\n",
      "Epoch 75/100\n",
      "541/541 [==============================] - 58s 108ms/step - loss: 0.0266 - classification_output_loss: 0.0498 - regression_output_loss: 0.0035 - final_output_loss: 0.0184 - classification_output_accuracy: 0.9788 - classification_output_auc: 0.9984 - regression_output_mse: 6.8235 - regression_output_mae: 1.8647 - regression_output_rmse: 2.6014 - regression_output_custom_mape: 60.8557 - final_output_mse: 0.0072 - final_output_mae: 0.0609 - final_output_rmse: 0.0826 - final_output_custom_mape: 60.9257 - val_loss: 0.0314 - val_classification_output_loss: 0.0692 - val_regression_output_loss: 0.0054 - val_final_output_loss: 0.0183 - val_classification_output_accuracy: 0.9728 - val_classification_output_auc: 0.9963 - val_regression_output_mse: 6.7276 - val_regression_output_mae: 1.8331 - val_regression_output_rmse: 2.5784 - val_regression_output_custom_mape: 61.4127 - val_final_output_mse: 0.0076 - val_final_output_mae: 0.0603 - val_final_output_rmse: 0.0848 - val_final_output_custom_mape: 61.4123 - lr: 2.5000e-05\n",
      "Epoch 76/100\n",
      "541/541 [==============================] - 55s 102ms/step - loss: 0.0259 - classification_output_loss: 0.0485 - regression_output_loss: 0.0029 - final_output_loss: 0.0182 - classification_output_accuracy: 0.9793 - classification_output_auc: 0.9985 - regression_output_mse: 6.8060 - regression_output_mae: 1.8586 - regression_output_rmse: 2.5978 - regression_output_custom_mape: 60.6619 - final_output_mse: 0.0070 - final_output_mae: 0.0601 - final_output_rmse: 0.0815 - final_output_custom_mape: 60.7422 - val_loss: 0.0325 - val_classification_output_loss: 0.0705 - val_regression_output_loss: 0.0072 - val_final_output_loss: 0.0185 - val_classification_output_accuracy: 0.9713 - val_classification_output_auc: 0.9962 - val_regression_output_mse: 6.6720 - val_regression_output_mae: 1.8182 - val_regression_output_rmse: 2.5674 - val_regression_output_custom_mape: 61.5169 - val_final_output_mse: 0.0076 - val_final_output_mae: 0.0608 - val_final_output_rmse: 0.0849 - val_final_output_custom_mape: 61.6409 - lr: 2.5000e-05\n",
      "Epoch 77/100\n",
      "541/541 [==============================] - 57s 105ms/step - loss: 0.0261 - classification_output_loss: 0.0495 - regression_output_loss: 0.0031 - final_output_loss: 0.0182 - classification_output_accuracy: 0.9785 - classification_output_auc: 0.9984 - regression_output_mse: 6.7979 - regression_output_mae: 1.8569 - regression_output_rmse: 2.5965 - regression_output_custom_mape: 60.6774 - final_output_mse: 0.0070 - final_output_mae: 0.0603 - final_output_rmse: 0.0815 - final_output_custom_mape: 60.7930 - val_loss: 0.0312 - val_classification_output_loss: 0.0681 - val_regression_output_loss: 0.0057 - val_final_output_loss: 0.0188 - val_classification_output_accuracy: 0.9739 - val_classification_output_auc: 0.9964 - val_regression_output_mse: 6.6716 - val_regression_output_mae: 1.8195 - val_regression_output_rmse: 2.5681 - val_regression_output_custom_mape: 61.4893 - val_final_output_mse: 0.0079 - val_final_output_mae: 0.0619 - val_final_output_rmse: 0.0865 - val_final_output_custom_mape: 61.6521 - lr: 2.5000e-05\n",
      "Epoch 78/100\n",
      "541/541 [==============================] - 56s 104ms/step - loss: 0.0258 - classification_output_loss: 0.0489 - regression_output_loss: 0.0030 - final_output_loss: 0.0182 - classification_output_accuracy: 0.9790 - classification_output_auc: 0.9984 - regression_output_mse: 6.8116 - regression_output_mae: 1.8599 - regression_output_rmse: 2.5993 - regression_output_custom_mape: 60.6759 - final_output_mse: 0.0071 - final_output_mae: 0.0601 - final_output_rmse: 0.0814 - final_output_custom_mape: 60.7954 - val_loss: 0.0317 - val_classification_output_loss: 0.0688 - val_regression_output_loss: 0.0069 - val_final_output_loss: 0.0181 - val_classification_output_accuracy: 0.9725 - val_classification_output_auc: 0.9963 - val_regression_output_mse: 6.7023 - val_regression_output_mae: 1.8251 - val_regression_output_rmse: 2.5736 - val_regression_output_custom_mape: 61.2798 - val_final_output_mse: 0.0073 - val_final_output_mae: 0.0596 - val_final_output_rmse: 0.0833 - val_final_output_custom_mape: 61.3668 - lr: 2.5000e-05\n",
      "Epoch 79/100\n",
      "541/541 [==============================] - 54s 99ms/step - loss: 0.0256 - classification_output_loss: 0.0491 - regression_output_loss: 0.0027 - final_output_loss: 0.0181 - classification_output_accuracy: 0.9790 - classification_output_auc: 0.9984 - regression_output_mse: 6.8148 - regression_output_mae: 1.8608 - regression_output_rmse: 2.6000 - regression_output_custom_mape: 60.6601 - final_output_mse: 0.0069 - final_output_mae: 0.0598 - final_output_rmse: 0.0806 - final_output_custom_mape: 60.7576 - val_loss: 0.0317 - val_classification_output_loss: 0.0696 - val_regression_output_loss: 0.0068 - val_final_output_loss: 0.0181 - val_classification_output_accuracy: 0.9719 - val_classification_output_auc: 0.9963 - val_regression_output_mse: 6.7091 - val_regression_output_mae: 1.8271 - val_regression_output_rmse: 2.5748 - val_regression_output_custom_mape: 61.2307 - val_final_output_mse: 0.0073 - val_final_output_mae: 0.0595 - val_final_output_rmse: 0.0830 - val_final_output_custom_mape: 61.2448 - lr: 2.5000e-05\n",
      "Epoch 80/100\n",
      "541/541 [==============================] - 56s 104ms/step - loss: 0.0259 - classification_output_loss: 0.0489 - regression_output_loss: 0.0035 - final_output_loss: 0.0182 - classification_output_accuracy: 0.9795 - classification_output_auc: 0.9985 - regression_output_mse: 6.7880 - regression_output_mae: 1.8545 - regression_output_rmse: 2.5947 - regression_output_custom_mape: 60.6354 - final_output_mse: 0.0069 - final_output_mae: 0.0603 - final_output_rmse: 0.0812 - final_output_custom_mape: 60.7862 - val_loss: 0.0316 - val_classification_output_loss: 0.0719 - val_regression_output_loss: 0.0061 - val_final_output_loss: 0.0177 - val_classification_output_accuracy: 0.9700 - val_classification_output_auc: 0.9961 - val_regression_output_mse: 6.6953 - val_regression_output_mae: 1.8237 - val_regression_output_rmse: 2.5712 - val_regression_output_custom_mape: 60.8627 - val_final_output_mse: 0.0070 - val_final_output_mae: 0.0583 - val_final_output_rmse: 0.0812 - val_final_output_custom_mape: 60.9105 - lr: 2.5000e-05\n",
      "Epoch 81/100\n",
      "541/541 [==============================] - ETA: 0s - loss: 0.0262 - classification_output_loss: 0.0505 - regression_output_loss: 0.0034 - final_output_loss: 0.0184 - classification_output_accuracy: 0.9789 - classification_output_auc: 0.9984 - regression_output_mse: 6.7970 - regression_output_mae: 1.8573 - regression_output_rmse: 2.5964 - regression_output_custom_mape: 60.7878 - final_output_mse: 0.0072 - final_output_mae: 0.0608 - final_output_rmse: 0.0823 - final_output_custom_mape: 60.8657\n",
      "Epoch 81 Detailed Metrics:\n",
      "541/541 [==============================] - 52s 95ms/step - loss: 0.0262 - classification_output_loss: 0.0505 - regression_output_loss: 0.0034 - final_output_loss: 0.0184 - classification_output_accuracy: 0.9789 - classification_output_auc: 0.9984 - regression_output_mse: 6.7970 - regression_output_mae: 1.8573 - regression_output_rmse: 2.5964 - regression_output_custom_mape: 60.7878 - final_output_mse: 0.0072 - final_output_mae: 0.0608 - final_output_rmse: 0.0823 - final_output_custom_mape: 60.8657 - val_loss: 0.0309 - val_classification_output_loss: 0.0678 - val_regression_output_loss: 0.0059 - val_final_output_loss: 0.0184 - val_classification_output_accuracy: 0.9726 - val_classification_output_auc: 0.9965 - val_regression_output_mse: 6.7524 - val_regression_output_mae: 1.8408 - val_regression_output_rmse: 2.5829 - val_regression_output_custom_mape: 61.7875 - val_final_output_mse: 0.0076 - val_final_output_mae: 0.0607 - val_final_output_rmse: 0.0848 - val_final_output_custom_mape: 61.7851 - lr: 2.5000e-05\n",
      "Epoch 82/100\n",
      "541/541 [==============================] - 52s 97ms/step - loss: 0.0256 - classification_output_loss: 0.0487 - regression_output_loss: 0.0031 - final_output_loss: 0.0183 - classification_output_accuracy: 0.9789 - classification_output_auc: 0.9985 - regression_output_mse: 6.8378 - regression_output_mae: 1.8668 - regression_output_rmse: 2.6042 - regression_output_custom_mape: 60.8146 - final_output_mse: 0.0070 - final_output_mae: 0.0605 - final_output_rmse: 0.0816 - final_output_custom_mape: 60.8914 - val_loss: 0.0354 - val_classification_output_loss: 0.0695 - val_regression_output_loss: 0.0140 - val_final_output_loss: 0.0189 - val_classification_output_accuracy: 0.9720 - val_classification_output_auc: 0.9964 - val_regression_output_mse: 6.7485 - val_regression_output_mae: 1.8401 - val_regression_output_rmse: 2.5822 - val_regression_output_custom_mape: 62.0076 - val_final_output_mse: 0.0079 - val_final_output_mae: 0.0622 - val_final_output_rmse: 0.0868 - val_final_output_custom_mape: 61.9773 - lr: 2.5000e-05\n",
      "Epoch 83/100\n",
      "541/541 [==============================] - 54s 100ms/step - loss: 0.0263 - classification_output_loss: 0.0488 - regression_output_loss: 0.0043 - final_output_loss: 0.0187 - classification_output_accuracy: 0.9794 - classification_output_auc: 0.9985 - regression_output_mse: 6.8124 - regression_output_mae: 1.8617 - regression_output_rmse: 2.5990 - regression_output_custom_mape: 60.8479 - final_output_mse: 0.0075 - final_output_mae: 0.0616 - final_output_rmse: 0.0838 - final_output_custom_mape: 60.9637 - val_loss: 0.0335 - val_classification_output_loss: 0.0685 - val_regression_output_loss: 0.0107 - val_final_output_loss: 0.0191 - val_classification_output_accuracy: 0.9724 - val_classification_output_auc: 0.9963 - val_regression_output_mse: 6.6785 - val_regression_output_mae: 1.8221 - val_regression_output_rmse: 2.5690 - val_regression_output_custom_mape: 62.2821 - val_final_output_mse: 0.0082 - val_final_output_mae: 0.0628 - val_final_output_rmse: 0.0883 - val_final_output_custom_mape: 62.4273 - lr: 2.5000e-05\n",
      "Epoch 84/100\n",
      "541/541 [==============================] - 54s 100ms/step - loss: 0.0253 - classification_output_loss: 0.0486 - regression_output_loss: 0.0028 - final_output_loss: 0.0182 - classification_output_accuracy: 0.9786 - classification_output_auc: 0.9985 - regression_output_mse: 6.7787 - regression_output_mae: 1.8528 - regression_output_rmse: 2.5927 - regression_output_custom_mape: 60.6893 - final_output_mse: 0.0070 - final_output_mae: 0.0601 - final_output_rmse: 0.0811 - final_output_custom_mape: 60.8322 - val_loss: 0.0320 - val_classification_output_loss: 0.0699 - val_regression_output_loss: 0.0076 - val_final_output_loss: 0.0184 - val_classification_output_accuracy: 0.9719 - val_classification_output_auc: 0.9962 - val_regression_output_mse: 6.6612 - val_regression_output_mae: 1.8178 - val_regression_output_rmse: 2.5656 - val_regression_output_custom_mape: 61.7606 - val_final_output_mse: 0.0075 - val_final_output_mae: 0.0606 - val_final_output_rmse: 0.0845 - val_final_output_custom_mape: 61.8305 - lr: 2.5000e-05\n",
      "Epoch 85/100\n",
      "541/541 [==============================] - 54s 100ms/step - loss: 0.0249 - classification_output_loss: 0.0481 - regression_output_loss: 0.0023 - final_output_loss: 0.0181 - classification_output_accuracy: 0.9792 - classification_output_auc: 0.9985 - regression_output_mse: 6.8113 - regression_output_mae: 1.8599 - regression_output_rmse: 2.5989 - regression_output_custom_mape: 60.7074 - final_output_mse: 0.0069 - final_output_mae: 0.0598 - final_output_rmse: 0.0808 - final_output_custom_mape: 60.7955 - val_loss: 0.0304 - val_classification_output_loss: 0.0693 - val_regression_output_loss: 0.0049 - val_final_output_loss: 0.0180 - val_classification_output_accuracy: 0.9726 - val_classification_output_auc: 0.9964 - val_regression_output_mse: 6.7028 - val_regression_output_mae: 1.8257 - val_regression_output_rmse: 2.5734 - val_regression_output_custom_mape: 60.9782 - val_final_output_mse: 0.0072 - val_final_output_mae: 0.0592 - val_final_output_rmse: 0.0828 - val_final_output_custom_mape: 61.0249 - lr: 2.5000e-05\n",
      "Epoch 86/100\n",
      "541/541 [==============================] - 54s 100ms/step - loss: 0.0249 - classification_output_loss: 0.0487 - regression_output_loss: 0.0023 - final_output_loss: 0.0180 - classification_output_accuracy: 0.9793 - classification_output_auc: 0.9985 - regression_output_mse: 6.8052 - regression_output_mae: 1.8587 - regression_output_rmse: 2.5978 - regression_output_custom_mape: 60.6683 - final_output_mse: 0.0067 - final_output_mae: 0.0597 - final_output_rmse: 0.0800 - final_output_custom_mape: 60.7617 - val_loss: 0.0318 - val_classification_output_loss: 0.0694 - val_regression_output_loss: 0.0074 - val_final_output_loss: 0.0187 - val_classification_output_accuracy: 0.9717 - val_classification_output_auc: 0.9962 - val_regression_output_mse: 6.7475 - val_regression_output_mae: 1.8414 - val_regression_output_rmse: 2.5822 - val_regression_output_custom_mape: 62.4116 - val_final_output_mse: 0.0077 - val_final_output_mae: 0.0615 - val_final_output_rmse: 0.0857 - val_final_output_custom_mape: 62.3664 - lr: 2.5000e-05\n",
      "Epoch 87/100\n",
      "541/541 [==============================] - ETA: 0s - loss: 0.0249 - classification_output_loss: 0.0487 - regression_output_loss: 0.0023 - final_output_loss: 0.0180 - classification_output_accuracy: 0.9785 - classification_output_auc: 0.9985 - regression_output_mse: 6.8167 - regression_output_mae: 1.8610 - regression_output_rmse: 2.6001 - regression_output_custom_mape: 60.7153 - final_output_mse: 0.0068 - final_output_mae: 0.0596 - final_output_rmse: 0.0803 - final_output_custom_mape: 60.7929\n",
      "Epoch 87: ReduceLROnPlateau reducing learning rate to 1.249999968422344e-05.\n",
      "541/541 [==============================] - 56s 104ms/step - loss: 0.0249 - classification_output_loss: 0.0487 - regression_output_loss: 0.0023 - final_output_loss: 0.0180 - classification_output_accuracy: 0.9785 - classification_output_auc: 0.9985 - regression_output_mse: 6.8167 - regression_output_mae: 1.8610 - regression_output_rmse: 2.6001 - regression_output_custom_mape: 60.7153 - final_output_mse: 0.0068 - final_output_mae: 0.0596 - final_output_rmse: 0.0803 - final_output_custom_mape: 60.7929 - val_loss: 0.0304 - val_classification_output_loss: 0.0699 - val_regression_output_loss: 0.0049 - val_final_output_loss: 0.0181 - val_classification_output_accuracy: 0.9722 - val_classification_output_auc: 0.9961 - val_regression_output_mse: 6.7331 - val_regression_output_mae: 1.8344 - val_regression_output_rmse: 2.5791 - val_regression_output_custom_mape: 61.3001 - val_final_output_mse: 0.0073 - val_final_output_mae: 0.0595 - val_final_output_rmse: 0.0831 - val_final_output_custom_mape: 61.3393 - lr: 2.5000e-05\n",
      "Epoch 88/100\n",
      "541/541 [==============================] - 54s 99ms/step - loss: 0.0246 - classification_output_loss: 0.0491 - regression_output_loss: 0.0018 - final_output_loss: 0.0177 - classification_output_accuracy: 0.9784 - classification_output_auc: 0.9984 - regression_output_mse: 6.8066 - regression_output_mae: 1.8585 - regression_output_rmse: 2.5979 - regression_output_custom_mape: 60.5242 - final_output_mse: 0.0065 - final_output_mae: 0.0587 - final_output_rmse: 0.0789 - final_output_custom_mape: 60.5975 - val_loss: 0.0309 - val_classification_output_loss: 0.0703 - val_regression_output_loss: 0.0056 - val_final_output_loss: 0.0183 - val_classification_output_accuracy: 0.9724 - val_classification_output_auc: 0.9960 - val_regression_output_mse: 6.7678 - val_regression_output_mae: 1.8449 - val_regression_output_rmse: 2.5861 - val_regression_output_custom_mape: 61.8515 - val_final_output_mse: 0.0075 - val_final_output_mae: 0.0604 - val_final_output_rmse: 0.0843 - val_final_output_custom_mape: 61.8132 - lr: 1.2500e-05\n",
      "Epoch 89/100\n",
      "541/541 [==============================] - 55s 101ms/step - loss: 0.0243 - classification_output_loss: 0.0484 - regression_output_loss: 0.0016 - final_output_loss: 0.0177 - classification_output_accuracy: 0.9788 - classification_output_auc: 0.9985 - regression_output_mse: 6.8169 - regression_output_mae: 1.8608 - regression_output_rmse: 2.5999 - regression_output_custom_mape: 60.5160 - final_output_mse: 0.0065 - final_output_mae: 0.0586 - final_output_rmse: 0.0786 - final_output_custom_mape: 60.5783 - val_loss: 0.0308 - val_classification_output_loss: 0.0727 - val_regression_output_loss: 0.0047 - val_final_output_loss: 0.0180 - val_classification_output_accuracy: 0.9712 - val_classification_output_auc: 0.9959 - val_regression_output_mse: 6.7522 - val_regression_output_mae: 1.8404 - val_regression_output_rmse: 2.5829 - val_regression_output_custom_mape: 61.3068 - val_final_output_mse: 0.0072 - val_final_output_mae: 0.0594 - val_final_output_rmse: 0.0827 - val_final_output_custom_mape: 61.2601 - lr: 1.2500e-05\n",
      "Epoch 90/100\n",
      "541/541 [==============================] - 55s 102ms/step - loss: 0.0244 - classification_output_loss: 0.0490 - regression_output_loss: 0.0016 - final_output_loss: 0.0177 - classification_output_accuracy: 0.9787 - classification_output_auc: 0.9985 - regression_output_mse: 6.8088 - regression_output_mae: 1.8586 - regression_output_rmse: 2.5983 - regression_output_custom_mape: 60.4771 - final_output_mse: 0.0065 - final_output_mae: 0.0586 - final_output_rmse: 0.0786 - final_output_custom_mape: 60.5605 - val_loss: 0.0306 - val_classification_output_loss: 0.0704 - val_regression_output_loss: 0.0052 - val_final_output_loss: 0.0182 - val_classification_output_accuracy: 0.9721 - val_classification_output_auc: 0.9961 - val_regression_output_mse: 6.7138 - val_regression_output_mae: 1.8292 - val_regression_output_rmse: 2.5759 - val_regression_output_custom_mape: 61.5655 - val_final_output_mse: 0.0073 - val_final_output_mae: 0.0599 - val_final_output_rmse: 0.0833 - val_final_output_custom_mape: 61.6074 - lr: 1.2500e-05\n",
      "Epoch 91/100\n",
      "541/541 [==============================] - ETA: 0s - loss: 0.0240 - classification_output_loss: 0.0485 - regression_output_loss: 0.0012 - final_output_loss: 0.0176 - classification_output_accuracy: 0.9792 - classification_output_auc: 0.9985 - regression_output_mse: 6.8007 - regression_output_mae: 1.8559 - regression_output_rmse: 2.5968 - regression_output_custom_mape: 60.4504 - final_output_mse: 0.0064 - final_output_mae: 0.0583 - final_output_rmse: 0.0780 - final_output_custom_mape: 60.5501\n",
      "Epoch 91 Detailed Metrics:\n",
      "541/541 [==============================] - 55s 102ms/step - loss: 0.0240 - classification_output_loss: 0.0485 - regression_output_loss: 0.0012 - final_output_loss: 0.0176 - classification_output_accuracy: 0.9792 - classification_output_auc: 0.9985 - regression_output_mse: 6.8007 - regression_output_mae: 1.8559 - regression_output_rmse: 2.5968 - regression_output_custom_mape: 60.4504 - final_output_mse: 0.0064 - final_output_mae: 0.0583 - final_output_rmse: 0.0780 - final_output_custom_mape: 60.5501 - val_loss: 0.0300 - val_classification_output_loss: 0.0695 - val_regression_output_loss: 0.0046 - val_final_output_loss: 0.0179 - val_classification_output_accuracy: 0.9721 - val_classification_output_auc: 0.9962 - val_regression_output_mse: 6.7305 - val_regression_output_mae: 1.8337 - val_regression_output_rmse: 2.5790 - val_regression_output_custom_mape: 61.3197 - val_final_output_mse: 0.0071 - val_final_output_mae: 0.0590 - val_final_output_rmse: 0.0820 - val_final_output_custom_mape: 61.3129 - lr: 1.2500e-05\n",
      "Epoch 92/100\n",
      "541/541 [==============================] - 56s 104ms/step - loss: 0.0241 - classification_output_loss: 0.0482 - regression_output_loss: 0.0015 - final_output_loss: 0.0176 - classification_output_accuracy: 0.9792 - classification_output_auc: 0.9986 - regression_output_mse: 6.8190 - regression_output_mae: 1.8612 - regression_output_rmse: 2.6005 - regression_output_custom_mape: 60.5316 - final_output_mse: 0.0064 - final_output_mae: 0.0585 - final_output_rmse: 0.0783 - final_output_custom_mape: 60.5853 - val_loss: 0.0303 - val_classification_output_loss: 0.0688 - val_regression_output_loss: 0.0054 - val_final_output_loss: 0.0183 - val_classification_output_accuracy: 0.9722 - val_classification_output_auc: 0.9962 - val_regression_output_mse: 6.7579 - val_regression_output_mae: 1.8427 - val_regression_output_rmse: 2.5845 - val_regression_output_custom_mape: 61.9738 - val_final_output_mse: 0.0074 - val_final_output_mae: 0.0605 - val_final_output_rmse: 0.0841 - val_final_output_custom_mape: 61.9213 - lr: 1.2500e-05\n",
      "Epoch 93/100\n",
      "541/541 [==============================] - 58s 106ms/step - loss: 0.0240 - classification_output_loss: 0.0480 - regression_output_loss: 0.0015 - final_output_loss: 0.0177 - classification_output_accuracy: 0.9792 - classification_output_auc: 0.9985 - regression_output_mse: 6.8070 - regression_output_mae: 1.8578 - regression_output_rmse: 2.5980 - regression_output_custom_mape: 60.4706 - final_output_mse: 0.0065 - final_output_mae: 0.0585 - final_output_rmse: 0.0784 - final_output_custom_mape: 60.5622 - val_loss: 0.0301 - val_classification_output_loss: 0.0695 - val_regression_output_loss: 0.0050 - val_final_output_loss: 0.0181 - val_classification_output_accuracy: 0.9728 - val_classification_output_auc: 0.9961 - val_regression_output_mse: 6.7989 - val_regression_output_mae: 1.8531 - val_regression_output_rmse: 2.5925 - val_regression_output_custom_mape: 61.7481 - val_final_output_mse: 0.0072 - val_final_output_mae: 0.0595 - val_final_output_rmse: 0.0828 - val_final_output_custom_mape: 61.6569 - lr: 1.2500e-05\n",
      "Epoch 94/100\n",
      "541/541 [==============================] - 55s 102ms/step - loss: 0.0239 - classification_output_loss: 0.0480 - regression_output_loss: 0.0013 - final_output_loss: 0.0177 - classification_output_accuracy: 0.9792 - classification_output_auc: 0.9985 - regression_output_mse: 6.8109 - regression_output_mae: 1.8589 - regression_output_rmse: 2.5988 - regression_output_custom_mape: 60.4838 - final_output_mse: 0.0065 - final_output_mae: 0.0585 - final_output_rmse: 0.0785 - final_output_custom_mape: 60.5632 - val_loss: 0.0301 - val_classification_output_loss: 0.0701 - val_regression_output_loss: 0.0047 - val_final_output_loss: 0.0180 - val_classification_output_accuracy: 0.9723 - val_classification_output_auc: 0.9961 - val_regression_output_mse: 6.7354 - val_regression_output_mae: 1.8344 - val_regression_output_rmse: 2.5800 - val_regression_output_custom_mape: 61.3804 - val_final_output_mse: 0.0071 - val_final_output_mae: 0.0593 - val_final_output_rmse: 0.0824 - val_final_output_custom_mape: 61.3907 - lr: 1.2500e-05\n",
      "Epoch 95/100\n",
      "541/541 [==============================] - ETA: 0s - loss: 0.0236 - classification_output_loss: 0.0476 - regression_output_loss: 0.0012 - final_output_loss: 0.0176 - classification_output_accuracy: 0.9793 - classification_output_auc: 0.9986 - regression_output_mse: 6.8029 - regression_output_mae: 1.8564 - regression_output_rmse: 2.5971 - regression_output_custom_mape: 60.4213 - final_output_mse: 0.0063 - final_output_mae: 0.0582 - final_output_rmse: 0.0778 - final_output_custom_mape: 60.5216Restoring model weights from the end of the best epoch: 80.\n",
      "\n",
      "Epoch 95: ReduceLROnPlateau reducing learning rate to 6.24999984211172e-06.\n",
      "541/541 [==============================] - 52s 96ms/step - loss: 0.0236 - classification_output_loss: 0.0476 - regression_output_loss: 0.0012 - final_output_loss: 0.0176 - classification_output_accuracy: 0.9793 - classification_output_auc: 0.9986 - regression_output_mse: 6.8029 - regression_output_mae: 1.8564 - regression_output_rmse: 2.5971 - regression_output_custom_mape: 60.4213 - final_output_mse: 0.0063 - final_output_mae: 0.0582 - final_output_rmse: 0.0778 - final_output_custom_mape: 60.5216 - val_loss: 0.0304 - val_classification_output_loss: 0.0712 - val_regression_output_loss: 0.0051 - val_final_output_loss: 0.0179 - val_classification_output_accuracy: 0.9720 - val_classification_output_auc: 0.9961 - val_regression_output_mse: 6.6979 - val_regression_output_mae: 1.8236 - val_regression_output_rmse: 2.5727 - val_regression_output_custom_mape: 61.3934 - val_final_output_mse: 0.0070 - val_final_output_mae: 0.0591 - val_final_output_rmse: 0.0818 - val_final_output_custom_mape: 61.4749 - lr: 1.2500e-05\n",
      "Epoch 95: early stopping\n",
      "\n",
      "Training completed successfully!\n",
      "\n",
      "Classification Metrics:\n",
      "Accuracy: 97.00%\n",
      "AUC-ROC: 0.9968\n",
      "\n",
      "Confusion Matrix:\n",
      "[[12503   504]\n",
      " [  275 12651]]\n",
      "\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "        Zero     0.9785    0.9613    0.9698     13007\n",
      "    Non-Zero     0.9617    0.9787    0.9701     12926\n",
      "\n",
      "    accuracy                         0.9700     25933\n",
      "   macro avg     0.9701    0.9700    0.9700     25933\n",
      "weighted avg     0.9701    0.9700    0.9700     25933\n",
      "\n",
      "\n",
      "Regression Metrics (non-zero values):\n",
      "Out of range: 0 predictions\n",
      "MAPE: 23.39%\n",
      "Within ±10%: 54.66%\n",
      "MAE: 0.11\n",
      "RMSE: 0.29\n",
      "\n",
      "Final Combined Output Metrics:\n",
      "Out of range: 0 predictions\n",
      "MAPE: 60.97%\n",
      "Within ±10%: 27.97%\n",
      "MAE: 0.06\n",
      "RMSE: 0.08\n"
     ]
    }
   ],
   "source": [
    "# Model creation\n",
    "print(\"\\n2. Creating model...\")\n",
    "input_shape = (X_train_seq.shape[1], X_train_seq.shape[2])\n",
    "\n",
    "max_val = df['solarradiation'].max()\n",
    "min_val_scaled = scaler_y.transform([[0]])[0][0]\n",
    "max_val_scaled = scaler_y.transform([[max_val]])[0][0]\n",
    "\n",
    "print(f\"\\nMax dataset solar radiation : {max_val} - Scaled Version : {max_val_scaled}\")\n",
    "\n",
    "increase_percentage = 15\n",
    "\n",
    "max_val = max_val * (1 + increase_percentage / 100)\n",
    "max_val_scaled = max_val_scaled * (1 + increase_percentage / 100)\n",
    "\n",
    "print(f\"Max dataset solar radiation increased by {increase_percentage}% : {max_val} - Scaled Version : {max_val_scaled}\")\n",
    "\n",
    "# Create the hybrid model\n",
    "model = create_solarradiation_model(\n",
    "    input_shape=input_shape, \n",
    "    folder_name=folder_name, \n",
    "    min_output=min_val_scaled, \n",
    "    max_output=max_val_scaled\n",
    ")\n",
    "\n",
    "# Prepare binary targets for classification\n",
    "y_train_binary = (y_train > 0).astype(float)\n",
    "y_test_binary = (y_test > 0).astype(float)\n",
    "\n",
    "print(\"\\nClass distribution in training set:\")\n",
    "print(f\"Zeros: {np.sum(y_train_binary == 0)} ({np.mean(y_train_binary == 0)*100:.2f}%)\")\n",
    "print(f\"Non-zeros: {np.sum(y_train_binary == 1)} ({np.mean(y_train_binary == 1)*100:.2f}%)\")\n",
    "\n",
    "print(\"\\nClass distribution in test set:\")\n",
    "print(f\"Zeros: {np.sum(y_test_binary == 0)} ({np.mean(y_test_binary == 0)*100:.2f}%)\")\n",
    "print(f\"Non-zeros: {np.sum(y_test_binary == 1)} ({np.mean(y_test_binary == 1)*100:.2f}%)\")\n",
    "\n",
    "# Get the exact output names from the model\n",
    "output_names = [output.name.split('/')[0] for output in model.outputs]\n",
    "print(\"\\nModel output names:\", output_names)\n",
    "\n",
    "print(\"\\n4. Starting training...\")\n",
    "history = train_hybrid_model(\n",
    "    model=model,\n",
    "    X_train=X_train_seq,\n",
    "    y_train=y_train,\n",
    "    X_test=X_test_seq,\n",
    "    y_test=y_test,\n",
    "    epochs=100,\n",
    "    batch_size=192,\n",
    "    folder_name=folder_name,\n",
    "    min_output=min_val_scaled,\n",
    "    max_output=max_val_scaled\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "958d78b99e8898d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "5. Generating predictions...\n",
      "811/811 [==============================] - 13s 15ms/step\n",
      "\n",
      "6. Evaluating model...\n",
      "\n",
      "Solar Radiation Prediction Metrics:\n",
      "\n",
      "Absolute Metrics:\n",
      "MAE: 19.32 W/m²\n",
      "RMSE: 27.95 W/m²\n",
      "R² Score: 0.989\n",
      "MAPE: 16.92%\n",
      "\n",
      "Accuracy Metrics:\n",
      "Within ±5 W/m²: 8.8%\n",
      "Within ±10 W/m²: 16.3%\n",
      "Within ±20 W/m²: 74.5%\n",
      "\n",
      "Level Accuracy:\n",
      "Level Accuracy: 92.6%\n",
      "\n",
      "Confusion Matrix for Radiation Levels:\n",
      "           Very Low   Low  Moderate  High  Very High  Extreme\n",
      "Very Low          0     0         0     0         10        0\n",
      "Low               0  1494         0   174        153        0\n",
      "Moderate          0     0      2041   413          0      407\n",
      "High              0   215       156  1925          0        0\n",
      "Very High         0    99         0     0       1038        0\n",
      "Extreme           0     0       298     0          0    17510\n",
      "\n",
      "Plot saved as: 2024-11-26_05-41_radiation_analysis.png\n",
      "\n",
      "Error Statistics:\n",
      "Mean error: 4.431\n",
      "Error standard deviation: 27.596\n",
      "Median error: 12.000\n",
      "95th percentile absolute error: 57.806\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n5. Generating predictions...\")\n",
    "predictions = model.predict(X_test_seq)\n",
    "classification_pred, regression_pred, final_pred = predictions\n",
    "\n",
    "# Clip solo le predizioni di regressione e finali\n",
    "regression_pred = np.clip(regression_pred, 0, 11)\n",
    "final_pred = np.clip(final_pred, 0, 11)\n",
    "\n",
    "# Inverse transform per tornare ai valori originali\n",
    "regression_pred_original = scaler_y.inverse_transform(regression_pred)\n",
    "final_pred_original = scaler_y.inverse_transform(final_pred)\n",
    "y_test_original = scaler_y.inverse_transform(y_test)\n",
    "\n",
    "print(\"\\n6. Evaluating model...\")\n",
    "# Valutazione delle predizioni finali\n",
    "metrics = evaluate_solarradiation_predictions(y_test_original, final_pred_original, folder_name=folder_name)\n",
    "\n",
    "# Create results dictionary con metriche aggiuntive per il modello ibrido\n",
    "training_results = {\n",
    "    'model_params': {\n",
    "        'input_shape': input_shape,\n",
    "        'n_features': len(features),\n",
    "        'sequence_length': X_train_seq.shape[1]\n",
    "    },\n",
    "    'training_params': {\n",
    "        'batch_size': 192,\n",
    "        'total_epochs': len(history.history['loss']),\n",
    "        'best_epoch': np.argmin(history.history['val_final_output_loss']) + 1\n",
    "    },\n",
    "    'performance_metrics': {\n",
    "        'classification': {\n",
    "            'final_loss': float(history.history['val_classification_output_loss'][-1]),\n",
    "            'final_accuracy': float(history.history['val_classification_output_accuracy'][-1]),\n",
    "            'final_auc': float(history.history['val_classification_output_auc'][-1])\n",
    "        },\n",
    "        'regression': {\n",
    "            'final_loss': float(history.history['val_regression_output_loss'][-1]),\n",
    "            'final_mae': float(history.history['val_regression_output_mae'][-1]),\n",
    "            'out_of_range_predictions': int(np.sum((regression_pred < 0) | (regression_pred > 11)))\n",
    "        },\n",
    "        'final_output': {\n",
    "            'final_loss': float(history.history['val_final_output_loss'][-1]),\n",
    "            'final_mae': float(history.history['val_final_output_mae'][-1]),\n",
    "            'best_val_loss': float(min(history.history['val_final_output_loss'])),\n",
    "            'out_of_range_predictions': int(np.sum((final_pred < 0) | (final_pred > 11)))\n",
    "        }\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5c05d1d03336b1e4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "7. Predicting missing data...\n",
      "7122/7122 [==============================] - 112s 16ms/step\n",
      "\n",
      "8. Integrating predictions into original dataset...\n",
      "\n",
      "Prediction Integration Statistics:\n",
      "Added 227879 predictions to dataset\n",
      "Rows with solar radiation after integration: 357615\n",
      "\n",
      "Filled Values Analysis:\n",
      "Zero predictions (classification < 0.5): 113630\n",
      "Non-zero predictions (classification >= 0.5): 114249\n",
      "\n",
      "Non-zero predictions statistics:\n",
      "Mean: 181.31\n",
      "Median: 12.00\n",
      "Std: 254.32\n",
      "\n",
      "Prediction Statistics:\n",
      "Total predictions added: 227879\n",
      "\n",
      "Classification Statistics:\n",
      "Predicted zeros: 113630 (49.86%)\n",
      "Predicted non-zeros: 114249 (50.14%)\n",
      "Mean classification confidence: 0.4989\n",
      "\n",
      "Final Predictions Statistics:\n",
      "Mean solar radiation: 181.31\n",
      "Min solar radiation: 12.00\n",
      "Max solar radiation: 966.98\n",
      "Zero predictions: 0 (0.00%)\n",
      "\n",
      "Training completed successfully!\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n7. Predicting missing data...\")\n",
    "to_predict_predictions = model.predict(X_to_predict_seq)\n",
    "classification_pred, regression_pred, final_pred = to_predict_predictions\n",
    "\n",
    "# Clip solo le predizioni finali che useremo per l'integrazione\n",
    "final_pred = np.clip(final_pred, 0, 11)\n",
    "final_pred_original = scaler_y.inverse_transform(final_pred)\n",
    "\n",
    "print(\"\\n8. Integrating predictions into original dataset...\")\n",
    "df_updated = integrate_predictions(df.copy(), predictions=(classification_pred, regression_pred, final_pred_original))\n",
    "\n",
    "df_updated.to_parquet('../../sources/weather_data_solarradiation.parquet')\n",
    "\n",
    "# Add prediction statistics to training_results\n",
    "training_results['prediction_stats'] = {\n",
    "    'n_predictions_added': len(final_pred_original),\n",
    "    'classification_stats': {\n",
    "        'predicted_zeros': int(np.sum(classification_pred < 0.5)),\n",
    "        'predicted_non_zeros': int(np.sum(classification_pred >= 0.5)),\n",
    "        'mean_confidence': float(classification_pred.mean()),\n",
    "    },\n",
    "    'regression_stats': {\n",
    "        'mean_predicted_value': float(regression_pred.mean()),\n",
    "        'min_predicted_value': float(regression_pred.min()),\n",
    "        'max_predicted_value': float(regression_pred.max()),\n",
    "    },\n",
    "    'final_predictions': {\n",
    "        'mean_predicted_solarradiation': float(final_pred_original.mean()),\n",
    "        'min_predicted_solarradiation': float(final_pred_original.min()),\n",
    "        'max_predicted_solarradiation': float(final_pred_original.max()),\n",
    "        'zero_predictions': int(np.sum(final_pred_original == 0)),\n",
    "        'non_zero_predictions': int(np.sum(final_pred_original > 0)),\n",
    "    }\n",
    "}\n",
    "\n",
    "print(\"\\nPrediction Statistics:\")\n",
    "print(f\"Total predictions added: {training_results['prediction_stats']['n_predictions_added']}\")\n",
    "print(\"\\nClassification Statistics:\")\n",
    "print(f\"Predicted zeros: {training_results['prediction_stats']['classification_stats']['predicted_zeros']} \"\n",
    "      f\"({training_results['prediction_stats']['classification_stats']['predicted_zeros']/len(final_pred_original)*100:.2f}%)\")\n",
    "print(f\"Predicted non-zeros: {training_results['prediction_stats']['classification_stats']['predicted_non_zeros']} \"\n",
    "      f\"({training_results['prediction_stats']['classification_stats']['predicted_non_zeros']/len(final_pred_original)*100:.2f}%)\")\n",
    "print(f\"Mean classification confidence: {training_results['prediction_stats']['classification_stats']['mean_confidence']:.4f}\")\n",
    "\n",
    "print(\"\\nFinal Predictions Statistics:\")\n",
    "print(f\"Mean solar radiation: {training_results['prediction_stats']['final_predictions']['mean_predicted_solarradiation']:.2f}\")\n",
    "print(f\"Min solar radiation: {training_results['prediction_stats']['final_predictions']['min_predicted_solarradiation']:.2f}\")\n",
    "print(f\"Max solar radiation: {training_results['prediction_stats']['final_predictions']['max_predicted_solarradiation']:.2f}\")\n",
    "print(f\"Zero predictions: {training_results['prediction_stats']['final_predictions']['zero_predictions']} \"\n",
    "      f\"({training_results['prediction_stats']['final_predictions']['zero_predictions']/len(final_pred_original)*100:.2f}%)\")\n",
    "\n",
    "print(\"\\nTraining completed successfully!\")\n",
    "\n",
    "tf.keras.backend.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ef29b3ecdf12c6db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 134288 characters]",
      "text/plain": [
       "<Figure size 2000x1200 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 70984 characters]",
      "text/plain": [
       "<Figure size 1500x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 25648 characters]",
      "text/plain": [
       "<Figure size 1200x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Statistiche principali di Solar Radiation:\n",
      "--------------------------------------------------\n",
      "count          : 357,679.0000\n",
      "missing        : 64.0000\n",
      "zeros          : 59,357.0000\n",
      "mean           : 183.8441\n",
      "median         : 12.0000\n",
      "std            : 259.8156\n",
      "min            : 0.0000\n",
      "max            : 1,113.0000\n",
      "skewness       : 1.3491\n",
      "kurtosis       : 0.5914\n",
      "percentile_1   : 0.0000\n",
      "percentile_5   : 0.0000\n",
      "percentile_10  : 0.0000\n",
      "percentile_25  : 12.0000\n",
      "percentile_50  : 12.0000\n",
      "percentile_75  : 321.3083\n",
      "percentile_90  : 624.6504\n",
      "percentile_95  : 776.0000\n",
      "percentile_99  : 907.6779\n",
      "\n",
      "Suggerimenti per la normalizzazione:\n",
      "--------------------------------------------------\n",
      "- La distribuzione è fortemente asimmetrica (skewness > 1)\n",
      "- Considerare una trasformazione logaritmica: np.log1p(x)\n",
      "- Alta presenza di zeri (16.60%)\n",
      "- Considerare un modello in due parti: classificazione degli zeri + regressione sui valori non-zero\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'count': 357679,\n",
       " 'missing': 64,\n",
       " 'zeros': 59357,\n",
       " 'mean': 183.84409789852336,\n",
       " 'median': 12.0,\n",
       " 'std': 259.8156425752193,\n",
       " 'min': 0.0,\n",
       " 'max': 1113.0,\n",
       " 'skewness': 1.3490904735404219,\n",
       " 'kurtosis': 0.5914208419781612,\n",
       " 'percentile_1': 0.0,\n",
       " 'percentile_5': 0.0,\n",
       " 'percentile_10': 0.0,\n",
       " 'percentile_25': 12.0,\n",
       " 'percentile_50': 12.0,\n",
       " 'percentile_75': 321.3082580566406,\n",
       " 'percentile_90': 624.6503662109386,\n",
       " 'percentile_95': 776.0,\n",
       " 'percentile_99': 907.677912597656}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analyze_distribution(df_updated, 'solarradiation', 'Solar Radiation')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e884cc287364c4ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Plot saved as: 2024-11-26_05-41_error_analysis.png\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 520716 characters]",
      "text/plain": [
       "<Figure size 2000x1500 with 10 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Classification Statistics:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.98      0.96      0.97     13007\n",
      "         1.0       0.96      0.98      0.97     12926\n",
      "\n",
      "    accuracy                           0.97     25933\n",
      "   macro avg       0.97      0.97      0.97     25933\n",
      "weighted avg       0.97      0.97      0.97     25933\n",
      "\n",
      "AUC-ROC: 0.9968\n",
      "\n",
      "Regression Statistics (Non-zero values):\n",
      "MAE: 0.1056\n",
      "RMSE: 0.2896\n",
      "Mean error: 0.0143\n",
      "Error std: 0.2892\n",
      "\n",
      "Final Prediction Statistics:\n",
      "MAE: 0.0583\n",
      "RMSE: 0.0835\n",
      "Mean error: 0.0113\n",
      "Error std: 0.0827\n",
      "\n",
      "Error Thresholds (Final Predictions):\n",
      "Predictions within ±0.5: 99.9%\n",
      "Predictions within ±1.0: 100.0%\n",
      "Predictions within ±1.5: 100.0%\n",
      "Predictions within ±2.0: 100.0%\n"
     ]
    }
   ],
   "source": [
    "def plot_error_analysis(y_true, predictions, folder_name=None):\n",
    "    \"\"\"\n",
    "    Function to visualize prediction error analysis for the hybrid model\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    y_true : array-like\n",
    "        Actual values\n",
    "    predictions : tuple\n",
    "        Tuple containing (classification_pred, regression_pred, final_pred)\n",
    "    folder_name : str, optional\n",
    "        Directory to save plots. If None, plots are only displayed\n",
    "\n",
    "    Generates:\n",
    "    ----------\n",
    "    - Classification analysis plots\n",
    "    - Regression error analysis plots\n",
    "    - Final prediction error analysis plots\n",
    "    \"\"\"\n",
    "    from sklearn.metrics import roc_curve\n",
    "\n",
    "    # Unpack predictions\n",
    "    classification_pred, regression_pred, final_pred = predictions\n",
    "\n",
    "    # Convert to 1D numpy arrays if needed\n",
    "    y_true = np.ravel(y_true)\n",
    "    classification_pred = np.ravel(classification_pred)\n",
    "    regression_pred = np.ravel(regression_pred)\n",
    "    final_pred = np.ravel(final_pred)\n",
    "\n",
    "    # Create binary ground truth\n",
    "    y_true_binary = (y_true > 0).astype(float)\n",
    "\n",
    "    # Calculate errors for regression and final predictions\n",
    "    regression_errors = regression_pred - y_true\n",
    "    final_errors = final_pred - y_true\n",
    "\n",
    "    # Create main figure\n",
    "    plt.figure(figsize=(20, 15))\n",
    "\n",
    "    # Classification Analysis (Top Row)\n",
    "    # Plot 1: Classification Distribution\n",
    "    plt.subplot(3, 3, 1)\n",
    "    plt.hist(classification_pred, bins=50, alpha=0.7)\n",
    "    plt.axvline(x=0.5, color='r', linestyle='--')\n",
    "    plt.title('Classification Probability Distribution')\n",
    "    plt.xlabel('Classification Probability')\n",
    "    plt.ylabel('Frequency')\n",
    "\n",
    "    # Plot 2: ROC Curve\n",
    "    plt.subplot(3, 3, 2)\n",
    "    fpr, tpr, _ = roc_curve(y_true_binary, classification_pred)\n",
    "    plt.plot(fpr, tpr)\n",
    "    plt.plot([0, 1], [0, 1], 'r--')\n",
    "    plt.title(f'ROC Curve (AUC = {roc_auc_score(y_true_binary, classification_pred):.4f})')\n",
    "    plt.xlabel('False Positive Rate')\n",
    "    plt.ylabel('True Positive Rate')\n",
    "\n",
    "    # Plot 3: Classification Confusion Matrix\n",
    "    plt.subplot(3, 3, 3)\n",
    "    cm = confusion_matrix(y_true_binary, classification_pred > 0.5)\n",
    "    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')\n",
    "    plt.title('Classification Confusion Matrix')\n",
    "    plt.xlabel('Predicted')\n",
    "    plt.ylabel('Actual')\n",
    "\n",
    "    # Regression Analysis (Middle Row)\n",
    "    # Plot 4: Regression Error Distribution\n",
    "    plt.subplot(3, 3, 4)\n",
    "    plt.hist(regression_errors[y_true > 0], bins=50, alpha=0.7)\n",
    "    plt.title('Regression Error Distribution (Non-zero Values)')\n",
    "    plt.xlabel('Error')\n",
    "    plt.ylabel('Frequency')\n",
    "\n",
    "    # Plot 5: Actual vs Predicted (Regression)\n",
    "    plt.subplot(3, 3, 5)\n",
    "    mask_nonzero = y_true > 0\n",
    "    plt.scatter(y_true[mask_nonzero], regression_pred[mask_nonzero], alpha=0.5)\n",
    "    plt.plot([y_true[mask_nonzero].min(), y_true[mask_nonzero].max()],\n",
    "             [y_true[mask_nonzero].min(), y_true[mask_nonzero].max()], 'r--', lw=2)\n",
    "    plt.title('Actual vs Predicted (Regression, Non-zero Values)')\n",
    "    plt.xlabel('Actual Values')\n",
    "    plt.ylabel('Predicted Values')\n",
    "\n",
    "    # Plot 6: Regression Errors vs Actual Values\n",
    "    plt.subplot(3, 3, 6)\n",
    "    plt.scatter(y_true[mask_nonzero], regression_errors[mask_nonzero], alpha=0.5)\n",
    "    plt.axhline(y=0, color='r', linestyle='--')\n",
    "    plt.title('Regression Errors vs Actual Values (Non-zero Values)')\n",
    "    plt.xlabel('Actual Values')\n",
    "    plt.ylabel('Error')\n",
    "\n",
    "    # Final Predictions Analysis (Bottom Row)\n",
    "    # Plot 7: Final Error Distribution\n",
    "    plt.subplot(3, 3, 7)\n",
    "    plt.hist(final_errors, bins=50, alpha=0.7)\n",
    "    plt.title('Final Prediction Error Distribution')\n",
    "    plt.xlabel('Error')\n",
    "    plt.ylabel('Frequency')\n",
    "\n",
    "    # Plot 8: Actual vs Predicted (Final)\n",
    "    plt.subplot(3, 3, 8)\n",
    "    plt.scatter(y_true, final_pred, alpha=0.5)\n",
    "    plt.plot([y_true.min(), y_true.max()], [y_true.min(), y_true.max()], 'r--', lw=2)\n",
    "    plt.title('Actual vs Predicted (Final)')\n",
    "    plt.xlabel('Actual Values')\n",
    "    plt.ylabel('Predicted Values')\n",
    "\n",
    "    # Plot 9: Final Errors vs Actual Values\n",
    "    plt.subplot(3, 3, 9)\n",
    "    plt.scatter(y_true, final_errors, alpha=0.5)\n",
    "    plt.axhline(y=0, color='r', linestyle='--')\n",
    "    plt.title('Final Errors vs Actual Values')\n",
    "    plt.xlabel('Actual Values')\n",
    "    plt.ylabel('Error')\n",
    "\n",
    "    plt.tight_layout()\n",
    "\n",
    "    # Save plot if directory is specified\n",
    "    if folder_name is not None:\n",
    "        try:\n",
    "            filename = f'{folder_name}_error_analysis.png'\n",
    "            plt.savefig(filename, dpi=300, bbox_inches='tight')\n",
    "            print(f\"\\nPlot saved as: {filename}\")\n",
    "        except Exception as e:\n",
    "            print(f\"\\nError saving plot: {str(e)}\")\n",
    "\n",
    "    plt.show()\n",
    "\n",
    "    # Print comprehensive statistics\n",
    "    print(\"\\nClassification Statistics:\")\n",
    "    print(classification_report(y_true_binary, classification_pred > 0.5))\n",
    "    print(f\"AUC-ROC: {roc_auc_score(y_true_binary, classification_pred):.4f}\")\n",
    "\n",
    "    print(\"\\nRegression Statistics (Non-zero values):\")\n",
    "    mask_nonzero = y_true > 0\n",
    "    if np.any(mask_nonzero):\n",
    "        print(f\"MAE: {np.mean(np.abs(regression_errors[mask_nonzero])):.4f}\")\n",
    "        print(f\"RMSE: {np.sqrt(np.mean(regression_errors[mask_nonzero] ** 2)):.4f}\")\n",
    "        print(f\"Mean error: {np.mean(regression_errors[mask_nonzero]):.4f}\")\n",
    "        print(f\"Error std: {np.std(regression_errors[mask_nonzero]):.4f}\")\n",
    "\n",
    "    print(\"\\nFinal Prediction Statistics:\")\n",
    "    print(f\"MAE: {np.mean(np.abs(final_errors)):.4f}\")\n",
    "    print(f\"RMSE: {np.sqrt(np.mean(final_errors ** 2)):.4f}\")\n",
    "    print(f\"Mean error: {np.mean(final_errors):.4f}\")\n",
    "    print(f\"Error std: {np.std(final_errors):.4f}\")\n",
    "\n",
    "    # Calculate percentage of errors within thresholds\n",
    "    thresholds = [0.5, 1.0, 1.5, 2.0]\n",
    "    print(\"\\nError Thresholds (Final Predictions):\")\n",
    "    for threshold in thresholds:\n",
    "        within_threshold = np.mean(np.abs(final_errors) <= threshold) * 100\n",
    "        print(f\"Predictions within ±{threshold}: {within_threshold:.1f}%\")\n",
    "\n",
    "# Example usage\n",
    "plot_error_analysis(y_test, predictions, folder_name=folder_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd5197ea71becfc6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f982c92c-ba99-4df6-b3c8-df92426679db",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
