584 KiB
584 KiB
In [1]:
from opt_einsum.paths import branch_1
!apt-get update
!apt-get install graphviz -y
!pip install tensorflow==2.13.0
!pip install numpy
!pip install pandas
!pip install keras==2.13.1
!pip install scikit-learn
!pip install matplotlib
!pip install joblib
!pip install pyarrow
!pip install fastparquet
!pip install scipy
!pip install seaborn
!pip install tqdm
!pip install pydot
!pip install tensorflow-io
!pip install tensorflow-addonsGet:1 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB] Hit:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 InRelease Hit:3 http://archive.ubuntu.com/ubuntu jammy InRelease Get:4 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] Hit:5 http://archive.ubuntu.com/ubuntu jammy-backports InRelease Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2732 kB] Fetched 2989 kB in 1s (2026 kB/s) Reading package lists... Done Reading package lists... Done Building dependency tree... Done Reading state information... Done graphviz is already the newest version (2.42.2-6ubuntu0.1). 0 upgraded, 0 newly installed, 0 to remove and 121 not upgraded. Requirement already satisfied: tensorflow==2.13.0 in /usr/local/lib/python3.11/dist-packages (2.13.0) Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (2.0.0) Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (1.6.3) Requirement already satisfied: flatbuffers>=23.1.21 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (23.5.26) Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (0.4.0) Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (0.2.0) Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (1.58.0) Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (3.9.0) Requirement already satisfied: keras<2.14,>=2.13.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (2.13.1) Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (16.0.6) Requirement already satisfied: numpy<=1.24.3,>=1.22 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (1.24.3) Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (3.3.0) Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (23.1) Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (4.24.3) Requirement already satisfied: setuptools in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (68.2.2) Requirement already satisfied: six>=1.12.0 in /usr/lib/python3/dist-packages (from tensorflow==2.13.0) (1.16.0) Requirement already satisfied: tensorboard<2.14,>=2.13 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (2.13.0) Requirement already satisfied: tensorflow-estimator<2.14,>=2.13.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (2.13.0) Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (2.3.0) Requirement already satisfied: typing-extensions<4.6.0,>=3.6.6 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (4.5.0) Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (1.14.1) Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow==2.13.0) (0.37.1) Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.11/dist-packages (from astunparse>=1.6.0->tensorflow==2.13.0) (0.41.2) Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2.23.1) Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (1.0.0) Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (3.4.4) Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2.31.0) Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (0.7.1) Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2.3.7) Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (5.3.1) Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (0.3.0) Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (4.9) Requirement already satisfied: urllib3>=2.0.5 in /usr/local/lib/python3.11/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2.0.5) Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (1.3.1) Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (3.2.0) Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (3.4) Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2023.7.22) Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.11/dist-packages (from werkzeug>=1.0.1->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2.1.3) Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.11/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (0.5.0) Requirement already satisfied: oauthlib>=3.0.0 in /usr/lib/python3/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (3.2.0) [33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv[0m[33m [0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.2.1[0m[39;49m -> [0m[32;49m24.3.1[0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpython3 -m pip install --upgrade pip[0m Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (1.24.3) [33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv[0m[33m [0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.2.1[0m[39;49m -> [0m[32;49m24.3.1[0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpython3 -m pip install --upgrade pip[0m Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (2.2.3) Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (1.24.3) Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (2.8.2) Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2) Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2) Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas) (1.16.0) [33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv[0m[33m [0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.2.1[0m[39;49m -> [0m[32;49m24.3.1[0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpython3 -m pip install --upgrade pip[0m Requirement already satisfied: keras==2.13.1 in /usr/local/lib/python3.11/dist-packages (2.13.1) [33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv[0m[33m [0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.2.1[0m[39;49m -> [0m[32;49m24.3.1[0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpython3 -m pip install --upgrade pip[0m Requirement already satisfied: scikit-learn in /usr/local/lib/python3.11/dist-packages (1.5.2) Requirement already satisfied: numpy>=1.19.5 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.24.3) Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.14.1) Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.4.2) Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (3.5.0) [33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv[0m[33m [0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.2.1[0m[39;49m -> [0m[32;49m24.3.1[0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpython3 -m pip install --upgrade pip[0m Requirement already satisfied: matplotlib in /usr/local/lib/python3.11/dist-packages (3.8.0) Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.1.1) Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (0.11.0) Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (4.42.1) Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.4.5) Requirement already satisfied: numpy<2,>=1.21 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.24.3) Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (23.1) Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (10.0.1) Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (3.2.0) Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (2.8.2) Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib) (1.16.0) [33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv[0m[33m [0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.2.1[0m[39;49m -> [0m[32;49m24.3.1[0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpython3 -m pip install --upgrade pip[0m Requirement already satisfied: joblib in /usr/local/lib/python3.11/dist-packages (1.4.2) [33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv[0m[33m [0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.2.1[0m[39;49m -> [0m[32;49m24.3.1[0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpython3 -m pip install --upgrade pip[0m Requirement already satisfied: pyarrow in /usr/local/lib/python3.11/dist-packages (18.0.0) [33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv[0m[33m [0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.2.1[0m[39;49m -> [0m[32;49m24.3.1[0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpython3 -m pip install --upgrade pip[0m Requirement already satisfied: fastparquet in /usr/local/lib/python3.11/dist-packages (2024.11.0) Requirement already satisfied: pandas>=1.5.0 in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2.2.3) Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (from fastparquet) (1.24.3) Requirement already satisfied: cramjam>=2.3 in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2.9.0) Requirement already satisfied: fsspec in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2024.10.0) Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from fastparquet) (23.1) Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2.8.2) Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2024.2) Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2024.2) Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas>=1.5.0->fastparquet) (1.16.0) [33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv[0m[33m [0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.2.1[0m[39;49m -> [0m[32;49m24.3.1[0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpython3 -m pip install --upgrade pip[0m Requirement already satisfied: scipy in /usr/local/lib/python3.11/dist-packages (1.14.1) Requirement already satisfied: numpy<2.3,>=1.23.5 in /usr/local/lib/python3.11/dist-packages (from scipy) (1.24.3) [33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv[0m[33m [0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.2.1[0m[39;49m -> [0m[32;49m24.3.1[0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpython3 -m pip install --upgrade pip[0m Requirement already satisfied: seaborn in /usr/local/lib/python3.11/dist-packages (0.13.2) Requirement already satisfied: numpy!=1.24.0,>=1.20 in /usr/local/lib/python3.11/dist-packages (from seaborn) (1.24.3) Requirement already satisfied: pandas>=1.2 in /usr/local/lib/python3.11/dist-packages (from seaborn) (2.2.3) Requirement already satisfied: matplotlib!=3.6.1,>=3.4 in /usr/local/lib/python3.11/dist-packages (from seaborn) (3.8.0) Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.1.1) Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (0.11.0) Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (4.42.1) Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.4.5) Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (23.1) Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (10.0.1) Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (3.2.0) Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.11/dist-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (2.8.2) Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.2->seaborn) (2024.2) Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.2->seaborn) (2024.2) Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib!=3.6.1,>=3.4->seaborn) (1.16.0) [33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv[0m[33m [0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.2.1[0m[39;49m -> [0m[32;49m24.3.1[0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpython3 -m pip install --upgrade pip[0m Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (4.67.0) [33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv[0m[33m [0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.2.1[0m[39;49m -> [0m[32;49m24.3.1[0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpython3 -m pip install --upgrade pip[0m Requirement already satisfied: pydot in /usr/local/lib/python3.11/dist-packages (3.0.2) Requirement already satisfied: pyparsing>=3.0.9 in /usr/local/lib/python3.11/dist-packages (from pydot) (3.2.0) [33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv[0m[33m [0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.2.1[0m[39;49m -> [0m[32;49m24.3.1[0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpython3 -m pip install --upgrade pip[0m Requirement already satisfied: tensorflow-io in /usr/local/lib/python3.11/dist-packages (0.37.1) Requirement already satisfied: tensorflow-io-gcs-filesystem==0.37.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow-io) (0.37.1) [33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv[0m[33m [0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.2.1[0m[39;49m -> [0m[32;49m24.3.1[0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpython3 -m pip install --upgrade pip[0m Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.11/dist-packages (0.23.0) Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from tensorflow-addons) (23.1) Requirement already satisfied: typeguard<3.0.0,>=2.7 in /usr/local/lib/python3.11/dist-packages (from tensorflow-addons) (2.13.3) [33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv[0m[33m [0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.2.1[0m[39;49m -> [0m[32;49m24.3.1[0m [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpython3 -m pip install --upgrade pip[0m
In [2]:
import tensorflow as tf
from tensorflow.keras.layers import Dense, LSTM, MultiHeadAttention, Dropout, BatchNormalization, LayerNormalization, Input, Activation, Lambda, Bidirectional, Add, MaxPooling1D, Conv1D, GlobalAveragePooling1D
from tensorflow.keras import regularizers
from tensorflow.keras.models import Model
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import RobustScaler
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint
from tensorflow.keras.optimizers import AdamW
import json
from datetime import datetime
import matplotlib.pyplot as plt
from tensorflow.keras.utils import plot_model
import tensorflow_addons as tfa
import os
import joblib
import seaborn as sns
from sklearn.metrics import confusion_matrix, mean_absolute_error, mean_squared_error, r2_score
folder_name = datetime.now().strftime("%Y-%m-%d_%H-%M")
random_state_value = None2024-11-21 08:23:10.586264: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. /usr/local/lib/python3.11/dist-packages/tensorflow_addons/utils/tfa_eol_msg.py:23: UserWarning: TensorFlow Addons (TFA) has ended development and introduction of new features. TFA has entered a minimal maintenance and release mode until a planned end of life in May 2024. Please modify downstream libraries to take dependencies from other repositories in our TensorFlow community (e.g. Keras, Keras-CV, and Keras-NLP). For more information see: https://github.com/tensorflow/addons/issues/2807 warnings.warn(
In [3]:
def get_season(date):
month = date.month
day = date.day
if (month == 12 and day >= 21) or (month <= 3 and day < 20):
return 'Winter'
elif (month == 3 and day >= 20) or (month <= 6 and day < 21):
return 'Spring'
elif (month == 6 and day >= 21) or (month <= 9 and day < 23):
return 'Summer'
elif (month == 9 and day >= 23) or (month <= 12 and day < 21):
return 'Autumn'
else:
return 'Unknown'
def get_time_period(hour):
if 5 <= hour < 12:
return 'Morning'
elif 12 <= hour < 17:
return 'Afternoon'
elif 17 <= hour < 21:
return 'Evening'
else:
return 'Night'
def add_time_features(df):
df['datetime'] = pd.to_datetime(df['datetime'])
df['timestamp'] = df['datetime'].astype(np.int64) // 10 ** 9
df['year'] = df['datetime'].dt.year
df['month'] = df['datetime'].dt.month
df['day'] = df['datetime'].dt.day
df['hour'] = df['datetime'].dt.hour
df['minute'] = df['datetime'].dt.minute
df['hour_sin'] = np.sin(df['hour'] * (2 * np.pi / 24))
df['hour_cos'] = np.cos(df['hour'] * (2 * np.pi / 24))
df['day_of_week'] = df['datetime'].dt.dayofweek
df['day_of_year'] = df['datetime'].dt.dayofyear
df['week_of_year'] = df['datetime'].dt.isocalendar().week.astype(int)
df['quarter'] = df['datetime'].dt.quarter
df['is_month_end'] = df['datetime'].dt.is_month_end.astype(int)
df['is_quarter_end'] = df['datetime'].dt.is_quarter_end.astype(int)
df['is_year_end'] = df['datetime'].dt.is_year_end.astype(int)
df['month_sin'] = np.sin(df['month'] * (2 * np.pi / 12))
df['month_cos'] = np.cos(df['month'] * (2 * np.pi / 12))
df['day_of_year_sin'] = np.sin(df['day_of_year'] * (2 * np.pi / 365.25))
df['day_of_year_cos'] = np.cos(df['day_of_year'] * (2 * np.pi / 365.25))
df['season'] = df['datetime'].apply(get_season)
df['time_period'] = df['hour'].apply(get_time_period)
return df
def add_solar_features(df):
# Calculate solar angle
df['solar_angle'] = np.sin(df['day_of_year'] * (2 * np.pi / 365.25)) * np.sin(df['hour'] * (2 * np.pi / 24))
# Interactions between relevant features
df['cloud_temp_interaction'] = df['cloudcover'] * df['temp']
df['visibility_cloud_interaction'] = df['visibility'] * (100 - df['cloudcover'])
# Derived features
df['clear_sky_index'] = (100 - df['cloudcover']) / 100
df['temp_gradient'] = df['temp'] - df['tempmin']
return df
def add_solar_specific_features(df):
# Solar angle and day length calculations
df['day_length'] = 12 + 3 * np.sin(2 * np.pi * (df['day_of_year'] - 81) / 365.25)
df['solar_noon'] = 12 - df['hour']
df['solar_elevation'] = np.sin(2 * np.pi * df['day_of_year'] / 365.25) * np.cos(2 * np.pi * df['solar_noon'] / 24)
# Feature interactions
df['cloud_elevation'] = df['cloudcover'] * df['solar_elevation']
df['visibility_elevation'] = df['visibility'] * df['solar_elevation']
# Extended window rolling features
df['cloud_rolling_12h'] = df['cloudcover'].rolling(window=12).mean()
df['temp_rolling_12h'] = df['temp'].rolling(window=12).mean()
return df
def add_uv_specific_features(df):
# Solar zenith angle calculation
lat = 41.9 # assuming constant latitude for the dataset - Rome's latitude
df['solar_zenith'] = 90 - np.degrees(
np.arcsin(
np.sin(np.radians(lat)) * np.sin(df['solar_elevation']) +
np.cos(np.radians(lat)) * np.cos(df['solar_elevation']) * np.cos(df['hour'] * 15)
)
)
# UV peak hours indicator (10:00-16:00)
df['is_uv_peak_hours'] = ((df['hour'] >= 10) & (df['hour'] <= 16)).astype(int)
# Atmospheric attenuation factor
df['atmospheric_attenuation'] = (100 - df['cloudcover']) * (df['visibility'] / 100) * (1 - df['humidity'] / 200)
# Seasonal UV factor
df['uv_seasonal_factor'] = np.where(df['season_Summer'], 1.0,
np.where(df['season_Spring'], 0.7,
np.where(df['season_Autumn'], 0.5, 0.3)))
# Solar elevation and atmospheric transparency interaction
df['solar_clarity_index'] = df['solar_elevation'] * df['atmospheric_attenuation'] / 100
# UV-specific rolling features
df['clarity_rolling_3h'] = df['atmospheric_attenuation'].rolling(window=3).mean()
df['temp_uv_interaction'] = df['temp'] * df['solar_clarity_index']
return df
def add_advanced_features(df):
"""
Add all advanced features in the correct order
"""
# 1. First add basic time features
df = add_time_features(df)
# 2. One-hot encoding for categorical features
df = pd.get_dummies(df, columns=['season', 'time_period'])
# 3. Add solar and specific features
df = add_solar_features(df)
df = add_solar_specific_features(df)
# 4. Ensure datetime index
if not isinstance(df.index, pd.DatetimeIndex):
df.index = pd.to_datetime(df.index)
# 5. Add weather variable interactions
df['temp_humidity'] = df['temp'] * df['humidity']
df['temp_cloudcover'] = df['temp'] * df['cloudcover']
df['visibility_cloudcover'] = df['visibility'] * df['cloudcover']
# 6. Add solar radiation derived features
df['clear_sky_factor'] = (100 - df['cloudcover']) / 100
df['day_length'] = np.sin(df['day_of_year_sin']) * 12 + 12
# 7. Add lag features
df['temp_1h_lag'] = df['temp'].shift(1)
df['cloudcover_1h_lag'] = df['cloudcover'].shift(1)
df['humidity_1h_lag'] = df['humidity'].shift(1)
# 8. Add rolling means
df['temp_rolling_mean_6h'] = df['temp'].rolling(window=6).mean()
df['cloudcover_rolling_mean_6h'] = df['cloudcover'].rolling(window=6).mean()
df['temp_humidity_interaction'] = df['temp'] * df['humidity'] / 100
# 9. Add atmospheric stability
df['atmospheric_stability'] = df.groupby(df.index.date)['pressure'].transform(
lambda x: x.std()
).fillna(0)
# 10. Add extreme conditions indicator
df['extreme_conditions'] = ((df['temp'] > df['temp'].quantile(0.75)) &
(df['humidity'] < df['humidity'].quantile(0.25))).astype(int)
# 11. Add atmospheric transparency
df['atmospheric_transparency'] = (100 - df['cloudcover']) * (df['visibility'] / 10)
# 12. Add transitional seasons indicator
df['is_transition_season'] = ((df['season_Spring'] | df['season_Autumn'])).astype(int)
# 13. Add solar cloud effect
if 'solar_elevation' in df.columns:
df['solar_cloud_effect'] = df['solar_elevation'] * (100 - df['cloudcover']) / 100
# 14. Finally add UV specific features
df = add_uv_specific_features(df)
return df
def prepare_advanced_data(df):
"""
Prepares data for UV index prediction model with advanced feature engineering
and optimized preprocessing.
Args:
df: DataFrame with meteorological data
Returns:
tuple: (X_train_scaled, X_test_scaled, y_train, y_test, scaler, final_features, X_to_predict_scaled)
"""
# Apply feature engineering functions
df = add_advanced_features(df)
# Optimized feature selection for UV index
selected_features = {
# Primary meteorological features
'atmospheric': [
'temp', 'humidity', 'cloudcover', 'visibility',
'clear_sky_index', 'atmospheric_transparency'
],
# Essential temporal features
'temporal': [
'hour_sin', 'hour_cos',
'day_of_year_sin', 'day_of_year_cos'
],
# Solar features
'solar': [
'solar_angle', 'solar_elevation',
'day_length', 'solar_noon',
'solar_cloud_effect'
],
# Key interactions
'interactions': [
'cloud_temp_interaction',
'visibility_cloud_interaction',
'temp_humidity_interaction',
'solar_clarity_index'
],
# Rolling features
'rolling': [
'cloud_rolling_12h',
'temp_rolling_mean_6h'
]
}
# Flatten feature list
base_features = [item for sublist in selected_features.values() for item in sublist]
# Add categorical features (one-hot encoded)
categorical_columns = [col for col in df.columns if col.startswith(('season_', 'time_period_'))]
final_features = base_features + categorical_columns
# Temporal preprocessing
df = df.sort_values('datetime')
df.set_index('datetime', inplace=True)
# Advanced interpolation for missing values
for column in final_features:
if column in df.columns:
if df[column].isnull().any():
if column in selected_features['rolling']:
df[column] = df[column].ffill().bfill()
else:
df[column] = df[column].interpolate(method='time', limit_direction='both')
# Temporal data split
data_after_2010 = df[df.index.year >= 2010].copy()
data_before_2010 = df[df.index.year < 2010].copy()
print(f"\nTemporal distribution of data:")
print(f"Records after 2010: {len(data_after_2010):,}")
print(f"Records before 2010: {len(data_before_2010):,}")
# Feature and target preparation
X = data_after_2010[final_features]
y = data_after_2010['uvindex']
X_to_predict = data_before_2010[final_features]
# Data validation
if X.isnull().any().any() or y.isnull().any():
print("\nWarning: Found missing values after preprocessing")
print("Features with missing values:", X.columns[X.isnull().any()].tolist())
X = X.fillna(X.mean())
y = y.fillna(y.mean())
# Stratified data split
X_train, X_test, y_train, y_test = train_test_split(
X, y,
test_size=0.5,
random_state=random_state_value,
stratify=pd.qcut(y, q=5, duplicates='drop', labels=False)
)
# Robust feature scaling
feature_scaler = RobustScaler()
X_train_scaled = feature_scaler.fit_transform(X_train)
X_test_scaled = feature_scaler.transform(X_test)
X_to_predict_scaled = feature_scaler.transform(X_to_predict)
target_scaler = RobustScaler()
y_train_scaled = target_scaler.fit_transform(y_train.values.reshape(-1, 1)).ravel()
y_test_scaled = target_scaler.transform(y_test.values.reshape(-1, 1)).ravel()
# Final validation
assert not np.isnan(X_train_scaled).any(), "Found NaN in X_train_scaled"
assert not np.isnan(X_test_scaled).any(), "Found NaN in X_test_scaled"
assert not np.isnan(X_to_predict_scaled).any(), "Found NaN in X_to_predict_scaled"
# Print feature information
print("\nNumber of features used:", len(final_features))
print("\nFeature categories:")
for category, features in selected_features.items():
print(f"{category}: {len(features)} features")
print(f"Categorical: {len(categorical_columns)} features")
return (X_train_scaled, X_test_scaled, y_train_scaled, y_test_scaled,
feature_scaler, target_scaler, final_features, X_to_predict_scaled)
def create_sequence_data(X, sequence_length=24):
"""
Converts data into sequences for LSTM input
sequence_length represents how many previous hours to consider
"""
sequences = []
for i in range(len(X) - sequence_length + 1):
sequences.append(X[i:i + sequence_length])
return np.array(sequences)
def prepare_hybrid_data(df):
# Use existing data preparation
X_train_scaled, X_test_scaled, y_train, y_test, feature_scaler, target_scaler, features, X_to_predict_scaled = prepare_advanced_data(df)
# Convert data to sequences
sequence_length = 24 # 24 hours of historical data
X_train_seq = create_sequence_data(X_train_scaled, sequence_length)
X_test_seq = create_sequence_data(X_test_scaled, sequence_length)
# Adjust y by removing the first (sequence_length-1) elements
y_train = y_train[sequence_length - 1:]
y_test = y_test[sequence_length - 1:]
X_to_predict_seq = create_sequence_data(X_to_predict_scaled, sequence_length)
return X_train_seq, X_test_seq, y_train, y_test, feature_scaler, target_scaler, features, X_to_predict_seqIn [4]:
def create_residual_lstm_layer(x, units, dropout_rate, l2_reg=0.01,
survival_probability=0.8, return_sequences=True):
"""LSTM layer with stochastic depth"""
residual = x
# Main path
x = Bidirectional(LSTM(units, return_sequences=return_sequences,
kernel_regularizer=regularizers.l2(l2_reg)))(x)
x = LayerNormalization()(x)
x = Dropout(dropout_rate)(x)
# Adjust residual dimension if needed
if return_sequences:
# For Bidirectional LSTM, the output dimension is 2 * units
target_dim = 2 * units
if int(residual.shape[-1]) != target_dim:
# Use Dense layer instead of Conv1D for better dimension matching
residual = Dense(target_dim)(residual)
# Apply stochastic depth only if dimensions match
if x.shape[-1] == residual.shape[-1]:
x = tfa.layers.StochasticDepth(survival_probability)([x, residual])
else:
print(f"Warning: Dimension mismatch - x: {x.shape}, residual: {residual.shape}")
# Skip residual connection if dimensions don't match
pass
return x
def attention_block(x, units, num_heads=8, survival_probability=0.8):
"""
Attention block with stochastic depth.
"""
original_x = x
# Compute self-attention
attention = MultiHeadAttention(num_heads=num_heads, key_dim=units)(x, x)
# Ensure dimensions match before applying stochastic depth
if attention.shape[-1] != original_x.shape[-1]:
original_x = Dense(attention.shape[-1])(original_x)
# Apply stochastic depth to the attention path
x = tfa.layers.StochasticDepth(survival_probability)([attention, original_x])
x = LayerNormalization()(x)
# Store the input to the FFN
ffn_input = x
# FFN block
x = Dense(units * 4, activation='swish')(x)
x = Dense(ffn_input.shape[-1])(x) # Match the input dimension
# Apply stochastic depth to the FFN
x = tfa.layers.StochasticDepth(survival_probability)([x, ffn_input])
x = LayerNormalization()(x)
return x
def create_uv_index_model(input_shape, folder_name, l2_lambda=0.005, max_output=11):
inputs = Input(shape=input_shape)
# Further adjusted hyperparameters
survival_probs = [0.98, 0.95, 0.92] # Even higher survival probabilities
attention_survival_probs = [0.95, 0.92, 0.9]
# First LSTM block
x = create_residual_lstm_layer(
inputs, 64, dropout_rate=0.2, # Further reduced dropout
l2_reg=l2_lambda,
survival_probability=survival_probs[0],
return_sequences=True
)
x = attention_block(x, 128, num_heads=2, # Reduced heads
survival_probability=attention_survival_probs[0])
# Second LSTM block
x = create_residual_lstm_layer(
x, 32, dropout_rate=0.15,
l2_reg=l2_lambda,
survival_probability=survival_probs[1],
return_sequences=True
)
x = attention_block(x, 64, num_heads=2,
survival_probability=attention_survival_probs[1])
# Third LSTM block
x = create_residual_lstm_layer(
x, 16, dropout_rate=0.1,
l2_reg=l2_lambda,
survival_probability=survival_probs[2],
return_sequences=True
)
x = attention_block(x, 32, num_heads=2,
survival_probability=attention_survival_probs[2])
# Global attention with reduced complexity
x_input = x
x = MultiHeadAttention(num_heads=2, key_dim=32)(x, x)
if x.shape[-1] != x_input.shape[-1]:
x_input = Dense(x.shape[-1])(x_input)
x = tfa.layers.StochasticDepth(survival_probability=0.95)([x, x_input])
x = LayerNormalization()(x)
# Simplified dense layers
x = GlobalAveragePooling1D()(x)
# Gradual dimension reduction
x = Dense(32, activation='swish', kernel_regularizer=regularizers.l2(l2_lambda / 2), kernel_constraint=tf.keras.constraints.MaxNorm(3))(x)
x = BatchNormalization()(x)
x = Dropout(0.05)(x) # Minimal dropout
x = Dense(16, activation='swish',
kernel_regularizer=regularizers.l2(l2_lambda / 2))(x)
x = BatchNormalization()(x)
# Modified output layer
x = Dense(8, activation='swish')(x)
outputs = Dense(1, activation='sigmoid')(x) # Sigmoid activation
outputs = Lambda(lambda x: x * max_output)(outputs) # Scale to [0, 11] range
model = Model(inputs=inputs, outputs=outputs, name="UvModel")
# More stable learning rate schedule
initial_learning_rate = 0.0001 # Further reduced
warmup_steps = 1000
decay_steps = 5000
# Corretto learning rate schedule
class CustomLRSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):
def __init__(self, initial_lr=0.0001, warmup_steps=1000, decay_steps=5000):
super().__init__()
self.initial_lr = initial_lr
self.warmup_steps = warmup_steps
self.decay_steps = decay_steps
def __call__(self, step):
# Convert to float32
step_f = tf.cast(step, tf.float32)
warmup_steps_f = tf.cast(self.warmup_steps, tf.float32)
decay_steps_f = tf.cast(self.decay_steps, tf.float32)
# Warmup phase
warmup_progress = step_f / warmup_steps_f
warmup_lr = self.initial_lr * warmup_progress
# Decay phase
decay_progress = (step_f - warmup_steps_f) / decay_steps_f
decay_factor = 0.5 * (1.0 + tf.cos(tf.constant(np.pi) * decay_progress))
decay_lr = self.initial_lr * decay_factor
# Combine phases
lr = tf.where(step_f < warmup_steps_f, warmup_lr, decay_lr)
return lr
def get_config(self):
return {
"initial_lr": self.initial_lr,
"warmup_steps": self.warmup_steps,
"decay_steps": self.decay_steps
}
# Utilizzo dello schedule corretto
lr_schedule = CustomLRSchedule(
initial_lr=initial_learning_rate,
warmup_steps=warmup_steps,
decay_steps=decay_steps
)
optimizer = AdamW(
learning_rate=lr_schedule,
weight_decay=0.0005,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-7
)
# Improved loss function
def smooth_uv_loss(y_true, y_pred):
# Basic MSE with smoothing
mse = tf.square(y_true - y_pred)
# Smooth L1 component for better stability
abs_diff = tf.abs(y_true - y_pred)
smooth_l1 = tf.where(abs_diff < 1.0,
0.5 * tf.square(abs_diff),
abs_diff - 0.5)
# Combined loss with dynamic weighting
combined_loss = 0.7 * mse + 0.3 * smooth_l1
# Gentle weighting for high UV values
high_uv_weight = tf.where(y_true >= 8.0, 1.2, 1.0)
# Smooth peak hours weight
time_of_day = tf.cast(tf.math.floormod(tf.range(tf.shape(y_true)[0]), 24),
tf.float32)
peak_weight = 1.0 + 0.2 * tf.math.sigmoid((time_of_day - 10.0) * 0.5) * \
tf.math.sigmoid((16.0 - time_of_day) * 0.5)
total_weight = high_uv_weight * peak_weight
return tf.reduce_mean(combined_loss * total_weight)
# Improved MAPE metric
def smooth_mape(y_true, y_pred):
epsilon = 1e-7
diff = tf.abs(y_true - y_pred)
scale = tf.maximum(tf.abs(y_true) + epsilon, 0.5) # Minimum scale of 0.5
return tf.reduce_mean(diff / scale) * 100
model.compile(
optimizer=optimizer,
loss=smooth_uv_loss,
metrics=[
'mae',
'mse',
tf.keras.metrics.RootMeanSquaredError(),
smooth_mape
]
)
model.summary()
plot_model(model,
to_file=f'{folder_name}_model_architecture.png',
show_shapes=True,
show_layer_names=True,
dpi=150,
show_layer_activations=True)
return model
def evaluate_uv_predictions(y_true, y_pred, folder_name=None):
"""
Comprehensive evaluation of UV index predictions with detailed analysis and visualizations.
Parameters:
-----------
y_true : array-like
Actual UV index values
y_pred : array-like
Predicted UV index values
folder_name : str, optional
Folder to save analysis plots
Returns:
--------
dict
Dictionary containing all calculated metrics
"""
# Initialize plot paths
main_plot_path = None
conf_matrix_path = None
# Data preprocessing
y_true = np.array(y_true).ravel()
y_pred = np.array(y_pred).ravel()
# Rounding and clipping predictions
y_pred_rounded = np.round(y_pred * 2) / 2 # Round to nearest 0.5
y_pred_clipped = np.clip(y_pred_rounded, 0, 11)
# Calculate errors
errors = y_pred - y_true
errors_rounded = y_pred_clipped - y_true
# Function to determine UV risk level
def get_uv_risk_level(values):
levels = np.full_like(values, 'Low', dtype=object)
levels[(values > 2) & (values <= 5)] = 'Moderate'
levels[(values > 5) & (values <= 7)] = 'High'
levels[(values > 7) & (values <= 10)] = 'Very High'
levels[values > 10] = 'Extreme'
return levels
# Calculate basic metrics
metrics = {
'raw': {
'mae': mean_absolute_error(y_true, y_pred),
'rmse': np.sqrt(mean_squared_error(y_true, y_pred)),
'r2': r2_score(y_true, y_pred),
'mean_error': np.mean(errors),
'std_error': np.std(errors),
'median_error': np.median(errors),
'p95_abs_error': np.percentile(np.abs(errors), 95)
},
'rounded': {
'mae': mean_absolute_error(y_true, y_pred_clipped),
'rmse': np.sqrt(mean_squared_error(y_true, y_pred_clipped)),
'r2': r2_score(y_true, y_pred_clipped)
}
}
# Calculate accuracies for different margins
for data_type, errors_data in [('raw', errors), ('rounded', errors_rounded)]:
metrics[data_type].update({
'within_05': np.mean(np.abs(errors_data) <= 0.5) * 100,
'within_1': np.mean(np.abs(errors_data) <= 1.0) * 100,
'within_15': np.mean(np.abs(errors_data) <= 1.5) * 100,
'within_2': np.mean(np.abs(errors_data) <= 2.0) * 100
})
# Analysis by UV risk level
y_true_risk = get_uv_risk_level(y_true)
y_pred_risk = get_uv_risk_level(y_pred_clipped)
# Calculate confusion matrix with handling for missing classes
risk_levels = ['Low', 'Moderate', 'High', 'Very High', 'Extreme']
# Get unique labels present in the data
present_labels = np.unique(np.concatenate([y_true_risk, y_pred_risk]))
# Calculate confusion matrix for present labels
cm = confusion_matrix(y_true_risk, y_pred_risk, labels=present_labels)
# Create full confusion matrix with zeros
full_cm = np.zeros((len(risk_levels), len(risk_levels)))
# Map present labels to their positions in the full matrix
label_positions = {label: i for i, label in enumerate(risk_levels)}
for i, true_label in enumerate(present_labels):
for j, pred_label in enumerate(present_labels):
full_cm[label_positions[true_label], label_positions[pred_label]] = cm[i, j]
# Create DataFrame with all risk levels
cm_df = pd.DataFrame(full_cm, columns=risk_levels, index=risk_levels)
# Analysis by UV range
uv_ranges = [
(0, 2, 'Low'),
(2, 5, 'Moderate'),
(5, 7, 'High'),
(7, 10, 'Very High'),
(10, 11, 'Extreme')
]
range_analysis = {}
for low, high, label in uv_ranges:
mask = (y_true >= low) & (y_true < high)
if mask.any():
range_analysis[label] = {
'mae': mean_absolute_error(y_true[mask], y_pred[mask]),
'count': np.sum(mask),
'accuracy_within_05': np.mean(np.abs(errors[mask]) <= 0.5) * 100,
'accuracy_within_1': np.mean(np.abs(errors[mask]) <= 1.0) * 100
}
# Visualizations
if folder_name is not None:
try:
# Main figure with 4 subplots
fig = plt.figure(figsize=(20, 15))
# 1. Error distribution
plt.subplot(2, 2, 1)
plt.hist(errors, bins=50, alpha=0.7)
plt.title('Prediction Error Distribution')
plt.xlabel('Error')
plt.ylabel('Frequency')
# 2. Actual vs Predicted scatter plot
plt.subplot(2, 2, 2)
plt.scatter(y_true, y_pred, alpha=0.5)
plt.plot([0, 11], [0, 11], 'r--', lw=2)
plt.title('Actual vs Predicted Values')
plt.xlabel('Actual Values')
plt.ylabel('Predicted Values')
# 3. Errors vs Actual Values
plt.subplot(2, 2, 3)
plt.scatter(y_true, errors, alpha=0.5)
plt.axhline(y=0, color='r', linestyle='--')
plt.title('Errors vs Actual Values')
plt.xlabel('Actual Values')
plt.ylabel('Error')
# 4. Accuracy and MAE by range
ax = plt.subplot(2, 2, 4)
x_labels = [f"{label}\n({low}-{high})" for low, high, label in uv_ranges]
accuracies = [range_analysis[label]['accuracy_within_05']
for _, _, label in uv_ranges if label in range_analysis]
mae_values = [range_analysis[label]['mae']
for _, _, label in uv_ranges if label in range_analysis]
bars = plt.bar(x_labels, accuracies, alpha=0.6)
plt.ylabel('Accuracy within ±0.5 (%)')
plt.title('Accuracy and MAE by UV Range')
# Add MAE as line
ax2 = ax.twinx()
ax2.plot(x_labels, mae_values, 'r-o', label='MAE')
ax2.set_ylabel('MAE', color='red')
plt.tight_layout()
# Save main figure
main_plot_path = f'{folder_name}_uv_analysis.png'
plt.savefig(main_plot_path, dpi=300, bbox_inches='tight')
# Confusion matrix as separate plot
plt.figure(figsize=(10, 8))
sns.heatmap(cm_df, annot=True, fmt='d', cmap='Blues')
plt.title('Confusion Matrix for UV Risk Levels')
conf_matrix_path = f'{folder_name}_confusion_matrix.png'
plt.savefig(conf_matrix_path, dpi=300, bbox_inches='tight')
plt.close('all')
except Exception as e:
print(f"\nError saving plots: {str(e)}")
main_plot_path = None
conf_matrix_path = None
# Print detailed report
print("\nUV Index Prediction Analysis:")
print("\nRaw Metrics:")
for key, value in metrics['raw'].items():
print(f"{key}: {value:.3f}")
print("\nRounded Metrics:")
for key, value in metrics['rounded'].items():
print(f"{key}: {value:.3f}")
print("\nAnalysis by UV Range:")
for label, stats in range_analysis.items():
print(f"\n{label}:")
for key, value in stats.items():
print(f" {key}: {value:.3f}")
print("\nConfusion Matrix:")
print(cm_df)
# Add range analysis and confusion matrix to metrics dictionary
metrics.update({
'range_analysis': range_analysis,
'confusion_matrix': cm_df.to_dict(),
'plot_paths': {
'main_analysis': main_plot_path,
'confusion_matrix': conf_matrix_path
}
})
return metrics
def plot_training_history(history, folder_name=None):
"""
Visualize and save the loss and metrics plots during training
Parameters:
-----------
history : tensorflow.keras.callbacks.History
The history object returned by model training
folder_name : str
Folder where to save the plot
"""
try:
# Create the figure
plt.figure(figsize=(12, 4))
# Loss Plot
plt.subplot(1, 2, 1)
plt.plot(history.history['loss'], label='Training Loss')
plt.plot(history.history['val_loss'], label='Validation Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.grid(True)
# MAE Plot
plt.subplot(1, 2, 2)
plt.plot(history.history['mae'], label='Training MAE')
plt.plot(history.history['val_mae'], label='Validation MAE')
plt.title('Model MAE')
plt.xlabel('Epoch')
plt.ylabel('MAE')
plt.legend()
plt.grid(True)
plt.tight_layout()
if folder_name is not None:
os.makedirs(folder_name, exist_ok=True)
# Generate filename with timestamp
filename = os.path.join(folder_name, 'training_history.png')
# Save the figure
plt.savefig(filename, dpi=300, bbox_inches='tight')
print(f"\nTraining history plot saved as: {filename}")
# Also save numerical data in CSV format
history_df = pd.DataFrame({
'epoch': range(1, len(history.history['loss']) + 1),
'training_loss': history.history['loss'],
'validation_loss': history.history['val_loss'],
'training_mae': history.history['mae'],
'validation_mae': history.history['val_mae']
})
if folder_name is not None:
csv_filename = os.path.join(folder_name, 'training_history.csv')
history_df.to_csv(csv_filename, index=False)
print(f"Training history data saved as: {csv_filename}")
# Calculate and save final statistics
final_stats = {
'final_training_loss': history.history['loss'][-1],
'final_validation_loss': history.history['val_loss'][-1],
'final_training_mae': history.history['mae'][-1],
'final_validation_mae': history.history['val_mae'][-1],
'best_validation_loss': min(history.history['val_loss']),
'best_validation_mae': min(history.history['val_mae']),
'epochs': len(history.history['loss']),
}
if folder_name is not None:
# Save statistics in JSON format
stats_filename = os.path.join(folder_name, 'training_stats.json')
with open(stats_filename, 'w') as f:
json.dump(final_stats, f, indent=4)
print(f"Final statistics saved as: {stats_filename}")
# Print main statistics
print("\nFinal training statistics:")
print(f"Final Loss (train/val): {final_stats['final_training_loss']:.4f}/{final_stats['final_validation_loss']:.4f}")
print(f"Final MAE (train/val): {final_stats['final_training_mae']:.4f}/{final_stats['final_validation_mae']:.4f}")
print(f"Best validation loss: {final_stats['best_validation_loss']:.4f}")
print(f"Best validation MAE: {final_stats['best_validation_mae']:.4f}")
plt.show()
except Exception as e:
print(f"\nError during plot creation or saving: {str(e)}")
def train_hybrid_model(model, X_train, y_train, X_test, y_test, epochs=100, batch_size=32, folder_name='uv_index'):
"""
Advanced training function for the hybrid UV index model with detailed monitoring
and training management.
Parameters:
-----------
model : keras.Model
The compiled hybrid model
X_train : numpy.ndarray
Training data
y_train : numpy.ndarray
Training targets
X_test : numpy.ndarray
Validation data
y_test : numpy.ndarray
Validation targets
epochs : int, optional
Maximum number of training epochs
batch_size : int, optional
Batch size
Returns:
--------
history : keras.callbacks.History
Training history with all metrics
"""
# Advanced callbacks for training
callbacks = [
# Advanced Early Stopping
EarlyStopping(
monitor='mae',
patience=15,
restore_best_weights=True,
mode='min',
verbose=1,
min_delta=1e-6
),
ReduceLROnPlateau(
monitor='mae',
factor=0.05,
patience=3,
verbose=1,
mode='min',
min_delta=1e-6,
cooldown=2,
min_lr=1e-7
),
ReduceLROnPlateau(
monitor='val_loss',
factor=0.2,
patience=2,
verbose=1,
mode='min',
min_delta=1e-6,
cooldown=1,
min_lr=1e-7
),
tf.keras.callbacks.ModelCheckpoint(
filepath=f'{folder_name}_best_uv_model.h5',
monitor='mae',
save_best_only=True,
mode='min'
),
tf.keras.callbacks.TensorBoard(
log_dir=f'./{folder_name}_logs',
histogram_freq=1,
write_graph=True,
update_freq='epoch'
),
tf.keras.callbacks.LambdaCallback(
on_epoch_end=lambda epoch, logs: print(
f"\nEpoch {epoch + 1}: Out of range predictions: "
f"{np.sum((model.predict(X_test) < 0) | (model.predict(X_test) > 11))}"
) if epoch % 20 == 0 else None
)
]
try:
history = model.fit(
X_train, y_train,
validation_data=(X_test, y_test),
epochs=epochs,
batch_size=batch_size,
callbacks=callbacks,
verbose=1,
shuffle=False,
validation_freq=1,
)
# Post-training analysis
print("\nTraining completed successfully!")
return history
except Exception as e:
print(f"\nError during training: {str(e)}")
raise
finally:
# Memory cleanup
tf.keras.backend.clear_session()
def integrate_predictions(df, predictions, sequence_length=24):
"""
Integrate UV index predictions into the original dataset for pre-2010 data.
Parameters:
-----------
df : pandas.DataFrame
Original dataset
predictions : numpy.ndarray
Array of UV index predictions
sequence_length : int
Sequence length used for predictions
Returns:
--------
pandas.DataFrame
Updated dataset with UV index predictions
"""
# Convert datetime to datetime format if not already
df['datetime'] = pd.to_datetime(df['datetime'])
# Identify pre-2010 rows
mask_pre_2010 = df['datetime'].dt.year < 2010
# Create temporary DataFrame with predictions
dates_pre_2010 = df[mask_pre_2010]['datetime'].iloc[sequence_length - 1:]
predictions_df = pd.DataFrame({
'datetime': dates_pre_2010,
'uvindex_predicted': predictions.flatten()
})
# Merge with original dataset
df = df.merge(predictions_df, on='datetime', how='left')
# Update uvindex column where missing
df['uvindex'] = df['uvindex'].fillna(df['uvindex_predicted'])
# Remove temporary column
df = df.drop('uvindex_predicted', axis=1)
print(f"Added {len(predictions)} predictions to dataset")
print(f"Rows with UV index after integration: {df['uvindex'].notna().sum()}")
return dfIn [6]:
df = pd.read_parquet('../../sources/weather_data.parquet')
print("Initializing UV index model training...")
# Data preparation
print("\n1. Preparing data...")
X_train_seq, X_test_seq, y_train, y_test, feature_scaler, target_scaler, features, X_to_predict_seq = prepare_hybrid_data(df)
print(f"Training data shape: {X_train_seq.shape}")
print(f"Test data shape: {X_test_seq.shape}")
# Save or load scaler and features
feature_scaler_path = f'{folder_name}_feature_scaler.joblib'
target_scaler_path = f'{folder_name}_target_scaler.joblib'
features_path = f'{folder_name}_features.json'
model_path = f'{folder_name}_best_model.h5'
history_path = f'{folder_name}_training_history.json'
if os.path.exists(feature_scaler_path):
print(f"Loading existing scaler from: {feature_scaler_path}")
scaler = joblib.load(feature_scaler_path)
else:
print(f"Saving scaler to: {feature_scaler_path}")
joblib.dump(feature_scaler, feature_scaler_path)
if os.path.exists(target_scaler_path):
print(f"Loading existing scaler from: {target_scaler_path}")
scaler = joblib.load(target_scaler_path)
else:
print(f"Saving scaler to: {target_scaler_path}")
joblib.dump(target_scaler, target_scaler_path)
if os.path.exists(features_path):
print(f"Loading existing features from: {features_path}")
with open(features_path, 'r') as f:
features = json.load(f)
else:
print(f"Saving features to: {features_path}")
with open(features_path, 'w') as f:
json.dump(features, f)
# Data quality verification
if np.isnan(X_train_seq).any() or np.isnan(y_train).any():
raise ValueError("Found NaN values in training data")Initializing UV index model training... 1. Preparing data... Temporal distribution of data: Records after 2010: 129,777 Records before 2010: 227,902 Warning: Found missing values after preprocessing Features with missing values: [] Number of features used: 30 Feature categories: atmospheric: 6 features temporal: 4 features solar: 5 features interactions: 4 features rolling: 2 features Categorical: 9 features Training data shape: (64865, 24, 30) Test data shape: (64866, 24, 30) Saving scaler to: 2024-11-21_08-23_feature_scaler.joblib Saving scaler to: 2024-11-21_08-23_target_scaler.joblib Saving features to: 2024-11-21_08-23_features.json
In [7]:
# Model creation or loading
print("\n2. Model initialization...")
input_shape = (X_train_seq.shape[1], X_train_seq.shape[2])
MAX_UVINDEX = 11
max_val_scaled = target_scaler.transform([[MAX_UVINDEX]])[0][0]
if os.path.exists(model_path):
print(f"Loading existing model from: {model_path}")
model = tf.keras.models.load_model(model_path)
# Load existing history if available
if os.path.exists(history_path):
print(f"Loading existing training history from: {history_path}")
with open(history_path, 'r') as f:
history_dict = json.load(f)
history = type('History', (), {'history': history_dict})()
else:
history = type('History', (), {'history': {}})()
else:
print("Creating new model...")
model = create_uv_index_model(input_shape=input_shape, folder_name=folder_name, max_output=max_val_scaled)
print("\n3. Starting training...")
history = train_hybrid_model(
model=model,
X_train=X_train_seq,
y_train=y_train,
X_test=X_test_seq,
y_test=y_test,
epochs=100,
batch_size=128,
folder_name=folder_name
)2. Model initialization... Creating new model...
2024-11-21 08:26:35.683631: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 43404 MB memory: -> device: 0, name: NVIDIA L40, pci bus id: 0000:25:00.0, compute capability: 8.9
Model: "UvModel"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
input_1 (InputLayer) [(None, 24, 30)] 0 []
bidirectional (Bidirection (None, 24, 128) 48640 ['input_1[0][0]']
al)
layer_normalization (Layer (None, 24, 128) 256 ['bidirectional[0][0]']
Normalization)
dropout (Dropout) (None, 24, 128) 0 ['layer_normalization[0][0]']
dense (Dense) (None, 24, 128) 3968 ['input_1[0][0]']
stochastic_depth (Stochast (None, 24, 128) 0 ['dropout[0][0]',
icDepth) 'dense[0][0]']
multi_head_attention (Mult (None, 24, 128) 131968 ['stochastic_depth[0][0]',
iHeadAttention) 'stochastic_depth[0][0]']
stochastic_depth_1 (Stocha (None, 24, 128) 0 ['multi_head_attention[0][0]',
sticDepth) 'stochastic_depth[0][0]']
layer_normalization_1 (Lay (None, 24, 128) 256 ['stochastic_depth_1[0][0]']
erNormalization)
dense_1 (Dense) (None, 24, 512) 66048 ['layer_normalization_1[0][0]'
]
dense_2 (Dense) (None, 24, 128) 65664 ['dense_1[0][0]']
stochastic_depth_2 (Stocha (None, 24, 128) 0 ['dense_2[0][0]',
sticDepth) 'layer_normalization_1[0][0]'
]
layer_normalization_2 (Lay (None, 24, 128) 256 ['stochastic_depth_2[0][0]']
erNormalization)
bidirectional_1 (Bidirecti (None, 24, 64) 41216 ['layer_normalization_2[0][0]'
onal) ]
layer_normalization_3 (Lay (None, 24, 64) 128 ['bidirectional_1[0][0]']
erNormalization)
dropout_1 (Dropout) (None, 24, 64) 0 ['layer_normalization_3[0][0]'
]
dense_3 (Dense) (None, 24, 64) 8256 ['layer_normalization_2[0][0]'
]
stochastic_depth_3 (Stocha (None, 24, 64) 0 ['dropout_1[0][0]',
sticDepth) 'dense_3[0][0]']
multi_head_attention_1 (Mu (None, 24, 64) 33216 ['stochastic_depth_3[0][0]',
ltiHeadAttention) 'stochastic_depth_3[0][0]']
stochastic_depth_4 (Stocha (None, 24, 64) 0 ['multi_head_attention_1[0][0]
sticDepth) ',
'stochastic_depth_3[0][0]']
layer_normalization_4 (Lay (None, 24, 64) 128 ['stochastic_depth_4[0][0]']
erNormalization)
dense_4 (Dense) (None, 24, 256) 16640 ['layer_normalization_4[0][0]'
]
dense_5 (Dense) (None, 24, 64) 16448 ['dense_4[0][0]']
stochastic_depth_5 (Stocha (None, 24, 64) 0 ['dense_5[0][0]',
sticDepth) 'layer_normalization_4[0][0]'
]
layer_normalization_5 (Lay (None, 24, 64) 128 ['stochastic_depth_5[0][0]']
erNormalization)
bidirectional_2 (Bidirecti (None, 24, 32) 10368 ['layer_normalization_5[0][0]'
onal) ]
layer_normalization_6 (Lay (None, 24, 32) 64 ['bidirectional_2[0][0]']
erNormalization)
dropout_2 (Dropout) (None, 24, 32) 0 ['layer_normalization_6[0][0]'
]
dense_6 (Dense) (None, 24, 32) 2080 ['layer_normalization_5[0][0]'
]
stochastic_depth_6 (Stocha (None, 24, 32) 0 ['dropout_2[0][0]',
sticDepth) 'dense_6[0][0]']
multi_head_attention_2 (Mu (None, 24, 32) 8416 ['stochastic_depth_6[0][0]',
ltiHeadAttention) 'stochastic_depth_6[0][0]']
stochastic_depth_7 (Stocha (None, 24, 32) 0 ['multi_head_attention_2[0][0]
sticDepth) ',
'stochastic_depth_6[0][0]']
layer_normalization_7 (Lay (None, 24, 32) 64 ['stochastic_depth_7[0][0]']
erNormalization)
dense_7 (Dense) (None, 24, 128) 4224 ['layer_normalization_7[0][0]'
]
dense_8 (Dense) (None, 24, 32) 4128 ['dense_7[0][0]']
stochastic_depth_8 (Stocha (None, 24, 32) 0 ['dense_8[0][0]',
sticDepth) 'layer_normalization_7[0][0]'
]
layer_normalization_8 (Lay (None, 24, 32) 64 ['stochastic_depth_8[0][0]']
erNormalization)
multi_head_attention_3 (Mu (None, 24, 32) 8416 ['layer_normalization_8[0][0]'
ltiHeadAttention) , 'layer_normalization_8[0][0]
']
stochastic_depth_9 (Stocha (None, 24, 32) 0 ['multi_head_attention_3[0][0]
sticDepth) ',
'layer_normalization_8[0][0]'
]
layer_normalization_9 (Lay (None, 24, 32) 64 ['stochastic_depth_9[0][0]']
erNormalization)
global_average_pooling1d ( (None, 32) 0 ['layer_normalization_9[0][0]'
GlobalAveragePooling1D) ]
dense_9 (Dense) (None, 32) 1056 ['global_average_pooling1d[0][
0]']
batch_normalization (Batch (None, 32) 128 ['dense_9[0][0]']
Normalization)
dropout_3 (Dropout) (None, 32) 0 ['batch_normalization[0][0]']
dense_10 (Dense) (None, 16) 528 ['dropout_3[0][0]']
batch_normalization_1 (Bat (None, 16) 64 ['dense_10[0][0]']
chNormalization)
dense_11 (Dense) (None, 8) 136 ['batch_normalization_1[0][0]'
]
dense_12 (Dense) (None, 1) 9 ['dense_11[0][0]']
lambda (Lambda) (None, 1) 0 ['dense_12[0][0]']
==================================================================================================
Total params: 473025 (1.80 MB)
Trainable params: 472929 (1.80 MB)
Non-trainable params: 96 (384.00 Byte)
__________________________________________________________________________________________________
3. Starting training...
Epoch 1/100
2024-11-21 08:26:51.620818: I tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:606] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once. 2024-11-21 08:26:51.695976: I tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:432] Loaded cuDNN version 8905 2024-11-21 08:26:51.911310: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0xd713390 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2024-11-21 08:26:51.911349: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): NVIDIA L40, Compute Capability 8.9 2024-11-21 08:26:51.921786: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:255] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable. 2024-11-21 08:26:52.001781: I tensorflow/tsl/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory 2024-11-21 08:26:52.063791: I ./tensorflow/compiler/jit/device_compiler.h:186] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.
507/507 [==============================] - ETA: 0s - loss: 4.4444 - mae: 1.3032 - mse: 2.1820 - root_mean_squared_error: 1.4772 - smooth_mape: 226.3000
/usr/local/lib/python3.11/dist-packages/keras/src/engine/training.py:3000: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.
saving_api.save_model(
2028/2028 [==============================] - 25s 11ms/step 2028/2028 [==============================] - 19s 9ms/step Epoch 1: Out of range predictions: 0 507/507 [==============================] - 95s 151ms/step - loss: 4.4444 - mae: 1.3032 - mse: 2.1820 - root_mean_squared_error: 1.4772 - smooth_mape: 226.3000 - val_loss: 3.3901 - val_mae: 0.7726 - val_mse: 1.0065 - val_root_mean_squared_error: 1.0033 - val_smooth_mape: 96.0347 - lr: 5.0600e-05 Epoch 2/100 507/507 [==============================] - 24s 48ms/step - loss: 3.4513 - mae: 0.9422 - mse: 1.2050 - root_mean_squared_error: 1.0977 - smooth_mape: 144.6453 - val_loss: 3.0292 - val_mae: 0.6905 - val_mse: 0.9136 - val_root_mean_squared_error: 0.9558 - val_smooth_mape: 72.3569 - lr: 9.9998e-05 Epoch 3/100 507/507 [==============================] - 27s 53ms/step - loss: 2.8616 - mae: 0.7835 - mse: 0.9255 - root_mean_squared_error: 0.9620 - smooth_mape: 105.5174 - val_loss: 2.6105 - val_mae: 0.6461 - val_mse: 0.8668 - val_root_mean_squared_error: 0.9310 - val_smooth_mape: 62.3054 - lr: 9.7355e-05 Epoch 4/100 507/507 [==============================] - 27s 52ms/step - loss: 2.2615 - mae: 0.6304 - mse: 0.6460 - root_mean_squared_error: 0.8038 - smooth_mape: 84.4009 - val_loss: 1.9317 - val_mae: 0.4135 - val_mse: 0.4540 - val_root_mean_squared_error: 0.6738 - val_smooth_mape: 35.5852 - lr: 8.9946e-05 Epoch 5/100 507/507 [==============================] - 26s 52ms/step - loss: 1.6904 - mae: 0.4345 - mse: 0.3313 - root_mean_squared_error: 0.5756 - smooth_mape: 59.1463 - val_loss: 1.4432 - val_mae: 0.2685 - val_mse: 0.1950 - val_root_mean_squared_error: 0.4416 - val_smooth_mape: 27.2014 - lr: 7.8518e-05 Epoch 6/100 507/507 [==============================] - 27s 54ms/step - loss: 1.3637 - mae: 0.3450 - mse: 0.2236 - root_mean_squared_error: 0.4729 - smooth_mape: 45.5589 - val_loss: 1.2321 - val_mae: 0.2588 - val_mse: 0.1828 - val_root_mean_squared_error: 0.4276 - val_smooth_mape: 25.4509 - lr: 6.4221e-05 Epoch 7/100 507/507 [==============================] - 27s 53ms/step - loss: 1.1519 - mae: 0.2973 - mse: 0.1789 - root_mean_squared_error: 0.4229 - smooth_mape: 38.0580 - val_loss: 1.0643 - val_mae: 0.2375 - val_mse: 0.1577 - val_root_mean_squared_error: 0.3971 - val_smooth_mape: 23.5643 - lr: 4.8492e-05 Epoch 8/100 507/507 [==============================] - 28s 55ms/step - loss: 1.0083 - mae: 0.2665 - mse: 0.1546 - root_mean_squared_error: 0.3932 - smooth_mape: 32.9333 - val_loss: 0.9257 - val_mae: 0.2038 - val_mse: 0.1143 - val_root_mean_squared_error: 0.3380 - val_smooth_mape: 21.4354 - lr: 3.2915e-05 Epoch 9/100 507/507 [==============================] - 28s 55ms/step - loss: 0.9148 - mae: 0.2470 - mse: 0.1407 - root_mean_squared_error: 0.3751 - smooth_mape: 29.7929 - val_loss: 0.8520 - val_mae: 0.1890 - val_mse: 0.1027 - val_root_mean_squared_error: 0.3204 - val_smooth_mape: 20.0954 - lr: 1.9058e-05 Epoch 10/100 507/507 [==============================] - 27s 54ms/step - loss: 0.8584 - mae: 0.2366 - mse: 0.1317 - root_mean_squared_error: 0.3628 - smooth_mape: 28.3349 - val_loss: 0.8144 - val_mae: 0.1853 - val_mse: 0.0991 - val_root_mean_squared_error: 0.3148 - val_smooth_mape: 19.7012 - lr: 8.3134e-06 Epoch 11/100 507/507 [==============================] - 28s 55ms/step - loss: 0.8331 - mae: 0.2331 - mse: 0.1292 - root_mean_squared_error: 0.3594 - smooth_mape: 27.8952 - val_loss: 0.7991 - val_mae: 0.1829 - val_mse: 0.0966 - val_root_mean_squared_error: 0.3108 - val_smooth_mape: 19.7143 - lr: 1.7639e-06 Epoch 12/100 507/507 [==============================] - 28s 55ms/step - loss: 0.8266 - mae: 0.2326 - mse: 0.1289 - root_mean_squared_error: 0.3591 - smooth_mape: 27.6276 - val_loss: 0.8002 - val_mae: 0.1851 - val_mse: 0.0996 - val_root_mean_squared_error: 0.3155 - val_smooth_mape: 19.6762 - lr: 6.7976e-08 Epoch 13/100 507/507 [==============================] - 27s 53ms/step - loss: 0.8256 - mae: 0.2332 - mse: 0.1297 - root_mean_squared_error: 0.3601 - smooth_mape: 27.8768 - val_loss: 0.7938 - val_mae: 0.1827 - val_mse: 0.0977 - val_root_mean_squared_error: 0.3126 - val_smooth_mape: 19.5952 - lr: 3.3964e-06 Epoch 14/100 507/507 [==============================] - 27s 52ms/step - loss: 0.8108 - mae: 0.2311 - mse: 0.1283 - root_mean_squared_error: 0.3582 - smooth_mape: 27.5029 - val_loss: 0.7689 - val_mae: 0.1841 - val_mse: 0.0983 - val_root_mean_squared_error: 0.3135 - val_smooth_mape: 19.2959 - lr: 1.1414e-05 Epoch 15/100 507/507 [==============================] - 26s 52ms/step - loss: 0.7666 - mae: 0.2260 - mse: 0.1253 - root_mean_squared_error: 0.3539 - smooth_mape: 26.7909 - val_loss: 0.7071 - val_mae: 0.1791 - val_mse: 0.0942 - val_root_mean_squared_error: 0.3069 - val_smooth_mape: 18.7677 - lr: 2.3315e-05 Epoch 16/100 507/507 [==============================] - 26s 52ms/step - loss: 0.6896 - mae: 0.2200 - mse: 0.1214 - root_mean_squared_error: 0.3484 - smooth_mape: 25.8531 - val_loss: 0.6141 - val_mae: 0.1740 - val_mse: 0.0874 - val_root_mean_squared_error: 0.2956 - val_smooth_mape: 18.6023 - lr: 3.7901e-05 Epoch 17/100 507/507 [==============================] - 25s 50ms/step - loss: 0.5905 - mae: 0.2116 - mse: 0.1160 - root_mean_squared_error: 0.3406 - smooth_mape: 24.5564 - val_loss: 0.5156 - val_mae: 0.1681 - val_mse: 0.0874 - val_root_mean_squared_error: 0.2956 - val_smooth_mape: 17.4665 - lr: 5.3704e-05 Epoch 18/100 507/507 [==============================] - 27s 54ms/step - loss: 0.4901 - mae: 0.2037 - mse: 0.1116 - root_mean_squared_error: 0.3341 - smooth_mape: 23.3802 - val_loss: 0.4216 - val_mae: 0.1617 - val_mse: 0.0852 - val_root_mean_squared_error: 0.2919 - val_smooth_mape: 17.2917 - lr: 6.9134e-05 Epoch 19/100 507/507 [==============================] - 27s 53ms/step - loss: 0.3984 - mae: 0.1930 - mse: 0.1038 - root_mean_squared_error: 0.3221 - smooth_mape: 21.8713 - val_loss: 0.3391 - val_mae: 0.1637 - val_mse: 0.0790 - val_root_mean_squared_error: 0.2810 - val_smooth_mape: 17.4952 - lr: 8.2639e-05 Epoch 20/100 507/507 [==============================] - 27s 54ms/step - loss: 0.3280 - mae: 0.1882 - mse: 0.1019 - root_mean_squared_error: 0.3192 - smooth_mape: 21.1186 - val_loss: 0.2774 - val_mae: 0.1532 - val_mse: 0.0766 - val_root_mean_squared_error: 0.2767 - val_smooth_mape: 16.5548 - lr: 9.2860e-05 Epoch 21/100 2028/2028 [==============================] - 22s 11ms/step 2028/2028 [==============================] - 23s 12ms/step Epoch 21: Out of range predictions: 0 507/507 [==============================] - 75s 148ms/step - loss: 0.2717 - mae: 0.1800 - mse: 0.0959 - root_mean_squared_error: 0.3097 - smooth_mape: 19.9770 - val_loss: 0.2327 - val_mae: 0.1514 - val_mse: 0.0756 - val_root_mean_squared_error: 0.2750 - val_smooth_mape: 16.9079 - lr: 9.8768e-05 Epoch 22/100 507/507 [==============================] - 27s 52ms/step - loss: 0.2290 - mae: 0.1732 - mse: 0.0907 - root_mean_squared_error: 0.3011 - smooth_mape: 19.0873 - val_loss: 0.1969 - val_mae: 0.1482 - val_mse: 0.0722 - val_root_mean_squared_error: 0.2687 - val_smooth_mape: 16.2890 - lr: 9.9769e-05 Epoch 23/100 507/507 [==============================] - 26s 50ms/step - loss: 0.1994 - mae: 0.1705 - mse: 0.0889 - root_mean_squared_error: 0.2982 - smooth_mape: 18.7545 - val_loss: 0.1750 - val_mae: 0.1452 - val_mse: 0.0739 - val_root_mean_squared_error: 0.2719 - val_smooth_mape: 15.6235 - lr: 9.5762e-05 Epoch 24/100 507/507 [==============================] - 27s 53ms/step - loss: 0.1768 - mae: 0.1661 - mse: 0.0861 - root_mean_squared_error: 0.2934 - smooth_mape: 18.1383 - val_loss: 0.1559 - val_mae: 0.1482 - val_mse: 0.0716 - val_root_mean_squared_error: 0.2676 - val_smooth_mape: 16.7181 - lr: 8.7150e-05 Epoch 25/100 507/507 [==============================] - 26s 52ms/step - loss: 0.1601 - mae: 0.1629 - mse: 0.0835 - root_mean_squared_error: 0.2890 - smooth_mape: 17.7203 - val_loss: 0.1425 - val_mae: 0.1434 - val_mse: 0.0702 - val_root_mean_squared_error: 0.2649 - val_smooth_mape: 15.5010 - lr: 7.4800e-05 Epoch 26/100 507/507 [==============================] - 27s 54ms/step - loss: 0.1472 - mae: 0.1586 - mse: 0.0806 - root_mean_squared_error: 0.2839 - smooth_mape: 17.2387 - val_loss: 0.1347 - val_mae: 0.1454 - val_mse: 0.0710 - val_root_mean_squared_error: 0.2665 - val_smooth_mape: 16.1275 - lr: 5.9955e-05 Epoch 27/100 507/507 [==============================] - 27s 54ms/step - loss: 0.1399 - mae: 0.1584 - mse: 0.0803 - root_mean_squared_error: 0.2834 - smooth_mape: 17.2506 - val_loss: 0.1270 - val_mae: 0.1401 - val_mse: 0.0687 - val_root_mean_squared_error: 0.2621 - val_smooth_mape: 15.1573 - lr: 4.4108e-05 Epoch 28/100 507/507 [==============================] - 27s 53ms/step - loss: 0.1344 - mae: 0.1563 - mse: 0.0792 - root_mean_squared_error: 0.2815 - smooth_mape: 16.9289 - val_loss: 0.1229 - val_mae: 0.1394 - val_mse: 0.0682 - val_root_mean_squared_error: 0.2611 - val_smooth_mape: 15.1774 - lr: 2.8853e-05 Epoch 29/100 507/507 [==============================] - 26s 52ms/step - loss: 0.1293 - mae: 0.1536 - mse: 0.0767 - root_mean_squared_error: 0.2770 - smooth_mape: 16.6836 - val_loss: 0.1206 - val_mae: 0.1383 - val_mse: 0.0679 - val_root_mean_squared_error: 0.2606 - val_smooth_mape: 14.8600 - lr: 1.5727e-05 Epoch 30/100 507/507 [==============================] - 27s 54ms/step - loss: 0.1275 - mae: 0.1526 - mse: 0.0763 - root_mean_squared_error: 0.2763 - smooth_mape: 16.5544 - val_loss: 0.1198 - val_mae: 0.1375 - val_mse: 0.0683 - val_root_mean_squared_error: 0.2613 - val_smooth_mape: 14.7848 - lr: 6.0491e-06 Epoch 31/100 507/507 [==============================] - 27s 53ms/step - loss: 0.1259 - mae: 0.1517 - mse: 0.0753 - root_mean_squared_error: 0.2744 - smooth_mape: 16.4806 - val_loss: 0.1192 - val_mae: 0.1370 - val_mse: 0.0678 - val_root_mean_squared_error: 0.2605 - val_smooth_mape: 14.5789 - lr: 7.9394e-07 Epoch 32/100 507/507 [==============================] - 25s 50ms/step - loss: 0.1263 - mae: 0.1522 - mse: 0.0759 - root_mean_squared_error: 0.2754 - smooth_mape: 16.5490 - val_loss: 0.1192 - val_mae: 0.1368 - val_mse: 0.0679 - val_root_mean_squared_error: 0.2606 - val_smooth_mape: 14.5403 - lr: 4.9000e-07 Epoch 33/100 507/507 [==============================] - 26s 52ms/step - loss: 0.1258 - mae: 0.1518 - mse: 0.0754 - root_mean_squared_error: 0.2745 - smooth_mape: 16.4660 - val_loss: 0.1189 - val_mae: 0.1376 - val_mse: 0.0678 - val_root_mean_squared_error: 0.2605 - val_smooth_mape: 14.7214 - lr: 5.1679e-06 Epoch 34/100 506/507 [============================>.] - ETA: 0s - loss: 0.1255 - mae: 0.1520 - mse: 0.0756 - root_mean_squared_error: 0.2749 - smooth_mape: 16.4794 Epoch 34: ReduceLROnPlateau reducing learning rate to 7.178531632234808e-07. 507/507 [==============================] - 27s 54ms/step - loss: 0.1255 - mae: 0.1520 - mse: 0.0756 - root_mean_squared_error: 0.2749 - smooth_mape: 16.4811 - val_loss: 0.1179 - val_mae: 0.1389 - val_mse: 0.0676 - val_root_mean_squared_error: 0.2601 - val_smooth_mape: 14.8385 - lr: 7.1785e-07 Epoch 35/100 507/507 [==============================] - 27s 52ms/step - loss: 0.1245 - mae: 0.1527 - mse: 0.0760 - root_mean_squared_error: 0.2756 - smooth_mape: 16.5704 - val_loss: 0.1169 - val_mae: 0.1410 - val_mse: 0.0685 - val_root_mean_squared_error: 0.2618 - val_smooth_mape: 15.6455 - lr: 2.7133e-05 Epoch 36/100 507/507 [==============================] - 27s 54ms/step - loss: 0.1227 - mae: 0.1525 - mse: 0.0766 - root_mean_squared_error: 0.2767 - smooth_mape: 16.5028 - val_loss: 0.1139 - val_mae: 0.1381 - val_mse: 0.0683 - val_root_mean_squared_error: 0.2614 - val_smooth_mape: 14.6552 - lr: 4.2209e-05 Epoch 37/100 507/507 [==============================] - 27s 54ms/step - loss: 0.1198 - mae: 0.1535 - mse: 0.0769 - root_mean_squared_error: 0.2773 - smooth_mape: 16.6359 - val_loss: 0.1107 - val_mae: 0.1409 - val_mse: 0.0686 - val_root_mean_squared_error: 0.2619 - val_smooth_mape: 14.7102 - lr: 5.8070e-05 Epoch 38/100 507/507 [==============================] - ETA: 0s - loss: 0.1160 - mae: 0.1532 - mse: 0.0769 - root_mean_squared_error: 0.2772 - smooth_mape: 16.5710 Epoch 38: ReduceLROnPlateau reducing learning rate to 3.6559198633767668e-06. 507/507 [==============================] - 26s 51ms/step - loss: 0.1160 - mae: 0.1532 - mse: 0.0769 - root_mean_squared_error: 0.2772 - smooth_mape: 16.5710 - val_loss: 0.1057 - val_mae: 0.1380 - val_mse: 0.0675 - val_root_mean_squared_error: 0.2599 - val_smooth_mape: 14.8251 - lr: 3.6559e-06 Epoch 39/100 507/507 [==============================] - 28s 55ms/step - loss: 0.1113 - mae: 0.1525 - mse: 0.0762 - root_mean_squared_error: 0.2761 - smooth_mape: 16.4623 - val_loss: 0.1040 - val_mae: 0.1391 - val_mse: 0.0703 - val_root_mean_squared_error: 0.2652 - val_smooth_mape: 14.3343 - lr: 8.5841e-05 Epoch 40/100 507/507 [==============================] - 27s 53ms/step - loss: 0.1080 - mae: 0.1531 - mse: 0.0770 - root_mean_squared_error: 0.2775 - smooth_mape: 16.5384 - val_loss: 0.0995 - val_mae: 0.1393 - val_mse: 0.0697 - val_root_mean_squared_error: 0.2640 - val_smooth_mape: 15.2621 - lr: 9.4957e-05 Epoch 41/100 2028/2028 [==============================] - 22s 11ms/step 2028/2028 [==============================] - 23s 11ms/step Epoch 41: Out of range predictions: 0 507/507 [==============================] - 73s 144ms/step - loss: 0.1038 - mae: 0.1523 - mse: 0.0766 - root_mean_squared_error: 0.2767 - smooth_mape: 16.4194 - val_loss: 0.0982 - val_mae: 0.1425 - val_mse: 0.0723 - val_root_mean_squared_error: 0.2689 - val_smooth_mape: 14.8658 - lr: 9.9549e-05 Epoch 42/100 506/507 [============================>.] - ETA: 0s - loss: 0.1026 - mae: 0.1539 - mse: 0.0783 - root_mean_squared_error: 0.2798 - smooth_mape: 16.6916 Epoch 42: ReduceLROnPlateau reducing learning rate to 4.957754936185666e-06. 507/507 [==============================] - 27s 53ms/step - loss: 0.1026 - mae: 0.1539 - mse: 0.0783 - root_mean_squared_error: 0.2798 - smooth_mape: 16.6892 - val_loss: 0.0915 - val_mae: 0.1368 - val_mse: 0.0677 - val_root_mean_squared_error: 0.2602 - val_smooth_mape: 14.4180 - lr: 4.9578e-06 Epoch 43/100 507/507 [==============================] - 28s 55ms/step - loss: 0.0964 - mae: 0.1500 - mse: 0.0749 - root_mean_squared_error: 0.2736 - smooth_mape: 16.2060 - val_loss: 0.0881 - val_mae: 0.1362 - val_mse: 0.0671 - val_root_mean_squared_error: 0.2591 - val_smooth_mape: 14.6337 - lr: 9.3815e-05 Epoch 44/100 507/507 [==============================] - 27s 53ms/step - loss: 0.0926 - mae: 0.1483 - mse: 0.0735 - root_mean_squared_error: 0.2711 - smooth_mape: 15.9664 - val_loss: 0.0856 - val_mae: 0.1355 - val_mse: 0.0668 - val_root_mean_squared_error: 0.2585 - val_smooth_mape: 14.4813 - lr: 8.4067e-05 Epoch 45/100 507/507 [==============================] - 26s 52ms/step - loss: 0.0904 - mae: 0.1477 - mse: 0.0732 - root_mean_squared_error: 0.2706 - smooth_mape: 15.9141 - val_loss: 0.0839 - val_mae: 0.1354 - val_mse: 0.0669 - val_root_mean_squared_error: 0.2587 - val_smooth_mape: 14.4705 - lr: 7.0890e-05 Epoch 46/100 507/507 [==============================] - 27s 54ms/step - loss: 0.0876 - mae: 0.1461 - mse: 0.0718 - root_mean_squared_error: 0.2680 - smooth_mape: 15.7518 - val_loss: 0.0825 - val_mae: 0.1373 - val_mse: 0.0668 - val_root_mean_squared_error: 0.2585 - val_smooth_mape: 14.4117 - lr: 5.5612e-05 Epoch 47/100 507/507 [==============================] - 27s 53ms/step - loss: 0.0853 - mae: 0.1446 - mse: 0.0706 - root_mean_squared_error: 0.2658 - smooth_mape: 15.5602 - val_loss: 0.0805 - val_mae: 0.1370 - val_mse: 0.0658 - val_root_mean_squared_error: 0.2566 - val_smooth_mape: 14.8418 - lr: 3.9768e-05 Epoch 48/100 507/507 [==============================] - 28s 55ms/step - loss: 0.0851 - mae: 0.1449 - mse: 0.0713 - root_mean_squared_error: 0.2671 - smooth_mape: 15.6229 - val_loss: 0.0801 - val_mae: 0.1363 - val_mse: 0.0660 - val_root_mean_squared_error: 0.2570 - val_smooth_mape: 14.7760 - lr: 2.4955e-05 Epoch 49/100 507/507 [==============================] - 26s 51ms/step - loss: 0.0833 - mae: 0.1432 - mse: 0.0699 - root_mean_squared_error: 0.2643 - smooth_mape: 15.4598 - val_loss: 0.0791 - val_mae: 0.1348 - val_mse: 0.0654 - val_root_mean_squared_error: 0.2557 - val_smooth_mape: 14.3369 - lr: 1.2661e-05 Epoch 50/100 507/507 [==============================] - 27s 53ms/step - loss: 0.0819 - mae: 0.1421 - mse: 0.0687 - root_mean_squared_error: 0.2620 - smooth_mape: 15.3369 - val_loss: 0.0790 - val_mae: 0.1342 - val_mse: 0.0655 - val_root_mean_squared_error: 0.2558 - val_smooth_mape: 14.2211 - lr: 4.1248e-06 Epoch 51/100 507/507 [==============================] - 27s 53ms/step - loss: 0.0818 - mae: 0.1421 - mse: 0.0686 - root_mean_squared_error: 0.2619 - smooth_mape: 15.3209 - val_loss: 0.0791 - val_mae: 0.1332 - val_mse: 0.0656 - val_root_mean_squared_error: 0.2562 - val_smooth_mape: 14.1215 - lr: 2.0452e-07 Epoch 52/100 507/507 [==============================] - ETA: 0s - loss: 0.0820 - mae: 0.1420 - mse: 0.0689 - root_mean_squared_error: 0.2625 - smooth_mape: 15.3575 Epoch 52: ReduceLROnPlateau reducing learning rate to 2.589756149973255e-07. 507/507 [==============================] - 28s 55ms/step - loss: 0.0820 - mae: 0.1420 - mse: 0.0689 - root_mean_squared_error: 0.2625 - smooth_mape: 15.3575 - val_loss: 0.0791 - val_mae: 0.1334 - val_mse: 0.0656 - val_root_mean_squared_error: 0.2561 - val_smooth_mape: 14.1793 - lr: 1.2949e-06 Epoch 53/100 507/507 [==============================] - 28s 55ms/step - loss: 0.0818 - mae: 0.1417 - mse: 0.0687 - root_mean_squared_error: 0.2621 - smooth_mape: 15.2994 - val_loss: 0.0790 - val_mae: 0.1349 - val_mse: 0.0655 - val_root_mean_squared_error: 0.2560 - val_smooth_mape: 14.1680 - lr: 7.2861e-06 Epoch 54/100 507/507 [==============================] - 27s 53ms/step - loss: 0.0818 - mae: 0.1421 - mse: 0.0689 - root_mean_squared_error: 0.2625 - smooth_mape: 15.3289 - val_loss: 0.0786 - val_mae: 0.1360 - val_mse: 0.0654 - val_root_mean_squared_error: 0.2558 - val_smooth_mape: 14.6983 - lr: 1.7575e-05 Epoch 55/100 507/507 [==============================] - 26s 51ms/step - loss: 0.0826 - mae: 0.1438 - mse: 0.0701 - root_mean_squared_error: 0.2648 - smooth_mape: 15.5292 - val_loss: 0.0784 - val_mae: 0.1367 - val_mse: 0.0656 - val_root_mean_squared_error: 0.2562 - val_smooth_mape: 14.7914 - lr: 3.1127e-05 Epoch 56/100 507/507 [==============================] - ETA: 0s - loss: 0.0820 - mae: 0.1433 - mse: 0.0700 - root_mean_squared_error: 0.2647 - smooth_mape: 15.4452 Epoch 56: ReduceLROnPlateau reducing learning rate to 2.3289143427973617e-06. 507/507 [==============================] - 27s 53ms/step - loss: 0.0820 - mae: 0.1433 - mse: 0.0700 - root_mean_squared_error: 0.2647 - smooth_mape: 15.4452 - val_loss: 0.0777 - val_mae: 0.1360 - val_mse: 0.0656 - val_root_mean_squared_error: 0.2561 - val_smooth_mape: 14.4766 - lr: 2.3289e-06 Epoch 57/100 507/507 [==============================] - 27s 53ms/step - loss: 0.0823 - mae: 0.1451 - mse: 0.0711 - root_mean_squared_error: 0.2667 - smooth_mape: 15.6261 - val_loss: 0.0785 - val_mae: 0.1375 - val_mse: 0.0674 - val_root_mean_squared_error: 0.2596 - val_smooth_mape: 15.0211 - lr: 6.2374e-05 Epoch 58/100 507/507 [==============================] - 26s 52ms/step - loss: 0.0818 - mae: 0.1454 - mse: 0.0715 - root_mean_squared_error: 0.2675 - smooth_mape: 15.6269 - val_loss: 0.0772 - val_mae: 0.1366 - val_mse: 0.0669 - val_root_mean_squared_error: 0.2586 - val_smooth_mape: 14.1091 - lr: 7.6924e-05 Epoch 59/100 507/507 [==============================] - 27s 54ms/step - loss: 0.0817 - mae: 0.1461 - mse: 0.0723 - root_mean_squared_error: 0.2689 - smooth_mape: 15.7262 - val_loss: 0.0761 - val_mae: 0.1370 - val_mse: 0.0663 - val_root_mean_squared_error: 0.2575 - val_smooth_mape: 14.8032 - lr: 8.8765e-05 Epoch 60/100 507/507 [==============================] - ETA: 0s - loss: 0.0805 - mae: 0.1461 - mse: 0.0719 - root_mean_squared_error: 0.2681 - smooth_mape: 15.7620 Epoch 60: ReduceLROnPlateau reducing learning rate to 4.835262006963604e-06. 507/507 [==============================] - 26s 52ms/step - loss: 0.0805 - mae: 0.1461 - mse: 0.0719 - root_mean_squared_error: 0.2681 - smooth_mape: 15.7620 - val_loss: 0.0759 - val_mae: 0.1358 - val_mse: 0.0674 - val_root_mean_squared_error: 0.2597 - val_smooth_mape: 14.4084 - lr: 4.8353e-06 Epoch 61/100 2028/2028 [==============================] - 22s 11ms/step 2028/2028 [==============================] - 23s 11ms/step Epoch 61: Out of range predictions: 0 507/507 [==============================] - 75s 147ms/step - loss: 0.0787 - mae: 0.1447 - mse: 0.0711 - root_mean_squared_error: 0.2666 - smooth_mape: 15.5565 - val_loss: 0.0737 - val_mae: 0.1352 - val_mse: 0.0661 - val_root_mean_squared_error: 0.2571 - val_smooth_mape: 14.3196 - lr: 9.9946e-05 Epoch 62/100 507/507 [==============================] - 27s 54ms/step - loss: 0.0781 - mae: 0.1451 - mse: 0.0715 - root_mean_squared_error: 0.2674 - smooth_mape: 15.5903 - val_loss: 0.0726 - val_mae: 0.1344 - val_mse: 0.0658 - val_root_mean_squared_error: 0.2565 - val_smooth_mape: 14.3699 - lr: 9.8161e-05 Epoch 63/100 507/507 [==============================] - 27s 53ms/step - loss: 0.0771 - mae: 0.1446 - mse: 0.0713 - root_mean_squared_error: 0.2670 - smooth_mape: 15.5551 - val_loss: 0.0721 - val_mae: 0.1350 - val_mse: 0.0661 - val_root_mean_squared_error: 0.2571 - val_smooth_mape: 14.3152 - lr: 9.1530e-05 Epoch 64/100 506/507 [============================>.] - ETA: 0s - loss: 0.0757 - mae: 0.1441 - mse: 0.0705 - root_mean_squared_error: 0.2655 - smooth_mape: 15.5067 Epoch 64: ReduceLROnPlateau reducing learning rate to 4.035990059492178e-06. 507/507 [==============================] - 27s 54ms/step - loss: 0.0757 - mae: 0.1441 - mse: 0.0705 - root_mean_squared_error: 0.2655 - smooth_mape: 15.5081 - val_loss: 0.0716 - val_mae: 0.1347 - val_mse: 0.0663 - val_root_mean_squared_error: 0.2574 - val_smooth_mape: 14.7350 - lr: 4.0360e-06 Epoch 65/100 507/507 [==============================] - 26s 50ms/step - loss: 0.0745 - mae: 0.1431 - mse: 0.0698 - root_mean_squared_error: 0.2642 - smooth_mape: 15.3922 - val_loss: 0.0710 - val_mae: 0.1360 - val_mse: 0.0661 - val_root_mean_squared_error: 0.2571 - val_smooth_mape: 14.1923 - lr: 6.6819e-05 Epoch 66/100 507/507 [==============================] - 27s 54ms/step - loss: 0.0729 - mae: 0.1413 - mse: 0.0686 - root_mean_squared_error: 0.2619 - smooth_mape: 15.1677 - val_loss: 0.0697 - val_mae: 0.1349 - val_mse: 0.0652 - val_root_mean_squared_error: 0.2553 - val_smooth_mape: 14.2538 - lr: 5.1225e-05 Epoch 67/100 507/507 [==============================] - 28s 55ms/step - loss: 0.0722 - mae: 0.1408 - mse: 0.0682 - root_mean_squared_error: 0.2612 - smooth_mape: 15.0982 - val_loss: 0.0691 - val_mae: 0.1346 - val_mse: 0.0650 - val_root_mean_squared_error: 0.2549 - val_smooth_mape: 14.2971 - lr: 3.5508e-05 Epoch 68/100 507/507 [==============================] - 25s 48ms/step - loss: 0.0707 - mae: 0.1389 - mse: 0.0669 - root_mean_squared_error: 0.2587 - smooth_mape: 14.9200 - val_loss: 0.0686 - val_mae: 0.1339 - val_mse: 0.0646 - val_root_mean_squared_error: 0.2542 - val_smooth_mape: 14.3511 - lr: 2.1250e-05 Epoch 69/100 507/507 [==============================] - 27s 53ms/step - loss: 0.0705 - mae: 0.1388 - mse: 0.0669 - root_mean_squared_error: 0.2586 - smooth_mape: 14.9215 - val_loss: 0.0684 - val_mae: 0.1325 - val_mse: 0.0645 - val_root_mean_squared_error: 0.2540 - val_smooth_mape: 14.0225 - lr: 9.8841e-06 Epoch 70/100 507/507 [==============================] - 27s 53ms/step - loss: 0.0698 - mae: 0.1379 - mse: 0.0662 - root_mean_squared_error: 0.2573 - smooth_mape: 14.8352 - val_loss: 0.0684 - val_mae: 0.1320 - val_mse: 0.0646 - val_root_mean_squared_error: 0.2542 - val_smooth_mape: 13.9647 - lr: 2.5551e-06 Epoch 71/100 507/507 [==============================] - 25s 49ms/step - loss: 0.0695 - mae: 0.1374 - mse: 0.0658 - root_mean_squared_error: 0.2565 - smooth_mape: 14.7516 - val_loss: 0.0685 - val_mae: 0.1317 - val_mse: 0.0648 - val_root_mean_squared_error: 0.2545 - val_smooth_mape: 13.8579 - lr: 1.5795e-10 Epoch 72/100 506/507 [============================>.] - ETA: 0s - loss: 0.0696 - mae: 0.1376 - mse: 0.0660 - root_mean_squared_error: 0.2569 - smooth_mape: 14.7707 Epoch 72: ReduceLROnPlateau reducing learning rate to 4.95273479828029e-07. 507/507 [==============================] - 25s 50ms/step - loss: 0.0696 - mae: 0.1376 - mse: 0.0660 - root_mean_squared_error: 0.2569 - smooth_mape: 14.7708 - val_loss: 0.0684 - val_mae: 0.1318 - val_mse: 0.0646 - val_root_mean_squared_error: 0.2542 - val_smooth_mape: 13.9681 - lr: 2.4764e-06 Epoch 73/100 507/507 [==============================] - 27s 54ms/step - loss: 0.0694 - mae: 0.1373 - mse: 0.0658 - root_mean_squared_error: 0.2565 - smooth_mape: 14.7647 - val_loss: 0.0683 - val_mae: 0.1327 - val_mse: 0.0646 - val_root_mean_squared_error: 0.2541 - val_smooth_mape: 13.9221 - lr: 9.7346e-06 Epoch 74/100 507/507 [==============================] - 24s 47ms/step - loss: 0.0698 - mae: 0.1380 - mse: 0.0663 - root_mean_squared_error: 0.2575 - smooth_mape: 14.8505 - val_loss: 0.0683 - val_mae: 0.1332 - val_mse: 0.0647 - val_root_mean_squared_error: 0.2543 - val_smooth_mape: 14.4362 - lr: 2.1044e-05 Epoch 75/100 507/507 [==============================] - 25s 50ms/step - loss: 0.0697 - mae: 0.1381 - mse: 0.0664 - root_mean_squared_error: 0.2577 - smooth_mape: 14.8281 - val_loss: 0.0680 - val_mae: 0.1335 - val_mse: 0.0646 - val_root_mean_squared_error: 0.2541 - val_smooth_mape: 14.2706 - lr: 3.5268e-05 Epoch 76/100 506/507 [============================>.] - ETA: 0s - loss: 0.0713 - mae: 0.1407 - mse: 0.0685 - root_mean_squared_error: 0.2616 - smooth_mape: 15.0952 Epoch 76: ReduceLROnPlateau reducing learning rate to 2.548691554693505e-06. 507/507 [==============================] - 27s 53ms/step - loss: 0.0713 - mae: 0.1407 - mse: 0.0685 - root_mean_squared_error: 0.2616 - smooth_mape: 15.0982 - val_loss: 0.0680 - val_mae: 0.1339 - val_mse: 0.0648 - val_root_mean_squared_error: 0.2546 - val_smooth_mape: 14.0539 - lr: 2.5487e-06 Epoch 77/100 506/507 [============================>.] - ETA: 0s - loss: 0.0706 - mae: 0.1401 - mse: 0.0679 - root_mean_squared_error: 0.2606 - smooth_mape: 15.0332 Epoch 77: ReduceLROnPlateau reducing learning rate to 1.3316345575731248e-05. 507/507 [==============================] - 25s 49ms/step - loss: 0.0706 - mae: 0.1401 - mse: 0.0679 - root_mean_squared_error: 0.2606 - smooth_mape: 15.0357 - val_loss: 0.0682 - val_mae: 0.1346 - val_mse: 0.0653 - val_root_mean_squared_error: 0.2556 - val_smooth_mape: 14.3903 - lr: 6.6582e-05 Epoch 78/100 507/507 [==============================] - 25s 50ms/step - loss: 0.0707 - mae: 0.1408 - mse: 0.0683 - root_mean_squared_error: 0.2614 - smooth_mape: 15.1099 - val_loss: 0.0680 - val_mae: 0.1327 - val_mse: 0.0656 - val_root_mean_squared_error: 0.2561 - val_smooth_mape: 14.0229 - lr: 8.0521e-05 Epoch 79/100 507/507 [==============================] - 24s 47ms/step - loss: 0.0710 - mae: 0.1417 - mse: 0.0691 - root_mean_squared_error: 0.2629 - smooth_mape: 15.2021 - val_loss: 0.0677 - val_mae: 0.1354 - val_mse: 0.0656 - val_root_mean_squared_error: 0.2561 - val_smooth_mape: 14.2230 - lr: 9.1389e-05 Epoch 80/100 507/507 [==============================] - ETA: 0s - loss: 0.0726 - mae: 0.1444 - mse: 0.0712 - root_mean_squared_error: 0.2668 - smooth_mape: 15.5279 Epoch 80: ReduceLROnPlateau reducing learning rate to 4.904638990410604e-06. 507/507 [==============================] - 27s 54ms/step - loss: 0.0726 - mae: 0.1444 - mse: 0.0712 - root_mean_squared_error: 0.2668 - smooth_mape: 15.5279 - val_loss: 0.0682 - val_mae: 0.1343 - val_mse: 0.0661 - val_root_mean_squared_error: 0.2570 - val_smooth_mape: 14.2839 - lr: 4.9046e-06 Epoch 81/100 506/507 [============================>.] - ETA: 0s - loss: 0.0713 - mae: 0.1438 - mse: 0.0699 - root_mean_squared_error: 0.2645 - smooth_mape: 15.4713 Epoch 81: ReduceLROnPlateau reducing learning rate to 1.9991402223240587e-05. 2028/2028 [==============================] - 23s 11ms/step 2028/2028 [==============================] - 23s 11ms/step Epoch 81: Out of range predictions: 0 507/507 [==============================] - 75s 148ms/step - loss: 0.0713 - mae: 0.1438 - mse: 0.0700 - root_mean_squared_error: 0.2645 - smooth_mape: 15.4736 - val_loss: 0.0690 - val_mae: 0.1379 - val_mse: 0.0677 - val_root_mean_squared_error: 0.2602 - val_smooth_mape: 15.5488 - lr: 9.9957e-05 Epoch 82/100 507/507 [==============================] - 27s 53ms/step - loss: 0.0696 - mae: 0.1419 - mse: 0.0686 - root_mean_squared_error: 0.2619 - smooth_mape: 15.2732 - val_loss: 0.0667 - val_mae: 0.1334 - val_mse: 0.0656 - val_root_mean_squared_error: 0.2562 - val_smooth_mape: 14.2230 - lr: 9.6794e-05 Epoch 83/100 507/507 [==============================] - 26s 51ms/step - loss: 0.0688 - mae: 0.1409 - mse: 0.0682 - root_mean_squared_error: 0.2612 - smooth_mape: 15.1230 - val_loss: 0.0659 - val_mae: 0.1343 - val_mse: 0.0652 - val_root_mean_squared_error: 0.2553 - val_smooth_mape: 14.2547 - lr: 8.8923e-05 Epoch 84/100 506/507 [============================>.] - ETA: 0s - loss: 0.0674 - mae: 0.1393 - mse: 0.0670 - root_mean_squared_error: 0.2589 - smooth_mape: 14.9698 Epoch 84: ReduceLROnPlateau reducing learning rate to 3.8567635783692825e-06. 507/507 [==============================] - 25s 50ms/step - loss: 0.0674 - mae: 0.1393 - mse: 0.0670 - root_mean_squared_error: 0.2589 - smooth_mape: 14.9716 - val_loss: 0.0655 - val_mae: 0.1338 - val_mse: 0.0650 - val_root_mean_squared_error: 0.2550 - val_smooth_mape: 14.2066 - lr: 3.8568e-06 Epoch 85/100 507/507 [==============================] - 26s 52ms/step - loss: 0.0673 - mae: 0.1395 - mse: 0.0672 - root_mean_squared_error: 0.2593 - smooth_mape: 14.9972 - val_loss: 0.0680 - val_mae: 0.1333 - val_mse: 0.0682 - val_root_mean_squared_error: 0.2611 - val_smooth_mape: 13.8129 - lr: 6.2617e-05 Epoch 86/100 507/507 [==============================] - 25s 49ms/step - loss: 0.0669 - mae: 0.1389 - mse: 0.0670 - root_mean_squared_error: 0.2588 - smooth_mape: 14.9195 - val_loss: 0.0647 - val_mae: 0.1343 - val_mse: 0.0645 - val_root_mean_squared_error: 0.2539 - val_smooth_mape: 14.2430 - lr: 4.6829e-05 Epoch 87/100 507/507 [==============================] - 23s 46ms/step - loss: 0.0656 - mae: 0.1372 - mse: 0.0657 - root_mean_squared_error: 0.2563 - smooth_mape: 14.7366 - val_loss: 0.0643 - val_mae: 0.1317 - val_mse: 0.0644 - val_root_mean_squared_error: 0.2537 - val_smooth_mape: 14.1979 - lr: 3.1360e-05 Epoch 88/100 507/507 [==============================] - 22s 44ms/step - loss: 0.0649 - mae: 0.1365 - mse: 0.0651 - root_mean_squared_error: 0.2552 - smooth_mape: 14.6326 - val_loss: 0.0638 - val_mae: 0.1331 - val_mse: 0.0639 - val_root_mean_squared_error: 0.2528 - val_smooth_mape: 14.3781 - lr: 1.7767e-05 Epoch 89/100 507/507 [==============================] - 23s 45ms/step - loss: 0.0647 - mae: 0.1361 - mse: 0.0649 - root_mean_squared_error: 0.2548 - smooth_mape: 14.6184 - val_loss: 0.0637 - val_mae: 0.1317 - val_mse: 0.0638 - val_root_mean_squared_error: 0.2526 - val_smooth_mape: 13.9675 - lr: 7.4173e-06 Epoch 90/100 507/507 [==============================] - 26s 51ms/step - loss: 0.0642 - mae: 0.1356 - mse: 0.0644 - root_mean_squared_error: 0.2538 - smooth_mape: 14.5656 - val_loss: 0.0638 - val_mae: 0.1309 - val_mse: 0.0640 - val_root_mean_squared_error: 0.2530 - val_smooth_mape: 13.8811 - lr: 1.3523e-06 Epoch 91/100 507/507 [==============================] - ETA: 0s - loss: 0.0636 - mae: 0.1346 - mse: 0.0637 - root_mean_squared_error: 0.2524 - smooth_mape: 14.4922 Epoch 91: ReduceLROnPlateau reducing learning rate to 1e-07. 507/507 [==============================] - 24s 47ms/step - loss: 0.0636 - mae: 0.1346 - mse: 0.0637 - root_mean_squared_error: 0.2524 - smooth_mape: 14.4922 - val_loss: 0.0638 - val_mae: 0.1311 - val_mse: 0.0640 - val_root_mean_squared_error: 0.2530 - val_smooth_mape: 13.8034 - lr: 1.8244e-07 Epoch 92/100 507/507 [==============================] - 25s 49ms/step - loss: 0.0634 - mae: 0.1342 - mse: 0.0635 - root_mean_squared_error: 0.2520 - smooth_mape: 14.4498 - val_loss: 0.0637 - val_mae: 0.1312 - val_mse: 0.0639 - val_root_mean_squared_error: 0.2527 - val_smooth_mape: 13.8449 - lr: 4.0254e-06 Epoch 93/100 507/507 [==============================] - 26s 51ms/step - loss: 0.0636 - mae: 0.1346 - mse: 0.0638 - root_mean_squared_error: 0.2526 - smooth_mape: 14.4781 - val_loss: 0.0636 - val_mae: 0.1318 - val_mse: 0.0638 - val_root_mean_squared_error: 0.2526 - val_smooth_mape: 14.0036 - lr: 1.2494e-05 Epoch 94/100 507/507 [==============================] - 26s 51ms/step - loss: 0.0639 - mae: 0.1350 - mse: 0.0642 - root_mean_squared_error: 0.2534 - smooth_mape: 14.4772 - val_loss: 0.0635 - val_mae: 0.1324 - val_mse: 0.0638 - val_root_mean_squared_error: 0.2526 - val_smooth_mape: 14.1743 - lr: 2.4737e-05 Epoch 95/100 507/507 [==============================] - ETA: 0s - loss: 0.0648 - mae: 0.1366 - mse: 0.0653 - root_mean_squared_error: 0.2556 - smooth_mape: 14.6348 Epoch 95: ReduceLROnPlateau reducing learning rate to 1.976121529878583e-06. 507/507 [==============================] - 25s 50ms/step - loss: 0.0648 - mae: 0.1366 - mse: 0.0653 - root_mean_squared_error: 0.2556 - smooth_mape: 14.6348 - val_loss: 0.0638 - val_mae: 0.1331 - val_mse: 0.0643 - val_root_mean_squared_error: 0.2535 - val_smooth_mape: 14.4393 - lr: 1.9761e-06 Epoch 96/100 507/507 [==============================] - ETA: 0s - loss: 0.0652 - mae: 0.1374 - mse: 0.0659 - root_mean_squared_error: 0.2567 - smooth_mape: 14.7371 Epoch 96: ReduceLROnPlateau reducing learning rate to 1.1072350753238425e-05. 507/507 [==============================] - 25s 50ms/step - loss: 0.0652 - mae: 0.1374 - mse: 0.0659 - root_mean_squared_error: 0.2567 - smooth_mape: 14.7371 - val_loss: 0.0644 - val_mae: 0.1353 - val_mse: 0.0650 - val_root_mean_squared_error: 0.2550 - val_smooth_mape: 14.2866 - lr: 5.5362e-05 Epoch 97/100 507/507 [==============================] - 27s 53ms/step - loss: 0.0660 - mae: 0.1386 - mse: 0.0668 - root_mean_squared_error: 0.2585 - smooth_mape: 14.8711 - val_loss: 0.0640 - val_mae: 0.1324 - val_mse: 0.0647 - val_root_mean_squared_error: 0.2544 - val_smooth_mape: 14.0692 - lr: 7.0662e-05 Epoch 98/100 507/507 [==============================] - ETA: 0s - loss: 0.0653 - mae: 0.1380 - mse: 0.0662 - root_mean_squared_error: 0.2574 - smooth_mape: 14.7726 Epoch 98: ReduceLROnPlateau reducing learning rate to 1.6776460688561202e-05. 507/507 [==============================] - 26s 51ms/step - loss: 0.0653 - mae: 0.1380 - mse: 0.0662 - root_mean_squared_error: 0.2574 - smooth_mape: 14.7726 - val_loss: 0.0652 - val_mae: 0.1359 - val_mse: 0.0662 - val_root_mean_squared_error: 0.2572 - val_smooth_mape: 14.0106 - lr: 8.3882e-05 Epoch 99/100 506/507 [============================>.] - ETA: 0s - loss: 0.0660 - mae: 0.1395 - mse: 0.0672 - root_mean_squared_error: 0.2593 - smooth_mape: 14.9081 Epoch 99: ReduceLROnPlateau reducing learning rate to 4.684684972744436e-06. 507/507 [==============================] - 24s 47ms/step - loss: 0.0660 - mae: 0.1395 - mse: 0.0672 - root_mean_squared_error: 0.2593 - smooth_mape: 14.9098 - val_loss: 0.0642 - val_mae: 0.1338 - val_mse: 0.0653 - val_root_mean_squared_error: 0.2555 - val_smooth_mape: 14.2021 - lr: 4.6847e-06 Epoch 100/100 506/507 [============================>.] - ETA: 0s - loss: 0.0649 - mae: 0.1379 - mse: 0.0662 - root_mean_squared_error: 0.2574 - smooth_mape: 14.7397 Epoch 100: ReduceLROnPlateau reducing learning rate to 1.9821693422272803e-05. 507/507 [==============================] - 24s 46ms/step - loss: 0.0649 - mae: 0.1379 - mse: 0.0662 - root_mean_squared_error: 0.2574 - smooth_mape: 14.7407 - val_loss: 0.0652 - val_mae: 0.1356 - val_mse: 0.0667 - val_root_mean_squared_error: 0.2582 - val_smooth_mape: 13.8309 - lr: 9.9108e-05 Training completed successfully!
In [12]:
print("\n4. Generating predictions...")
predictions = model.predict(X_test_seq)
predictions = np.clip(predictions, 0, max_val_scaled)
predictions_original = target_scaler.inverse_transform(predictions.reshape(-1, 1))
y_test_original = target_scaler.inverse_transform(y_test.reshape(-1, 1))
print("\n5. Model evaluation...")
metrics = evaluate_uv_predictions(y_test_original, predictions_original, folder_name=folder_name)
# Save training results only if new training was performed
if not os.path.exists(model_path):
training_results = {
'model_params': {
'input_shape': input_shape,
'n_features': len(features),
'sequence_length': X_train_seq.shape[1]
},
'training_params': {
'batch_size': 128,
'total_epochs': len(history.history['loss']),
'best_epoch': np.argmin(history.history['val_loss']) + 1,
},
'performance_metrics': {
'final_loss': float(history.history['val_loss'][-1]),
'final_mae': float(history.history['val_mae'][-1]),
'best_val_loss': float(min(history.history['val_loss'])),
'out_of_range_predictions': int(np.sum((predictions < 0) | (predictions > 11)))
}
}
# Save training history
with open(history_path, 'w') as f:
history_dict = {key: [float(val) for val in values]
for key, values in history.history.items()}
json.dump(history_dict, f, indent=4)
else:
# Load existing training results if available
results_path = f'{folder_name}_training_results.json'
if os.path.exists(results_path):
with open(results_path, 'r') as f:
training_results = json.load(f)
else:
training_results = {}
tf.keras.backend.clear_session()
4. Generating predictions...
2028/2028 [==============================] - 18s 9ms/step
5. Model evaluation...
Error saving plots: Unknown format code 'd' for object of type 'float'
UV Index Prediction Analysis:
Raw Metrics:
mae: 0.407
rmse: 0.775
r2: 0.918
mean_error: -0.076
std_error: 0.771
median_error: 0.012
p95_abs_error: 1.745
within_05: 71.379
within_1: 86.040
within_15: 92.984
within_2: 96.562
Rounded Metrics:
mae: 0.393
rmse: 0.782
r2: 0.916
within_05: 78.975
within_1: 90.160
within_15: 95.037
within_2: 97.478
Analysis by UV Range:
Low:
mae: 0.133
count: 41407.000
accuracy_within_05: 90.828
accuracy_within_1: 97.341
Moderate:
mae: 0.874
count: 11467.000
accuracy_within_05: 36.418
accuracy_within_1: 66.713
High:
mae: 0.871
count: 5415.000
accuracy_within_05: 37.876
accuracy_within_1: 65.614
Very High:
mae: 0.905
count: 6343.000
accuracy_within_05: 38.862
accuracy_within_1: 66.404
Extreme:
mae: 1.649
count: 234.000
accuracy_within_05: 0.000
accuracy_within_1: 38.462
Confusion Matrix:
Low Moderate High Very High Extreme
Low 43040.0 2283.0 97.0 17.0 0.0
Moderate 1336.0 7625.0 1181.0 107.0 0.0
High 10.0 1155.0 3149.0 576.0 0.0
Very High 0.0 114.0 1110.0 3066.0 0.0
Extreme 0.0 0.0 0.0 0.0 0.0
In [15]:
print("\n6. Predicting missing data...")
to_predict_predictions = model.predict(X_to_predict_seq)
to_predict_predictions = np.clip(to_predict_predictions, 0, max_val_scaled)
to_predict_predictions_original = target_scaler.inverse_transform(to_predict_predictions.reshape(-1, 1))
print("\n7. Integrating predictions into dataset...")
df_updated = integrate_predictions(df.copy(), to_predict_predictions_original)
output_path = f'../../sources/weather_data_uvindex.parquet'
df_updated.to_parquet(output_path)
print(f"Updated dataset saved to: {output_path}")
# Add prediction statistics
prediction_stats = {
'n_predictions_added': len(to_predict_predictions),
'mean_predicted_uv': float(to_predict_predictions.mean()),
'min_predicted_uv': float(to_predict_predictions.min()),
'max_predicted_uv': float(to_predict_predictions.max()),
}
def convert_to_serializable(obj):
"""Convert numpy types to Python standard types for JSON serialization"""
if isinstance(obj, (np.int_, np.intc, np.intp, np.int8,
np.int16, np.int32, np.int64, np.uint8,
np.uint16, np.uint32, np.uint64)):
return int(obj)
elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)):
return float(obj)
elif isinstance(obj, (np.ndarray,)):
return obj.tolist()
elif isinstance(obj, dict):
return {key: convert_to_serializable(value) for key, value in obj.items()}
elif isinstance(obj, list):
return [convert_to_serializable(item) for item in obj]
return obj
if not os.path.exists(model_path):
training_results['prediction_stats'] = prediction_stats
training_results = convert_to_serializable(training_results)
# Save final results
results_path = f'{folder_name}_training_results.json'
with open(results_path, 'w') as f:
json.dump(training_results, f, indent=4)
print(f"\nAll files saved with prefix: {folder_name}")6. Predicting missing data... 7122/7122 [==============================] - 74s 10ms/step 7. Integrating predictions into dataset... Added 227879 predictions to dataset Rows with UV index after integration: 357615 Updated dataset saved to: ../../sources/weather_data_uvindex.parquet All files saved with prefix: 2024-11-21_08-23
In [16]:
def plot_error_analysis(y_true, y_pred, folder_name=None):
"""
Function to visualize prediction error analysis
Parameters:
-----------
y_true : array-like
Actual values
y_pred : array-like
Predicted values
folder_name : str, optional
Folder to save plots. If None, plots are not saved.
"""
# Convert to 1D numpy array if necessary
if isinstance(y_true, pd.Series):
y_true = y_true.values
if isinstance(y_pred, pd.Series):
y_pred = y_pred.values
y_true = y_true.ravel()
y_pred = y_pred.ravel()
# Calculate errors
errors = y_pred - y_true
# Create main figure
fig = plt.figure(figsize=(15, 5))
# Plot 1: Error Distribution
plt.subplot(1, 3, 1)
plt.hist(errors, bins=50, alpha=0.7)
plt.title('Prediction Error Distribution')
plt.xlabel('Error')
plt.ylabel('Frequency')
# Plot 2: Actual vs Predicted
plt.subplot(1, 3, 2)
plt.scatter(y_true, y_pred, alpha=0.5)
plt.plot([y_true.min(), y_true.max()], [y_true.min(), y_true.max()], 'r--', lw=2)
plt.title('Actual vs Predicted Values')
plt.xlabel('Actual Values')
plt.ylabel('Predicted Values')
# Plot 3: Errors vs Actual Values
plt.subplot(1, 3, 3)
plt.scatter(y_true, errors, alpha=0.5)
plt.axhline(y=0, color='r', linestyle='--')
plt.title('Errors vs Actual Values')
plt.xlabel('Actual Values')
plt.ylabel('Error')
plt.tight_layout()
# Save plot if folder is specified
if folder_name is not None:
try:
# Create folder if it doesn't exist
os.makedirs(folder_name, exist_ok=True)
# Generate filename with timestamp
filename = os.path.join(folder_name, 'error_analysis.png')
# Save figure
plt.savefig(filename, dpi=300, bbox_inches='tight')
print(f"\nPlot saved as: {filename}")
except Exception as e:
print(f"\nError saving plot: {str(e)}")
plt.show()
# Print error statistics
print("\nError statistics:")
print(f"MAE: {np.mean(np.abs(errors)):.4f}")
print(f"MSE: {np.mean(errors ** 2):.4f}")
print(f"RMSE: {np.sqrt(np.mean(errors ** 2)):.4f}")
print(f"Mean errors: {np.mean(errors):.4f}")
print(f"Std errors: {np.std(errors):.4f}")
# Calculate percentage of errors within thresholds
thresholds = [0.5, 1.0, 1.5, 2.0]
for threshold in thresholds:
within_threshold = np.mean(np.abs(errors) <= threshold) * 100
print(f"Predictions within ±{threshold}: {within_threshold:.1f}%")
plot_error_analysis(y_test, predictions, folder_name=folder_name)Plot saved as: 2024-11-21_08-23/error_analysis.png
Error statistics: MAE: 0.1356 MSE: 0.0667 RMSE: 0.2582 Mean errors: -0.0252 Std errors: 0.2569 Predictions within ±0.5: 93.0% Predictions within ±1.0: 99.0% Predictions within ±1.5: 99.9% Predictions within ±2.0: 100.0%
In [ ]: