add scaler to target data
This commit is contained in:
parent
88cc8f7936
commit
2dcbff3004
247
src/models/uv_index/uv_index_model.ipynb
Normal file → Executable file
247
src/models/uv_index/uv_index_model.ipynb
Normal file → Executable file
@ -66,39 +66,39 @@
|
||||
"Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.11/dist-packages (from werkzeug>=1.0.1->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (2.1.3)\n",
|
||||
"Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.11/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (0.5.0)\n",
|
||||
"Requirement already satisfied: oauthlib>=3.0.0 in /usr/lib/python3/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard<2.14,>=2.13->tensorflow==2.13.0) (3.2.0)\n",
|
||||
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
|
||||
"\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
|
||||
"\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
|
||||
"\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
|
||||
"Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (1.24.3)\n",
|
||||
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
|
||||
"\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
|
||||
"\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
|
||||
"\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
|
||||
"Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (2.2.3)\n",
|
||||
"Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (1.24.3)\n",
|
||||
"Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (2.8.2)\n",
|
||||
"Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2)\n",
|
||||
"Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2)\n",
|
||||
"Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
|
||||
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
|
||||
"\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
|
||||
"\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
|
||||
"\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
|
||||
"Requirement already satisfied: keras==2.13.1 in /usr/local/lib/python3.11/dist-packages (2.13.1)\n",
|
||||
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
|
||||
"\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
|
||||
"\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
|
||||
"\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
|
||||
"Requirement already satisfied: scikit-learn in /usr/local/lib/python3.11/dist-packages (1.5.2)\n",
|
||||
"Requirement already satisfied: numpy>=1.19.5 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.24.3)\n",
|
||||
"Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.14.1)\n",
|
||||
"Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.4.2)\n",
|
||||
"Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (3.5.0)\n",
|
||||
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
|
||||
"\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
|
||||
"\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
|
||||
"\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
|
||||
"Requirement already satisfied: matplotlib in /usr/local/lib/python3.11/dist-packages (3.8.0)\n",
|
||||
"Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.1.1)\n",
|
||||
"Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (0.11.0)\n",
|
||||
@ -110,20 +110,20 @@
|
||||
"Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (3.2.0)\n",
|
||||
"Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (2.8.2)\n",
|
||||
"Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n",
|
||||
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
|
||||
"\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
|
||||
"\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
|
||||
"\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
|
||||
"Requirement already satisfied: joblib in /usr/local/lib/python3.11/dist-packages (1.4.2)\n",
|
||||
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
|
||||
"\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
|
||||
"\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
|
||||
"\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
|
||||
"Requirement already satisfied: pyarrow in /usr/local/lib/python3.11/dist-packages (18.0.0)\n",
|
||||
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
|
||||
"\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
|
||||
"\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
|
||||
"\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
|
||||
"Requirement already satisfied: fastparquet in /usr/local/lib/python3.11/dist-packages (2024.11.0)\n",
|
||||
"Requirement already satisfied: pandas>=1.5.0 in /usr/local/lib/python3.11/dist-packages (from fastparquet) (2.2.3)\n",
|
||||
"Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (from fastparquet) (1.24.3)\n",
|
||||
@ -134,16 +134,16 @@
|
||||
"Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2024.2)\n",
|
||||
"Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.5.0->fastparquet) (2024.2)\n",
|
||||
"Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas>=1.5.0->fastparquet) (1.16.0)\n",
|
||||
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
|
||||
"\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
|
||||
"\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
|
||||
"\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
|
||||
"Requirement already satisfied: scipy in /usr/local/lib/python3.11/dist-packages (1.14.1)\n",
|
||||
"Requirement already satisfied: numpy<2.3,>=1.23.5 in /usr/local/lib/python3.11/dist-packages (from scipy) (1.24.3)\n",
|
||||
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
|
||||
"\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
|
||||
"\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
|
||||
"\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
|
||||
"Requirement already satisfied: seaborn in /usr/local/lib/python3.11/dist-packages (0.13.2)\n",
|
||||
"Requirement already satisfied: numpy!=1.24.0,>=1.20 in /usr/local/lib/python3.11/dist-packages (from seaborn) (1.24.3)\n",
|
||||
"Requirement already satisfied: pandas>=1.2 in /usr/local/lib/python3.11/dist-packages (from seaborn) (2.2.3)\n",
|
||||
@ -159,34 +159,34 @@
|
||||
"Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.2->seaborn) (2024.2)\n",
|
||||
"Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas>=1.2->seaborn) (2024.2)\n",
|
||||
"Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib!=3.6.1,>=3.4->seaborn) (1.16.0)\n",
|
||||
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
|
||||
"\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
|
||||
"\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
|
||||
"\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
|
||||
"Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (4.67.0)\n",
|
||||
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
|
||||
"\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
|
||||
"\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
|
||||
"\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
|
||||
"Requirement already satisfied: pydot in /usr/local/lib/python3.11/dist-packages (3.0.2)\n",
|
||||
"Requirement already satisfied: pyparsing>=3.0.9 in /usr/local/lib/python3.11/dist-packages (from pydot) (3.2.0)\n",
|
||||
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
|
||||
"\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
|
||||
"\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
|
||||
"\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
|
||||
"Requirement already satisfied: tensorflow-io in /usr/local/lib/python3.11/dist-packages (0.37.1)\n",
|
||||
"Requirement already satisfied: tensorflow-io-gcs-filesystem==0.37.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow-io) (0.37.1)\n",
|
||||
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
|
||||
"\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
|
||||
"\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
|
||||
"\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n",
|
||||
"Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.11/dist-packages (0.23.0)\n",
|
||||
"Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from tensorflow-addons) (23.1)\n",
|
||||
"Requirement already satisfied: typeguard<3.0.0,>=2.7 in /usr/local/lib/python3.11/dist-packages (from tensorflow-addons) (2.13.3)\n",
|
||||
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
|
||||
"\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
|
||||
"\u001B[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001B[0m\u001B[33m\n",
|
||||
"\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\n",
|
||||
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3 -m pip install --upgrade pip\u001B[0m\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -260,7 +260,6 @@
|
||||
"import os\n",
|
||||
"import joblib\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"folder_name = datetime.now().strftime(\"%Y-%m-%d_%H-%M\")\n",
|
||||
"random_state_value = None"
|
||||
]
|
||||
@ -396,11 +395,11 @@
|
||||
"\n",
|
||||
" # 2. One-hot encoding for categorical features\n",
|
||||
" df = pd.get_dummies(df, columns=['season', 'time_period'])\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # 3. Add solar and specific features\n",
|
||||
" df = add_solar_features(df)\n",
|
||||
" df = add_solar_specific_features(df)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # 4. Ensure datetime index\n",
|
||||
" if not isinstance(df.index, pd.DatetimeIndex):\n",
|
||||
" df.index = pd.to_datetime(df.index)\n",
|
||||
@ -470,20 +469,20 @@
|
||||
" 'temp', 'humidity', 'cloudcover', 'visibility',\n",
|
||||
" 'clear_sky_index', 'atmospheric_transparency'\n",
|
||||
" ],\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Essential temporal features\n",
|
||||
" 'temporal': [\n",
|
||||
" 'hour_sin', 'hour_cos',\n",
|
||||
" 'day_of_year_sin', 'day_of_year_cos'\n",
|
||||
" ],\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Solar features\n",
|
||||
" 'solar': [\n",
|
||||
" 'solar_angle', 'solar_elevation',\n",
|
||||
" 'day_length', 'solar_noon',\n",
|
||||
" 'solar_cloud_effect'\n",
|
||||
" ],\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Key interactions\n",
|
||||
" 'interactions': [\n",
|
||||
" 'cloud_temp_interaction',\n",
|
||||
@ -491,7 +490,7 @@
|
||||
" 'temp_humidity_interaction',\n",
|
||||
" 'solar_clarity_index'\n",
|
||||
" ],\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Rolling features\n",
|
||||
" 'rolling': [\n",
|
||||
" 'cloud_rolling_12h',\n",
|
||||
@ -548,10 +547,14 @@
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Robust feature scaling\n",
|
||||
" scaler = RobustScaler()\n",
|
||||
" X_train_scaled = scaler.fit_transform(X_train)\n",
|
||||
" X_test_scaled = scaler.transform(X_test)\n",
|
||||
" X_to_predict_scaled = scaler.transform(X_to_predict)\n",
|
||||
" feature_scaler = RobustScaler()\n",
|
||||
" X_train_scaled = feature_scaler.fit_transform(X_train)\n",
|
||||
" X_test_scaled = feature_scaler.transform(X_test)\n",
|
||||
" X_to_predict_scaled = feature_scaler.transform(X_to_predict)\n",
|
||||
"\n",
|
||||
" target_scaler = RobustScaler()\n",
|
||||
" y_train_scaled = target_scaler.fit_transform(y_train.values.reshape(-1, 1)).ravel()\n",
|
||||
" y_test_scaled = target_scaler.transform(y_test.values.reshape(-1, 1)).ravel()\n",
|
||||
"\n",
|
||||
" # Final validation\n",
|
||||
" assert not np.isnan(X_train_scaled).any(), \"Found NaN in X_train_scaled\"\n",
|
||||
@ -565,8 +568,8 @@
|
||||
" print(f\"{category}: {len(features)} features\")\n",
|
||||
" print(f\"Categorical: {len(categorical_columns)} features\")\n",
|
||||
"\n",
|
||||
" return (X_train_scaled, X_test_scaled, y_train, y_test,\n",
|
||||
" scaler, final_features, X_to_predict_scaled)\n",
|
||||
" return (X_train_scaled, X_test_scaled, y_train_scaled, y_test_scaled,\n",
|
||||
" feature_scaler, target_scaler, final_features, X_to_predict_scaled)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def create_sequence_data(X, sequence_length=24):\n",
|
||||
@ -582,7 +585,7 @@
|
||||
"\n",
|
||||
"def prepare_hybrid_data(df):\n",
|
||||
" # Use existing data preparation\n",
|
||||
" X_train_scaled, X_test_scaled, y_train, y_test, scaler, features, X_to_predict_scaled = prepare_advanced_data(df)\n",
|
||||
" X_train_scaled, X_test_scaled, y_train, y_test, feature_scaler, target_scaler, features, X_to_predict_scaled = prepare_advanced_data(df)\n",
|
||||
"\n",
|
||||
" # Convert data to sequences\n",
|
||||
" sequence_length = 24 # 24 hours of historical data\n",
|
||||
@ -596,7 +599,7 @@
|
||||
"\n",
|
||||
" X_to_predict_seq = create_sequence_data(X_to_predict_scaled, sequence_length)\n",
|
||||
"\n",
|
||||
" return X_train_seq, X_test_seq, y_train, y_test, scaler, features, X_to_predict_seq"
|
||||
" return X_train_seq, X_test_seq, y_train, y_test, feature_scaler, target_scaler, features, X_to_predict_seq"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -610,7 +613,7 @@
|
||||
" survival_probability=0.8, return_sequences=True):\n",
|
||||
" \"\"\"LSTM layer with stochastic depth\"\"\"\n",
|
||||
" residual = x\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Main path\n",
|
||||
" x = Bidirectional(LSTM(units, return_sequences=return_sequences,\n",
|
||||
" kernel_regularizer=regularizers.l2(l2_reg)))(x)\n",
|
||||
@ -624,7 +627,7 @@
|
||||
" if int(residual.shape[-1]) != target_dim:\n",
|
||||
" # Use Dense layer instead of Conv1D for better dimension matching\n",
|
||||
" residual = Dense(target_dim)(residual)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Apply stochastic depth only if dimensions match\n",
|
||||
" if x.shape[-1] == residual.shape[-1]:\n",
|
||||
" x = tfa.layers.StochasticDepth(survival_probability)([x, residual])\n",
|
||||
@ -641,25 +644,25 @@
|
||||
" Attention block with stochastic depth.\n",
|
||||
" \"\"\"\n",
|
||||
" original_x = x\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Compute self-attention\n",
|
||||
" attention = MultiHeadAttention(num_heads=num_heads, key_dim=units)(x, x)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Ensure dimensions match before applying stochastic depth\n",
|
||||
" if attention.shape[-1] != original_x.shape[-1]:\n",
|
||||
" original_x = Dense(attention.shape[-1])(original_x)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Apply stochastic depth to the attention path\n",
|
||||
" x = tfa.layers.StochasticDepth(survival_probability)([attention, original_x])\n",
|
||||
" x = LayerNormalization()(x)\n",
|
||||
"\n",
|
||||
" # Store the input to the FFN\n",
|
||||
" ffn_input = x\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # FFN block\n",
|
||||
" x = Dense(units * 4, activation='swish')(x)\n",
|
||||
" x = Dense(ffn_input.shape[-1])(x) # Match the input dimension\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Apply stochastic depth to the FFN\n",
|
||||
" x = tfa.layers.StochasticDepth(survival_probability)([x, ffn_input])\n",
|
||||
" x = LayerNormalization()(x)\n",
|
||||
@ -707,23 +710,23 @@
|
||||
" # Global attention with reduced complexity\n",
|
||||
" x_input = x\n",
|
||||
" x = MultiHeadAttention(num_heads=2, key_dim=32)(x, x)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" if x.shape[-1] != x_input.shape[-1]:\n",
|
||||
" x_input = Dense(x.shape[-1])(x_input)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" x = tfa.layers.StochasticDepth(survival_probability=0.95)([x, x_input])\n",
|
||||
" x = LayerNormalization()(x)\n",
|
||||
"\n",
|
||||
" # Simplified dense layers\n",
|
||||
" x = GlobalAveragePooling1D()(x)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Gradual dimension reduction\n",
|
||||
" x = Dense(32, activation='swish', kernel_regularizer=regularizers.l2(l2_lambda/2), kernel_constraint=tf.keras.constraints.MaxNorm(3))(x)\n",
|
||||
" x = Dense(32, activation='swish', kernel_regularizer=regularizers.l2(l2_lambda / 2), kernel_constraint=tf.keras.constraints.MaxNorm(3))(x)\n",
|
||||
" x = BatchNormalization()(x)\n",
|
||||
" x = Dropout(0.05)(x) # Minimal dropout\n",
|
||||
"\n",
|
||||
" x = Dense(16, activation='swish',\n",
|
||||
" kernel_regularizer=regularizers.l2(l2_lambda/2))(x)\n",
|
||||
" kernel_regularizer=regularizers.l2(l2_lambda / 2))(x)\n",
|
||||
" x = BatchNormalization()(x)\n",
|
||||
"\n",
|
||||
" # Modified output layer\n",
|
||||
@ -791,27 +794,27 @@
|
||||
" def smooth_uv_loss(y_true, y_pred):\n",
|
||||
" # Basic MSE with smoothing\n",
|
||||
" mse = tf.square(y_true - y_pred)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Smooth L1 component for better stability\n",
|
||||
" abs_diff = tf.abs(y_true - y_pred)\n",
|
||||
" smooth_l1 = tf.where(abs_diff < 1.0,\n",
|
||||
" 0.5 * tf.square(abs_diff),\n",
|
||||
" abs_diff - 0.5)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Combined loss with dynamic weighting\n",
|
||||
" combined_loss = 0.7 * mse + 0.3 * smooth_l1\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Gentle weighting for high UV values\n",
|
||||
" high_uv_weight = tf.where(y_true >= 8.0, 1.2, 1.0)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Smooth peak hours weight\n",
|
||||
" time_of_day = tf.cast(tf.math.floormod(tf.range(tf.shape(y_true)[0]), 24),\n",
|
||||
" tf.float32)\n",
|
||||
" peak_weight = 1.0 + 0.2 * tf.math.sigmoid((time_of_day - 10.0) * 0.5) * \\\n",
|
||||
" tf.math.sigmoid((16.0 - time_of_day) * 0.5)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" total_weight = high_uv_weight * peak_weight\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" return tf.reduce_mean(combined_loss * total_weight)\n",
|
||||
"\n",
|
||||
" # Improved MAPE metric\n",
|
||||
@ -832,16 +835,6 @@
|
||||
" ]\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" model.summary()\n",
|
||||
"\n",
|
||||
" # Save model architecture visualization\n",
|
||||
" plot_model(model,\n",
|
||||
" to_file=f'{folder_name}_model_architecture.png',\n",
|
||||
" show_shapes=True,\n",
|
||||
" show_layer_names=True,\n",
|
||||
" dpi=150,\n",
|
||||
" show_layer_activations=True)\n",
|
||||
"\n",
|
||||
" return model\n",
|
||||
"\n",
|
||||
"\n",
|
||||
@ -926,22 +919,22 @@
|
||||
"\n",
|
||||
" # Calculate confusion matrix with handling for missing classes\n",
|
||||
" risk_levels = ['Low', 'Moderate', 'High', 'Very High', 'Extreme']\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Get unique labels present in the data\n",
|
||||
" present_labels = np.unique(np.concatenate([y_true_risk, y_pred_risk]))\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Calculate confusion matrix for present labels\n",
|
||||
" cm = confusion_matrix(y_true_risk, y_pred_risk, labels=present_labels)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Create full confusion matrix with zeros\n",
|
||||
" full_cm = np.zeros((len(risk_levels), len(risk_levels)))\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Map present labels to their positions in the full matrix\n",
|
||||
" label_positions = {label: i for i, label in enumerate(risk_levels)}\n",
|
||||
" for i, true_label in enumerate(present_labels):\n",
|
||||
" for j, pred_label in enumerate(present_labels):\n",
|
||||
" full_cm[label_positions[true_label], label_positions[pred_label]] = cm[i, j]\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Create DataFrame with all risk levels\n",
|
||||
" cm_df = pd.DataFrame(full_cm, columns=risk_levels, index=risk_levels)\n",
|
||||
"\n",
|
||||
@ -1063,6 +1056,7 @@
|
||||
"\n",
|
||||
" return metrics\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def plot_training_history(history, folder_name=None):\n",
|
||||
" \"\"\"\n",
|
||||
" Visualize and save the loss and metrics plots during training\n",
|
||||
@ -1351,23 +1345,31 @@
|
||||
"\n",
|
||||
"# Data preparation\n",
|
||||
"print(\"\\n1. Preparing data...\")\n",
|
||||
"X_train_seq, X_test_seq, y_train, y_test, scaler, features, X_to_predict_seq = prepare_hybrid_data(df)\n",
|
||||
"X_train_seq, X_test_seq, y_train, y_test, scaler, feature_scaler, target_scaler, features, X_to_predict_seq = prepare_hybrid_data(df)\n",
|
||||
"\n",
|
||||
"print(f\"Training data shape: {X_train_seq.shape}\")\n",
|
||||
"print(f\"Test data shape: {X_test_seq.shape}\")\n",
|
||||
"\n",
|
||||
"# Save or load scaler and features\n",
|
||||
"scaler_path = f'{folder_name}_scaler.joblib'\n",
|
||||
"feature_scaler_path = f'{folder_name}_feature_scaler.joblib'\n",
|
||||
"target_scaler_path = f'{folder_name}_target_scaler.joblib'\n",
|
||||
"features_path = f'{folder_name}_features.json'\n",
|
||||
"model_path = f'{folder_name}_best_model.h5'\n",
|
||||
"history_path = f'{folder_name}_training_history.json'\n",
|
||||
"\n",
|
||||
"if os.path.exists(scaler_path):\n",
|
||||
" print(f\"Loading existing scaler from: {scaler_path}\")\n",
|
||||
" scaler = joblib.load(scaler_path)\n",
|
||||
"if os.path.exists(feature_scaler_path):\n",
|
||||
" print(f\"Loading existing scaler from: {feature_scaler_path}\")\n",
|
||||
" scaler = joblib.load(feature_scaler_path)\n",
|
||||
"else:\n",
|
||||
" print(f\"Saving scaler to: {scaler_path}\")\n",
|
||||
" joblib.dump(scaler, scaler_path)\n",
|
||||
" print(f\"Saving scaler to: {feature_scaler_path}\")\n",
|
||||
" joblib.dump(feature_scaler, feature_scaler_path)\n",
|
||||
"\n",
|
||||
"if os.path.exists(target_scaler_path):\n",
|
||||
" print(f\"Loading existing scaler from: {target_scaler_path}\")\n",
|
||||
" scaler = joblib.load(target_scaler_path)\n",
|
||||
"else:\n",
|
||||
" print(f\"Saving scaler to: {target_scaler_path}\")\n",
|
||||
" joblib.dump(target_scaler, target_scaler_path)\n",
|
||||
"\n",
|
||||
"if os.path.exists(features_path):\n",
|
||||
" print(f\"Loading existing features from: {features_path}\")\n",
|
||||
@ -1705,7 +1707,7 @@
|
||||
"if os.path.exists(model_path):\n",
|
||||
" print(f\"Loading existing model from: {model_path}\")\n",
|
||||
" model = tf.keras.models.load_model(model_path)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Load existing history if available\n",
|
||||
" if os.path.exists(history_path):\n",
|
||||
" print(f\"Loading existing training history from: {history_path}\")\n",
|
||||
@ -1717,7 +1719,7 @@
|
||||
"else:\n",
|
||||
" print(\"Creating new model...\")\n",
|
||||
" model = create_uv_index_model(input_shape, folder_name)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" print(\"\\n3. Starting training...\")\n",
|
||||
" history = train_hybrid_model(\n",
|
||||
" model=model,\n",
|
||||
@ -1840,8 +1842,11 @@
|
||||
"predictions = model.predict(X_test_seq)\n",
|
||||
"predictions = np.clip(predictions, 0, 11)\n",
|
||||
"\n",
|
||||
"predictions_original = target_scaler.inverse_transform(predictions)\n",
|
||||
"y_test_original = target_scaler.inverse_transform(y_test)\n",
|
||||
"\n",
|
||||
"print(\"\\n5. Model evaluation...\")\n",
|
||||
"metrics = evaluate_uv_predictions(y_test, predictions, folder_name=folder_name)\n",
|
||||
"metrics = evaluate_uv_predictions(y_test_original, predictions_original, folder_name=folder_name)\n",
|
||||
"\n",
|
||||
"# Save training results only if new training was performed\n",
|
||||
"if not os.path.exists(model_path):\n",
|
||||
@ -1866,7 +1871,7 @@
|
||||
"\n",
|
||||
" # Save training history\n",
|
||||
" with open(history_path, 'w') as f:\n",
|
||||
" history_dict = {key: [float(val) for val in values] \n",
|
||||
" history_dict = {key: [float(val) for val in values]\n",
|
||||
" for key, values in history.history.items()}\n",
|
||||
" json.dump(history_dict, f, indent=4)\n",
|
||||
"else:\n",
|
||||
@ -1912,7 +1917,7 @@
|
||||
"print(\"\\n7. Integrating predictions into dataset...\")\n",
|
||||
"df_updated = integrate_predictions(df.copy(), to_predict_predictions)\n",
|
||||
"\n",
|
||||
"output_path = f'{folder_name}_weather_data_uvindex.parquet'\n",
|
||||
"output_path = f'../../sources/weather_data_uvindex.parquet'\n",
|
||||
"df_updated.to_parquet(output_path)\n",
|
||||
"print(f\"Updated dataset saved to: {output_path}\")\n",
|
||||
"\n",
|
||||
@ -1924,6 +1929,7 @@
|
||||
" 'max_predicted_uv': float(to_predict_predictions.max()),\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def convert_to_serializable(obj):\n",
|
||||
" \"\"\"Convert numpy types to Python standard types for JSON serialization\"\"\"\n",
|
||||
" if isinstance(obj, (np.int_, np.intc, np.intp, np.int8,\n",
|
||||
@ -1940,6 +1946,7 @@
|
||||
" return [convert_to_serializable(item) for item in obj]\n",
|
||||
" return obj\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"if not os.path.exists(model_path):\n",
|
||||
" training_results['prediction_stats'] = prediction_stats\n",
|
||||
"\n",
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user