VictorMylle/Thesis
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed crps + more inputs

Browse Source
This commit is contained in:
Victor Mylle
2023-12-05 00:08:17 +00:00
parent 120b6aa5bd
commit d3bf04d68c
13 changed files with 128426 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
Result-Reports/December.md
View File

@@ -160,4 +160,20 @@ Estimated Total Size (MB): 219.17
|---|---|---| |---|---|---|
| Linear Model | 104.82491272720578 | 77.90755403958835 | | Linear Model | 104.82491272720578 | 77.90755403958835 |
| Non Linear Model | 103.89383283348461 | 77.7099763430082 | | Non Linear Model | 103.89383283348461 | 77.7099763430082 |
| LSTM/GRU Model | 103.57896084611653 | 79.2824327805463 | | LSTM/GRU Model | 103.57896084611653 | 79.2824327805463 |
# Tasks
- [ ] Quantiles zelf breder maken na fitten, literatuur bekijken (overconfident voor ondergrens)
- [x] !!! Historgram quantile plot volledige dag
- [ ] CRPS en MAE, over 96 wanneer wordt het slecht
- [x] crps loss fixen
- [ ] More input parameters
- [ ] Non autoregressive (baseline for linear)
(
- NRV to prices (bid ladder from yesterday) -> Elia
- Simple policy
)
- [ ] Diffusion vs Variable Selection Network

2
Result-Reports/November_1.md
View File

@@ -16,7 +16,7 @@
- [x] Non autoregressive Quantile Regression - [x] Non autoregressive Quantile Regression
- [x] Fix debug plots for quantile regression -> predict quantiles and look if true value is below a quantile, if so 1 else 0 and average these over all samples - [x] Fix debug plots for quantile regression -> predict quantiles and look if true value is below a quantile, if so 1 else 0 and average these over all samples
- [ ] Full day debug plots for quantile regression - [x] Full day debug plots for quantile regression
- [x] CPRS Metrics - [x] CPRS Metrics
- [x] Time as input parameter: - [x] Time as input parameter:
- [x] Probabilistic Baseline -> Quantiles on Training Data -> Breedte bekijken -> Gebruiken voor CPRS en plotjes - [x] Probabilistic Baseline -> Quantiles on Training Data -> Breedte bekijken -> Gebruiken voor CPRS en plotjes

128186
data/nominal_net_position.csv Normal file
View File

File diff suppressed because it is too large Load Diff

11
src/data/dataset.py
View File

@@ -29,6 +29,7 @@ class NrvDataset(Dataset):
range(len(dataframe) - self.sequence_length - self.predict_sequence_length) range(len(dataframe) - self.sequence_length - self.predict_sequence_length)
) )
self.valid_indices = sorted(list(total_indices - set(self.samples_to_skip))) self.valid_indices = sorted(list(total_indices - set(self.samples_to_skip)))
print(len(self.valid_indices))
self.history_features = [] self.history_features = []
if self.data_config.LOAD_HISTORY: if self.data_config.LOAD_HISTORY:
@@ -36,7 +37,9 @@ class NrvDataset(Dataset):
if self.data_config.PV_HISTORY: if self.data_config.PV_HISTORY:
self.history_features.append("pv_gen_forecast") self.history_features.append("pv_gen_forecast")
if self.data_config.WIND_HISTORY: if self.data_config.WIND_HISTORY:
self.history_features.append("wind_gen_forecast") self.history_features.append("wind_history")
if self.data_config.NOMINAL_NET_POSITION:
self.history_features.append("nominal_net_position")
self.forecast_features = [] self.forecast_features = []
if self.data_config.LOAD_FORECAST: if self.data_config.LOAD_FORECAST:
@@ -44,7 +47,9 @@ class NrvDataset(Dataset):
if self.data_config.PV_FORECAST: if self.data_config.PV_FORECAST:
self.forecast_features.append("pv_gen_forecast") self.forecast_features.append("pv_gen_forecast")
if self.data_config.WIND_FORECAST: if self.data_config.WIND_FORECAST:
self.forecast_features.append("wind_gen_forecast") self.forecast_features.append("wind_forecast")
if self.data_config.NOMINAL_NET_POSITION:
self.forecast_features.append("nominal_net_position")
# add time feature to dataframe # add time feature to dataframe
time_feature = np.array([0] * len(dataframe)) time_feature = np.array([0] * len(dataframe))
@@ -73,7 +78,7 @@ class NrvDataset(Dataset):
skip_indices = [ skip_indices = [
list( list(
range( range(
idx - self.sequence_length - self.predict_sequence_length, idx + 1 idx - self.sequence_length - 96, idx + 1
) )
) )
for idx in nan_indices for idx in nan_indices

65
src/data/preprocessing.py
View File

@@ -11,6 +11,7 @@ history_data_path = "data/history-quarter-hour-data.csv"
forecast_data_path = "data/load_forecast.csv" forecast_data_path = "data/load_forecast.csv"
pv_forecast_data_path = "data/pv_gen_forecast.csv" pv_forecast_data_path = "data/pv_gen_forecast.csv"
wind_forecast_data_path = "data/wind_gen_forecast.csv" wind_forecast_data_path = "data/wind_gen_forecast.csv"
nominal_net_position_data_path = "data/nominal_net_position.csv"
class DataConfig: class DataConfig:
@@ -29,6 +30,9 @@ class DataConfig:
self.WIND_FORECAST: bool = False self.WIND_FORECAST: bool = False
self.WIND_HISTORY: bool = False self.WIND_HISTORY: bool = False
### NET POSITION ###
self.NOMINAL_NET_POSITION: bool = False
### TIME ### ### TIME ###
self.YEAR: bool = False self.YEAR: bool = False
self.DAY_OF_WEEK: bool = False self.DAY_OF_WEEK: bool = False
@@ -51,19 +55,24 @@ class DataProcessor:
self.history_features = self.get_nrv_history() self.history_features = self.get_nrv_history()
self.future_features = self.get_load_forecast() self.future_features = self.get_load_forecast()
self.pv_forecast = self.get_pv_forecast() # self.pv_forecast = self.get_pv_forecast()
self.wind_forecast = self.get_wind_forecast() self.wind_forecast = self.get_wind_forecast()
self.all_features = self.history_features.merge( self.all_features = self.history_features.merge(
self.future_features, on="datetime", how="left" self.future_features, on="datetime", how="left"
) )
self.all_features = self.all_features.merge( # self.all_features = self.all_features.merge(
self.pv_forecast, on="datetime", how="left" # self.pv_forecast, on="datetime", how="left"
) # )
self.all_features = self.all_features.merge( self.all_features = self.all_features.merge(
self.wind_forecast, on="datetime", how="left" self.wind_forecast, on="datetime", how="left"
) )
self.all_features = self.all_features.merge(
self.get_nominal_net_position(), on="datetime", how="left"
)
self.all_features["quarter"] = ( self.all_features["quarter"] = (
self.all_features["datetime"].dt.hour * 4 self.all_features["datetime"].dt.hour * 4
+ self.all_features["datetime"].dt.minute / 15 + self.all_features["datetime"].dt.minute / 15
@@ -77,6 +86,8 @@ class DataProcessor:
self.nrv_scaler = MinMaxScaler(feature_range=(-1, 1)) self.nrv_scaler = MinMaxScaler(feature_range=(-1, 1))
self.load_forecast_scaler = MinMaxScaler(feature_range=(-1, 1)) self.load_forecast_scaler = MinMaxScaler(feature_range=(-1, 1))
self.wind_forecast_scaler = MinMaxScaler(feature_range=(-1, 1))
self.nominal_net_position_scaler = MinMaxScaler(feature_range=(-1, 1))
self.full_day_skip = False self.full_day_skip = False
@@ -154,9 +165,9 @@ class DataProcessor:
df = pd.read_csv(self.path + wind_forecast_data_path, delimiter=";") df = pd.read_csv(self.path + wind_forecast_data_path, delimiter=";")
df = df.rename( df = df.rename(
columns={"dayaheadforecast": "wind_forecast", "datetime": "datetime"} columns={"measured": "wind_history", "dayaheadforecast": "wind_forecast", "datetime": "datetime"}
) )
df = df[["datetime", "wind_forecast"]] df = df[["datetime", "wind_forecast", "wind_history"]]
# remove nan rows # remove nan rows
df = df[~df["wind_forecast"].isnull()] df = df[~df["wind_forecast"].isnull()]
@@ -166,6 +177,29 @@ class DataProcessor:
df.sort_values(by="datetime", inplace=True) df.sort_values(by="datetime", inplace=True)
return df return df
def get_nominal_net_position(self):
df = pd.read_csv(self.path + nominal_net_position_data_path, delimiter=";")
# remove Resulotion column
df = df.drop(columns=["Resolution code"])
# rename columns
df = df.rename(
columns={
"Datetime": "datetime",
"Implicit net position": "nominal_net_position",
}
)
# to pandas datetime
df["datetime"] = pd.to_datetime(df["datetime"], utc=True)
# make sure all rows are quarter-hourly, if some are not, copy the previous value
df = df.set_index("datetime").resample("15min").ffill().reset_index()
return df
def set_batch_size(self, batch_size: int): def set_batch_size(self, batch_size: int):
self.batch_size = batch_size self.batch_size = batch_size
@@ -199,6 +233,15 @@ class DataProcessor:
train_df["total_load"] = self.load_forecast_scaler.transform( train_df["total_load"] = self.load_forecast_scaler.transform(
train_df["total_load"].values.reshape(-1, 1) train_df["total_load"].values.reshape(-1, 1)
).reshape(-1) ).reshape(-1)
train_df["wind_forecast"] = self.wind_forecast_scaler.fit_transform(
train_df["wind_forecast"].values.reshape(-1, 1)
).reshape(-1)
train_df["wind_history"] = self.wind_forecast_scaler.transform(
train_df["wind_history"].values.reshape(-1, 1)
).reshape(-1)
train_df["nominal_net_position"] = self.nominal_net_position_scaler.fit_transform(
train_df["nominal_net_position"].values.reshape(-1, 1)
).reshape(-1)
train_dataset = NrvDataset( train_dataset = NrvDataset(
train_df, train_df,
@@ -230,6 +273,16 @@ class DataProcessor:
test_df["total_load"] = self.load_forecast_scaler.transform( test_df["total_load"] = self.load_forecast_scaler.transform(
test_df["total_load"].values.reshape(-1, 1) test_df["total_load"].values.reshape(-1, 1)
).reshape(-1) ).reshape(-1)
test_df["wind_forecast"] = self.wind_forecast_scaler.transform(
test_df["wind_forecast"].values.reshape(-1, 1)
).reshape(-1)
test_df["wind_history"] = self.wind_forecast_scaler.transform(
test_df["wind_history"].values.reshape(-1, 1)
).reshape(-1)
test_df["nominal_net_position"] = self.nominal_net_position_scaler.transform(
test_df["nominal_net_position"].values.reshape(-1, 1)
).reshape(-1)
test_dataset = NrvDataset( test_dataset = NrvDataset(
test_df, test_df,

38
src/losses/crps_metric.py
View File

@@ -1,12 +1,13 @@
import torch import torch
from torch import nn from torch import nn
import torch import numpy as np
from properscoring import crps_ensemble from scipy.interpolate import CubicSpline
class CRPSLoss(nn.Module): class CRPSLoss(nn.Module):
def __init__(self): def __init__(self, quantiles):
super(CRPSLoss, self).__init__() super(CRPSLoss, self).__init__()
self.quantiles = quantiles
def forward(self, preds, target): def forward(self, preds, target):
# if tensor, to cpu # if tensor, to cpu
@@ -16,13 +17,30 @@ class CRPSLoss(nn.Module):
if isinstance(target, torch.Tensor): if isinstance(target, torch.Tensor):
target = target.detach().cpu() target = target.detach().cpu()
# target squeeze -1 # if preds more than 2 dimensions, flatten to 2
target = target.squeeze(-1) if len(preds.shape) > 2:
preds = preds.reshape(-1, preds.shape[-1])
# target will be reshaped from (1024, 96, 15) to (1024*96, 15)
# our target (1024, 96) also needs to be reshaped to (1024*96, 1)
target = target.reshape(-1, 1)
# preds shape: [batch_size, num_quantiles] # preds and target as numpy
scores = crps_ensemble(target, preds) preds = preds.numpy()
target = target.numpy()
# mean over batch
crps = scores.mean()
return crps n_x = 101
probs = np.linspace(0, 1, n_x)
spline = CubicSpline(self.quantiles, preds, axis=1)
imbalances = spline(probs)
larger_than_label = imbalances > target
tiled_probs = np.tile(probs, (len(imbalances), 1))
tiled_probs[larger_than_label] -= 1
crps_per_sample = np.trapz(tiled_probs ** 2, imbalances, axis=-1)
crps = np.mean(crps_per_sample)
return crps

1
src/models/time_embedding_layer.py
View File

@@ -5,7 +5,6 @@ class TimeEmbedding(nn.Module):
def __init__(self, time_features: int, embedding_dim: int): def __init__(self, time_features: int, embedding_dim: int):
super().__init__() super().__init__()
self.time_features = time_features self.time_features = time_features
print(time_features)
self.embedding = nn.Embedding(time_features, embedding_dim) self.embedding = nn.Embedding(time_features, embedding_dim)
def forward(self, x): def forward(self, x):

104
src/notebooks/training.ipynb
View File

@@ -42,7 +42,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 5, "execution_count": 3,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -52,14 +52,16 @@
"data_config.LOAD_HISTORY = True\n", "data_config.LOAD_HISTORY = True\n",
"data_config.LOAD_FORECAST = True\n", "data_config.LOAD_FORECAST = True\n",
"\n", "\n",
"data_config.WIND_FORECAST = False\n", "data_config.WIND_FORECAST = True\n",
"data_config.WIND_HISTORY = False\n", "data_config.WIND_HISTORY = True\n",
"\n", "\n",
"data_config.QUARTER = True\n", "data_config.QUARTER = True\n",
"data_config.DAY_OF_WEEK = False\n", "data_config.DAY_OF_WEEK = True\n",
"\n",
"data_config.NOMINAL_NET_POSITION = True\n",
"\n", "\n",
"data_processor = DataProcessor(data_config, path=\"../../\")\n", "data_processor = DataProcessor(data_config, path=\"../../\")\n",
"data_processor.set_batch_size(1024)\n", "data_processor.set_batch_size(1024*10)\n",
"data_processor.set_full_day_skip(False)" "data_processor.set_full_day_skip(False)"
] ]
}, },
@@ -72,22 +74,24 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 11, "execution_count": null,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"ename": "KeyboardInterrupt", "name": "stderr",
"evalue": "Interrupted by user", "output_type": "stream",
"output_type": "error", "text": [
"traceback": [ "InsecureRequestWarning: Certificate verification is disabled! Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n"
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", ]
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", },
"\u001b[1;32m/workspaces/Thesis/src/notebooks/training.ipynb Cell 5\u001b[0m line \u001b[0;36m7\n\u001b[1;32m <a href='vscode-notebook-cell://dev-container%2B7b22686f737450617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f546865736973222c226c6f63616c446f636b6572223a66616c73652c22636f6e66696746696c65223a7b22246d6964223a312c2270617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f5468657369732f2e646576636f6e7461696e65722f646576636f6e7461696e65722e6a736f6e222c22736368656d65223a227673636f64652d66696c65486f7374227d7d@ssh-remote%2Bvictormylle.be/workspaces/Thesis/src/notebooks/training.ipynb#W4sdnNjb2RlLXJlbW90ZQ%3D%3D?line=2'>3</a>\u001b[0m trainer \u001b[39m=\u001b[39m ProbabilisticBaselineTrainer(\n\u001b[1;32m <a href='vscode-notebook-cell://dev-container%2B7b22686f737450617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f546865736973222c226c6f63616c446f636b6572223a66616c73652c22636f6e66696746696c65223a7b22246d6964223a312c2270617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f5468657369732f2e646576636f6e7461696e65722f646576636f6e7461696e65722e6a736f6e222c22736368656d65223a227673636f64652d66696c65486f7374227d7d@ssh-remote%2Bvictormylle.be/workspaces/Thesis/src/notebooks/training.ipynb#W4sdnNjb2RlLXJlbW90ZQ%3D%3D?line=3'>4</a>\u001b[0m quantiles\u001b[39m=\u001b[39mquantiles, data_processor\u001b[39m=\u001b[39mdata_processor, clearml_helper\u001b[39m=\u001b[39mclearml_helper\n\u001b[1;32m <a href='vscode-notebook-cell://dev-container%2B7b22686f737450617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f546865736973222c226c6f63616c446f636b6572223a66616c73652c22636f6e66696746696c65223a7b22246d6964223a312c2270617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f5468657369732f2e646576636f6e7461696e65722f646576636f6e7461696e65722e6a736f6e222c22736368656d65223a227673636f64652d66696c65486f7374227d7d@ssh-remote%2Bvictormylle.be/workspaces/Thesis/src/notebooks/training.ipynb#W4sdnNjb2RlLXJlbW90ZQ%3D%3D?line=4'>5</a>\u001b[0m )\n\u001b[1;32m <a href='vscode-notebook-cell://dev-container%2B7b22686f737450617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f546865736973222c226c6f63616c446f636b6572223a66616c73652c22636f6e66696746696c65223a7b22246d6964223a312c2270617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f5468657369732f2e646576636f6e7461696e65722f646576636f6e7461696e65722e6a736f6e222c22736368656d65223a227673636f64652d66696c65486f7374227d7d@ssh-remote%2Bvictormylle.be/workspaces/Thesis/src/notebooks/training.ipynb#W4sdnNjb2RlLXJlbW90ZQ%3D%3D?line=5'>6</a>\u001b[0m trainer\u001b[39m.\u001b[39madd_metrics_to_track([CRPSLoss()])\n\u001b[0;32m----> <a href='vscode-notebook-cell://dev-container%2B7b22686f737450617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f546865736973222c226c6f63616c446f636b6572223a66616c73652c22636f6e66696746696c65223a7b22246d6964223a312c2270617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f5468657369732f2e646576636f6e7461696e65722f646576636f6e7461696e65722e6a736f6e222c22736368656d65223a227673636f64652d66696c65486f7374227d7d@ssh-remote%2Bvictormylle.be/workspaces/Thesis/src/notebooks/training.ipynb#W4sdnNjb2RlLXJlbW90ZQ%3D%3D?line=6'>7</a>\u001b[0m trainer\u001b[39m.\u001b[39;49mtrain()\n", {
"File \u001b[0;32m/workspaces/Thesis/src/notebooks/../../src/trainers/probabilistic_baseline.py:43\u001b[0m, in \u001b[0;36mProbabilisticBaselineTrainer.train\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mtrain\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[0;32m---> 43\u001b[0m task \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49minit_clearml_task()\n\u001b[1;32m 44\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 45\u001b[0m time_steps \u001b[39m=\u001b[39m [[] \u001b[39mfor\u001b[39;00m _ \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39m96\u001b[39m)]\n", "name": "stdout",
"File \u001b[0;32m/workspaces/Thesis/src/notebooks/../../src/trainers/probabilistic_baseline.py:28\u001b[0m, in \u001b[0;36mProbabilisticBaselineTrainer.init_clearml_task\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mclearml_helper:\n\u001b[1;32m 26\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m---> 28\u001b[0m task_name \u001b[39m=\u001b[39m \u001b[39minput\u001b[39;49m(\u001b[39m\"\u001b[39;49m\u001b[39mEnter a task name: \u001b[39;49m\u001b[39m\"\u001b[39;49m)\n\u001b[1;32m 29\u001b[0m \u001b[39mif\u001b[39;00m task_name \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[1;32m 30\u001b[0m task_name \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mUntitled Task\u001b[39m\u001b[39m\"\u001b[39m\n", "output_type": "stream",
"File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/ipykernel/kernelbase.py:1251\u001b[0m, in \u001b[0;36mKernel.raw_input\u001b[0;34m(self, prompt)\u001b[0m\n\u001b[1;32m 1249\u001b[0m msg \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mraw_input was called, but this frontend does not support input requests.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 1250\u001b[0m \u001b[39mraise\u001b[39;00m StdinNotImplementedError(msg)\n\u001b[0;32m-> 1251\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_input_request(\n\u001b[1;32m 1252\u001b[0m \u001b[39mstr\u001b[39;49m(prompt),\n\u001b[1;32m 1253\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_parent_ident[\u001b[39m\"\u001b[39;49m\u001b[39mshell\u001b[39;49m\u001b[39m\"\u001b[39;49m],\n\u001b[1;32m 1254\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mget_parent(\u001b[39m\"\u001b[39;49m\u001b[39mshell\u001b[39;49m\u001b[39m\"\u001b[39;49m),\n\u001b[1;32m 1255\u001b[0m password\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 1256\u001b[0m )\n", "text": [
"File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/ipykernel/kernelbase.py:1295\u001b[0m, in \u001b[0;36mKernel._input_request\u001b[0;34m(self, prompt, ident, parent, password)\u001b[0m\n\u001b[1;32m 1292\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mKeyboardInterrupt\u001b[39;00m:\n\u001b[1;32m 1293\u001b[0m \u001b[39m# re-raise KeyboardInterrupt, to truncate traceback\u001b[39;00m\n\u001b[1;32m 1294\u001b[0m msg \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mInterrupted by user\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m-> 1295\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mKeyboardInterrupt\u001b[39;00m(msg) \u001b[39mfrom\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 1296\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m:\n\u001b[1;32m 1297\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mlog\u001b[39m.\u001b[39mwarning(\u001b[39m\"\u001b[39m\u001b[39mInvalid Message:\u001b[39m\u001b[39m\"\u001b[39m, exc_info\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n", "ClearML Task: created new task id=f9ff29ae49a24f4eba90f88f5ef9725f\n",
"\u001b[0;31mKeyboardInterrupt\u001b[0m: Interrupted by user" "ClearML results page: http://192.168.1.182:8080/projects/2e46d4af6f1e4c399cf9f5aa30bc8795/experiments/f9ff29ae49a24f4eba90f88f5ef9725f/output/log\n",
"2023-12-04 15:00:16,255 - clearml.Task - INFO - Storing jupyter notebook directly as code\n",
"2023-12-04 15:00:20,444 - clearml - WARNING - JSON serialization of artifact 'dictionary' failed, reverting to pickle\n"
] ]
} }
], ],
@@ -97,7 +101,7 @@
"trainer = ProbabilisticBaselineTrainer(\n", "trainer = ProbabilisticBaselineTrainer(\n",
" quantiles=quantiles, data_processor=data_processor, clearml_helper=clearml_helper\n", " quantiles=quantiles, data_processor=data_processor, clearml_helper=clearml_helper\n",
")\n", ")\n",
"trainer.add_metrics_to_track([CRPSLoss()])\n", "trainer.add_metrics_to_track([CRPSLoss(quantiles=quantiles)])\n",
"trainer.train()" "trainer.train()"
] ]
}, },
@@ -183,35 +187,79 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 6, "execution_count": 5,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"ClearML Task: created new task id=1182d39a984b478c9301aafb4a81ff1b\n", "ClearML Task: created new task id=520015e1e6ec4a83821e0bcb5411445f\n",
"ClearML results page: http://192.168.1.182:8080/projects/2e46d4af6f1e4c399cf9f5aa30bc8795/experiments/1182d39a984b478c9301aafb4a81ff1b/output/log\n", "ClearML results page: http://192.168.1.182:8080/projects/2e46d4af6f1e4c399cf9f5aa30bc8795/experiments/520015e1e6ec4a83821e0bcb5411445f/output/log\n",
"96\n" "151780\n"
] ]
}, },
{ {
"name": "stderr", "name": "stderr",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"Connecting multiple input models with the same name: `checkpoint`. This might result in the wrong model being used when executing remotely\n" "ERROR: Unexpected bus error encountered in worker. This might be caused by insufficient shared memory (shm).\n",
"\u0000ERROR: Unexpected bus error encountered in worker. This might be caused by insufficient shared memory (shm).\n",
"\u0000Exception ignored in: <function _MultiProcessingDataLoaderIter.__del__ at 0x7fd4cc70bac0>\n",
"Traceback (most recent call last):\n",
" File \"/opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1478, in __del__\n",
" self._shutdown_workers()\n",
" File \"/opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1442, in _shutdown_workers\n",
" w.join(timeout=_utils.MP_STATUS_CHECK_INTERVAL)\n",
" File \"/opt/conda/lib/python3.10/multiprocessing/process.py\", line 149, in join\n",
" res = self._popen.wait(timeout)\n",
" File \"/opt/conda/lib/python3.10/multiprocessing/popen_fork.py\", line 40, in wait\n",
" if not wait([self.sentinel], timeout):\n",
" File \"/opt/conda/lib/python3.10/multiprocessing/connection.py\", line 931, in wait\n",
" ready = selector.select(timeout)\n",
" File \"/opt/conda/lib/python3.10/selectors.py\", line 416, in select\n",
" fd_event_list = self._selector.poll(timeout)\n",
" File \"/opt/conda/lib/python3.10/site-packages/torch/utils/data/_utils/signal_handling.py\", line 66, in handler\n",
" _error_if_any_worker_fails()\n",
"RuntimeError: DataLoader worker (pid 1930288) is killed by signal: Bus error. It is possible that dataloader's workers are out of shared memory. Please try to raise your shared memory limit.\n"
] ]
}, },
{ {
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"Early stopping triggered\n" "151780\n",
"24979\n",
"151780\n",
"24979\n",
"151780\n",
"24979\n"
]
},
{
"ename": "KeyboardInterrupt",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m/workspaces/Thesis/src/notebooks/training.ipynb Cell 11\u001b[0m line \u001b[0;36m4\n\u001b[1;32m <a href='vscode-notebook-cell://dev-container%2B7b22686f737450617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f546865736973222c226c6f63616c446f636b6572223a66616c73652c22636f6e66696746696c65223a7b22246d6964223a312c2270617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f5468657369732f2e646576636f6e7461696e65722f646576636f6e7461696e65722e6a736f6e222c22736368656d65223a227673636f64652d66696c65486f7374227d7d@ssh-remote%2Bvictormylle.be/workspaces/Thesis/src/notebooks/training.ipynb#X13sdnNjb2RlLXJlbW90ZQ%3D%3D?line=45'>46</a>\u001b[0m trainer\u001b[39m.\u001b[39mearly_stopping(patience\u001b[39m=\u001b[39m\u001b[39m10\u001b[39m)\n\u001b[1;32m <a href='vscode-notebook-cell://dev-container%2B7b22686f737450617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f546865736973222c226c6f63616c446f636b6572223a66616c73652c22636f6e66696746696c65223a7b22246d6964223a312c2270617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f5468657369732f2e646576636f6e7461696e65722f646576636f6e7461696e65722e6a736f6e222c22736368656d65223a227673636f64652d66696c65486f7374227d7d@ssh-remote%2Bvictormylle.be/workspaces/Thesis/src/notebooks/training.ipynb#X13sdnNjb2RlLXJlbW90ZQ%3D%3D?line=46'>47</a>\u001b[0m trainer\u001b[39m.\u001b[39mplot_every(\u001b[39m15\u001b[39m)\n\u001b[0;32m---> <a href='vscode-notebook-cell://dev-container%2B7b22686f737450617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f546865736973222c226c6f63616c446f636b6572223a66616c73652c22636f6e66696746696c65223a7b22246d6964223a312c2270617468223a222f686f6d652f766963746f726d796c6c652f53656144726976652f4d79204c69627261726965732f4750552d7365727665722f5468657369732f2e646576636f6e7461696e65722f646576636f6e7461696e65722e6a736f6e222c22736368656d65223a227673636f64652d66696c65486f7374227d7d@ssh-remote%2Bvictormylle.be/workspaces/Thesis/src/notebooks/training.ipynb#X13sdnNjb2RlLXJlbW90ZQ%3D%3D?line=47'>48</a>\u001b[0m trainer\u001b[39m.\u001b[39;49mtrain(task\u001b[39m=\u001b[39;49mtask, epochs\u001b[39m=\u001b[39;49mepochs, remotely\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m)\n",
"File \u001b[0;32m/workspaces/Thesis/src/notebooks/../../src/trainers/trainer.py:168\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, epochs, remotely, task)\u001b[0m\n\u001b[1;32m 165\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdebug_plots(task, \u001b[39mFalse\u001b[39;00m, test_loader, test_samples, epoch)\n\u001b[1;32m 167\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mhasattr\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mplot_quantile_percentages\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[0;32m--> 168\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mplot_quantile_percentages(\n\u001b[1;32m 169\u001b[0m task, train_loader, \u001b[39mTrue\u001b[39;49;00m, epoch, \u001b[39mFalse\u001b[39;49;00m\n\u001b[1;32m 170\u001b[0m )\n\u001b[1;32m 171\u001b[0m \u001b[39m# self.plot_quantile_percentages(\u001b[39;00m\n\u001b[1;32m 172\u001b[0m \u001b[39m# task, train_loader, True, epoch, True\u001b[39;00m\n\u001b[1;32m 173\u001b[0m \u001b[39m# )\u001b[39;00m\n\u001b[1;32m 174\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mplot_quantile_percentages(\n\u001b[1;32m 175\u001b[0m task, test_loader, \u001b[39mFalse\u001b[39;00m, epoch, \u001b[39mFalse\u001b[39;00m\n\u001b[1;32m 176\u001b[0m )\n",
"File \u001b[0;32m/workspaces/Thesis/src/notebooks/../../src/trainers/quantile_trainer.py:283\u001b[0m, in \u001b[0;36mAutoRegressiveQuantileTrainer.plot_quantile_percentages\u001b[0;34m(self, task, data_loader, train, iteration, full_day)\u001b[0m\n\u001b[1;32m 280\u001b[0m \u001b[39mwith\u001b[39;00m torch\u001b[39m.\u001b[39mno_grad():\n\u001b[1;32m 281\u001b[0m total_samples \u001b[39m=\u001b[39m \u001b[39mlen\u001b[39m(data_loader\u001b[39m.\u001b[39mdataset) \u001b[39m-\u001b[39m \u001b[39m96\u001b[39m\n\u001b[0;32m--> 283\u001b[0m \u001b[39mfor\u001b[39;00m inputs, targets, idx_batch \u001b[39min\u001b[39;00m data_loader:\n\u001b[1;32m 284\u001b[0m idx_batch \u001b[39m=\u001b[39m [idx \u001b[39mfor\u001b[39;00m idx \u001b[39min\u001b[39;00m idx_batch \u001b[39mif\u001b[39;00m idx \u001b[39m<\u001b[39m total_samples]\n\u001b[1;32m 286\u001b[0m \u001b[39mif\u001b[39;00m full_day:\n",
"File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py:633\u001b[0m, in \u001b[0;36m_BaseDataLoaderIter.__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 630\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_sampler_iter \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 631\u001b[0m \u001b[39m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[39;00m\n\u001b[1;32m 632\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_reset() \u001b[39m# type: ignore[call-arg]\u001b[39;00m\n\u001b[0;32m--> 633\u001b[0m data \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_next_data()\n\u001b[1;32m 634\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_num_yielded \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m \u001b[39m1\u001b[39m\n\u001b[1;32m 635\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_dataset_kind \u001b[39m==\u001b[39m _DatasetKind\u001b[39m.\u001b[39mIterable \u001b[39mand\u001b[39;00m \\\n\u001b[1;32m 636\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_IterableDataset_len_called \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m \\\n\u001b[1;32m 637\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_num_yielded \u001b[39m>\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_IterableDataset_len_called:\n",
"File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py:1317\u001b[0m, in \u001b[0;36m_MultiProcessingDataLoaderIter._next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1314\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 1315\u001b[0m \u001b[39m# no valid `self._rcvd_idx` is found (i.e., didn't break)\u001b[39;00m\n\u001b[1;32m 1316\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_persistent_workers:\n\u001b[0;32m-> 1317\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_shutdown_workers()\n\u001b[1;32m 1318\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mStopIteration\u001b[39;00m\n\u001b[1;32m 1320\u001b[0m \u001b[39m# Now `self._rcvd_idx` is the batch index we want to fetch\u001b[39;00m\n\u001b[1;32m 1321\u001b[0m \n\u001b[1;32m 1322\u001b[0m \u001b[39m# Check if the next sample has already been generated\u001b[39;00m\n",
"File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py:1442\u001b[0m, in \u001b[0;36m_MultiProcessingDataLoaderIter._shutdown_workers\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1437\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_mark_worker_as_unavailable(worker_id, shutdown\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n\u001b[1;32m 1438\u001b[0m \u001b[39mfor\u001b[39;00m w \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_workers:\n\u001b[1;32m 1439\u001b[0m \u001b[39m# We should be able to join here, but in case anything went\u001b[39;00m\n\u001b[1;32m 1440\u001b[0m \u001b[39m# wrong, we set a timeout and if the workers fail to join,\u001b[39;00m\n\u001b[1;32m 1441\u001b[0m \u001b[39m# they are killed in the `finally` block.\u001b[39;00m\n\u001b[0;32m-> 1442\u001b[0m w\u001b[39m.\u001b[39;49mjoin(timeout\u001b[39m=\u001b[39;49m_utils\u001b[39m.\u001b[39;49mMP_STATUS_CHECK_INTERVAL)\n\u001b[1;32m 1443\u001b[0m \u001b[39mfor\u001b[39;00m q \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_index_queues:\n\u001b[1;32m 1444\u001b[0m q\u001b[39m.\u001b[39mcancel_join_thread()\n",
"File \u001b[0;32m/opt/conda/lib/python3.10/multiprocessing/process.py:149\u001b[0m, in \u001b[0;36mBaseProcess.join\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_parent_pid \u001b[39m==\u001b[39m os\u001b[39m.\u001b[39mgetpid(), \u001b[39m'\u001b[39m\u001b[39mcan only join a child process\u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 148\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_popen \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m, \u001b[39m'\u001b[39m\u001b[39mcan only join a started process\u001b[39m\u001b[39m'\u001b[39m\n\u001b[0;32m--> 149\u001b[0m res \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_popen\u001b[39m.\u001b[39;49mwait(timeout)\n\u001b[1;32m 150\u001b[0m \u001b[39mif\u001b[39;00m res \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 151\u001b[0m _children\u001b[39m.\u001b[39mdiscard(\u001b[39mself\u001b[39m)\n",
"File \u001b[0;32m/opt/conda/lib/python3.10/multiprocessing/popen_fork.py:40\u001b[0m, in \u001b[0;36mPopen.wait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 38\u001b[0m \u001b[39mif\u001b[39;00m timeout \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 39\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mmultiprocessing\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mconnection\u001b[39;00m \u001b[39mimport\u001b[39;00m wait\n\u001b[0;32m---> 40\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m wait([\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msentinel], timeout):\n\u001b[1;32m 41\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 42\u001b[0m \u001b[39m# This shouldn't block if wait() returned successfully.\u001b[39;00m\n",
"File \u001b[0;32m/opt/conda/lib/python3.10/multiprocessing/connection.py:931\u001b[0m, in \u001b[0;36mwait\u001b[0;34m(object_list, timeout)\u001b[0m\n\u001b[1;32m 928\u001b[0m deadline \u001b[39m=\u001b[39m time\u001b[39m.\u001b[39mmonotonic() \u001b[39m+\u001b[39m timeout\n\u001b[1;32m 930\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[0;32m--> 931\u001b[0m ready \u001b[39m=\u001b[39m selector\u001b[39m.\u001b[39;49mselect(timeout)\n\u001b[1;32m 932\u001b[0m \u001b[39mif\u001b[39;00m ready:\n\u001b[1;32m 933\u001b[0m \u001b[39mreturn\u001b[39;00m [key\u001b[39m.\u001b[39mfileobj \u001b[39mfor\u001b[39;00m (key, events) \u001b[39min\u001b[39;00m ready]\n",
"File \u001b[0;32m/opt/conda/lib/python3.10/selectors.py:416\u001b[0m, in \u001b[0;36m_PollLikeSelector.select\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 414\u001b[0m ready \u001b[39m=\u001b[39m []\n\u001b[1;32m 415\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 416\u001b[0m fd_event_list \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_selector\u001b[39m.\u001b[39;49mpoll(timeout)\n\u001b[1;32m 417\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mInterruptedError\u001b[39;00m:\n\u001b[1;32m 418\u001b[0m \u001b[39mreturn\u001b[39;00m ready\n",
"File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/clearml/task.py:4313\u001b[0m, in \u001b[0;36mTask.__register_at_exit.<locals>.ExitHooks.signal_handler\u001b[0;34m(self, sig, frame)\u001b[0m\n\u001b[1;32m 4310\u001b[0m \u001b[39m# if this is a sig term, we wait until __at_exit is called (basically do nothing)\u001b[39;00m\n\u001b[1;32m 4311\u001b[0m \u001b[39mif\u001b[39;00m sig \u001b[39m==\u001b[39m signal\u001b[39m.\u001b[39mSIGINT:\n\u001b[1;32m 4312\u001b[0m \u001b[39m# return original handler result\u001b[39;00m\n\u001b[0;32m-> 4313\u001b[0m \u001b[39mreturn\u001b[39;00m org_handler \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mcallable\u001b[39m(org_handler) \u001b[39melse\u001b[39;00m org_handler(sig, frame)\n\u001b[1;32m 4315\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_signal_recursion_protection_flag:\n\u001b[1;32m 4316\u001b[0m \u001b[39m# call original\u001b[39;00m\n\u001b[1;32m 4317\u001b[0m os\u001b[39m.\u001b[39mkill(os\u001b[39m.\u001b[39mgetpid(), sig)\n",
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
] ]
} }
], ],
"source": [ "source": [
"task = clearml_helper.get_task(task_name=\"Autoregressive Quantile Regression + Quarter + Load Forecast + Load History\")\n", "task = clearml_helper.get_task(task_name=\"Autoregressive Quantile Regression + Quarter + Nominal Net Position + Wind + Load\")\n",
"data_config = task.connect(data_config, name=\"data_features\")\n", "data_config = task.connect(data_config, name=\"data_features\")\n",
"\n", "\n",
"#### Hyperparameters ####\n", "#### Hyperparameters ####\n",
@@ -254,10 +302,10 @@
" debug=False,\n", " debug=False,\n",
")\n", ")\n",
"trainer.add_metrics_to_track(\n", "trainer.add_metrics_to_track(\n",
" [PinballLoss(quantiles), MSELoss(), L1Loss(), CRPSLoss()]\n", " [PinballLoss(quantiles), MSELoss(), L1Loss(), CRPSLoss(quantiles)]\n",
")\n", ")\n",
"trainer.early_stopping(patience=10)\n", "trainer.early_stopping(patience=10)\n",
"trainer.plot_every(5)\n", "trainer.plot_every(15)\n",
"trainer.train(task=task, epochs=epochs, remotely=False)" "trainer.train(task=task, epochs=epochs, remotely=False)"
] ]
}, },

3
src/trainers/autoregressive_trainer.py
View File

@@ -94,7 +94,6 @@ class AutoRegressiveTrainer(Trainer):
target_full.append(target) target_full.append(target)
with torch.no_grad(): with torch.no_grad():
print(prev_features.shape)
prediction = self.model(prev_features.unsqueeze(0)) prediction = self.model(prev_features.unsqueeze(0))
predictions_full.append(prediction.squeeze(-1)) predictions_full.append(prediction.squeeze(-1))
@@ -107,8 +106,6 @@ class AutoRegressiveTrainer(Trainer):
dim=0, dim=0,
) )
print(new_features.shape)
# get the other needed features # get the other needed features
other_features, new_target = data_loader.dataset.random_day_autoregressive( other_features, new_target = data_loader.dataset.random_day_autoregressive(
idx + i + 1 idx + i + 1

2
src/trainers/probabilistic_baseline.py
View File

@@ -80,7 +80,7 @@ class ProbabilisticBaselineTrainer(Trainer):
raise raise
def log_final_metrics(self, task, dataloader, quantile_values, train: bool = True): def log_final_metrics(self, task, dataloader, quantile_values, train: bool = True):
metric = CRPSLoss() metric = CRPSLoss(quantiles=self.quantiles)
crps_values = [] crps_values = []
crps_inversed_values = [] crps_inversed_values = []

39
src/trainers/quantile_trainer.py
View File

@@ -270,7 +270,7 @@ class AutoRegressiveQuantileTrainer(AutoRegressiveTrainer):
) )
def plot_quantile_percentages( def plot_quantile_percentages(
self, task, data_loader, train: bool = True, iteration: int = None self, task, data_loader, train: bool = True, iteration: int = None, full_day: bool = False
): ):
quantiles = self.quantiles quantiles = self.quantiles
total = 0 total = 0
@@ -278,16 +278,34 @@ class AutoRegressiveQuantileTrainer(AutoRegressiveTrainer):
self.model.eval() self.model.eval()
with torch.no_grad(): with torch.no_grad():
for inputs, targets, _ in data_loader: total_samples = len(data_loader.dataset) - 96
inputs = inputs.to(self.device)
output = self.model(inputs).cpu().numpy() for inputs, targets, idx_batch in data_loader:
targets = targets.squeeze(-1).cpu().numpy() idx_batch = [idx for idx in idx_batch if idx < total_samples]
if full_day:
_, outputs, samples, targets = self.auto_regressive(
data_loader.dataset, idx_batch=idx_batch
)
# outputs: (batch, sequence_length, num_quantiles)
# targets: (batch, sequence_length, 1)
# reshape to (batch_size * sequence_length, num_quantiles)
outputs = outputs.reshape(-1, len(quantiles))
targets = targets.reshape(-1)
# to cpu
outputs = outputs.cpu().numpy()
targets = targets.cpu().numpy()
else:
inputs = inputs.to(self.device)
outputs = self.model(inputs).cpu().numpy() # (batch_size, num_quantiles)
targets = targets.squeeze(-1).cpu().numpy() # (batch_size, 1)
# output shape: (batch_size, num_quantiles)
# target shape: (batch_size, 1)
for i, q in enumerate(quantiles): for i, q in enumerate(quantiles):
quantile_counter[q] += np.sum( quantile_counter[q] += np.sum(
targets < output[:, i] targets < outputs[:, i]
) )
total += len(targets) total += len(targets)
@@ -322,18 +340,19 @@ class AutoRegressiveQuantileTrainer(AutoRegressiveTrainer):
) # Format the number as a percentage ) # Format the number as a percentage
series_name = "Training Set" if train else "Test Set" series_name = "Training Set" if train else "Test Set"
full_day_str = "Full Day" if full_day else "Single Step"
# Adding labels and title # Adding labels and title
ax.set_xlabel("Quantile") ax.set_xlabel("Quantile")
ax.set_ylabel("Fraction of data under quantile forecast") ax.set_ylabel("Fraction of data under quantile forecast")
ax.set_title(f"Quantile Performance Comparison ({series_name})") ax.set_title(f"{series_name} {full_day_str} Quantile Performance Comparison")
ax.set_xticks(index + bar_width / 2) ax.set_xticks(index + bar_width / 2)
ax.set_xticklabels(quantiles) ax.set_xticklabels(quantiles)
ax.legend() ax.legend()
task.get_logger().report_matplotlib_figure( task.get_logger().report_matplotlib_figure(
title="Quantile Performance Comparison", title="Quantile Performance Comparison",
series=series_name, series=f"{series_name} {full_day_str}",
report_image=True, report_image=True,
figure=plt, figure=plt,
iteration=iteration, iteration=iteration,

10
src/trainers/trainer.py
View File

@@ -166,11 +166,17 @@ class Trainer:
if hasattr(self, "plot_quantile_percentages"): if hasattr(self, "plot_quantile_percentages"):
self.plot_quantile_percentages( self.plot_quantile_percentages(
task, train_loader, True, epoch task, train_loader, True, epoch, False
) )
# self.plot_quantile_percentages(
# task, train_loader, True, epoch, True
# )
self.plot_quantile_percentages( self.plot_quantile_percentages(
task, test_loader, False, epoch task, test_loader, False, epoch, False
) )
# self.plot_quantile_percentages(
# task, test_loader, False, epoch, True
# )
if task: if task:
self.finish_training(task=task) self.finish_training(task=task)

17
src/training_scripts/autoregressive_quantiles.py
View File

@@ -17,16 +17,25 @@ from src.models.time_embedding_layer import TimeEmbedding
#### ClearML #### #### ClearML ####
clearml_helper = ClearMLHelper(project_name="Thesis/NrvForecast") clearml_helper = ClearMLHelper(project_name="Thesis/NrvForecast")
task = clearml_helper.get_task(task_name="Autoregressive Quantile Regression: GRU + Quarter + Load Forecast") task = clearml_helper.get_task(task_name="Autoregressive Quantile Regression: GRU + Quarter + Load + Wind")
#### Data Processor #### #### Data Processor ####
data_config = DataConfig() data_config = DataConfig()
data_config.NRV_HISTORY = True data_config.NRV_HISTORY = True
data_config.LOAD_HISTORY = True
data_config.LOAD_FORECAST = True data_config.LOAD_FORECAST = True
data_config.WIND_FORECAST = True
data_config.WIND_HISTORY = True
data_config.QUARTER = True data_config.QUARTER = True
data_config.DAY_OF_WEEK = False data_config.DAY_OF_WEEK = True
data_config.NOMINAL_NET_POSITION = False
data_config = task.connect(data_config, name="data_features") data_config = task.connect(data_config, name="data_features")
@@ -52,7 +61,7 @@ else:
quantiles = eval(quantiles) quantiles = eval(quantiles)
model_parameters = { model_parameters = {
"learning_rate": 0.0001, "learning_rate": 0.0002,
"hidden_size": 512, "hidden_size": 512,
"num_layers": 2, "num_layers": 2,
"dropout": 0.2, "dropout": 0.2,
@@ -79,7 +88,7 @@ trainer = AutoRegressiveQuantileTrainer(
) )
trainer.add_metrics_to_track( trainer.add_metrics_to_track(
[PinballLoss(quantiles), MSELoss(), L1Loss(), CRPSLoss()] [PinballLoss(quantiles), MSELoss(), L1Loss(), CRPSLoss(quantiles)]
) )
trainer.early_stopping(patience=30) trainer.early_stopping(patience=30)
trainer.plot_every(5) trainer.plot_every(5)