VictorMylle/Thesis
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Plots to compare between quantile regression and diffusion

Browse Source
This commit is contained in:
Victor Mylle
2024-02-18 19:21:59 +01:00
parent bd250a664b
commit 174a82fab2
5 changed files with 123 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/data/dataset.py
View File

@@ -25,12 +25,19 @@ class NrvDataset(Dataset):
self.sequence_length = sequence_length self.sequence_length = sequence_length
self.predict_sequence_length = predict_sequence_length self.predict_sequence_length = predict_sequence_length
self.samples_to_skip = self.skip_samples(dataframe=dataframe) self.samples_to_skip = self.skip_samples(dataframe=dataframe, full_day_skip=self.full_day_skip)
total_indices = set( total_indices = set(
range(len(dataframe) - self.sequence_length - self.predict_sequence_length) range(len(dataframe) - self.sequence_length - self.predict_sequence_length)
) )
self.valid_indices = sorted(list(total_indices - set(self.samples_to_skip))) self.valid_indices = sorted(list(total_indices - set(self.samples_to_skip)))
# full day indices
full_day_skipped_samples = self.skip_samples(dataframe=dataframe, full_day_skip=True)
full_day_total_indices = set(
range(len(dataframe) - self.sequence_length - self.predict_sequence_length)
)
self.full_day_valid_indices = sorted(list(full_day_total_indices - set(full_day_skipped_samples)))
self.history_features = [] self.history_features = []
if self.data_config.LOAD_HISTORY: if self.data_config.LOAD_HISTORY:
self.history_features.append("total_load") self.history_features.append("total_load")
@@ -73,7 +80,7 @@ class NrvDataset(Dataset):
self.history_features, self.forecast_features = self.preprocess_data(dataframe) self.history_features, self.forecast_features = self.preprocess_data(dataframe)
def skip_samples(self, dataframe): def skip_samples(self, dataframe, full_day_skip):
nan_rows = dataframe[dataframe.isnull().any(axis=1)] nan_rows = dataframe[dataframe.isnull().any(axis=1)]
nan_indices = nan_rows.index nan_indices = nan_rows.index
skip_indices = [ skip_indices = [
@@ -91,7 +98,7 @@ class NrvDataset(Dataset):
# add indices that are not the start of a day (00:15) to the skip indices (use datetime column) # add indices that are not the start of a day (00:15) to the skip indices (use datetime column)
# get indices of all 00:15 timestamps # get indices of all 00:15 timestamps
if self.full_day_skip: if full_day_skip:
start_of_day_indices = dataframe[ start_of_day_indices = dataframe[
dataframe["datetime"].dt.time != pd.Timestamp("00:00:00").time() dataframe["datetime"].dt.time != pd.Timestamp("00:00:00").time()
].index ].index

62
src/trainers/autoregressive_trainer.py
View File

@@ -33,68 +33,30 @@ class AutoRegressiveTrainer(Trainer):
self.model.output_size = 1 self.model.output_size = 1
def debug_plots(self, task, train: bool, data_loader, sample_indices, epoch): def debug_plots(self, task, train: bool, data_loader, sample_indices, epoch):
num_samples = len(sample_indices) for actual_idx, idx in sample_indices.items():
rows = num_samples # One row per sample since we only want one column auto_regressive_output = self.auto_regressive(data_loader.dataset, [idx]*1000)
# check if self has get_plot_error
if hasattr(self, "get_plot_error"):
cols = 2
print("Using get_plot_error")
else:
cols = 1
print("Using get_plot")
fig = make_subplots(
rows=rows,
cols=cols,
subplot_titles=[f"Sample {i+1}" for i in range(num_samples)],
)
for i, idx in enumerate(sample_indices):
auto_regressive_output = self.auto_regressive(data_loader.dataset, [idx])
if len(auto_regressive_output) == 3: if len(auto_regressive_output) == 3:
initial, predictions, target = auto_regressive_output initial, predictions, target = auto_regressive_output
else: else:
initial, predictions, _, target = auto_regressive_output initial, _, predictions, target = auto_regressive_output
initial = initial.squeeze(0)
predictions = predictions.squeeze(0)
target = target.squeeze(0)
sub_fig = self.get_plot(initial, target, predictions, show_legend=(i == 0)) # keep one initial
initial = initial[0]
target = target[0]
row = i + 1 predictions = predictions
col = 1
for trace in sub_fig.data: fig = self.get_plot(initial, target, predictions, show_legend=(0 == 0))
fig.add_trace(trace, row=row, col=col)
if cols == 2: task.get_logger().report_matplotlib_figure(
error_sub_fig = self.get_plot_error( title="Training" if train else "Testing",
target, predictions series=f'Sample {actual_idx}',
)
for trace in error_sub_fig.data:
fig.add_trace(trace, row=row, col=col + 1)
loss = self.criterion(
predictions.to(self.device), target.to(self.device)
).item()
fig["layout"]["annotations"][i].update(
text=f"{self.criterion.__class__.__name__}: {loss:.6f}"
)
# y axis same for all plots
# fig.update_yaxes(range=[-1, 1], col=1)
fig.update_layout(height=1000 * rows)
task.get_logger().report_plotly(
title=f"{'Training' if train else 'Test'} Samples",
series="full_day",
iteration=epoch, iteration=epoch,
figure=fig, figure=fig,
) )
def auto_regressive(self, data_loader, idx, sequence_length: int = 96): def auto_regressive(self, data_loader, idx, sequence_length: int = 96):
self.model.eval() self.model.eval()
target_full = [] target_full = []

17
src/trainers/diffusion_trainer.py
View File

@@ -96,7 +96,16 @@ class DiffusionTrainer:
else: else:
loader = test_loader loader = test_loader
indices = np.random.randint(0, len(loader.dataset) - 1, size=num_samples) # set seed
np.random.seed(42)
actual_indices = np.random.choice(loader.dataset.full_day_valid_indices, num_samples, replace=False)
indices = {}
for i in actual_indices:
indices[i] = loader.dataset.valid_indices.index(i)
print(actual_indices)
return indices return indices
def init_clearml_task(self, task): def init_clearml_task(self, task):
@@ -171,7 +180,7 @@ class DiffusionTrainer:
def debug_plots(self, task, training: bool, data_loader, sample_indices, epoch): def debug_plots(self, task, training: bool, data_loader, sample_indices, epoch):
for i, idx in enumerate(sample_indices): for actual_idx, idx in sample_indices.items():
features, target, _ = data_loader.dataset[idx] features, target, _ = data_loader.dataset[idx]
features = features.to(self.device) features = features.to(self.device)
@@ -180,6 +189,8 @@ class DiffusionTrainer:
self.model.eval() self.model.eval()
with torch.no_grad(): with torch.no_grad():
samples = self.sample(self.model, 100, features).cpu().numpy() samples = self.sample(self.model, 100, features).cpu().numpy()
samples = self.data_processor.inverse_transform(samples)
target = self.data_processor.inverse_transform(target)
ci_99_upper = np.quantile(samples, 0.995, axis=0) ci_99_upper = np.quantile(samples, 0.995, axis=0)
ci_99_lower = np.quantile(samples, 0.005, axis=0) ci_99_lower = np.quantile(samples, 0.005, axis=0)
@@ -218,7 +229,7 @@ class DiffusionTrainer:
task.get_logger().report_matplotlib_figure( task.get_logger().report_matplotlib_figure(
title="Training" if training else "Testing", title="Training" if training else "Testing",
series=f'Sample {i}', series=f'Sample {actual_idx}',
iteration=epoch, iteration=epoch,
figure=fig, figure=fig,
) )

113
src/trainers/quantile_trainer.py
View File

@@ -10,7 +10,9 @@ import plotly.graph_objects as go
import numpy as np import numpy as np
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
from scipy.interpolate import CubicSpline from scipy.interpolate import CubicSpline
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib.patches as mpatches
def sample_from_dist(quantiles, preds): def sample_from_dist(quantiles, preds):
if isinstance(preds, torch.Tensor): if isinstance(preds, torch.Tensor):
@@ -261,35 +263,35 @@ class AutoRegressiveQuantileTrainer(AutoRegressiveTrainer):
name="test_CRPS_from_samples_transformed", value=np.mean(crps_from_samples_metric) name="test_CRPS_from_samples_transformed", value=np.mean(crps_from_samples_metric)
) )
def get_plot_error( # def get_plot_error(
self, # self,
next_day, # next_day,
predictions, # predictions,
): # ):
metric = PinballLoss(quantiles=self.quantiles) # metric = PinballLoss(quantiles=self.quantiles)
fig = go.Figure() # fig = go.Figure()
next_day_np = next_day.view(-1).cpu().numpy() # next_day_np = next_day.view(-1).cpu().numpy()
predictions_np = predictions.cpu().numpy() # predictions_np = predictions.cpu().numpy()
if True: # if True:
next_day_np = self.data_processor.inverse_transform(next_day_np) # next_day_np = self.data_processor.inverse_transform(next_day_np)
predictions_np = self.data_processor.inverse_transform(predictions_np) # predictions_np = self.data_processor.inverse_transform(predictions_np)
# for each time step, calculate the error using the metric # # for each time step, calculate the error using the metric
errors = [] # errors = []
for i in range(96): # for i in range(96):
target_tensor = torch.tensor(next_day_np[i]).unsqueeze(0) # target_tensor = torch.tensor(next_day_np[i]).unsqueeze(0)
prediction_tensor = torch.tensor(predictions_np[i]).unsqueeze(0) # prediction_tensor = torch.tensor(predictions_np[i]).unsqueeze(0)
errors.append(metric(prediction_tensor, target_tensor)) # errors.append(metric(prediction_tensor, target_tensor))
# plot the error # # plot the error
fig.add_trace(go.Scatter(x=np.arange(96), y=errors, name=metric.__class__.__name__)) # fig.add_trace(go.Scatter(x=np.arange(96), y=errors, name=metric.__class__.__name__))
fig.update_layout(title=f"Error of {metric.__class__.__name__} for each time step") # fig.update_layout(title=f"Error of {metric.__class__.__name__} for each time step")
return fig # return fig
def get_plot( def get_plot(
@@ -312,26 +314,59 @@ class AutoRegressiveQuantileTrainer(AutoRegressiveTrainer):
next_day_np = self.data_processor.inverse_transform(next_day_np) next_day_np = self.data_processor.inverse_transform(next_day_np)
predictions_np = self.data_processor.inverse_transform(predictions_np) predictions_np = self.data_processor.inverse_transform(predictions_np)
ci_99_upper = np.quantile(predictions_np, 0.995, axis=0)
ci_99_lower = np.quantile(predictions_np, 0.005, axis=0)
ci_95_upper = np.quantile(predictions_np, 0.975, axis=0)
ci_95_lower = np.quantile(predictions_np, 0.025, axis=0)
ci_90_upper = np.quantile(predictions_np, 0.95, axis=0)
ci_90_lower = np.quantile(predictions_np, 0.05, axis=0)
ci_50_lower = np.quantile(predictions_np, 0.25, axis=0)
ci_50_upper = np.quantile(predictions_np, 0.75, axis=0)
# Add traces for current and next day # Add traces for current and next day
fig.add_trace(go.Scatter(x=np.arange(96), y=current_day_np, name="Current Day")) # fig.add_trace(go.Scatter(x=np.arange(96), y=current_day_np, name="Current Day"))
fig.add_trace(go.Scatter(x=96 + np.arange(96), y=next_day_np, name="Next Day")) # fig.add_trace(go.Scatter(x=96 + np.arange(96), y=next_day_np, name="Next Day"))
for i, q in enumerate(self.quantiles): # for i, q in enumerate(self.quantiles):
fig.add_trace( # fig.add_trace(
go.Scatter( # go.Scatter(
x=96 + np.arange(96), # x=96 + np.arange(96),
y=predictions_np[:, i], # y=predictions_np[:, i],
name=f"Prediction (Q={q})", # name=f"Prediction (Q={q})",
line=dict(dash="dash"), # line=dict(dash="dash"),
) # )
) # )
# Update the layout # # Update the layout
fig.update_layout( # fig.update_layout(
title="Predictions and Quantiles of the Linear Model", # title="Predictions and Quantiles of the Linear Model",
showlegend=show_legend, # showlegend=show_legend,
) # )
sns.set_theme()
time_steps = np.arange(0, 96)
fig, ax = plt.subplots(figsize=(20, 10))
ax.plot(time_steps, predictions_np.mean(axis=0), label="Mean of NRV samples", linewidth=3)
# ax.fill_between(time_steps, ci_lower, ci_upper, color='b', alpha=0.2, label='Full Interval')
ax.fill_between(time_steps, ci_99_lower, ci_99_upper, color='b', alpha=0.2, label='99% Interval')
ax.fill_between(time_steps, ci_95_lower, ci_95_upper, color='b', alpha=0.2, label='95% Interval')
ax.fill_between(time_steps, ci_90_lower, ci_90_upper, color='b', alpha=0.2, label='90% Interval')
ax.fill_between(time_steps, ci_50_lower, ci_50_upper, color='b', alpha=0.2, label='50% Interval')
ax.plot(next_day_np, label="Real NRV", linewidth=3)
# full_interval_patch = mpatches.Patch(color='b', alpha=0.2, label='Full Interval')
ci_99_patch = mpatches.Patch(color='b', alpha=0.3, label='99% Interval')
ci_95_patch = mpatches.Patch(color='b', alpha=0.4, label='95% Interval')
ci_90_patch = mpatches.Patch(color='b', alpha=0.5, label='90% Interval')
ci_50_patch = mpatches.Patch(color='b', alpha=0.6, label='50% Interval')
ax.legend(handles=[ci_99_patch, ci_95_patch, ci_90_patch, ci_50_patch, ax.lines[0], ax.lines[1]])
return fig return fig
def auto_regressive(self, dataset, idx_batch, sequence_length: int = 96): def auto_regressive(self, dataset, idx_batch, sequence_length: int = 96):

15
src/trainers/trainer.py
View File

@@ -86,7 +86,7 @@ class Trainer:
def random_samples(self, train: bool = True, num_samples: int = 10): def random_samples(self, train: bool = True, num_samples: int = 10):
train_loader, test_loader = self.data_processor.get_dataloaders( train_loader, test_loader = self.data_processor.get_dataloaders(
predict_sequence_length=self.model.output_size predict_sequence_length=96
) )
if train: if train:
@@ -94,7 +94,14 @@ class Trainer:
else: else:
loader = test_loader loader = test_loader
indices = np.random.randint(0, len(loader.dataset) - 1, size=num_samples) np.random.seed(42)
actual_indices = np.random.choice(loader.dataset.full_day_valid_indices, num_samples, replace=False)
indices = {}
for i in actual_indices:
indices[i] = loader.dataset.valid_indices.index(i)
print(actual_indices)
return indices return indices
def train(self, epochs: int, remotely: bool = False, task: Task = None): def train(self, epochs: int, remotely: bool = False, task: Task = None):
@@ -107,8 +114,8 @@ class Trainer:
predict_sequence_length=self.model.output_size predict_sequence_length=self.model.output_size
) )
train_samples = self.random_samples(train=True) train_samples = self.random_samples(train=True, num_samples=5)
test_samples = self.random_samples(train=False) test_samples = self.random_samples(train=False, num_samples=5)
self.init_clearml_task(task) self.init_clearml_task(task)