VictorMylle/Thesis
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated training scripts

Browse Source
This commit is contained in:
Victor Mylle
2024-03-18 12:15:06 +01:00
parent 34335cd9fe
commit 1a8e735cbc
10 changed files with 487 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/trainers/autoregressive_trainer.py
View File

@@ -9,6 +9,8 @@ import plotly.subplots as sp
from plotly.subplots import make_subplots
from src.trainers.trainer import Trainer
from tqdm import tqdm
import matplotlib.pyplot as plt
class AutoRegressiveTrainer(Trainer):
def __init__(
@@ -34,28 +36,41 @@ class AutoRegressiveTrainer(Trainer):
def debug_plots(self, task, train: bool, data_loader, sample_indices, epoch):
for actual_idx, idx in sample_indices.items():
auto_regressive_output = self.auto_regressive(data_loader.dataset, [idx]*1000)
print(f"Plotting sample {actual_idx}")
auto_regressive_output = self.auto_regressive(
data_loader.dataset, [idx] * 1000
)
if len(auto_regressive_output) == 3:
initial, predictions, target = auto_regressive_output
else:
initial, _, predictions, target = auto_regressive_output
# keep one initial
initial = initial[0]
target = target[0]
predictions = predictions
fig = self.get_plot(initial, target, predictions, show_legend=(0 == 0))
fig, fig2 = self.get_plot(
initial, target, predictions, show_legend=(0 == 0)
)
task.get_logger().report_matplotlib_figure(
title="Training" if train else "Testing",
series=f'Sample {actual_idx}',
series=f"Sample {actual_idx}",
iteration=epoch,
figure=fig,
)
task.get_logger().report_matplotlib_figure(
title="Training Samples" if train else "Testing Samples",
series=f"Sample {actual_idx} samples",
iteration=epoch,
figure=fig2,
report_interactive=False,
)
plt.close()
def auto_regressive(self, data_loader, idx, sequence_length: int = 96):
self.model.eval()

106
src/trainers/diffusion_trainer.py
View File

@@ -85,6 +85,8 @@ class DiffusionTrainer:
self.best_score = None
self.policy_evaluator = policy_evaluator
self.prev_optimal_penalty = 0
def noise_time_series(self, x: torch.tensor, t: int):
"""Add noise to time series
Args:
@@ -206,8 +208,8 @@ class DiffusionTrainer:
running_loss /= len(train_loader.dataset)
if epoch % 40 == 0 and epoch != 0:
crps = self.test(test_loader, epoch, task)
if epoch % 150 == 0 and epoch != 0:
crps, _ = self.test(test_loader, epoch, task)
if best_crps is None or crps < best_crps:
best_crps = crps
@@ -215,7 +217,7 @@ class DiffusionTrainer:
else:
early_stopping += 1
if early_stopping > 5:
if early_stopping > 15:
break
if task:
@@ -238,8 +240,32 @@ class DiffusionTrainer:
self.model = torch.load("checkpoint.pt")
self.model.to(self.device)
self.test(test_loader, None, task)
self.policy_evaluator.plot_profits_table()
_, generated_sampels = self.test(test_loader, None, task)
# self.policy_evaluator.plot_profits_table()
optimal_penalty, profit, charge_cycles = (
self.policy_evaluator.optimize_penalty_for_target_charge_cycles(
idx_samples=generated_sampels,
test_loader=test_loader,
initial_penalty=900,
target_charge_cycles=283,
learning_rate=1,
max_iterations=50,
tolerance=1,
)
)
print(
f"Optimal Penalty: {optimal_penalty}, Profit: {profit}, Charge Cycles: {charge_cycles}"
)
task.get_logger().report_single_value(
name="Optimal Penalty", value=optimal_penalty
)
task.get_logger().report_single_value(name="Optimal Profit", value=profit)
task.get_logger().report_single_value(
name="Optimal Charge Cycles", value=charge_cycles
)
if task:
task.close()
@@ -332,6 +358,8 @@ class DiffusionTrainer:
]
)
ax.set_ylim([-1500, 1500])
task.get_logger().report_matplotlib_figure(
title="Training" if training else "Testing",
series=f"Sample {actual_idx}",
@@ -341,6 +369,25 @@ class DiffusionTrainer:
plt.close()
# plot some samples for the nrv genearations (10 samples) (scale -1500 to 1500)
fig, ax = plt.subplots(figsize=(20, 10))
for i in range(10):
ax.plot(samples[i], label=f"Sample {i}")
ax.plot(target, label="Real NRV", linewidth=3)
ax.legend()
ax.set_ylim([-1500, 1500])
task.get_logger().report_matplotlib_figure(
title="Training Samples" if training else "Testing Samples",
series=f"Sample {actual_idx} samples",
iteration=epoch,
figure=fig,
report_interactive=False,
)
plt.close()
def test(
self, data_loader: torch.utils.data.DataLoader, epoch: int, task: Task = None
):
@@ -385,28 +432,39 @@ class DiffusionTrainer:
predict_sequence_length=self.ts_length, full_day_skip=True
)
self.policy_evaluator.evaluate_test_set(generated_samples, test_loader)
df = self.policy_evaluator.get_profits_as_scalars()
for idx, row in df.iterrows():
task.get_logger().report_scalar(
title="Profit",
series=f"penalty_{row['Penalty']}",
value=row["Total Profit"],
iteration=epoch,
optimal_penalty, profit, charge_cycles = (
self.policy_evaluator.optimize_penalty_for_target_charge_cycles(
idx_samples=generated_samples,
test_loader=test_loader,
initial_penalty=self.prev_optimal_penalty,
target_charge_cycles=283,
learning_rate=1,
max_iterations=50,
tolerance=1,
)
)
df = self.policy_evaluator.get_profits_till_400()
for idx, row in df.iterrows():
task.get_logger().report_scalar(
title="Profit_till_400",
series=f"penalty_{row['Penalty']}",
value=row["Profit_till_400"],
iteration=epoch,
)
self.prev_optimal_penalty = optimal_penalty
return mean_crps
task.get_logger().report_scalar(
title="Optimal Penalty",
series="test",
value=optimal_penalty,
iteration=epoch,
)
task.get_logger().report_scalar(
title="Optimal Profit", series="test", value=profit, iteration=epoch
)
task.get_logger().report_scalar(
title="Optimal Charge Cycles",
series="test",
value=charge_cycles,
iteration=epoch,
)
return mean_crps, generated_samples
def save_checkpoint(self, val_loss, task, iteration: int):
torch.save(self.model, "checkpoint.pt")

137
src/trainers/quantile_trainer.py
View File

@@ -155,18 +155,6 @@ class AutoRegressiveQuantileTrainer(AutoRegressiveTrainer):
generated_samples = {}
with torch.no_grad():
total_samples = len(dataloader.dataset)
print(
"Full day valid indices: ",
len(dataloader.dataset.full_day_valid_indices),
)
print(
"Valid indices: ",
len(dataloader.dataset.valid_indices),
)
print(dataloader.dataset.valid_indices)
for i in tqdm(dataloader.dataset.full_day_valid_indices):
idx = dataloader.dataset.valid_indices.index(i)
@@ -188,74 +176,64 @@ class AutoRegressiveQuantileTrainer(AutoRegressiveTrainer):
crps_from_samples_metric.append(crps[0].mean().item())
task.get_logger().report_scalar(
title="CRPS_from_samples",
series="test",
value=np.mean(crps_from_samples_metric),
iteration=epoch,
)
if epoch is not None:
task.get_logger().report_scalar(
title="CRPS_from_samples",
series="test",
value=np.mean(crps_from_samples_metric),
iteration=epoch,
)
# using the policy evaluator, evaluate the policy with the generated samples
if self.policy_evaluator is not None:
self.policy_evaluator.evaluate_test_set(generated_samples, dataloader)
df = self.policy_evaluator.get_profits_as_scalars()
# using the policy evaluator, evaluate the policy with the generated samples
if self.policy_evaluator is not None:
optimal_penalty, profit, charge_cycles = (
self.policy_evaluator.optimize_penalty_for_target_charge_cycles(
idx_samples=generated_samples,
test_loader=dataloader,
initial_penalty=900,
target_charge_cycles=283,
learning_rate=2,
max_iterations=100,
tolerance=1,
)
)
print(
f"Optimal Penalty: {optimal_penalty}, Profit: {profit}, Charge Cycles: {charge_cycles}"
)
# for each row, report the profits
for idx, row in df.iterrows():
task.get_logger().report_scalar(
title="Profit",
series=f"penalty_{row['Penalty']}",
value=row["Total Profit"],
title="Optimal Penalty",
series="test",
value=optimal_penalty,
iteration=epoch,
)
df = self.policy_evaluator.get_profits_till_400()
for idx, row in df.iterrows():
task.get_logger().report_scalar(
title="Profit_till_400",
series=f"penalty_{row['Penalty']}",
value=row["Profit_till_400"],
title="Optimal Profit", series="test", value=profit, iteration=epoch
)
task.get_logger().report_scalar(
title="Optimal Charge Cycles",
series="test",
value=charge_cycles,
iteration=epoch,
)
return np.mean(crps_from_samples_metric), generated_samples
def log_final_metrics(self, task, dataloader, train: bool = True):
metrics = {metric.__class__.__name__: 0.0 for metric in self.metrics_to_track}
transformed_metrics = {
metric.__class__.__name__: 0.0 for metric in self.metrics_to_track
}
crps_from_samples_metric = []
with torch.no_grad():
total_samples = len(dataloader.dataset) - 96
batches = 0
for _, _, idx_batch in tqdm(dataloader):
idx_batch = [idx for idx in idx_batch if idx < total_samples]
if len(idx_batch) == 0:
continue
if train == False:
for idx in tqdm(idx_batch):
computed_idx_batch = [idx] * 250
initial, outputs, samples, targets = self.auto_regressive(
dataloader.dataset, idx_batch=computed_idx_batch
)
# save the samples for the idx, these will be used for evaluating the policy
self.test_set_samples[idx.item()] = (
self.data_processor.inverse_transform(initial),
self.data_processor.inverse_transform(samples),
)
samples = samples.unsqueeze(0)
targets = targets.squeeze(-1)
targets = targets[0].unsqueeze(0)
crps = crps_from_samples(samples, targets)
crps_from_samples_metric.append(crps[0].mean().item())
_, outputs, samples, targets = self.auto_regressive(
dataloader.dataset, idx_batch=idx_batch
)
@@ -308,6 +286,9 @@ class AutoRegressiveQuantileTrainer(AutoRegressiveTrainer):
task.get_logger().report_single_value(name=name, value=metric_value)
if train == False:
crps_from_samples_metric, self.test_set_samples = (
self.calculate_crps_from_samples(None, dataloader, None)
)
task.get_logger().report_single_value(
name="test_CRPS_from_samples_transformed",
value=np.mean(crps_from_samples_metric),
@@ -320,6 +301,7 @@ class AutoRegressiveQuantileTrainer(AutoRegressiveTrainer):
predictions,
show_legend: bool = True,
retransform: bool = True,
task=None,
):
fig = go.Figure()
@@ -427,7 +409,19 @@ class AutoRegressiveQuantileTrainer(AutoRegressiveTrainer):
ax.lines[1],
]
)
return fig
ax.set_ylim(-1500, 1500)
fig2, ax2 = plt.subplots(figsize=(20, 10))
for i in range(10):
ax2.plot(predictions_np[i], label=f"Sample {i}")
ax2.plot(next_day_np, label="Real NRV", linewidth=3)
ax2.legend()
ax2.set_ylim(-1500, 1500)
return fig, fig2
def auto_regressive(self, dataset, idx_batch, sequence_length: int = 96):
return auto_regressive(
@@ -646,6 +640,21 @@ class NonAutoRegressiveQuantileRegression(Trainer):
figure=fig,
)
fig, ax = plt.subplots(figsize=(20, 10))
for i in range(10):
ax.plot(samples[i], label=f"Sample {i}")
ax.plot(target, label="Real NRV", linewidth=3)
ax.legend()
task.get_logger().report_matplotlib_figure(
title="Training" if train else "Testing",
series=f"Sample {actual_idx} Samples",
iteration=epoch,
figure=fig,
)
plt.close()
def get_plot(
self,
current_day,
@@ -740,7 +749,15 @@ class NonAutoRegressiveQuantileRegression(Trainer):
ax.lines[1],
]
)
return fig
fig2, ax2 = plt.subplots(figsize=(20, 10))
for i in range(10):
ax2.plot(predictions_np[i], label=f"Sample {i}")
ax2.plot(next_day_np, label="Real NRV", linewidth=3)
ax2.legend()
return fig, fig2
def calculate_crps_from_samples(self, task, dataloader, epoch: int):
crps_from_samples_metric = []

3
src/trainers/trainer.py
View File

@@ -261,8 +261,7 @@ class Trainer:
self.model.eval()
# set full day skip
self.data_processor.set_full_day_skip(True)
train_loader, test_loader = self.data_processor.get_dataloaders(
_, test_loader = self.data_processor.get_dataloaders(
predict_sequence_length=self.model.output_size
)