diff --git a/Reports/Thesis/sections/results/policies/nrv_samples_policy.tex b/Reports/Thesis/sections/results/policies/nrv_samples_policy.tex index 5a92e1a..2b7a481 100644 --- a/Reports/Thesis/sections/results/policies/nrv_samples_policy.tex +++ b/Reports/Thesis/sections/results/policies/nrv_samples_policy.tex @@ -1,5 +1,7 @@ \subsection{Policy using generated NRV samples} -The generated full-day samples can be used to improve the profit of the policy. For each day, the generated samples can be used to determine the buying and selling thresholds. Assume that there are 100 generated full-day NRV samples for the day for which the profit needs to be optimized. The thresholds are determined for each generated sample separately using a simple grid search. All these thresholds can then be reduced by taking the mean to get one value for the buying threshold and one value for the selling threshold. Again, the penalty parameter is optimized for the test set to make sure around 283 charge cycles are used for fair comparison. The policy is evaluated for the different types of models that were trained and discussed in the previous sections. To reduce the computational cost, the policy is only evaluated for the best-performing models based on the CRPS metric. +The generated full-day samples can be used to improve the profit of the policy. For each day, the generated samples can be used to determine the buying and selling thresholds. Assume that there are 100 generated full-day NRV samples for the day for which the profit needs to be optimized. The thresholds are determined for each generated sample separately using a simple grid search. All these thresholds can then be reduced by taking the mean to get one value for the buying threshold and one value for the selling threshold. Again, the penalty parameter is optimized for the test set to make sure around 283 charge cycles are used for fair comparison. + +A low CRPS value does not necessarily mean the policy will generate a high profit. Because of this, the CRPS metric can not be used to evaluate the model during the training phase and use this metric to do early stopping. To fairly evaluate and compare the models, a validation set is split off from the training set. The validation set is used to evaluate the profit of the policy during the training and use this to do early stopping. The last two months of the training set are used as the validation set. This range starts on 01-11-2022 and ends on 31-12-2022. Two months are chosen to make sure enough data is available to have a good estimate of the profit while making sure the validation set is not too large. The policy can be evaluated quite fast on the validation set which is feasible to do during the training after a certain number of epochs. \begin{table}[ht] \centering diff --git a/Reports/Thesis/verslag.aux b/Reports/Thesis/verslag.aux index 1859b53..1da153c 100644 --- a/Reports/Thesis/verslag.aux +++ b/Reports/Thesis/verslag.aux @@ -121,8 +121,8 @@ \@writefile{lot}{\contentsline {table}{\numberline {11}{\ignorespaces Results of the baseline policies on the test set. \relax }}{45}{table.caption.29}\protected@file@percent } \newlabel{tab:fixed_thresholds}{{11}{45}{Results of the baseline policies on the test set. \relax }{table.caption.29}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {7.2}Policy using generated NRV samples}{45}{subsection.7.2}\protected@file@percent } -\@writefile{lot}{\contentsline {table}{\numberline {12}{\ignorespaces Comparison of the different models using the profit metric. The best-performing models for a certain type are selected based on the profit.\relax }}{45}{table.caption.30}\protected@file@percent } -\newlabel{tab:policy_comparison}{{12}{45}{Comparison of the different models using the profit metric. The best-performing models for a certain type are selected based on the profit.\relax }{table.caption.30}{}} +\@writefile{lot}{\contentsline {table}{\numberline {12}{\ignorespaces Comparison of the different models using the profit metric. The best-performing models for a certain type are selected based on the profit.\relax }}{46}{table.caption.30}\protected@file@percent } +\newlabel{tab:policy_comparison}{{12}{46}{Comparison of the different models using the profit metric. The best-performing models for a certain type are selected based on the profit.\relax }{table.caption.30}{}} \abx@aux@page{6}{48} \abx@aux@page{7}{48} \abx@aux@page{8}{48} diff --git a/Reports/Thesis/verslag.log b/Reports/Thesis/verslag.log index c112019..ea5e1c4 100644 --- a/Reports/Thesis/verslag.log +++ b/Reports/Thesis/verslag.log @@ -1,4 +1,4 @@ -This is pdfTeX, Version 3.141592653-2.6-1.40.25 (TeX Live 2023) (preloaded format=pdflatex 2023.9.17) 15 MAY 2024 16:18 +This is pdfTeX, Version 3.141592653-2.6-1.40.25 (TeX Live 2023) (preloaded format=pdflatex 2023.9.17) 18 MAY 2024 11:50 entering extended mode restricted \write18 enabled. file:line:error style messages enabled. @@ -2007,7 +2007,7 @@ Here is how much of TeX's memory you used: 1141 hyphenation exceptions out of 8191 84i,16n,131p,2100b,5180s stack positions out of 10000i,1000n,20000p,200000b,200000s -Output written on verslag.pdf (51 pages, 8683971 bytes). +Output written on verslag.pdf (51 pages, 8684536 bytes). PDF statistics: 672 PDF objects out of 1000 (max. 8388607) 497 compressed objects within 5 object streams diff --git a/Reports/Thesis/verslag.pdf b/Reports/Thesis/verslag.pdf index 21388a3..6ecace6 100644 Binary files a/Reports/Thesis/verslag.pdf and b/Reports/Thesis/verslag.pdf differ diff --git a/Reports/Thesis/verslag.synctex(busy) b/Reports/Thesis/verslag.synctex(busy) deleted file mode 100644 index e69de29..0000000 diff --git a/Reports/Thesis/verslag.synctex.gz b/Reports/Thesis/verslag.synctex.gz index aa2b1cf..abfa8c0 100644 Binary files a/Reports/Thesis/verslag.synctex.gz and b/Reports/Thesis/verslag.synctex.gz differ diff --git a/src/trainers/trainer.py b/src/trainers/trainer.py index fa018c8..af3d106 100644 --- a/src/trainers/trainer.py +++ b/src/trainers/trainer.py @@ -114,8 +114,8 @@ class Trainer: predict_sequence_length=self.model.output_size, full_day_skip=True ) - train_loader, test_loader = self.data_processor.get_dataloaders( - predict_sequence_length=self.model.output_size + train_loader, val_loader, test_loader = self.data_processor.get_dataloaders( + predict_sequence_length=self.model.output_size, validation=True ) train_samples = self.random_samples(train=True, num_samples=5) @@ -146,7 +146,7 @@ class Trainer: running_loss += loss.item() running_loss /= len(train_loader.dataset) - test_loss = self.test(test_loader) + test_loss = self.test(val_loader) if self.patience is not None: if ( @@ -170,7 +170,7 @@ class Trainer: ) task.get_logger().report_scalar( title=self.criterion.__class__.__name__, - series="test", + series="val", value=test_loss, iteration=epoch, ) @@ -194,7 +194,7 @@ class Trainer: # ) if hasattr(self, "calculate_crps_from_samples"): - self.calculate_crps_from_samples(task, test_loader, epoch) + self.calculate_crps_from_samples(task, val_loader, epoch) if task: self.finish_training(task=task) diff --git a/src/training_scripts/diffusion_training.py b/src/training_scripts/diffusion_training.py index dbaa283..7859750 100644 --- a/src/training_scripts/diffusion_training.py +++ b/src/training_scripts/diffusion_training.py @@ -2,9 +2,9 @@ from src.utils.clearml import ClearMLHelper clearml_helper = ClearMLHelper(project_name="Thesis/NrvForecast") task = clearml_helper.get_task( - task_name="Diffusion Training: hidden_sizes=[512, 512] (100 steps), lr=0.0001, time_dim=8", + task_name="Diffusion Training: hidden_sizes=[1024, 1024] (300 steps), all features", ) -# task.execute_remotely(queue_name="default", exit_process=True) +task.execute_remotely(queue_name="default", exit_process=True) from src.models import * from src.losses import * @@ -42,7 +42,7 @@ print("Input dim: ", inputDim) model_parameters = { "epochs": 15000, "learning_rate": 0.0001, - "hidden_sizes": [512, 512], + "hidden_sizes": [1024, 1024], "time_dim": 8, } @@ -71,6 +71,6 @@ policy_evaluator = PolicyEvaluator(baseline_policy, task) #### Trainer #### trainer = DiffusionTrainer( - model, data_processor, "cuda", policy_evaluator=policy_evaluator, noise_steps=20 + model, data_processor, "cuda", policy_evaluator=policy_evaluator, noise_steps=300 ) trainer.train(model_parameters["epochs"], model_parameters["learning_rate"], task)