Updated thesis and added quantile performance plots to non autoregressive quantiles

src/trainers/quantile_trainer.py

@@ -884,3 +884,82 @@ class NonAutoRegressiveQuantileRegression(Trainer):
                 value=charge_cycles,
                 iteration=epoch,
             )
+
+    def plot_quantile_percentages(
+        self,
+        task,
+        data_loader,
+        train: bool = True,
+        iteration: int = None,
+        full_day: bool = False,
+    ):
+        quantiles = self.quantiles
+        total = 0
+        quantile_counter = {q: 0 for q in quantiles}
+
+        self.model.eval()
+        with torch.no_grad():
+            total_samples = len(data_loader.dataset) - 96
+
+            for inputs, targets, idx_batch in data_loader:
+                idx_batch = [idx for idx in idx_batch if idx < total_samples]
+
+                inputs = inputs.to(self.device)
+                outputs = (
+                    self.model(inputs).cpu().numpy()
+                )  # (batch_size, 96*num_quantiles)
+
+                # reshape to (batch_size, num_quantiles, 96)
+                outputs = outputs.reshape(-1, 96, len(quantiles))
+
+                targets = targets.squeeze(-1).cpu().numpy()  # (batch_size, 96)
+
+                for i, q in enumerate(quantiles):
+                    quantile_counter[q] += np.sum(targets < outputs[:, i, :])
+
+                total += len(targets)
+
+        # to numpy array of length len(quantiles)
+        percentages = np.array([quantile_counter[q] / total for q in quantiles])
+
+        bar_width = 0.35
+        index = np.arange(len(quantiles))
+
+        # Plotting the bars
+        fig, ax = plt.subplots(figsize=(15, 10))
+
+        bar1 = ax.bar(index, quantiles, bar_width, label="Ideal", color="brown")
+        bar2 = ax.bar(
+            index + bar_width, percentages, bar_width, label="NN model", color="blue"
+        )
+
+        # Adding the percentage values above the bars for bar2
+        for rect in bar2:
+            height = rect.get_height()
+            ax.text(
+                rect.get_x() + rect.get_width() / 2.0,
+                1.005 * height,
+                f"{height:.2}",
+                ha="center",
+                va="bottom",
+            )  # Format the number as a percentage
+
+        series_name = "Training Set" if train else "Test Set"
+        full_day_str = "Full Day" if full_day else "Single Step"
+
+        # Adding labels and title
+        ax.set_xlabel("Quantile")
+        ax.set_ylabel("Fraction of data under quantile forecast")
+        ax.set_title(f"{series_name} {full_day_str} Quantile Performance Comparison")
+        ax.set_xticks(index + bar_width / 2)
+        ax.set_xticklabels(quantiles)
+        ax.legend()
+
+        task.get_logger().report_matplotlib_figure(
+            title="Quantile Performance Comparison",
+            series=f"{series_name} {full_day_str}",
+            report_image=True,
+            figure=plt,
+            iteration=iteration,
+        )
+        plt.close()

src/training_scripts/non_autoregressive_quantiles.py

@@ -51,7 +51,7 @@ data_processor.set_full_day_skip(True)
 #### Hyperparameters ####
 data_processor.set_output_size(96)
 inputDim = data_processor.get_input_size()
-epochs = 300
+epochs = 2
 
 # add parameters to clearml
 quantiles = task.get_parameter("general/quantiles", cast=True)
@@ -117,7 +117,7 @@ trainer.add_metrics_to_track(
     [PinballLoss(quantiles), MSELoss(), L1Loss(), CRPSLoss(quantiles)]
 )
 trainer.early_stopping(patience=5)
-trainer.plot_every(20)
+trainer.plot_every(1)
 trainer.train(task=task, epochs=epochs, remotely=True)
 
 ### Policy Evaluation ###