Trying out training script for ClearML

src/notebooks/training.py

@@ -0,0 +1,69 @@
+import sys
+sys.path.append('..')
+from data import DataProcessor, DataConfig
+from trainers.quantile_trainer import AutoRegressiveQuantileTrainer, NonAutoRegressiveQuantileRegression
+from trainers.probabilistic_baseline import ProbabilisticBaselineTrainer
+from trainers.autoregressive_trainer import AutoRegressiveTrainer
+from trainers.trainer import Trainer
+from utils.clearml import ClearMLHelper
+from models import *
+from losses import *
+import torch
+import numpy as np
+from torch.nn import MSELoss, L1Loss
+from datetime import datetime
+import pytz
+import torch.nn as nn
+
+# auto reload
+%load_ext autoreload
+%autoreload 2
+
+#### ClearML ####
+clearml_helper = ClearMLHelper(project_name="Thesis/NrvForecast")
+
+#### Data Processor ####
+data_config = DataConfig()
+data_config.NRV_HISTORY = True
+data_config.LOAD_HISTORY = False
+data_config.LOAD_FORECAST = False
+
+data_config.WIND_FORECAST = False
+data_config.WIND_HISTORY = False
+
+data_processor = DataProcessor(data_config)
+data_processor.set_batch_size(1024)
+data_processor.set_full_day_skip(False)
+
+#### Hyperparameters ####
+data_processor.set_output_size(1)
+inputDim = data_processor.get_input_size()
+learningRate = 0.0001
+epochs = 100
+
+# quantiles = torch.tensor([0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99]).to("cuda")
+quantiles = torch.tensor(
+    [0.01, 0.05, 0.1, 0.15, 0.3, 0.4, 0.5, 0.6, 0.7, 0.85, 0.9, 0.95, 0.99]
+).to("cuda")
+
+# model = LinearRegression(inputDim, len(quantiles))
+model = NonLinearRegression(inputDim, len(quantiles), hiddenSize=1024, numLayers=5)
+optimizer = torch.optim.Adam(model.parameters(), lr=learningRate)
+
+#### Trainer ####
+trainer = AutoRegressiveQuantileTrainer(
+    model,
+    optimizer,
+    data_processor,
+    quantiles,
+    "cuda",
+    debug=True,
+    clearml_helper=clearml_helper,
+)
+trainer.add_metrics_to_track(
+    [PinballLoss(quantiles), MSELoss(), L1Loss(), CRPSLoss(quantiles)]
+)
+trainer.early_stopping(patience=10)
+trainer.plot_every(5)
+
+trainer.train(epochs=epochs, remotely=True)