VictorMylle/Thesis
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Plots to compare between quantile regression and diffusion

Browse Source
This commit is contained in:
Victor Mylle
2024-02-18 19:21:59 +01:00
parent bd250a664b
commit 174a82fab2
5 changed files with 123 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/data/dataset.py
View File

@@ -25,12 +25,19 @@ class NrvDataset(Dataset):
self.sequence_length = sequence_length
self.predict_sequence_length = predict_sequence_length
self.samples_to_skip = self.skip_samples(dataframe=dataframe)
self.samples_to_skip = self.skip_samples(dataframe=dataframe, full_day_skip=self.full_day_skip)
total_indices = set(
range(len(dataframe) - self.sequence_length - self.predict_sequence_length)
)
self.valid_indices = sorted(list(total_indices - set(self.samples_to_skip)))
# full day indices
full_day_skipped_samples = self.skip_samples(dataframe=dataframe, full_day_skip=True)
full_day_total_indices = set(
range(len(dataframe) - self.sequence_length - self.predict_sequence_length)
)
self.full_day_valid_indices = sorted(list(full_day_total_indices - set(full_day_skipped_samples)))
self.history_features = []
if self.data_config.LOAD_HISTORY:
self.history_features.append("total_load")
@@ -73,7 +80,7 @@ class NrvDataset(Dataset):
self.history_features, self.forecast_features = self.preprocess_data(dataframe)
def skip_samples(self, dataframe):
def skip_samples(self, dataframe, full_day_skip):
nan_rows = dataframe[dataframe.isnull().any(axis=1)]
nan_indices = nan_rows.index
skip_indices = [
@@ -91,7 +98,7 @@ class NrvDataset(Dataset):
# add indices that are not the start of a day (00:15) to the skip indices (use datetime column)
# get indices of all 00:15 timestamps
if self.full_day_skip:
if full_day_skip:
start_of_day_indices = dataframe[
dataframe["datetime"].dt.time != pd.Timestamp("00:00:00").time()
].index