Add MixLinear baseline to TFB benchmark

scripts/multivariate_forecast/ETTh1_script/MixLinear.sh

@@ -0,0 +1,7 @@
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ETTh1.csv" --strategy-args '{"horizon": 96}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.95, "lpf": 1, "batch_size": 256, "lr": 0.03, "num_epochs": 30, "patience": 10, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 96, "norm": true, "enc_in": 7, "features": "M"}' --gpus 3 --num-workers 1 --timeout 60000 --save-path "ETTh1/MixLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ETTh1.csv" --strategy-args '{"horizon": 192}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.95, "lpf": 5, "batch_size": 256, "lr": 0.03, "num_epochs": 30, "patience": 10, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 192, "norm": true, "enc_in": 7, "features": "M"}' --gpus 3 --num-workers 1 --timeout 60000 --save-path "ETTh1/MixLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ETTh1.csv" --strategy-args '{"horizon": 336}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.99, "lpf": 2, "batch_size": 256, "lr": 0.03, "num_epochs": 30, "patience": 10, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 336, "norm": true, "enc_in": 7, "features": "M"}' --gpus 3 --num-workers 1 --timeout 60000 --save-path "ETTh1/MixLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ETTh1.csv" --strategy-args '{"horizon": 720}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.99, "lpf": 19, "batch_size": 256, "lr": 0.03, "num_epochs": 30, "patience": 10, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 720, "norm": true, "enc_in": 7, "features": "M"}' --gpus 3 --num-workers 1 --timeout 60000 --save-path "ETTh1/MixLinear"

scripts/multivariate_forecast/ETTh2_script/MixLinear.sh

@@ -0,0 +1,7 @@
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ETTh2.csv" --strategy-args '{"horizon": 96}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.5, "lpf": 19, "batch_size": 256, "lr": 0.02, "num_epochs": 30, "patience": 5, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 96, "norm": true, "enc_in": 7, "features": "M"}' --gpus 3 --num-workers 1 --timeout 60000 --save-path "ETTh2/MixLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ETTh2.csv" --strategy-args '{"horizon": 192}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.01, "lpf": 19, "batch_size": 310, "lr": 0.02, "num_epochs": 30, "patience": 5, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 192, "norm": true, "enc_in": 7, "features": "M"}' --gpus 3 --num-workers 1 --timeout 60000 --save-path "ETTh2/MixLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ETTh2.csv" --strategy-args '{"horizon": 336}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.01, "lpf": 15, "batch_size": 128, "lr": 0.02, "num_epochs": 30, "patience": 5, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 336, "norm": true, "enc_in": 7, "features": "M"}' --gpus 3 --num-workers 1 --timeout 60000 --save-path "ETTh2/MixLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ETTh2.csv" --strategy-args '{"horizon": 720}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.5, "lpf": 19, "batch_size": 310, "lr": 0.02, "num_epochs": 30, "patience": 5, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 720, "norm": true, "enc_in": 7, "features": "M"}' --gpus 3 --num-workers 1 --timeout 60000 --save-path "ETTh2/MixLinear"

scripts/multivariate_forecast/Electricity_script/MixLinear.sh

@@ -0,0 +1,8 @@
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Electricity.csv" --strategy-args '{"horizon": 96}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.5, "lpf": 19, "batch_size": 64, "lr": 0.03, "num_epochs": 30, "patience": 10, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 96, "norm": true}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Electricity/MixLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Electricity.csv" --strategy-args '{"horizon": 192}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.99, "lpf": 15, "batch_size": 64, "lr": 0.03, "num_epochs": 30, "patience": 10, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 192, "norm": true}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Electricity/MixLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Electricity.csv" --strategy-args '{"horizon": 336}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.99, "lpf": 15, "batch_size": 64, "lr": 0.03, "num_epochs": 30, "patience": 10, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 336, "norm": true}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Electricity/MixLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Electricity.csv" --strategy-args '{"horizon": 720}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.5, "lpf": 15, "batch_size": 64, "lr": 0.03, "num_epochs": 30, "patience": 10, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 720, "norm": true}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Electricity/MixLinear"
+

scripts/multivariate_forecast/Traffic_script/MixLinear.sh

@@ -0,0 +1,7 @@
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Traffic.csv" --strategy-args '{"horizon": 96}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.5, "lpf": 19, "batch_size": 64, "lr": 0.03, "num_epochs": 30, "patience": 5, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 96, "norm": true}' --gpus 7 --num-workers 1 --timeout 60000 --save-path "Traffic/MixLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Traffic.csv" --strategy-args '{"horizon": 192}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.01, "lpf": 15, "batch_size": 64, "lr": 0.03, "num_epochs": 30, "patience": 5, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 192, "norm": true}' --gpus 7 --num-workers 1 --timeout 60000 --save-path "Traffic/MixLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Traffic.csv" --strategy-args '{"horizon": 336}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.5, "lpf": 19, "batch_size": 64, "lr": 0.03, "num_epochs": 30, "patience": 5, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 336, "norm": true}' --gpus 7 --num-workers 1 --timeout 60000 --save-path "Traffic/MixLinear"
+
+python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Traffic.csv" --strategy-args '{"horizon": 720}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.9, "lpf": 19, "batch_size": 64, "lr": 0.03, "num_epochs": 30, "patience": 5, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 720, "norm": true}' --gpus 7 --num-workers 1 --timeout 60000 --save-path "Traffic/MixLinear"

ts_benchmark/baselines/mixlinear/__init__.py

@@ -0,0 +1,5 @@
+__all__ = [
+    "MIXLINEAR",
+]
+
+from ts_benchmark.baselines.mixlinear.mixlinear import MIXLINEAR

ts_benchmark/baselines/mixlinear/layers/Embed.py

@@ -0,0 +1,207 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.nn.utils import weight_norm
+import math
+
+
+class PositionalEmbedding(nn.Module):
+    def __init__(self, d_model, max_len=5000):
+        super(PositionalEmbedding, self).__init__()
+        # Compute the positional encodings once in log space.
+        pe = torch.zeros(max_len, d_model).float()
+        pe.require_grad = False
+
+        position = torch.arange(0, max_len).float().unsqueeze(1)
+        div_term = (
+            torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model)
+        ).exp()
+
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+
+        pe = pe.unsqueeze(0)
+        self.register_buffer("pe", pe)
+
+    def forward(self, x):
+        return self.pe[:, : x.size(1)]
+
+
+class TokenEmbedding(nn.Module):
+    def __init__(self, c_in, d_model):
+        super(TokenEmbedding, self).__init__()
+        padding = 1 if torch.__version__ >= "1.5.0" else 2
+        self.tokenConv = nn.Conv1d(
+            in_channels=c_in,
+            out_channels=d_model,
+            kernel_size=3,
+            padding=padding,
+            padding_mode="circular",
+            bias=False,
+        )
+        for m in self.modules():
+            if isinstance(m, nn.Conv1d):
+                nn.init.kaiming_normal_(
+                    m.weight, mode="fan_in", nonlinearity="leaky_relu"
+                )
+
+    def forward(self, x):
+        x = self.tokenConv(x.permute(0, 2, 1)).transpose(1, 2)
+        return x
+
+
+class FixedEmbedding(nn.Module):
+    def __init__(self, c_in, d_model):
+        super(FixedEmbedding, self).__init__()
+
+        w = torch.zeros(c_in, d_model).float()
+        w.require_grad = False
+
+        position = torch.arange(0, c_in).float().unsqueeze(1)
+        div_term = (
+            torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model)
+        ).exp()
+
+        w[:, 0::2] = torch.sin(position * div_term)
+        w[:, 1::2] = torch.cos(position * div_term)
+
+        self.emb = nn.Embedding(c_in, d_model)
+        self.emb.weight = nn.Parameter(w, requires_grad=False)
+
+    def forward(self, x):
+        return self.emb(x).detach()
+
+
+class TemporalEmbedding(nn.Module):
+    def __init__(self, d_model, embed_type="fixed", freq="h"):
+        super(TemporalEmbedding, self).__init__()
+
+        minute_size = 4
+        hour_size = 24
+        weekday_size = 7
+        day_size = 32
+        month_size = 13
+
+        Embed = FixedEmbedding if embed_type == "fixed" else nn.Embedding
+        if freq == "t":
+            self.minute_embed = Embed(minute_size, d_model)
+        self.hour_embed = Embed(hour_size, d_model)
+        self.weekday_embed = Embed(weekday_size, d_model)
+        self.day_embed = Embed(day_size, d_model)
+        self.month_embed = Embed(month_size, d_model)
+
+    def forward(self, x):
+        x = x.long()
+        minute_x = (
+            self.minute_embed(x[:, :, 4]) if hasattr(self, "minute_embed") else 0.0
+        )
+        hour_x = self.hour_embed(x[:, :, 3])
+        weekday_x = self.weekday_embed(x[:, :, 2])
+        day_x = self.day_embed(x[:, :, 1])
+        month_x = self.month_embed(x[:, :, 0])
+
+        return hour_x + weekday_x + day_x + month_x + minute_x
+
+
+class TimeFeatureEmbedding(nn.Module):
+    def __init__(self, d_model, embed_type="timeF", freq="h"):
+        super(TimeFeatureEmbedding, self).__init__()
+
+        freq_map = {"h": 4, "t": 5, "s": 6, "m": 1, "a": 1, "w": 2, "d": 3, "b": 3}
+        d_inp = freq_map[freq]
+        self.embed = nn.Linear(d_inp, d_model, bias=False)
+
+    def forward(self, x):
+        return self.embed(x)
+
+
+class DataEmbedding(nn.Module):
+    def __init__(self, c_in, d_model, embed_type="fixed", freq="h", dropout=0.1):
+        super(DataEmbedding, self).__init__()
+
+        self.value_embedding = TokenEmbedding(c_in=c_in, d_model=d_model)
+        self.position_embedding = PositionalEmbedding(d_model=d_model)
+        self.temporal_embedding = (
+            TemporalEmbedding(d_model=d_model, embed_type=embed_type, freq=freq)
+            if embed_type != "timeF"
+            else TimeFeatureEmbedding(d_model=d_model, embed_type=embed_type, freq=freq)
+        )
+        self.dropout = nn.Dropout(p=dropout)
+
+    def forward(self, x, x_mark):
+        if x_mark is None:
+            x = self.value_embedding(x) + self.position_embedding(x)
+        else:
+            x = (
+                self.value_embedding(x)
+                + self.temporal_embedding(x_mark)
+                + self.position_embedding(x)
+            )
+        return self.dropout(x)
+
+
+class DataEmbedding_inverted(nn.Module):
+    def __init__(self, c_in, d_model, embed_type="fixed", freq="h", dropout=0.1):
+        super(DataEmbedding_inverted, self).__init__()
+        self.value_embedding = nn.Linear(c_in, d_model)
+        self.dropout = nn.Dropout(p=dropout)
+
+    def forward(self, x, x_mark):
+        x = x.permute(0, 2, 1)
+        # x: [Batch Variate Time]
+        if x_mark is None:
+            x = self.value_embedding(x)
+        else:
+            x = self.value_embedding(torch.cat([x, x_mark.permute(0, 2, 1)], 1))
+        # x: [Batch Variate d_model]
+        return self.dropout(x)
+
+
+class DataEmbedding_wo_pos(nn.Module):
+    def __init__(self, c_in, d_model, embed_type="fixed", freq="h", dropout=0.1):
+        super(DataEmbedding_wo_pos, self).__init__()
+
+        self.value_embedding = TokenEmbedding(c_in=c_in, d_model=d_model)
+        self.position_embedding = PositionalEmbedding(d_model=d_model)
+        self.temporal_embedding = (
+            TemporalEmbedding(d_model=d_model, embed_type=embed_type, freq=freq)
+            if embed_type != "timeF"
+            else TimeFeatureEmbedding(d_model=d_model, embed_type=embed_type, freq=freq)
+        )
+        self.dropout = nn.Dropout(p=dropout)
+
+    def forward(self, x, x_mark):
+        if x_mark is None:
+            x = self.value_embedding(x)
+        else:
+            x = self.value_embedding(x) + self.temporal_embedding(x_mark)
+        return self.dropout(x)
+
+
+class PatchEmbedding(nn.Module):
+    def __init__(self, d_model, patch_len, stride, padding, dropout):
+        super(PatchEmbedding, self).__init__()
+        # Patching
+        self.patch_len = patch_len
+        self.stride = stride
+        self.padding_patch_layer = nn.ReplicationPad1d((0, padding))
+
+        # Backbone, Input encoding: projection of feature vectors onto a d-dim vector space
+        self.value_embedding = nn.Linear(patch_len, d_model, bias=False)
+
+        # Positional embedding
+        self.position_embedding = PositionalEmbedding(d_model)
+
+        # Residual dropout
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x):
+        # do patching
+        n_vars = x.shape[1]
+        x = self.padding_patch_layer(x)
+        x = x.unfold(dimension=-1, size=self.patch_len, step=self.stride)
+        x = torch.reshape(x, (x.shape[0] * x.shape[1], x.shape[2], x.shape[3]))
+        # Input encoding
+        x = self.value_embedding(x) + self.position_embedding(x)
+        return self.dropout(x), n_vars
+

ts_benchmark/baselines/mixlinear/layers/__init__.py

@@ -0,0 +1 @@
+

ts_benchmark/baselines/mixlinear/mixlinear.py

@@ -0,0 +1,44 @@
+from ts_benchmark.baselines.deep_forecasting_model_base import DeepForecastingModelBase
+from ts_benchmark.baselines.mixlinear.mixlinear_model import MixlinearModel
+
+MODEL_HYPER_PARAMS = {
+    "alpha": 0.5,
+    "lpf": 1,
+    "kernel": 24,
+    "freq": "h",
+    "embed": "learned",
+    "lradj": "type3",
+    "factor": 1,
+    "activation": "gelu",
+    "dropout": 0.05,
+    "batch_size": 32,
+    "lr": 0.0001,
+    "num_epochs": 100,
+    "num_workers": 10,
+    "loss": "MSE",
+    "itr": 2,
+    "distil": True,
+    "patience": 3,
+    "period_len": 24,
+}
+
+class MIXLINEAR(DeepForecastingModelBase):
+
+    def __init__(self, **kwargs):
+        super(MIXLINEAR, self).__init__(MODEL_HYPER_PARAMS, **kwargs)
+
+    @property
+    def model_name(self):
+        return "MIXLINEAR"
+
+    def _init_model(self):
+        return MixlinearModel(self.config)
+
+    def _process(self, input, target, input_mark, target_mark):
+        """
+        input: shape [batch, seq_len, enc_in]
+        output: shape [batch, pred_len, enc_in]
+        """
+        output = self.model(input)
+        return {"output": output}
+

ts_benchmark/baselines/mixlinear/mixlinear_model.py

@@ -0,0 +1,85 @@
+import torch
+import torch.nn as nn
+import math
+import torch.nn.functional as F
+from ts_benchmark.baselines.mixlinear.layers.Embed import PositionalEmbedding
+
+
+class MixlinearModel(nn.Module):
+    def __init__(self, configs):
+        super(MixlinearModel, self).__init__()
+
+        # get parameters
+        self.seq_len = configs.seq_len
+        self.pred_len = configs.pred_len
+        self.enc_in = configs.enc_in
+        self.period_len = configs.period_len
+
+        self.kernel = self.period_len
+        self.lpf = configs.lpf
+        self.alpha = configs.alpha
+
+        self.seg_num_x = math.ceil(self.seq_len / self.period_len)
+        self.seg_num_y = math.ceil(self.pred_len / self.period_len)
+
+        self.sqrt_seg_num_x = math.ceil(math.sqrt(self.seq_len / self.period_len))
+        self.sqrt_seg_num_y = math.ceil(math.sqrt(self.pred_len / self.period_len))
+
+        # TLinear
+        self.TLinear1 = nn.Linear(self.sqrt_seg_num_x, self.sqrt_seg_num_y, bias=False)
+        self.TLinear2 = nn.Linear(self.sqrt_seg_num_x, self.sqrt_seg_num_y, bias=False)
+
+        self.conv1d = nn.Conv1d(in_channels=1, out_channels=1, kernel_size=self.kernel + 1,
+                                stride=1, padding=int(self.kernel / 2), padding_mode="zeros", bias=False)
+
+        # FLinear
+        self.FLinear1 = nn.Linear(self.lpf, 2, bias=False).to(torch.cfloat)
+        self.FLinear2 = nn.Linear(2, self.seg_num_y, bias=False).to(torch.cfloat)
+
+    def forward(self, x):
+        batch_size = x.shape[0]
+        # normalization and permute     b,s,c -> b,c,s
+        seq_mean = torch.mean(x, dim=1).unsqueeze(1)
+        x = (x - seq_mean).permute(0, 2, 1)
+        # print(x.shape)
+        x = self.conv1d(x.reshape(-1, 1, self.seq_len)).reshape(-1, self.enc_in, self.seq_len) + x
+        # print(x.shape)
+
+        # ->b,e,w,n
+        x = x.reshape(batch_size, self.enc_in, -1, self.period_len).permute(0, 1, 3, 2)
+
+        # Time Domain
+
+
+        # x_o = torch.zeros(batch_size, self.enc_in, self.period_len, self.sqrt_seg_num_x ** 2).to(x.device)
+        # x_o[:, :, :, :x.shape[-1]] = x[:, :, :, :]
+
+        x_o = F.pad(x, (0, self.sqrt_seg_num_x ** 2 - x.shape[-1], 0, 0, 0, 0))
+
+        x_o = x_o.reshape(batch_size, self.enc_in, self.period_len, self.sqrt_seg_num_x, self.sqrt_seg_num_x)
+
+        x_o = self.TLinear1(x_o).permute(0, 1, 2, 4, 3)
+        x_t = self.TLinear2(x_o).permute(0, 1, 2, 4, 3)
+
+        x_t = x_t.reshape(batch_size, self.enc_in, self.period_len, -1).permute(0, 1, 3, 2).reshape(batch_size,
+                                                                                                    self.enc_in,
+                                                                                                    -1).permute(0, 2, 1)
+
+        # Frequency Domain
+
+        x_fft = torch.fft.fft(x, dim=3)[:, :, :, :self.lpf]
+        # x_fft = x_fft.view(-1,self.lpf)
+        x_fft = self.FLinear1(x_fft)
+        x_fft = self.FLinear2(x_fft).reshape(batch_size, self.enc_in, self.period_len, -1)
+
+        x_rfft = torch.fft.ifft(x_fft, dim=3).float()
+        x_f = x_rfft.permute(0, 1, 3, 2).reshape(batch_size, self.enc_in, -1).permute(0, 2, 1)
+
+        #print("shape", x_t.shape, x_f.shape)
+
+        # Mix
+        #print("shape", x_t[:, :self.pred_len, :].shape, x_f[:, :self.pred_len, :].shape)
+
+        x = x_t[:, :self.pred_len, :] * self.alpha + seq_mean + x_f[:, :self.pred_len, :] * (1 - self.alpha)
+
+        return x[:, :self.pred_len, :]