diff --git a/scripts/multivariate_forecast/ETTh1_script/MixLinear.sh b/scripts/multivariate_forecast/ETTh1_script/MixLinear.sh new file mode 100644 index 00000000..eb5aa34b --- /dev/null +++ b/scripts/multivariate_forecast/ETTh1_script/MixLinear.sh @@ -0,0 +1,7 @@ +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ETTh1.csv" --strategy-args '{"horizon": 96}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.95, "lpf": 1, "batch_size": 256, "lr": 0.03, "num_epochs": 30, "patience": 10, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 96, "norm": true, "enc_in": 7, "features": "M"}' --gpus 3 --num-workers 1 --timeout 60000 --save-path "ETTh1/MixLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ETTh1.csv" --strategy-args '{"horizon": 192}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.95, "lpf": 5, "batch_size": 256, "lr": 0.03, "num_epochs": 30, "patience": 10, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 192, "norm": true, "enc_in": 7, "features": "M"}' --gpus 3 --num-workers 1 --timeout 60000 --save-path "ETTh1/MixLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ETTh1.csv" --strategy-args '{"horizon": 336}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.99, "lpf": 2, "batch_size": 256, "lr": 0.03, "num_epochs": 30, "patience": 10, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 336, "norm": true, "enc_in": 7, "features": "M"}' --gpus 3 --num-workers 1 --timeout 60000 --save-path "ETTh1/MixLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ETTh1.csv" --strategy-args '{"horizon": 720}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.99, "lpf": 19, "batch_size": 256, "lr": 0.03, "num_epochs": 30, "patience": 10, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 720, "norm": true, "enc_in": 7, "features": "M"}' --gpus 3 --num-workers 1 --timeout 60000 --save-path "ETTh1/MixLinear" \ No newline at end of file diff --git a/scripts/multivariate_forecast/ETTh2_script/MixLinear.sh b/scripts/multivariate_forecast/ETTh2_script/MixLinear.sh new file mode 100644 index 00000000..a6abe6e1 --- /dev/null +++ b/scripts/multivariate_forecast/ETTh2_script/MixLinear.sh @@ -0,0 +1,7 @@ +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ETTh2.csv" --strategy-args '{"horizon": 96}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.5, "lpf": 19, "batch_size": 256, "lr": 0.02, "num_epochs": 30, "patience": 5, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 96, "norm": true, "enc_in": 7, "features": "M"}' --gpus 3 --num-workers 1 --timeout 60000 --save-path "ETTh2/MixLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ETTh2.csv" --strategy-args '{"horizon": 192}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.01, "lpf": 19, "batch_size": 310, "lr": 0.02, "num_epochs": 30, "patience": 5, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 192, "norm": true, "enc_in": 7, "features": "M"}' --gpus 3 --num-workers 1 --timeout 60000 --save-path "ETTh2/MixLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ETTh2.csv" --strategy-args '{"horizon": 336}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.01, "lpf": 15, "batch_size": 128, "lr": 0.02, "num_epochs": 30, "patience": 5, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 336, "norm": true, "enc_in": 7, "features": "M"}' --gpus 3 --num-workers 1 --timeout 60000 --save-path "ETTh2/MixLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "ETTh2.csv" --strategy-args '{"horizon": 720}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.5, "lpf": 19, "batch_size": 310, "lr": 0.02, "num_epochs": 30, "patience": 5, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 720, "norm": true, "enc_in": 7, "features": "M"}' --gpus 3 --num-workers 1 --timeout 60000 --save-path "ETTh2/MixLinear" \ No newline at end of file diff --git a/scripts/multivariate_forecast/Electricity_script/MixLinear.sh b/scripts/multivariate_forecast/Electricity_script/MixLinear.sh new file mode 100644 index 00000000..eee16ce0 --- /dev/null +++ b/scripts/multivariate_forecast/Electricity_script/MixLinear.sh @@ -0,0 +1,8 @@ +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Electricity.csv" --strategy-args '{"horizon": 96}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.5, "lpf": 19, "batch_size": 64, "lr": 0.03, "num_epochs": 30, "patience": 10, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 96, "norm": true}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Electricity/MixLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Electricity.csv" --strategy-args '{"horizon": 192}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.99, "lpf": 15, "batch_size": 64, "lr": 0.03, "num_epochs": 30, "patience": 10, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 192, "norm": true}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Electricity/MixLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Electricity.csv" --strategy-args '{"horizon": 336}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.99, "lpf": 15, "batch_size": 64, "lr": 0.03, "num_epochs": 30, "patience": 10, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 336, "norm": true}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Electricity/MixLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Electricity.csv" --strategy-args '{"horizon": 720}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.5, "lpf": 15, "batch_size": 64, "lr": 0.03, "num_epochs": 30, "patience": 10, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 720, "norm": true}' --gpus 0 --num-workers 1 --timeout 60000 --save-path "Electricity/MixLinear" + diff --git a/scripts/multivariate_forecast/Traffic_script/MixLinear.sh b/scripts/multivariate_forecast/Traffic_script/MixLinear.sh new file mode 100644 index 00000000..51491569 --- /dev/null +++ b/scripts/multivariate_forecast/Traffic_script/MixLinear.sh @@ -0,0 +1,7 @@ +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Traffic.csv" --strategy-args '{"horizon": 96}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.5, "lpf": 19, "batch_size": 64, "lr": 0.03, "num_epochs": 30, "patience": 5, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 96, "norm": true}' --gpus 7 --num-workers 1 --timeout 60000 --save-path "Traffic/MixLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Traffic.csv" --strategy-args '{"horizon": 192}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.01, "lpf": 15, "batch_size": 64, "lr": 0.03, "num_epochs": 30, "patience": 5, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 192, "norm": true}' --gpus 7 --num-workers 1 --timeout 60000 --save-path "Traffic/MixLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Traffic.csv" --strategy-args '{"horizon": 336}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.5, "lpf": 19, "batch_size": 64, "lr": 0.03, "num_epochs": 30, "patience": 5, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 336, "norm": true}' --gpus 7 --num-workers 1 --timeout 60000 --save-path "Traffic/MixLinear" + +python ./scripts/run_benchmark.py --config-path "rolling_forecast_config.json" --data-name-list "Traffic.csv" --strategy-args '{"horizon": 720}' --model-name "mixlinear.MIXLINEAR" --model-hyper-params '{"alpha": 0.9, "lpf": 19, "batch_size": 64, "lr": 0.03, "num_epochs": 30, "patience": 5, "itr": 1, "period_len": 24, "seq_len": 720, "horizon": 720, "norm": true}' --gpus 7 --num-workers 1 --timeout 60000 --save-path "Traffic/MixLinear" \ No newline at end of file diff --git a/ts_benchmark/baselines/mixlinear/__init__.py b/ts_benchmark/baselines/mixlinear/__init__.py new file mode 100644 index 00000000..027fd1d9 --- /dev/null +++ b/ts_benchmark/baselines/mixlinear/__init__.py @@ -0,0 +1,5 @@ +__all__ = [ + "MIXLINEAR", +] + +from ts_benchmark.baselines.mixlinear.mixlinear import MIXLINEAR \ No newline at end of file diff --git a/ts_benchmark/baselines/mixlinear/layers/Embed.py b/ts_benchmark/baselines/mixlinear/layers/Embed.py new file mode 100644 index 00000000..7dad35f4 --- /dev/null +++ b/ts_benchmark/baselines/mixlinear/layers/Embed.py @@ -0,0 +1,207 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.utils import weight_norm +import math + + +class PositionalEmbedding(nn.Module): + def __init__(self, d_model, max_len=5000): + super(PositionalEmbedding, self).__init__() + # Compute the positional encodings once in log space. + pe = torch.zeros(max_len, d_model).float() + pe.require_grad = False + + position = torch.arange(0, max_len).float().unsqueeze(1) + div_term = ( + torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model) + ).exp() + + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + + pe = pe.unsqueeze(0) + self.register_buffer("pe", pe) + + def forward(self, x): + return self.pe[:, : x.size(1)] + + +class TokenEmbedding(nn.Module): + def __init__(self, c_in, d_model): + super(TokenEmbedding, self).__init__() + padding = 1 if torch.__version__ >= "1.5.0" else 2 + self.tokenConv = nn.Conv1d( + in_channels=c_in, + out_channels=d_model, + kernel_size=3, + padding=padding, + padding_mode="circular", + bias=False, + ) + for m in self.modules(): + if isinstance(m, nn.Conv1d): + nn.init.kaiming_normal_( + m.weight, mode="fan_in", nonlinearity="leaky_relu" + ) + + def forward(self, x): + x = self.tokenConv(x.permute(0, 2, 1)).transpose(1, 2) + return x + + +class FixedEmbedding(nn.Module): + def __init__(self, c_in, d_model): + super(FixedEmbedding, self).__init__() + + w = torch.zeros(c_in, d_model).float() + w.require_grad = False + + position = torch.arange(0, c_in).float().unsqueeze(1) + div_term = ( + torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model) + ).exp() + + w[:, 0::2] = torch.sin(position * div_term) + w[:, 1::2] = torch.cos(position * div_term) + + self.emb = nn.Embedding(c_in, d_model) + self.emb.weight = nn.Parameter(w, requires_grad=False) + + def forward(self, x): + return self.emb(x).detach() + + +class TemporalEmbedding(nn.Module): + def __init__(self, d_model, embed_type="fixed", freq="h"): + super(TemporalEmbedding, self).__init__() + + minute_size = 4 + hour_size = 24 + weekday_size = 7 + day_size = 32 + month_size = 13 + + Embed = FixedEmbedding if embed_type == "fixed" else nn.Embedding + if freq == "t": + self.minute_embed = Embed(minute_size, d_model) + self.hour_embed = Embed(hour_size, d_model) + self.weekday_embed = Embed(weekday_size, d_model) + self.day_embed = Embed(day_size, d_model) + self.month_embed = Embed(month_size, d_model) + + def forward(self, x): + x = x.long() + minute_x = ( + self.minute_embed(x[:, :, 4]) if hasattr(self, "minute_embed") else 0.0 + ) + hour_x = self.hour_embed(x[:, :, 3]) + weekday_x = self.weekday_embed(x[:, :, 2]) + day_x = self.day_embed(x[:, :, 1]) + month_x = self.month_embed(x[:, :, 0]) + + return hour_x + weekday_x + day_x + month_x + minute_x + + +class TimeFeatureEmbedding(nn.Module): + def __init__(self, d_model, embed_type="timeF", freq="h"): + super(TimeFeatureEmbedding, self).__init__() + + freq_map = {"h": 4, "t": 5, "s": 6, "m": 1, "a": 1, "w": 2, "d": 3, "b": 3} + d_inp = freq_map[freq] + self.embed = nn.Linear(d_inp, d_model, bias=False) + + def forward(self, x): + return self.embed(x) + + +class DataEmbedding(nn.Module): + def __init__(self, c_in, d_model, embed_type="fixed", freq="h", dropout=0.1): + super(DataEmbedding, self).__init__() + + self.value_embedding = TokenEmbedding(c_in=c_in, d_model=d_model) + self.position_embedding = PositionalEmbedding(d_model=d_model) + self.temporal_embedding = ( + TemporalEmbedding(d_model=d_model, embed_type=embed_type, freq=freq) + if embed_type != "timeF" + else TimeFeatureEmbedding(d_model=d_model, embed_type=embed_type, freq=freq) + ) + self.dropout = nn.Dropout(p=dropout) + + def forward(self, x, x_mark): + if x_mark is None: + x = self.value_embedding(x) + self.position_embedding(x) + else: + x = ( + self.value_embedding(x) + + self.temporal_embedding(x_mark) + + self.position_embedding(x) + ) + return self.dropout(x) + + +class DataEmbedding_inverted(nn.Module): + def __init__(self, c_in, d_model, embed_type="fixed", freq="h", dropout=0.1): + super(DataEmbedding_inverted, self).__init__() + self.value_embedding = nn.Linear(c_in, d_model) + self.dropout = nn.Dropout(p=dropout) + + def forward(self, x, x_mark): + x = x.permute(0, 2, 1) + # x: [Batch Variate Time] + if x_mark is None: + x = self.value_embedding(x) + else: + x = self.value_embedding(torch.cat([x, x_mark.permute(0, 2, 1)], 1)) + # x: [Batch Variate d_model] + return self.dropout(x) + + +class DataEmbedding_wo_pos(nn.Module): + def __init__(self, c_in, d_model, embed_type="fixed", freq="h", dropout=0.1): + super(DataEmbedding_wo_pos, self).__init__() + + self.value_embedding = TokenEmbedding(c_in=c_in, d_model=d_model) + self.position_embedding = PositionalEmbedding(d_model=d_model) + self.temporal_embedding = ( + TemporalEmbedding(d_model=d_model, embed_type=embed_type, freq=freq) + if embed_type != "timeF" + else TimeFeatureEmbedding(d_model=d_model, embed_type=embed_type, freq=freq) + ) + self.dropout = nn.Dropout(p=dropout) + + def forward(self, x, x_mark): + if x_mark is None: + x = self.value_embedding(x) + else: + x = self.value_embedding(x) + self.temporal_embedding(x_mark) + return self.dropout(x) + + +class PatchEmbedding(nn.Module): + def __init__(self, d_model, patch_len, stride, padding, dropout): + super(PatchEmbedding, self).__init__() + # Patching + self.patch_len = patch_len + self.stride = stride + self.padding_patch_layer = nn.ReplicationPad1d((0, padding)) + + # Backbone, Input encoding: projection of feature vectors onto a d-dim vector space + self.value_embedding = nn.Linear(patch_len, d_model, bias=False) + + # Positional embedding + self.position_embedding = PositionalEmbedding(d_model) + + # Residual dropout + self.dropout = nn.Dropout(dropout) + + def forward(self, x): + # do patching + n_vars = x.shape[1] + x = self.padding_patch_layer(x) + x = x.unfold(dimension=-1, size=self.patch_len, step=self.stride) + x = torch.reshape(x, (x.shape[0] * x.shape[1], x.shape[2], x.shape[3])) + # Input encoding + x = self.value_embedding(x) + self.position_embedding(x) + return self.dropout(x), n_vars + diff --git a/ts_benchmark/baselines/mixlinear/layers/__init__.py b/ts_benchmark/baselines/mixlinear/layers/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/ts_benchmark/baselines/mixlinear/layers/__init__.py @@ -0,0 +1 @@ + diff --git a/ts_benchmark/baselines/mixlinear/mixlinear.py b/ts_benchmark/baselines/mixlinear/mixlinear.py new file mode 100644 index 00000000..804e294b --- /dev/null +++ b/ts_benchmark/baselines/mixlinear/mixlinear.py @@ -0,0 +1,44 @@ +from ts_benchmark.baselines.deep_forecasting_model_base import DeepForecastingModelBase +from ts_benchmark.baselines.mixlinear.mixlinear_model import MixlinearModel + +MODEL_HYPER_PARAMS = { + "alpha": 0.5, + "lpf": 1, + "kernel": 24, + "freq": "h", + "embed": "learned", + "lradj": "type3", + "factor": 1, + "activation": "gelu", + "dropout": 0.05, + "batch_size": 32, + "lr": 0.0001, + "num_epochs": 100, + "num_workers": 10, + "loss": "MSE", + "itr": 2, + "distil": True, + "patience": 3, + "period_len": 24, +} + +class MIXLINEAR(DeepForecastingModelBase): + + def __init__(self, **kwargs): + super(MIXLINEAR, self).__init__(MODEL_HYPER_PARAMS, **kwargs) + + @property + def model_name(self): + return "MIXLINEAR" + + def _init_model(self): + return MixlinearModel(self.config) + + def _process(self, input, target, input_mark, target_mark): + """ + input: shape [batch, seq_len, enc_in] + output: shape [batch, pred_len, enc_in] + """ + output = self.model(input) + return {"output": output} + diff --git a/ts_benchmark/baselines/mixlinear/mixlinear_model.py b/ts_benchmark/baselines/mixlinear/mixlinear_model.py new file mode 100644 index 00000000..b4e7d765 --- /dev/null +++ b/ts_benchmark/baselines/mixlinear/mixlinear_model.py @@ -0,0 +1,85 @@ +import torch +import torch.nn as nn +import math +import torch.nn.functional as F +from ts_benchmark.baselines.mixlinear.layers.Embed import PositionalEmbedding + + +class MixlinearModel(nn.Module): + def __init__(self, configs): + super(MixlinearModel, self).__init__() + + # get parameters + self.seq_len = configs.seq_len + self.pred_len = configs.pred_len + self.enc_in = configs.enc_in + self.period_len = configs.period_len + + self.kernel = self.period_len + self.lpf = configs.lpf + self.alpha = configs.alpha + + self.seg_num_x = math.ceil(self.seq_len / self.period_len) + self.seg_num_y = math.ceil(self.pred_len / self.period_len) + + self.sqrt_seg_num_x = math.ceil(math.sqrt(self.seq_len / self.period_len)) + self.sqrt_seg_num_y = math.ceil(math.sqrt(self.pred_len / self.period_len)) + + # TLinear + self.TLinear1 = nn.Linear(self.sqrt_seg_num_x, self.sqrt_seg_num_y, bias=False) + self.TLinear2 = nn.Linear(self.sqrt_seg_num_x, self.sqrt_seg_num_y, bias=False) + + self.conv1d = nn.Conv1d(in_channels=1, out_channels=1, kernel_size=self.kernel + 1, + stride=1, padding=int(self.kernel / 2), padding_mode="zeros", bias=False) + + # FLinear + self.FLinear1 = nn.Linear(self.lpf, 2, bias=False).to(torch.cfloat) + self.FLinear2 = nn.Linear(2, self.seg_num_y, bias=False).to(torch.cfloat) + + def forward(self, x): + batch_size = x.shape[0] + # normalization and permute b,s,c -> b,c,s + seq_mean = torch.mean(x, dim=1).unsqueeze(1) + x = (x - seq_mean).permute(0, 2, 1) + # print(x.shape) + x = self.conv1d(x.reshape(-1, 1, self.seq_len)).reshape(-1, self.enc_in, self.seq_len) + x + # print(x.shape) + + # ->b,e,w,n + x = x.reshape(batch_size, self.enc_in, -1, self.period_len).permute(0, 1, 3, 2) + + # Time Domain + + + # x_o = torch.zeros(batch_size, self.enc_in, self.period_len, self.sqrt_seg_num_x ** 2).to(x.device) + # x_o[:, :, :, :x.shape[-1]] = x[:, :, :, :] + + x_o = F.pad(x, (0, self.sqrt_seg_num_x ** 2 - x.shape[-1], 0, 0, 0, 0)) + + x_o = x_o.reshape(batch_size, self.enc_in, self.period_len, self.sqrt_seg_num_x, self.sqrt_seg_num_x) + + x_o = self.TLinear1(x_o).permute(0, 1, 2, 4, 3) + x_t = self.TLinear2(x_o).permute(0, 1, 2, 4, 3) + + x_t = x_t.reshape(batch_size, self.enc_in, self.period_len, -1).permute(0, 1, 3, 2).reshape(batch_size, + self.enc_in, + -1).permute(0, 2, 1) + + # Frequency Domain + + x_fft = torch.fft.fft(x, dim=3)[:, :, :, :self.lpf] + # x_fft = x_fft.view(-1,self.lpf) + x_fft = self.FLinear1(x_fft) + x_fft = self.FLinear2(x_fft).reshape(batch_size, self.enc_in, self.period_len, -1) + + x_rfft = torch.fft.ifft(x_fft, dim=3).float() + x_f = x_rfft.permute(0, 1, 3, 2).reshape(batch_size, self.enc_in, -1).permute(0, 2, 1) + + #print("shape", x_t.shape, x_f.shape) + + # Mix + #print("shape", x_t[:, :self.pred_len, :].shape, x_f[:, :self.pred_len, :].shape) + + x = x_t[:, :self.pred_len, :] * self.alpha + seq_mean + x_f[:, :self.pred_len, :] * (1 - self.alpha) + + return x[:, :self.pred_len, :] \ No newline at end of file