From 0bb5e883de867fb7bf6396e050dc0c05b8e399f0 Mon Sep 17 00:00:00 2001 From: fareise Date: Wed, 10 Jul 2019 11:21:16 +0800 Subject: [PATCH 01/39] Create mv_lstm.json --- model_zoo/demo/mv_lstm.json | 146 ++++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 model_zoo/demo/mv_lstm.json diff --git a/model_zoo/demo/mv_lstm.json b/model_zoo/demo/mv_lstm.json new file mode 100644 index 0000000..f308592 --- /dev/null +++ b/model_zoo/demo/mv_lstm.json @@ -0,0 +1,146 @@ +{ + "license": "Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.", + "tool_version": "1.1.0", + "model_description": "This model is used for question answer matching task, and it achieved auc: 0.7673 in WikiQACorpus test set", + "inputs": { + "use_cache": true, + "dataset_type": "classification", + "data_paths": { + "train_data_path": "./dataset/WikiQACorpus/WikiQA-train.tsv", + "valid_data_path": "./dataset/WikiQACorpus/WikiQA-dev.tsv", + "test_data_path": "./dataset/WikiQACorpus/WikiQA-test.tsv" + }, + "file_with_col_header": true, + "add_start_end_for_seq": true, + "file_header": { + "question_id": 0, + "question_text": 1, + "document_id": 2, + "document_title": 3, + "passage_id": 4, + "passage_text": 5, + "label": 6 + }, + "model_inputs": { + "question": ["question_text"], + "passage": ["passage_text"] + }, + "target": ["label"] + }, + "outputs":{ + "save_base_dir": "./models/wikiqa_bilstm/", + "model_name": "model.nb", + "train_log_name": "train.log", + "test_log_name": "test.log", + "predict_log_name": "predict.log", + "predict_fields": ["prediction"], + "predict_output_name": "predict.tsv", + "cache_dir": ".cache.wikiqa/" + }, + "training_params": { + "vocabulary": { + "min_word_frequency": 1 + }, + "optimizer": { + "name": "Adam", + "params": { + } + }, + "lr_decay": 0.95, + "minimum_lr": 0.0001, + "epoch_start_lr_decay": 3, + "use_gpu": true, + "batch_size": 8, + "batch_num_to_show_results": 100, + "max_epoch": 2, + "valid_times_per_epoch": 5, + "max_lengths":{ + "question": 30, + "passage": 100 + } + }, + "architecture":[ + { + "layer": "Embedding", + "conf": { + "word": { + "cols": ["question_text", "passage_text"], + "dim": 300, + "fix_weight": true + } + } + }, + { + "layer_id": "question_dropout", + "layer": "Dropout", + "conf": { + "dropout": 0.2 + }, + "inputs": ["question"] + }, + { + "layer_id": "passage_dropout", + "layer": "Dropout", + "conf": { + "dropout": 0.2 + }, + "inputs": ["passage"] + }, + { + "layer_id": "question_1", + "layer": "BiLSTM", + "conf": { + "hidden_dim": 128, + "dropout": 0.2, + "num_layers": 2 + }, + "inputs": ["question_dropout"] + }, + { + "layer_id": "passage_1", + "layer": "question_1", + "inputs": ["passage_dropout"] + }, + { + "layer_id": "comb_qp", + "layer": "Combination2D", + "conf": { + "operations": ["cosine"] + }, + "inputs": ["question_1", "passage_1"] + }, + { + "layer_id": "pooltest", + "layer": "PoolingKmax2D", + "conf": { + "pool_type": "max", + "k": 50 + }, + "inputs": ["comb_qp"] + }, + { + "output_layer_flag": true, + "layer_id": "output", + "layer": "Linear", + "conf": { + "hidden_dim": [128,2], + "activation": "PReLU", + "last_hidden_activation": false + }, + "inputs": ["pooltest"] + } + ], + "loss": { + "losses": [ + { + "type": "CrossEntropyLoss", + "conf": { + "weight": [0.1,0.9], + "size_average": true + }, + "inputs": ["output","label"] + } + ] + }, + "metrics": ["auc","accuracy"] +} From 567f75544568eb749a6b88f263cb4a3b4575458d Mon Sep 17 00:00:00 2001 From: fareise Date: Wed, 10 Jul 2019 11:22:17 +0800 Subject: [PATCH 02/39] Create Combination2D --- block_zoo/op/Combination2D | 161 +++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 block_zoo/op/Combination2D diff --git a/block_zoo/op/Combination2D b/block_zoo/op/Combination2D new file mode 100644 index 0000000..4a6c120 --- /dev/null +++ b/block_zoo/op/Combination2D @@ -0,0 +1,161 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT license. + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable + +import numpy as np +import logging + +from block_zoo.BaseLayer import BaseConf +from utils.DocInherit import DocInherit +from utils.exceptions import ConfigurationError +import copy + +class Combination2DConf(BaseConf): + """ Configuration for combination layer + + Args: + operations (list): a subset of ["origin", "difference", "dot_multiply"]. + "origin" means to keep the original representations;\n + "difference" means abs(sequence1 - sequence2); + "dot_multiply" means element-wised product; + + """ + def __init__(self, **kwargs): + super(Combination2DConf, self).__init__(**kwargs) + if "bilinear" in self.operations: + #bilinear_trans = nn.Linear(self.output_dim, self.output_dim) + + #ones=torch.Tensor(np.ones([2,2,3,3])) + #w.weight=torch.nn.Parameter(ones) + + #w = torch.empty(3, 5) + #nn.init.uniform_(w) + + weight_bilinear = Parameter(torch.Tensor(self.output_dim, self.output_dim)) + + + @DocInherit + def default(self): + # supported operations: "origin", "difference", "dot_multiply" + self.operations = ["origin", "difference", "dot_multiply", "cosine", "bilinear", "tensor"] + + @DocInherit + def declare(self): + self.num_of_inputs = -1 + self.input_ranks = [-1] + + @DocInherit + def inference(self): + #说明上一层的dim就有问题！ + self.output_dim = [self.input_dims[0][0], self.input_dims[0][1], self.input_dims[0][1]] + #print("*&*&*") + #print(self.output_dim) #[[-1, -1, 256], [-1, -1, 256], [-1, -1, 256]] + #copy.deepcopy(self.input_dims[0]) + ''' + self.output_dim[-1] = 0 + if "origin" in self.operations: + self.output_dim[-1] += sum([input_dim[-1] for input_dim in self.input_dims]) + if "difference" in self.operations: + self.output_dim[-1] += int(np.mean([input_dim[-1] for input_dim in self.input_dims])) # difference operation requires dimension of all the inputs should be equal + if "dot_multiply" in self.operations: + self.output_dim[-1] += int(np.mean([input_dim[-1] for input_dim in self.input_dims])) # dot_multiply operation requires dimension of all the inputs should be equal + if "cosine" in self.operations: + self.output_dim[-1] + ''' + super(Combination2DConf, self).inference() + + @DocInherit + def verify(self): + super(Combination2DConf, self).verify() + + # to check if the ranks of all the inputs are equal + rank_equal_flag = True + for i in range(len(self.input_ranks)): + if self.input_ranks[i] != self.input_ranks[0]: + rank_equal_flag = False + break + if rank_equal_flag == False: + raise ConfigurationError("For layer Combination, the ranks of each inputs should be consistent!") + + if "difference" in self.operations: + assert len(self.input_dims) == 2, "Difference operation requires that there should be two inputs" + + if "difference" in self.operations or "dot_multiply" in self.operations: + input_dims = list(self.input_dims) + dim_equal_flag = True + for i in range(len(input_dims)): + if input_dims[i] != input_dims[0]: + dim_equal_flag = False + break + if dim_equal_flag == False: + raise Exception("Difference and dot multiply require that the input dimensions should be the same") + + +class Combination2D(nn.Module): + """ Combination layer to merge the representation of two sequence + + Args: + layer_conf (CombinationConf): configuration of a layer + """ + def __init__(self, layer_conf): + super(Combination2D, self).__init__() + self.layer_conf = layer_conf + #print("**************") + #print(layer_conf.input_dims) #[[-1, -1, 256], [-1, -1, 256]] + #self.linear_bi = nn.Linear(layer_conf.input_dims[0][2], layer_conf.input_dims[0][2]) + #self.linear_ten1 = nn.Linear(layer_conf.input_dims[0][2], layer_conf.input_dims[0][2]) + #self.linear_ten2 = nn.Linear(layer_conf.input_dims[0][1], layer_conf.input_dims[0][2]) + + logging.warning("The length Combination layer returns is the length of first input") + + def forward(self, *args): + """ process inputs + + Args: + args (list): [string, string_len, string2, string2_len, ...] + e.g. string (Variable): [batch_size, dim], string_len (ndarray): [batch_size] + + Returns: + Variable: [batch_size, output_dim], None + + """ + ''' + 输入两个string都是[batch_size, seq_len, hidden_size]的 + 生成的每个矩阵都是[batch_size, seq_len, seq_len]的 + 最后输出的应该是[batch_size, num_map, seq_len, seq_len]的 + ''' + #print("&*&*&*") + #print(args[0].size()) #[8, 15, 256] 每次长度都不一样 + #print(args[2].size()) #[8, 34, 256] + result = [] + if "cosine" in self.layer_conf.operations: + string1 = args[0] + string2 = args[2] + result_multiply = torch.matmul(string1, string2.transpose(1,2)) + result.append(torch.unsqueeze(result_multiply, 1)) + #result.append(torch.unsqueeze(result_multiply, 1)) + + if "bilinear" in self.layer_conf.operations: + string1 = args[0] + string2 = args[2] + string1 = linear_bi(string1) + result_multiply = torch.matmul(string1, string2.transpose(1,2)) + result.append(torch.unsqueeze(result_multiply, 1)) + + if "tensor" in self.layer_conf.operations: + string1 = args[0] + string2 = args[2] + c = 10 + for i in range(0, c): + string1 = linear_ten_1(string1) + result_multiply = torch.matmul(string1, string2.transpose(1,2)) + string_concat = torch.cat((string1, string2), 1) + + return torch.cat(result, 1), args[1] + + #return torch.cat(result, 1), args[1] + From 4a4b12ca37fcf816c06d3a24dfaf399007a823a1 Mon Sep 17 00:00:00 2001 From: fareise Date: Wed, 10 Jul 2019 11:24:49 +0800 Subject: [PATCH 03/39] Create PoolingKmax2D --- block_zoo/PoolingKmax2D | 1 + 1 file changed, 1 insertion(+) create mode 100644 block_zoo/PoolingKmax2D diff --git a/block_zoo/PoolingKmax2D b/block_zoo/PoolingKmax2D new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/block_zoo/PoolingKmax2D @@ -0,0 +1 @@ + From 50f8ca055712e02c416573db267f8673027428d3 Mon Sep 17 00:00:00 2001 From: fareise Date: Wed, 10 Jul 2019 11:25:11 +0800 Subject: [PATCH 04/39] Rename PoolingKmax2D to PoolingKmax2D.py --- block_zoo/{PoolingKmax2D => PoolingKmax2D.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename block_zoo/{PoolingKmax2D => PoolingKmax2D.py} (100%) diff --git a/block_zoo/PoolingKmax2D b/block_zoo/PoolingKmax2D.py similarity index 100% rename from block_zoo/PoolingKmax2D rename to block_zoo/PoolingKmax2D.py From 91bbe51dafa38206b1cf42608d06b4d412c353ff Mon Sep 17 00:00:00 2001 From: fareise Date: Wed, 10 Jul 2019 11:25:38 +0800 Subject: [PATCH 05/39] Rename Combination2D to Combination2D.py --- block_zoo/op/{Combination2D => Combination2D.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename block_zoo/op/{Combination2D => Combination2D.py} (100%) diff --git a/block_zoo/op/Combination2D b/block_zoo/op/Combination2D.py similarity index 100% rename from block_zoo/op/Combination2D rename to block_zoo/op/Combination2D.py From 9f250b4963d43f1186fdd7632d93d6866940d8f2 Mon Sep 17 00:00:00 2001 From: fareise Date: Wed, 10 Jul 2019 22:57:31 +0800 Subject: [PATCH 06/39] Update mv_lstm.json From 17f871403c1c23e7fe3b77a146ee66a4ed54a65f Mon Sep 17 00:00:00 2001 From: fareise Date: Wed, 10 Jul 2019 22:58:08 +0800 Subject: [PATCH 07/39] Update PoolingKmax2D.py --- block_zoo/PoolingKmax2D.py | 138 +++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/block_zoo/PoolingKmax2D.py b/block_zoo/PoolingKmax2D.py index 8b13789..0685f62 100644 --- a/block_zoo/PoolingKmax2D.py +++ b/block_zoo/PoolingKmax2D.py @@ -1 +1,139 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT license. + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import numpy as np + +from block_zoo.BaseLayer import BaseLayer, BaseConf +from utils.DocInherit import DocInherit + + +class PoolingKmax2DConf(BaseConf): + """ + + Args: + pool_type (str): 'max' or 'mean', default is 'max'. + stride (int): which axis to conduct pooling, default is 1. + padding (int): implicit zero paddings on both sides of the input. Can be a single number or a tuple (padH, padW). Default: 0 + window_size (int): the size of the pooling + activation (string): activation functions, e.g. ReLU + + """ + def __init__(self, **kwargs): + super(PoolingKmax2DConf, self).__init__(**kwargs) + + @DocInherit + def default(self): + self.pool_type = 'max' # Supported: ['max', mean'] + self.stride = 1 + self.padding = 0 + self.window_size = 3 + self.k = 50 + + @DocInherit + def declare(self): + self.num_of_inputs = 1 + #self.input_ranks = [4] + self.input_ranks = [3] + + + def check_size(self, value, attr): + res = value + if isinstance(value,int): + res = [value, value] + elif (isinstance(self.window_size, tuple) or isinstance(self.window_size, list)) and len(value)==2: + res = list(value) + else: + raise AttributeError("The Atrribute `%s' should be given an integer or a list/tuple with length of 2, instead of %s." %(attr,str(value))) + return res + + @DocInherit + def inference(self): + ''' + self.window_size = self.check_size(self.window_size, "window_size") + self.stride = self.check_size(self.stride, "stride") + self.padding = self.check_size(self.padding, "padding") + + self.output_dim = [self.input_dims[0][0]] + if self.input_dims[0][1] != -1: + self.output_dim.append((self.input_dims[0][1] + 2 * self.padding[0] - self.window_size[0]) // self.stride[0] + 1) + else: + self.output_dim.append(-1) + if self.input_dims[0][2] != -1: + self.output_dim.append((self.input_dims[0][2] + 2 * self.padding[1] - self.window_size[1]) // self.stride[1] + 1) + else: + self.output_dim.append(-1) + # print("pool",self.output_dim) + self.input_channel_num = self.input_dims[0][-1] + + self.output_dim.append(self.input_dims[0][-1]) + ''' + self.output_dim = [self.input_dims[0][0], -self.input_dims[0][1] * self.k] #?怎么设定维度，input_dims都是-1 + #print("&^&^&^") + #print(self.output_dim) + + # DON'T MODIFY THIS + self.output_rank = len(self.output_dim) + + @DocInherit + def verify(self): + super(PoolingKmax2DConf, self).verify() + + necessary_attrs_for_user = ['pool_type'] + for attr in necessary_attrs_for_user: + self.add_attr_exist_assertion_for_user(attr) + + self.add_attr_value_assertion('pool_type', ['max', 'mean']) + + #assert all([input_rank >= 4 for input_rank in self.input_ranks]), "Cannot apply a pooling layer on a tensor of which the rank is less than 4. Usually, a tensor whose rank is at least 4, e.g. [batch size, length, width, feature]" + + assert self.output_dim[-1] != -1, "The shape of input is %s , and the input channel number of pooling should not be -1." % (str(self.input_dims[0])) + +class PoolingKmax2D(BaseLayer): + """ Pooling layer + + Args: + layer_conf (PoolingConf): configuration of a layer + """ + def __init__(self, layer_conf): + super(PoolingKmax2D, self).__init__(layer_conf) + self.pool = None + if layer_conf.pool_type == "max": + self.pool = nn.MaxPool2d(kernel_size=layer_conf.window_size,stride=layer_conf.stride,padding=layer_conf.padding) + elif layer_conf.pool_type == "mean": + self.pool = nn.AvgPool2d(kernel_size=layer_conf.window_size,stride=layer_conf.stride,padding=layer_conf.padding) + self.k = layer_conf.k + + def forward(self, string, string_len=None): + """ process inputs + + Args: + string (Tensor): tensor with shape: [batch_size, length, width, feature_dim] + string_len (Tensor): [batch_size], default is None. + + Returns: + Tensor: Pooling result of string + + """ + #print("&&&&&&") + #print(string.size()) #[8, 11, 89] + + + #string = string.permute([0,3,1,2]).contiguous() + + #string = self.pool(string) + + #string = string.permute([0,2,3,1]).contiguous() + string = string.view(string.size()[0], string.size()[1], -1) + index = string.topk(self.k, dim=-1)[1].sort(dim=-1)[0] + string = string.gather(-1, index) + string = string.view(string.size()[0], -1) + #print("*****") + #print(string.size()) + + return string, string_len + From a6f3da1d7a914c82af8af24730dd814fff8ce4f5 Mon Sep 17 00:00:00 2001 From: fareise Date: Tue, 16 Jul 2019 13:50:08 +0800 Subject: [PATCH 08/39] Update PoolingKmax2D.py --- block_zoo/PoolingKmax2D.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/block_zoo/PoolingKmax2D.py b/block_zoo/PoolingKmax2D.py index 0685f62..f1009cd 100644 --- a/block_zoo/PoolingKmax2D.py +++ b/block_zoo/PoolingKmax2D.py @@ -72,8 +72,7 @@ def inference(self): self.output_dim.append(self.input_dims[0][-1]) ''' self.output_dim = [self.input_dims[0][0], -self.input_dims[0][1] * self.k] #?怎么设定维度，input_dims都是-1 - #print("&^&^&^") - #print(self.output_dim) + # DON'T MODIFY THIS self.output_rank = len(self.output_dim) @@ -118,21 +117,10 @@ def forward(self, string, string_len=None): Tensor: Pooling result of string """ - #print("&&&&&&") - #print(string.size()) #[8, 11, 89] - - - #string = string.permute([0,3,1,2]).contiguous() - - #string = self.pool(string) - - #string = string.permute([0,2,3,1]).contiguous() string = string.view(string.size()[0], string.size()[1], -1) index = string.topk(self.k, dim=-1)[1].sort(dim=-1)[0] string = string.gather(-1, index) string = string.view(string.size()[0], -1) - #print("*****") - #print(string.size()) return string, string_len From 7ea6dd24c34aba87976f36f90708506676ef7dae Mon Sep 17 00:00:00 2001 From: fareise Date: Tue, 16 Jul 2019 13:51:11 +0800 Subject: [PATCH 09/39] Update Combination2D.py --- block_zoo/op/Combination2D.py | 33 +++------------------------------ 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/block_zoo/op/Combination2D.py b/block_zoo/op/Combination2D.py index 4a6c120..904498f 100644 --- a/block_zoo/op/Combination2D.py +++ b/block_zoo/op/Combination2D.py @@ -40,7 +40,7 @@ def __init__(self, **kwargs): @DocInherit def default(self): - # supported operations: "origin", "difference", "dot_multiply" + # supported operations: "origin", "difference", "dot_multiply", "cosine" self.operations = ["origin", "difference", "dot_multiply", "cosine", "bilinear", "tensor"] @DocInherit @@ -50,22 +50,8 @@ def declare(self): @DocInherit def inference(self): - #说明上一层的dim就有问题！ self.output_dim = [self.input_dims[0][0], self.input_dims[0][1], self.input_dims[0][1]] - #print("*&*&*") - #print(self.output_dim) #[[-1, -1, 256], [-1, -1, 256], [-1, -1, 256]] - #copy.deepcopy(self.input_dims[0]) - ''' - self.output_dim[-1] = 0 - if "origin" in self.operations: - self.output_dim[-1] += sum([input_dim[-1] for input_dim in self.input_dims]) - if "difference" in self.operations: - self.output_dim[-1] += int(np.mean([input_dim[-1] for input_dim in self.input_dims])) # difference operation requires dimension of all the inputs should be equal - if "dot_multiply" in self.operations: - self.output_dim[-1] += int(np.mean([input_dim[-1] for input_dim in self.input_dims])) # dot_multiply operation requires dimension of all the inputs should be equal - if "cosine" in self.operations: - self.output_dim[-1] - ''' + super(Combination2DConf, self).inference() @DocInherit @@ -104,11 +90,6 @@ class Combination2D(nn.Module): def __init__(self, layer_conf): super(Combination2D, self).__init__() self.layer_conf = layer_conf - #print("**************") - #print(layer_conf.input_dims) #[[-1, -1, 256], [-1, -1, 256]] - #self.linear_bi = nn.Linear(layer_conf.input_dims[0][2], layer_conf.input_dims[0][2]) - #self.linear_ten1 = nn.Linear(layer_conf.input_dims[0][2], layer_conf.input_dims[0][2]) - #self.linear_ten2 = nn.Linear(layer_conf.input_dims[0][1], layer_conf.input_dims[0][2]) logging.warning("The length Combination layer returns is the length of first input") @@ -123,14 +104,7 @@ def forward(self, *args): Variable: [batch_size, output_dim], None """ - ''' - 输入两个string都是[batch_size, seq_len, hidden_size]的 - 生成的每个矩阵都是[batch_size, seq_len, seq_len]的 - 最后输出的应该是[batch_size, num_map, seq_len, seq_len]的 - ''' - #print("&*&*&*") - #print(args[0].size()) #[8, 15, 256] 每次长度都不一样 - #print(args[2].size()) #[8, 34, 256] + result = [] if "cosine" in self.layer_conf.operations: string1 = args[0] @@ -157,5 +131,4 @@ def forward(self, *args): return torch.cat(result, 1), args[1] - #return torch.cat(result, 1), args[1] From 1ab792473e9c9a68bf3bf7f317505bf516116007 Mon Sep 17 00:00:00 2001 From: fareise Date: Tue, 16 Jul 2019 15:18:41 +0800 Subject: [PATCH 10/39] Update PoolingKmax2D.py --- block_zoo/PoolingKmax2D.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/block_zoo/PoolingKmax2D.py b/block_zoo/PoolingKmax2D.py index f1009cd..ab139a1 100644 --- a/block_zoo/PoolingKmax2D.py +++ b/block_zoo/PoolingKmax2D.py @@ -15,7 +15,7 @@ class PoolingKmax2DConf(BaseConf): """ Args: - pool_type (str): 'max' or 'mean', default is 'max'. + pool_type (str): 'max' . stride (int): which axis to conduct pooling, default is 1. padding (int): implicit zero paddings on both sides of the input. Can be a single number or a tuple (padH, padW). Default: 0 window_size (int): the size of the pooling @@ -27,7 +27,7 @@ def __init__(self, **kwargs): @DocInherit def default(self): - self.pool_type = 'max' # Supported: ['max', mean'] + self.pool_type = 'max' # Supported: ['max'] self.stride = 1 self.padding = 0 self.window_size = 3 @@ -85,7 +85,7 @@ def verify(self): for attr in necessary_attrs_for_user: self.add_attr_exist_assertion_for_user(attr) - self.add_attr_value_assertion('pool_type', ['max', 'mean']) + self.add_attr_value_assertion('pool_type', ['max']) #assert all([input_rank >= 4 for input_rank in self.input_ranks]), "Cannot apply a pooling layer on a tensor of which the rank is less than 4. Usually, a tensor whose rank is at least 4, e.g. [batch size, length, width, feature]" @@ -102,8 +102,6 @@ def __init__(self, layer_conf): self.pool = None if layer_conf.pool_type == "max": self.pool = nn.MaxPool2d(kernel_size=layer_conf.window_size,stride=layer_conf.stride,padding=layer_conf.padding) - elif layer_conf.pool_type == "mean": - self.pool = nn.AvgPool2d(kernel_size=layer_conf.window_size,stride=layer_conf.stride,padding=layer_conf.padding) self.k = layer_conf.k def forward(self, string, string_len=None): From 3d95f7332e9ffd48eaa4a7f1e32f47239df984ce Mon Sep 17 00:00:00 2001 From: fareise Date: Tue, 16 Jul 2019 15:21:21 +0800 Subject: [PATCH 11/39] Update PoolingKmax2D.py --- block_zoo/PoolingKmax2D.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/block_zoo/PoolingKmax2D.py b/block_zoo/PoolingKmax2D.py index ab139a1..ad2663a 100644 --- a/block_zoo/PoolingKmax2D.py +++ b/block_zoo/PoolingKmax2D.py @@ -99,9 +99,6 @@ class PoolingKmax2D(BaseLayer): """ def __init__(self, layer_conf): super(PoolingKmax2D, self).__init__(layer_conf) - self.pool = None - if layer_conf.pool_type == "max": - self.pool = nn.MaxPool2d(kernel_size=layer_conf.window_size,stride=layer_conf.stride,padding=layer_conf.padding) self.k = layer_conf.k def forward(self, string, string_len=None): From 98d8b9968df0bdf6397506310e1c92f0d1124d4c Mon Sep 17 00:00:00 2001 From: fareise Date: Tue, 16 Jul 2019 15:48:08 +0800 Subject: [PATCH 12/39] Update PoolingKmax2D.py --- block_zoo/PoolingKmax2D.py | 1 + 1 file changed, 1 insertion(+) diff --git a/block_zoo/PoolingKmax2D.py b/block_zoo/PoolingKmax2D.py index ad2663a..2aaf32b 100644 --- a/block_zoo/PoolingKmax2D.py +++ b/block_zoo/PoolingKmax2D.py @@ -89,6 +89,7 @@ def verify(self): #assert all([input_rank >= 4 for input_rank in self.input_ranks]), "Cannot apply a pooling layer on a tensor of which the rank is less than 4. Usually, a tensor whose rank is at least 4, e.g. [batch size, length, width, feature]" + assert all([input_rank == 4 for input_rank in self.input_ranks]), "can only apply on a tensor which the rank is 4" assert self.output_dim[-1] != -1, "The shape of input is %s , and the input channel number of pooling should not be -1." % (str(self.input_dims[0])) class PoolingKmax2D(BaseLayer): From 96486309f3c4c61f7c7e02e35a468dd48b3db73f Mon Sep 17 00:00:00 2001 From: fareise Date: Wed, 17 Jul 2019 16:53:39 +0800 Subject: [PATCH 13/39] Update Combination2D.py --- block_zoo/op/Combination2D.py | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/block_zoo/op/Combination2D.py b/block_zoo/op/Combination2D.py index 904498f..bbd258e 100644 --- a/block_zoo/op/Combination2D.py +++ b/block_zoo/op/Combination2D.py @@ -18,30 +18,19 @@ class Combination2DConf(BaseConf): """ Configuration for combination layer Args: - operations (list): a subset of ["origin", "difference", "dot_multiply"]. - "origin" means to keep the original representations;\n - "difference" means abs(sequence1 - sequence2); - "dot_multiply" means element-wised product; + operations (list): a subset of ["cosine", "bilinear"]. """ def __init__(self, **kwargs): super(Combination2DConf, self).__init__(**kwargs) if "bilinear" in self.operations: - #bilinear_trans = nn.Linear(self.output_dim, self.output_dim) - - #ones=torch.Tensor(np.ones([2,2,3,3])) - #w.weight=torch.nn.Parameter(ones) - - #w = torch.empty(3, 5) - #nn.init.uniform_(w) weight_bilinear = Parameter(torch.Tensor(self.output_dim, self.output_dim)) @DocInherit def default(self): - # supported operations: "origin", "difference", "dot_multiply", "cosine" - self.operations = ["origin", "difference", "dot_multiply", "cosine", "bilinear", "tensor"] + self.operations = ["cosine", "bilinear"] @DocInherit def declare(self): @@ -120,15 +109,6 @@ def forward(self, *args): result_multiply = torch.matmul(string1, string2.transpose(1,2)) result.append(torch.unsqueeze(result_multiply, 1)) - if "tensor" in self.layer_conf.operations: - string1 = args[0] - string2 = args[2] - c = 10 - for i in range(0, c): - string1 = linear_ten_1(string1) - result_multiply = torch.matmul(string1, string2.transpose(1,2)) - string_concat = torch.cat((string1, string2), 1) - return torch.cat(result, 1), args[1] From 6df09a96a6d871b89ef7e588cd9d1526f03a863a Mon Sep 17 00:00:00 2001 From: fareise Date: Wed, 17 Jul 2019 16:54:07 +0800 Subject: [PATCH 14/39] Update Combination2D.py --- block_zoo/op/Combination2D.py | 1 - 1 file changed, 1 deletion(-) diff --git a/block_zoo/op/Combination2D.py b/block_zoo/op/Combination2D.py index bbd258e..e8e28e3 100644 --- a/block_zoo/op/Combination2D.py +++ b/block_zoo/op/Combination2D.py @@ -100,7 +100,6 @@ def forward(self, *args): string2 = args[2] result_multiply = torch.matmul(string1, string2.transpose(1,2)) result.append(torch.unsqueeze(result_multiply, 1)) - #result.append(torch.unsqueeze(result_multiply, 1)) if "bilinear" in self.layer_conf.operations: string1 = args[0] From 862964ff9dc9fa280c5f9820bdd7db85575da82f Mon Sep 17 00:00:00 2001 From: fareise Date: Wed, 17 Jul 2019 16:55:32 +0800 Subject: [PATCH 15/39] Update PoolingKmax2D.py --- block_zoo/PoolingKmax2D.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/block_zoo/PoolingKmax2D.py b/block_zoo/PoolingKmax2D.py index 2aaf32b..4831fff 100644 --- a/block_zoo/PoolingKmax2D.py +++ b/block_zoo/PoolingKmax2D.py @@ -28,15 +28,11 @@ def __init__(self, **kwargs): @DocInherit def default(self): self.pool_type = 'max' # Supported: ['max'] - self.stride = 1 - self.padding = 0 - self.window_size = 3 self.k = 50 @DocInherit def declare(self): self.num_of_inputs = 1 - #self.input_ranks = [4] self.input_ranks = [3] @@ -52,25 +48,6 @@ def check_size(self, value, attr): @DocInherit def inference(self): - ''' - self.window_size = self.check_size(self.window_size, "window_size") - self.stride = self.check_size(self.stride, "stride") - self.padding = self.check_size(self.padding, "padding") - - self.output_dim = [self.input_dims[0][0]] - if self.input_dims[0][1] != -1: - self.output_dim.append((self.input_dims[0][1] + 2 * self.padding[0] - self.window_size[0]) // self.stride[0] + 1) - else: - self.output_dim.append(-1) - if self.input_dims[0][2] != -1: - self.output_dim.append((self.input_dims[0][2] + 2 * self.padding[1] - self.window_size[1]) // self.stride[1] + 1) - else: - self.output_dim.append(-1) - # print("pool",self.output_dim) - self.input_channel_num = self.input_dims[0][-1] - - self.output_dim.append(self.input_dims[0][-1]) - ''' self.output_dim = [self.input_dims[0][0], -self.input_dims[0][1] * self.k] #?怎么设定维度，input_dims都是-1 @@ -87,7 +64,6 @@ def verify(self): self.add_attr_value_assertion('pool_type', ['max']) - #assert all([input_rank >= 4 for input_rank in self.input_ranks]), "Cannot apply a pooling layer on a tensor of which the rank is less than 4. Usually, a tensor whose rank is at least 4, e.g. [batch size, length, width, feature]" assert all([input_rank == 4 for input_rank in self.input_ranks]), "can only apply on a tensor which the rank is 4" assert self.output_dim[-1] != -1, "The shape of input is %s , and the input channel number of pooling should not be -1." % (str(self.input_dims[0])) From 1a1ddde66aa9082f8f4ef6f220560aaa4c83684b Mon Sep 17 00:00:00 2001 From: fareise Date: Sun, 21 Jul 2019 14:46:44 +0800 Subject: [PATCH 16/39] Update PoolingKmax2D.py --- block_zoo/PoolingKmax2D.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/block_zoo/PoolingKmax2D.py b/block_zoo/PoolingKmax2D.py index 4831fff..1cd1263 100644 --- a/block_zoo/PoolingKmax2D.py +++ b/block_zoo/PoolingKmax2D.py @@ -16,11 +16,7 @@ class PoolingKmax2DConf(BaseConf): Args: pool_type (str): 'max' . - stride (int): which axis to conduct pooling, default is 1. - padding (int): implicit zero paddings on both sides of the input. Can be a single number or a tuple (padH, padW). Default: 0 - window_size (int): the size of the pooling - activation (string): activation functions, e.g. ReLU - + k: how many elements does pooling contain """ def __init__(self, **kwargs): super(PoolingKmax2DConf, self).__init__(**kwargs) @@ -48,7 +44,7 @@ def check_size(self, value, attr): @DocInherit def inference(self): - self.output_dim = [self.input_dims[0][0], -self.input_dims[0][1] * self.k] #?怎么设定维度，input_dims都是-1 + self.output_dim = [self.input_dims[0][0], -self.input_dims[0][1] * self.k] # DON'T MODIFY THIS From 51b91279086162cd0efb8906efdadaf700644a44 Mon Sep 17 00:00:00 2001 From: fareise Date: Sun, 21 Jul 2019 14:48:23 +0800 Subject: [PATCH 17/39] Update Combination2D.py --- block_zoo/op/Combination2D.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/block_zoo/op/Combination2D.py b/block_zoo/op/Combination2D.py index e8e28e3..ff29f52 100644 --- a/block_zoo/op/Combination2D.py +++ b/block_zoo/op/Combination2D.py @@ -56,25 +56,12 @@ def verify(self): if rank_equal_flag == False: raise ConfigurationError("For layer Combination, the ranks of each inputs should be consistent!") - if "difference" in self.operations: - assert len(self.input_dims) == 2, "Difference operation requires that there should be two inputs" - - if "difference" in self.operations or "dot_multiply" in self.operations: - input_dims = list(self.input_dims) - dim_equal_flag = True - for i in range(len(input_dims)): - if input_dims[i] != input_dims[0]: - dim_equal_flag = False - break - if dim_equal_flag == False: - raise Exception("Difference and dot multiply require that the input dimensions should be the same") - - + class Combination2D(nn.Module): """ Combination layer to merge the representation of two sequence Args: - layer_conf (CombinationConf): configuration of a layer + layer_conf (Combination2DConf): configuration of a layer """ def __init__(self, layer_conf): super(Combination2D, self).__init__() From 232b7614dd49281436385c44e703d254f7d0443c Mon Sep 17 00:00:00 2001 From: fareise Date: Sun, 21 Jul 2019 16:16:36 +0800 Subject: [PATCH 18/39] Update PoolingKmax2D.py --- block_zoo/PoolingKmax2D.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/block_zoo/PoolingKmax2D.py b/block_zoo/PoolingKmax2D.py index 1cd1263..e4b3924 100644 --- a/block_zoo/PoolingKmax2D.py +++ b/block_zoo/PoolingKmax2D.py @@ -15,8 +15,9 @@ class PoolingKmax2DConf(BaseConf): """ Args: - pool_type (str): 'max' . - k: how many elements does pooling contain + pool_type (str): 'max', default is 'max'. + k (int): how many element to reserve. + """ def __init__(self, **kwargs): super(PoolingKmax2DConf, self).__init__(**kwargs) @@ -29,46 +30,34 @@ def default(self): @DocInherit def declare(self): self.num_of_inputs = 1 - self.input_ranks = [3] + self.input_ranks = [4] def check_size(self, value, attr): res = value - if isinstance(value,int): - res = [value, value] - elif (isinstance(self.window_size, tuple) or isinstance(self.window_size, list)) and len(value)==2: - res = list(value) - else: - raise AttributeError("The Atrribute `%s' should be given an integer or a list/tuple with length of 2, instead of %s." %(attr,str(value))) return res @DocInherit def inference(self): - self.output_dim = [self.input_dims[0][0], -self.input_dims[0][1] * self.k] - - - # DON'T MODIFY THIS + self.output_dim = [self.input_dims[0][0], self.input_dims[0][1] * self.k] self.output_rank = len(self.output_dim) @DocInherit def verify(self): super(PoolingKmax2DConf, self).verify() - necessary_attrs_for_user = ['pool_type'] for attr in necessary_attrs_for_user: self.add_attr_exist_assertion_for_user(attr) - self.add_attr_value_assertion('pool_type', ['max']) - - assert all([input_rank == 4 for input_rank in self.input_ranks]), "can only apply on a tensor which the rank is 4" + assert all([input_rank == 4 for input_rank in self.input_ranks]), "Cannot apply a pooling layer on a tensor of which the rank is not 4. Usually, a tensor whose rank is 4, e.g. [batch size, length, width, feature]" assert self.output_dim[-1] != -1, "The shape of input is %s , and the input channel number of pooling should not be -1." % (str(self.input_dims[0])) class PoolingKmax2D(BaseLayer): """ Pooling layer Args: - layer_conf (PoolingConf): configuration of a layer + layer_conf (PoolingKmax2DConf): configuration of a layer """ def __init__(self, layer_conf): super(PoolingKmax2D, self).__init__(layer_conf) From da7cac545c6cd9db56787b95eee3efa0f4fdef9c Mon Sep 17 00:00:00 2001 From: fareise Date: Sun, 21 Jul 2019 16:18:55 +0800 Subject: [PATCH 19/39] Update Combination2D.py --- block_zoo/op/Combination2D.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/block_zoo/op/Combination2D.py b/block_zoo/op/Combination2D.py index ff29f52..9d342f6 100644 --- a/block_zoo/op/Combination2D.py +++ b/block_zoo/op/Combination2D.py @@ -16,17 +16,11 @@ class Combination2DConf(BaseConf): """ Configuration for combination layer - Args: operations (list): a subset of ["cosine", "bilinear"]. - """ def __init__(self, **kwargs): super(Combination2DConf, self).__init__(**kwargs) - if "bilinear" in self.operations: - - weight_bilinear = Parameter(torch.Tensor(self.output_dim, self.output_dim)) - @DocInherit def default(self): @@ -39,7 +33,7 @@ def declare(self): @DocInherit def inference(self): - self.output_dim = [self.input_dims[0][0], self.input_dims[0][1], self.input_dims[0][1]] + self.output_dim = [self.input_dims[0][0], len(self.operations), self.input_dims[0][1], self.input_dims[1][1]] super(Combination2DConf, self).inference() @@ -58,8 +52,7 @@ def verify(self): class Combination2D(nn.Module): - """ Combination layer to merge the representation of two sequence - + """ Combination2D layer to merge the representation of two sequence Args: layer_conf (Combination2DConf): configuration of a layer """ @@ -67,18 +60,18 @@ def __init__(self, layer_conf): super(Combination2D, self).__init__() self.layer_conf = layer_conf + self.weight_bilinear = torch.nn.Linear(self.layer_conf.input_dims[0][-1], self.layer_conf.input_dims[0][-1]) + + logging.warning("The length Combination layer returns is the length of first input") def forward(self, *args): """ process inputs - Args: args (list): [string, string_len, string2, string2_len, ...] e.g. string (Variable): [batch_size, dim], string_len (ndarray): [batch_size] - Returns: Variable: [batch_size, output_dim], None - """ result = [] @@ -86,15 +79,19 @@ def forward(self, *args): string1 = args[0] string2 = args[2] result_multiply = torch.matmul(string1, string2.transpose(1,2)) + + # normalize + #norm_matrix = torch.matmul(torch.norm(string1, p=2, dim=-1).unsqueeze(-1), torch.norm(string2, p=2, dim=-1).unsqueeze(-1).transpose(1,2)) + #result_multiply = result_multiply / norm_matrix + result.append(torch.unsqueeze(result_multiply, 1)) if "bilinear" in self.layer_conf.operations: string1 = args[0] string2 = args[2] - string1 = linear_bi(string1) + string1 = self.weight_bilinear(string1) result_multiply = torch.matmul(string1, string2.transpose(1,2)) + result.append(torch.unsqueeze(result_multiply, 1)) return torch.cat(result, 1), args[1] - - From 8145a58be6d669c370726e003ff5635472669e19 Mon Sep 17 00:00:00 2001 From: fareise Date: Tue, 23 Jul 2019 15:37:21 +0800 Subject: [PATCH 20/39] Update PoolingKmax2D.py --- block_zoo/PoolingKmax2D.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/block_zoo/PoolingKmax2D.py b/block_zoo/PoolingKmax2D.py index e4b3924..1b9f38b 100644 --- a/block_zoo/PoolingKmax2D.py +++ b/block_zoo/PoolingKmax2D.py @@ -32,10 +32,6 @@ def declare(self): self.num_of_inputs = 1 self.input_ranks = [4] - - def check_size(self, value, attr): - res = value - return res @DocInherit def inference(self): From b4fcbc2a4648a1d527556bd1ade3be96171d0649 Mon Sep 17 00:00:00 2001 From: fareise Date: Tue, 23 Jul 2019 15:39:54 +0800 Subject: [PATCH 21/39] Update Combination2D.py --- block_zoo/op/Combination2D.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/block_zoo/op/Combination2D.py b/block_zoo/op/Combination2D.py index 9d342f6..6ed5cd6 100644 --- a/block_zoo/op/Combination2D.py +++ b/block_zoo/op/Combination2D.py @@ -17,14 +17,14 @@ class Combination2DConf(BaseConf): """ Configuration for combination layer Args: - operations (list): a subset of ["cosine", "bilinear"]. + operations (list): a subset of ["dot", "cosine" "bilinear"]. """ def __init__(self, **kwargs): super(Combination2DConf, self).__init__(**kwargs) @DocInherit def default(self): - self.operations = ["cosine", "bilinear"] + self.operations = ["dot", "cosine", "bilinear"] @DocInherit def declare(self): @@ -71,18 +71,25 @@ def forward(self, *args): args (list): [string, string_len, string2, string2_len, ...] e.g. string (Variable): [batch_size, dim], string_len (ndarray): [batch_size] Returns: - Variable: [batch_size, output_dim], None + Variable: [batch_size, width, height, dim], None """ result = [] + if "dot" in self.layer_conf.operations: + string1 = args[0] + string2 = args[2] + result_multiply = torch.matmul(string1, string2.transpose(1,2)) + + result.append(torch.unsqueeze(result_multiply, 1)) + if "cosine" in self.layer_conf.operations: string1 = args[0] string2 = args[2] result_multiply = torch.matmul(string1, string2.transpose(1,2)) # normalize - #norm_matrix = torch.matmul(torch.norm(string1, p=2, dim=-1).unsqueeze(-1), torch.norm(string2, p=2, dim=-1).unsqueeze(-1).transpose(1,2)) - #result_multiply = result_multiply / norm_matrix + norm_matrix = torch.matmul(torch.norm(string1, p=2, dim=-1).unsqueeze(-1), torch.norm(string2, p=2, dim=-1).unsqueeze(-1).transpose(1,2)) + result_multiply = result_multiply / norm_matrix result.append(torch.unsqueeze(result_multiply, 1)) From d2887528159e24e0a956c8c5dd1403632e5b726d Mon Sep 17 00:00:00 2001 From: fareise Date: Tue, 23 Jul 2019 15:40:39 +0800 Subject: [PATCH 22/39] Update mv_lstm.json --- model_zoo/demo/mv_lstm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model_zoo/demo/mv_lstm.json b/model_zoo/demo/mv_lstm.json index f308592..3c8afc2 100644 --- a/model_zoo/demo/mv_lstm.json +++ b/model_zoo/demo/mv_lstm.json @@ -105,7 +105,7 @@ "layer_id": "comb_qp", "layer": "Combination2D", "conf": { - "operations": ["cosine"] + "operations": ["dot"] }, "inputs": ["question_1", "passage_1"] }, From 05a5cc55bcab5981cef5e81a112e02e56fecf68b Mon Sep 17 00:00:00 2001 From: fareise Date: Tue, 23 Jul 2019 15:44:45 +0800 Subject: [PATCH 23/39] Update Combination2D.py --- block_zoo/op/Combination2D.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/block_zoo/op/Combination2D.py b/block_zoo/op/Combination2D.py index 6ed5cd6..830e282 100644 --- a/block_zoo/op/Combination2D.py +++ b/block_zoo/op/Combination2D.py @@ -100,5 +100,12 @@ def forward(self, *args): result_multiply = torch.matmul(string1, string2.transpose(1,2)) result.append(torch.unsqueeze(result_multiply, 1)) - + ''' + if "add" in self.layer_conf.operations: + string1 = args[0] + string2 = args[2] + x_new = torch.stack([string1]*string2.size()[1], 2) # [batch_size, x_max_len, y_max_len, dim] + y_new = torch.stack([string2]*string1.size()[1], 1) # [batch_size, x_max_len, y_max_len, dim] + result.append(x_new + y_new) + ''' return torch.cat(result, 1), args[1] From e7972903afaaa1f57ee00a36db7e06de067b7190 Mon Sep 17 00:00:00 2001 From: fareise Date: Wed, 24 Jul 2019 22:05:44 +0800 Subject: [PATCH 24/39] Update Combination2D.py --- block_zoo/op/Combination2D.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/block_zoo/op/Combination2D.py b/block_zoo/op/Combination2D.py index 830e282..63834aa 100644 --- a/block_zoo/op/Combination2D.py +++ b/block_zoo/op/Combination2D.py @@ -17,14 +17,14 @@ class Combination2DConf(BaseConf): """ Configuration for combination layer Args: - operations (list): a subset of ["dot", "cosine" "bilinear"]. + operations (list): a subset of ["dot", "bilinear", "add"]. """ def __init__(self, **kwargs): super(Combination2DConf, self).__init__(**kwargs) @DocInherit def default(self): - self.operations = ["dot", "cosine", "bilinear"] + self.operations = ["dot", "bilinear", "add"] @DocInherit def declare(self): @@ -34,6 +34,8 @@ def declare(self): @DocInherit def inference(self): self.output_dim = [self.input_dims[0][0], len(self.operations), self.input_dims[0][1], self.input_dims[1][1]] + if "add" in self.operations: + self.output_dim[1] = self.output_dim[1] + self.input_dims[0][-1] - 1 super(Combination2DConf, self).inference() @@ -82,16 +84,6 @@ def forward(self, *args): result.append(torch.unsqueeze(result_multiply, 1)) - if "cosine" in self.layer_conf.operations: - string1 = args[0] - string2 = args[2] - result_multiply = torch.matmul(string1, string2.transpose(1,2)) - - # normalize - norm_matrix = torch.matmul(torch.norm(string1, p=2, dim=-1).unsqueeze(-1), torch.norm(string2, p=2, dim=-1).unsqueeze(-1).transpose(1,2)) - result_multiply = result_multiply / norm_matrix - - result.append(torch.unsqueeze(result_multiply, 1)) if "bilinear" in self.layer_conf.operations: string1 = args[0] @@ -100,12 +92,12 @@ def forward(self, *args): result_multiply = torch.matmul(string1, string2.transpose(1,2)) result.append(torch.unsqueeze(result_multiply, 1)) - ''' + if "add" in self.layer_conf.operations: string1 = args[0] string2 = args[2] x_new = torch.stack([string1]*string2.size()[1], 2) # [batch_size, x_max_len, y_max_len, dim] y_new = torch.stack([string2]*string1.size()[1], 1) # [batch_size, x_max_len, y_max_len, dim] - result.append(x_new + y_new) - ''' + result.append((x_new + y_new).permute(0, 3, 1, 2)) + return torch.cat(result, 1), args[1] From 663fb471a2e871a89c5966a66e2398e70f514668 Mon Sep 17 00:00:00 2001 From: fareise Date: Thu, 25 Jul 2019 19:13:15 +0800 Subject: [PATCH 25/39] Update PoolingKmax2D.py --- block_zoo/PoolingKmax2D.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/block_zoo/PoolingKmax2D.py b/block_zoo/PoolingKmax2D.py index 1b9f38b..402a4dd 100644 --- a/block_zoo/PoolingKmax2D.py +++ b/block_zoo/PoolingKmax2D.py @@ -13,11 +13,9 @@ class PoolingKmax2DConf(BaseConf): """ - Args: pool_type (str): 'max', default is 'max'. k (int): how many element to reserve. - """ def __init__(self, **kwargs): super(PoolingKmax2DConf, self).__init__(**kwargs) @@ -35,7 +33,7 @@ def declare(self): @DocInherit def inference(self): - self.output_dim = [self.input_dims[0][0], self.input_dims[0][1] * self.k] + self.output_dim = [self.input_dims[0][0], self.input_dims[0][3] * self.k] self.output_rank = len(self.output_dim) @DocInherit @@ -51,7 +49,6 @@ def verify(self): class PoolingKmax2D(BaseLayer): """ Pooling layer - Args: layer_conf (PoolingKmax2DConf): configuration of a layer """ @@ -61,20 +58,16 @@ def __init__(self, layer_conf): def forward(self, string, string_len=None): """ process inputs - Args: string (Tensor): tensor with shape: [batch_size, length, width, feature_dim] string_len (Tensor): [batch_size], default is None. - Returns: Tensor: Pooling result of string - """ + string = string.permute(0, 3, 1, 2) string = string.view(string.size()[0], string.size()[1], -1) index = string.topk(self.k, dim=-1)[1].sort(dim=-1)[0] string = string.gather(-1, index) string = string.view(string.size()[0], -1) return string, string_len - - From bf966782e16a2fc11abea4f34f0693cece7d2630 Mon Sep 17 00:00:00 2001 From: fareise Date: Thu, 25 Jul 2019 19:13:54 +0800 Subject: [PATCH 26/39] Update Combination2D.py --- block_zoo/op/Combination2D.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/block_zoo/op/Combination2D.py b/block_zoo/op/Combination2D.py index 63834aa..d416d17 100644 --- a/block_zoo/op/Combination2D.py +++ b/block_zoo/op/Combination2D.py @@ -33,9 +33,9 @@ def declare(self): @DocInherit def inference(self): - self.output_dim = [self.input_dims[0][0], len(self.operations), self.input_dims[0][1], self.input_dims[1][1]] + self.output_dim = [self.input_dims[0][0], self.input_dims[0][1], self.input_dims[1][1], len(self.operations)] if "add" in self.operations: - self.output_dim[1] = self.output_dim[1] + self.input_dims[0][-1] - 1 + self.output_dim[-1] = self.output_dim[-1] + self.input_dims[0][-1] - 1 super(Combination2DConf, self).inference() @@ -82,8 +82,20 @@ def forward(self, *args): string2 = args[2] result_multiply = torch.matmul(string1, string2.transpose(1,2)) - result.append(torch.unsqueeze(result_multiply, 1)) + result.append(torch.unsqueeze(result_multiply, 3)) + ''' + if "cosine" in self.layer_conf.operations: + string1 = args[0] + string2 = args[2] + result_multiply = torch.matmul(string1, string2.transpose(1,2)) + + # normalize + norm_matrix = torch.matmul(torch.norm(string1, p=2, dim=-1).unsqueeze(-1), torch.norm(string2, p=2, dim=-1).unsqueeze(-1).transpose(1,2)) + result_multiply = result_multiply / norm_matrix + + result.append(torch.unsqueeze(result_multiply, 1)) + ''' if "bilinear" in self.layer_conf.operations: string1 = args[0] @@ -91,13 +103,13 @@ def forward(self, *args): string1 = self.weight_bilinear(string1) result_multiply = torch.matmul(string1, string2.transpose(1,2)) - result.append(torch.unsqueeze(result_multiply, 1)) + result.append(torch.unsqueeze(result_multiply, 3)) if "add" in self.layer_conf.operations: string1 = args[0] string2 = args[2] x_new = torch.stack([string1]*string2.size()[1], 2) # [batch_size, x_max_len, y_max_len, dim] y_new = torch.stack([string2]*string1.size()[1], 1) # [batch_size, x_max_len, y_max_len, dim] - result.append((x_new + y_new).permute(0, 3, 1, 2)) + result.append((x_new + y_new)) - return torch.cat(result, 1), args[1] + return torch.cat(result, 3), args[1] From 21d6dd834fe1cb7f413ab48c84029590ca87cb81 Mon Sep 17 00:00:00 2001 From: fareise Date: Thu, 25 Jul 2019 19:41:27 +0800 Subject: [PATCH 27/39] Update Combination2D.py --- block_zoo/op/Combination2D.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/block_zoo/op/Combination2D.py b/block_zoo/op/Combination2D.py index d416d17..99560b4 100644 --- a/block_zoo/op/Combination2D.py +++ b/block_zoo/op/Combination2D.py @@ -84,18 +84,6 @@ def forward(self, *args): result.append(torch.unsqueeze(result_multiply, 3)) - ''' - if "cosine" in self.layer_conf.operations: - string1 = args[0] - string2 = args[2] - result_multiply = torch.matmul(string1, string2.transpose(1,2)) - - # normalize - norm_matrix = torch.matmul(torch.norm(string1, p=2, dim=-1).unsqueeze(-1), torch.norm(string2, p=2, dim=-1).unsqueeze(-1).transpose(1,2)) - result_multiply = result_multiply / norm_matrix - - result.append(torch.unsqueeze(result_multiply, 1)) - ''' if "bilinear" in self.layer_conf.operations: string1 = args[0] From 031e9cc50aecc64dc328f33c76ae0fcf40a946d6 Mon Sep 17 00:00:00 2001 From: fareise Date: Sun, 28 Jul 2019 15:22:42 +0800 Subject: [PATCH 28/39] Delete Expand_plus.py --- block_zoo/op/Expand_plus.py | 76 ------------------------------------- 1 file changed, 76 deletions(-) delete mode 100644 block_zoo/op/Expand_plus.py diff --git a/block_zoo/op/Expand_plus.py b/block_zoo/op/Expand_plus.py deleted file mode 100644 index 17ebb47..0000000 --- a/block_zoo/op/Expand_plus.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT license. - -# Come from http://www.hangli-hl.com/uploads/3/1/6/8/3168008/hu-etal-nips2014.pdf [ARC-II] - -import torch -import torch.nn as nn -import copy - -from block_zoo.BaseLayer import BaseLayer, BaseConf -from utils.DocInherit import DocInherit -from utils.exceptions import ConfigurationError - -class Expand_plusConf(BaseConf): - """Configuration for Expand_plus layer - - """ - def __init__(self, **kwargs): - super(Expand_plusConf, self).__init__(**kwargs) - - @DocInherit - def default(self): - self.operation = 'Plus' - - @DocInherit - def declare(self): - self.num_of_inputs = 2 - self.input_ranks = [3, 3] - - @DocInherit - def inference(self): - self.output_dim = copy.deepcopy(self.input_dims[0]) - if self.input_dims[0][1] == -1 or self.input_dims[1][1] == -1: - raise ConfigurationError("For Expand_plus layer, the sequence length should be fixed") - self.output_dim.insert(2, self.input_dims[1][1]) # y_len - super(Expand_plusConf, self).inference() # PUT THIS LINE AT THE END OF inference() - - @DocInherit - def verify(self): - super(Expand_plusConf, self).verify() - - -class Expand_plus(BaseLayer): - """ Expand_plus layer - Given sequences X and Y, put X and Y expand_dim, and then add. - - Args: - layer_conf (Expand_plusConf): configuration of a layer - - """ - def __init__(self, layer_conf): - - super(Expand_plus, self).__init__(layer_conf) - assert layer_conf.input_dims[0][-1] == layer_conf.input_dims[1][-1] - - - def forward(self, x, x_len, y, y_len): - """ - - Args: - x: [batch_size, x_max_len, dim]. - x_len: [batch_size], default is None. - y: [batch_size, y_max_len, dim]. - y_len: [batch_size], default is None. - - Returns: - output: batch_size, x_max_len, y_max_len, dim]. - - """ - - x_new = torch.stack([x]*y.size()[1], 2) # [batch_size, x_max_len, y_max_len, dim] - y_new = torch.stack([y]*x.size()[1], 1) # [batch_size, x_max_len, y_max_len, dim] - - return x_new + y_new, None - - From 2eef3223b87666c19959a6936052ccc5a06bb184 Mon Sep 17 00:00:00 2001 From: fareise Date: Sun, 28 Jul 2019 15:24:17 +0800 Subject: [PATCH 29/39] Update conf_question_answer_matching_arcii.json --- .../conf_question_answer_matching_arcii.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/model_zoo/nlp_tasks/question_answer_matching/conf_question_answer_matching_arcii.json b/model_zoo/nlp_tasks/question_answer_matching/conf_question_answer_matching_arcii.json index e9bf0d9..c8d81f4 100644 --- a/model_zoo/nlp_tasks/question_answer_matching/conf_question_answer_matching_arcii.json +++ b/model_zoo/nlp_tasks/question_answer_matching/conf_question_answer_matching_arcii.json @@ -114,8 +114,9 @@ }, { "layer_id": "match", - "layer": "Expand_plus", + "layer": "Combination2D", "conf": { + "operations": ["add"] }, "inputs": ["s1_conv_1", "s2_conv_1"] }, @@ -209,4 +210,4 @@ ] }, "metrics": ["auc", "accuracy"] -} \ No newline at end of file +} From a1df7490e7683c4db1a7012992e15f3c333bc4d6 Mon Sep 17 00:00:00 2001 From: fareise Date: Sun, 28 Jul 2019 15:25:42 +0800 Subject: [PATCH 30/39] Create conf_question_answer_matching_mvlstm.json --- .../conf_question_answer_matching_mvlstm.json | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 model_zoo/nlp_tasks/question_answer_matching/conf_question_answer_matching_mvlstm.json diff --git a/model_zoo/nlp_tasks/question_answer_matching/conf_question_answer_matching_mvlstm.json b/model_zoo/nlp_tasks/question_answer_matching/conf_question_answer_matching_mvlstm.json new file mode 100644 index 0000000..3c8afc2 --- /dev/null +++ b/model_zoo/nlp_tasks/question_answer_matching/conf_question_answer_matching_mvlstm.json @@ -0,0 +1,146 @@ +{ + "license": "Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.", + "tool_version": "1.1.0", + "model_description": "This model is used for question answer matching task, and it achieved auc: 0.7673 in WikiQACorpus test set", + "inputs": { + "use_cache": true, + "dataset_type": "classification", + "data_paths": { + "train_data_path": "./dataset/WikiQACorpus/WikiQA-train.tsv", + "valid_data_path": "./dataset/WikiQACorpus/WikiQA-dev.tsv", + "test_data_path": "./dataset/WikiQACorpus/WikiQA-test.tsv" + }, + "file_with_col_header": true, + "add_start_end_for_seq": true, + "file_header": { + "question_id": 0, + "question_text": 1, + "document_id": 2, + "document_title": 3, + "passage_id": 4, + "passage_text": 5, + "label": 6 + }, + "model_inputs": { + "question": ["question_text"], + "passage": ["passage_text"] + }, + "target": ["label"] + }, + "outputs":{ + "save_base_dir": "./models/wikiqa_bilstm/", + "model_name": "model.nb", + "train_log_name": "train.log", + "test_log_name": "test.log", + "predict_log_name": "predict.log", + "predict_fields": ["prediction"], + "predict_output_name": "predict.tsv", + "cache_dir": ".cache.wikiqa/" + }, + "training_params": { + "vocabulary": { + "min_word_frequency": 1 + }, + "optimizer": { + "name": "Adam", + "params": { + } + }, + "lr_decay": 0.95, + "minimum_lr": 0.0001, + "epoch_start_lr_decay": 3, + "use_gpu": true, + "batch_size": 8, + "batch_num_to_show_results": 100, + "max_epoch": 2, + "valid_times_per_epoch": 5, + "max_lengths":{ + "question": 30, + "passage": 100 + } + }, + "architecture":[ + { + "layer": "Embedding", + "conf": { + "word": { + "cols": ["question_text", "passage_text"], + "dim": 300, + "fix_weight": true + } + } + }, + { + "layer_id": "question_dropout", + "layer": "Dropout", + "conf": { + "dropout": 0.2 + }, + "inputs": ["question"] + }, + { + "layer_id": "passage_dropout", + "layer": "Dropout", + "conf": { + "dropout": 0.2 + }, + "inputs": ["passage"] + }, + { + "layer_id": "question_1", + "layer": "BiLSTM", + "conf": { + "hidden_dim": 128, + "dropout": 0.2, + "num_layers": 2 + }, + "inputs": ["question_dropout"] + }, + { + "layer_id": "passage_1", + "layer": "question_1", + "inputs": ["passage_dropout"] + }, + { + "layer_id": "comb_qp", + "layer": "Combination2D", + "conf": { + "operations": ["dot"] + }, + "inputs": ["question_1", "passage_1"] + }, + { + "layer_id": "pooltest", + "layer": "PoolingKmax2D", + "conf": { + "pool_type": "max", + "k": 50 + }, + "inputs": ["comb_qp"] + }, + { + "output_layer_flag": true, + "layer_id": "output", + "layer": "Linear", + "conf": { + "hidden_dim": [128,2], + "activation": "PReLU", + "last_hidden_activation": false + }, + "inputs": ["pooltest"] + } + ], + "loss": { + "losses": [ + { + "type": "CrossEntropyLoss", + "conf": { + "weight": [0.1,0.9], + "size_average": true + }, + "inputs": ["output","label"] + } + ] + }, + "metrics": ["auc","accuracy"] +} From be94c7d13642cf04832953e4317356fd866b52fc Mon Sep 17 00:00:00 2001 From: fareise Date: Mon, 29 Jul 2019 18:24:42 +0800 Subject: [PATCH 31/39] Update conf_question_answer_matching_mvlstm.json --- .../conf_question_answer_matching_mvlstm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model_zoo/nlp_tasks/question_answer_matching/conf_question_answer_matching_mvlstm.json b/model_zoo/nlp_tasks/question_answer_matching/conf_question_answer_matching_mvlstm.json index 3c8afc2..674e0ac 100644 --- a/model_zoo/nlp_tasks/question_answer_matching/conf_question_answer_matching_mvlstm.json +++ b/model_zoo/nlp_tasks/question_answer_matching/conf_question_answer_matching_mvlstm.json @@ -1,7 +1,7 @@ { "license": "Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.", "tool_version": "1.1.0", - "model_description": "This model is used for question answer matching task, and it achieved auc: 0.7673 in WikiQACorpus test set", + "model_description": "This model is used for question answer matching task, and it achieved auc: 0.7144 in WikiQACorpus test set", "inputs": { "use_cache": true, "dataset_type": "classification", From 1aa64c96b83aad96eeca35064dfc9d03b1ef795a Mon Sep 17 00:00:00 2001 From: fareise Date: Mon, 29 Jul 2019 18:25:29 +0800 Subject: [PATCH 32/39] Update __init__.py --- block_zoo/op/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/block_zoo/op/__init__.py b/block_zoo/op/__init__.py index 896cef6..8628810 100644 --- a/block_zoo/op/__init__.py +++ b/block_zoo/op/__init__.py @@ -5,5 +5,4 @@ from .Combination import Combination, CombinationConf from .Match import Match, MatchConf from .Flatten import Flatten, FlattenConf -from .Expand_plus import Expand_plus, Expand_plusConf -from .CalculateDistance import CalculateDistance, CalculateDistanceConf \ No newline at end of file +from .CalculateDistance import CalculateDistance, CalculateDistanceConf From 83d879dd22a26a4d7b702f07b468e4b79e75e5d2 Mon Sep 17 00:00:00 2001 From: fareise Date: Mon, 29 Jul 2019 18:26:25 +0800 Subject: [PATCH 33/39] Update __init__.py --- block_zoo/op/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/block_zoo/op/__init__.py b/block_zoo/op/__init__.py index 8628810..eb67ce5 100644 --- a/block_zoo/op/__init__.py +++ b/block_zoo/op/__init__.py @@ -3,6 +3,7 @@ from .Concat2D import Concat2D, Concat2DConf from .Concat3D import Concat3D, Concat3DConf from .Combination import Combination, CombinationConf +from .Combination2D import Combination2D, Combination2DConf from .Match import Match, MatchConf from .Flatten import Flatten, FlattenConf from .CalculateDistance import CalculateDistance, CalculateDistanceConf From 31cade786cf00604d6f106a5ec21ea128a0e04ea Mon Sep 17 00:00:00 2001 From: fareise Date: Mon, 29 Jul 2019 18:28:34 +0800 Subject: [PATCH 34/39] Update __init__.py --- block_zoo/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/block_zoo/__init__.py b/block_zoo/__init__.py index 7351a69..2ccdde9 100644 --- a/block_zoo/__init__.py +++ b/block_zoo/__init__.py @@ -12,6 +12,7 @@ from .Conv import Conv, ConvConf from .Pooling import Pooling, PoolingConf from .ConvPooling import ConvPooling, ConvPoolingConf +from .PoolingKMax import PoolingKMax, PoolingKMaxConf from .Dropout import Dropout, DropoutConf From 9b97d738cf4ba631a0d0c4017a8cb1615d9d9a07 Mon Sep 17 00:00:00 2001 From: fareise Date: Mon, 29 Jul 2019 18:29:52 +0800 Subject: [PATCH 35/39] Update __init__.py --- block_zoo/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block_zoo/__init__.py b/block_zoo/__init__.py index 2ccdde9..837e706 100644 --- a/block_zoo/__init__.py +++ b/block_zoo/__init__.py @@ -12,7 +12,7 @@ from .Conv import Conv, ConvConf from .Pooling import Pooling, PoolingConf from .ConvPooling import ConvPooling, ConvPoolingConf -from .PoolingKMax import PoolingKMax, PoolingKMaxConf +from .PoolingKmax2D import PoolingKmax2D, PoolingKmax2DConf from .Dropout import Dropout, DropoutConf From f6ba77a28494e288385b0dcb9ccfa37b1930d046 Mon Sep 17 00:00:00 2001 From: fareise Date: Mon, 29 Jul 2019 22:35:50 +0800 Subject: [PATCH 36/39] Delete mv_lstm.json --- model_zoo/demo/mv_lstm.json | 146 ------------------------------------ 1 file changed, 146 deletions(-) delete mode 100644 model_zoo/demo/mv_lstm.json diff --git a/model_zoo/demo/mv_lstm.json b/model_zoo/demo/mv_lstm.json deleted file mode 100644 index 3c8afc2..0000000 --- a/model_zoo/demo/mv_lstm.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "license": "Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.", - "tool_version": "1.1.0", - "model_description": "This model is used for question answer matching task, and it achieved auc: 0.7673 in WikiQACorpus test set", - "inputs": { - "use_cache": true, - "dataset_type": "classification", - "data_paths": { - "train_data_path": "./dataset/WikiQACorpus/WikiQA-train.tsv", - "valid_data_path": "./dataset/WikiQACorpus/WikiQA-dev.tsv", - "test_data_path": "./dataset/WikiQACorpus/WikiQA-test.tsv" - }, - "file_with_col_header": true, - "add_start_end_for_seq": true, - "file_header": { - "question_id": 0, - "question_text": 1, - "document_id": 2, - "document_title": 3, - "passage_id": 4, - "passage_text": 5, - "label": 6 - }, - "model_inputs": { - "question": ["question_text"], - "passage": ["passage_text"] - }, - "target": ["label"] - }, - "outputs":{ - "save_base_dir": "./models/wikiqa_bilstm/", - "model_name": "model.nb", - "train_log_name": "train.log", - "test_log_name": "test.log", - "predict_log_name": "predict.log", - "predict_fields": ["prediction"], - "predict_output_name": "predict.tsv", - "cache_dir": ".cache.wikiqa/" - }, - "training_params": { - "vocabulary": { - "min_word_frequency": 1 - }, - "optimizer": { - "name": "Adam", - "params": { - } - }, - "lr_decay": 0.95, - "minimum_lr": 0.0001, - "epoch_start_lr_decay": 3, - "use_gpu": true, - "batch_size": 8, - "batch_num_to_show_results": 100, - "max_epoch": 2, - "valid_times_per_epoch": 5, - "max_lengths":{ - "question": 30, - "passage": 100 - } - }, - "architecture":[ - { - "layer": "Embedding", - "conf": { - "word": { - "cols": ["question_text", "passage_text"], - "dim": 300, - "fix_weight": true - } - } - }, - { - "layer_id": "question_dropout", - "layer": "Dropout", - "conf": { - "dropout": 0.2 - }, - "inputs": ["question"] - }, - { - "layer_id": "passage_dropout", - "layer": "Dropout", - "conf": { - "dropout": 0.2 - }, - "inputs": ["passage"] - }, - { - "layer_id": "question_1", - "layer": "BiLSTM", - "conf": { - "hidden_dim": 128, - "dropout": 0.2, - "num_layers": 2 - }, - "inputs": ["question_dropout"] - }, - { - "layer_id": "passage_1", - "layer": "question_1", - "inputs": ["passage_dropout"] - }, - { - "layer_id": "comb_qp", - "layer": "Combination2D", - "conf": { - "operations": ["dot"] - }, - "inputs": ["question_1", "passage_1"] - }, - { - "layer_id": "pooltest", - "layer": "PoolingKmax2D", - "conf": { - "pool_type": "max", - "k": 50 - }, - "inputs": ["comb_qp"] - }, - { - "output_layer_flag": true, - "layer_id": "output", - "layer": "Linear", - "conf": { - "hidden_dim": [128,2], - "activation": "PReLU", - "last_hidden_activation": false - }, - "inputs": ["pooltest"] - } - ], - "loss": { - "losses": [ - { - "type": "CrossEntropyLoss", - "conf": { - "weight": [0.1,0.9], - "size_average": true - }, - "inputs": ["output","label"] - } - ] - }, - "metrics": ["auc","accuracy"] -} From fcc44c44ab94fc2cd1fed662c621814a13b4dfd2 Mon Sep 17 00:00:00 2001 From: fareise Date: Tue, 6 Aug 2019 17:33:47 +0800 Subject: [PATCH 37/39] Fintune MV-LSTM and get 0.7736 AUC Fintune MV-LSTM and get 0.7736 AUC in WikiQACorpus test set --- .../conf_question_answer_matching_mvlstm.json | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/model_zoo/nlp_tasks/question_answer_matching/conf_question_answer_matching_mvlstm.json b/model_zoo/nlp_tasks/question_answer_matching/conf_question_answer_matching_mvlstm.json index 674e0ac..8d99868 100644 --- a/model_zoo/nlp_tasks/question_answer_matching/conf_question_answer_matching_mvlstm.json +++ b/model_zoo/nlp_tasks/question_answer_matching/conf_question_answer_matching_mvlstm.json @@ -1,14 +1,15 @@ { "license": "Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.", "tool_version": "1.1.0", - "model_description": "This model is used for question answer matching task, and it achieved auc: 0.7144 in WikiQACorpus test set", + "model_description": "This model is used for question answer matching task, and it achieved auc: 0.7736 in WikiQACorpus test set", "inputs": { "use_cache": true, "dataset_type": "classification", "data_paths": { "train_data_path": "./dataset/WikiQACorpus/WikiQA-train.tsv", "valid_data_path": "./dataset/WikiQACorpus/WikiQA-dev.tsv", - "test_data_path": "./dataset/WikiQACorpus/WikiQA-test.tsv" + "test_data_path": "./dataset/WikiQACorpus/WikiQA-test.tsv", + "pre_trained_emb": "./dataset/GloVe/glove.840B.300d.txt" }, "file_with_col_header": true, "add_start_end_for_seq": true, @@ -46,17 +47,17 @@ "params": { } }, - "lr_decay": 0.95, - "minimum_lr": 0.0001, + "lr_decay": 0.9, + "minimum_lr": 0.00005, "epoch_start_lr_decay": 3, "use_gpu": true, - "batch_size": 8, + "batch_size": 64, "batch_num_to_show_results": 100, - "max_epoch": 2, + "max_epoch": 10, "valid_times_per_epoch": 5, - "max_lengths":{ - "question": 30, - "passage": 100 + "fixed_lengths":{ + "question": 200, + "passage": 200 } }, "architecture":[ @@ -74,7 +75,7 @@ "layer_id": "question_dropout", "layer": "Dropout", "conf": { - "dropout": 0.2 + "dropout": 0.5 }, "inputs": ["question"] }, @@ -82,7 +83,7 @@ "layer_id": "passage_dropout", "layer": "Dropout", "conf": { - "dropout": 0.2 + "dropout": 0.5 }, "inputs": ["passage"] }, @@ -90,7 +91,7 @@ "layer_id": "question_1", "layer": "BiLSTM", "conf": { - "hidden_dim": 128, + "hidden_dim": 256, "dropout": 0.2, "num_layers": 2 }, @@ -105,7 +106,7 @@ "layer_id": "comb_qp", "layer": "Combination2D", "conf": { - "operations": ["dot"] + "operations": ["dot", "bilinear"] }, "inputs": ["question_1", "passage_1"] }, From 7244460fbcef23b873edf05d083cb6e090d54c36 Mon Sep 17 00:00:00 2001 From: fareise Date: Tue, 6 Aug 2019 17:47:52 +0800 Subject: [PATCH 38/39] add mv lstm result --- Tutorial.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Tutorial.md b/Tutorial.md index 8e98e9a..a0a064e 100644 --- a/Tutorial.md +++ b/Tutorial.md @@ -303,6 +303,7 @@ Question answer matching is a crucial subtask of the question answering problem, [ARC-I](https://arxiv.org/abs/1503.03244) (NeuronBlocks) | 0.7508 [ARC-II](https://arxiv.org/abs/1503.03244) (NeuronBlocks) | 0.7612 [MatchPyramid](https://arxiv.org/abs/1602.06359) (NeuronBlocks) | 0.763 + [MV-LSTM](https://arxiv.org/abs/1511.08277) (NeuronBlocks) | 0.774 BiLSTM+Match Attention (NeuronBlocks) | 0.786 From 725812d1aaa195d04d1488a689fada4bc0064702 Mon Sep 17 00:00:00 2001 From: fareise Date: Tue, 6 Aug 2019 17:49:16 +0800 Subject: [PATCH 39/39] add mv lstm result --- Tutorial_zh_CN.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Tutorial_zh_CN.md b/Tutorial_zh_CN.md index 7d93db9..e243958 100644 --- a/Tutorial_zh_CN.md +++ b/Tutorial_zh_CN.md @@ -292,6 +292,7 @@ Question answer matching is a crucial subtask of the question answering problem, [ARC-I](https://arxiv.org/abs/1503.03244) (NeuronBlocks) | 0.7508 [ARC-II](https://arxiv.org/abs/1503.03244) (NeuronBlocks) | 0.7612 [MatchPyramid](https://arxiv.org/abs/1602.06359) (NeuronBlocks) | 0.763 + [MV-LSTM](https://arxiv.org/abs/1511.08277) (NeuronBlocks) | 0.774 BiLSTM+Match Attention (NeuronBlocks) | 0.786