Second-Order-Unsupervised-Feature-Selection/model.py at main · brandeis-machine-learning/Second-Order-Unsupervised-Feature-Selection · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn import init
import numpy as np


class SOFT(nn.Module):
    def __init__(self, input_shape, nClass, USE_CUDA=True):
        super(SOFT, self).__init__()
        self.USE_CUDA = USE_CUDA
        self.input_shape = input_shape
        self.input_shape2 = input_shape//2
        self.nClass = nClass
        self.act = nn.ReLU()
        self.mask_weight = nn.Parameter(torch.from_numpy(np.random.normal(loc=0.0, scale=0.01, size=[input_shape, input_shape])).float() )
        self.pred_model = nn.Linear(self.input_shape, self.input_shape2)
        self.score_cl = nn.Linear(self.input_shape2, self.nClass)
        self.gcn1 = nn.Linear(self.input_shape, self.input_shape)
        self.gcn2 = nn.Linear(self.input_shape, self.input_shape)

        self.fc_pool = nn.Sequential(
            nn.Linear(self.input_shape, self.input_shape2, False),
            nn.ReLU(),
            nn.Linear(self.input_shape2, self.input_shape, False)
            )


    def apply_bn(self, x):
        bn_module = nn.BatchNorm1d(x.size()[1])
        if self.USE_CUDA:
            bn_module = bn_module.cuda()
        return bn_module(x)


    def forward(self, x, adj):
        # predict by original matrix
        adj_trans = adj
        c = torch.matmul(x, adj_trans)
        c = self.gcn1(c)
        c = self.act(c)
        c = self.apply_bn(c)
        c = torch.matmul(c, adj_trans)
        c = self.gcn2(c)
        c = self.act(c)
        c = self.apply_bn(c)
        pred = self.pred_model(c)
        pred = self.score_cl(pred)
        pred_cluster = pred
        pred = torch.softmax(pred, dim=1)
        pseudo_label = pred


        # create mask
        # gcam = self.mask_weight + self.mask_weight.T
        mean_val = torch.mean(c, dim=0, keepdim=True)
        max_val,_ = torch.max(c, dim=0, keepdim=True)
        mean_val = self.fc_pool(mean_val)
        max_val = self.fc_pool(max_val)
        first_attention = mean_val + max_val
        gcam = torch.matmul(first_attention.T, first_attention)

        mask = self.get_mask(gcam)
        masked_image = self.mask_image(adj_trans, mask)


        # predict by masked matrix
        c = torch.matmul(x, masked_image)
        c = self.gcn1(c)
        c = self.act(c)
        c = self.apply_bn(c)
        c = torch.matmul(c, masked_image)
        c = self.gcn2(c)
        c = self.act(c)
        c = self.apply_bn(c)
        masked_output = self.pred_model(c)
        masked_output = self.score_cl(masked_output)
        masked_output = F.sigmoid(masked_output)


        # get matrix after attention
        A_att = adj_trans - masked_image
        pred_att = masked_output


        # generate representations for psudo label by attentioned matrix
        c = torch.matmul(x, A_att)
        c = self.gcn1(c)
        c = self.act(c)
        c = self.apply_bn(c)
        c = torch.matmul(c, A_att)
        c = self.gcn2(c)
        c = self.act(c)
        c = self.apply_bn(c)
        flag = c

        return flag, pred_cluster, pseudo_label, A_att, pred, pred, pred_att, mask


    def get_mask(self, gcam, sigma=.5, w=8):
        mask = (gcam - torch.min(gcam)) / (torch.max(gcam) - torch.min(gcam))
        mask = F.sigmoid(w * (mask - sigma))
        return mask

    def mask_image(self, img, mask):
        return img - img * mask