-
Notifications
You must be signed in to change notification settings - Fork 2
/
models.py
135 lines (120 loc) · 4.55 KB
/
models.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
import torch
import torch.nn as nn
import numpy as np
# ----------- NEURAL NETWORKS ----------------
class TwoLayerNN(nn.Module):
def __init__(self, d, m, bias_status, train_outer=False, act='ReLU'):
super(TwoLayerNN, self).__init__()
if act == 'ReLU':
self.activation = nn.ReLU()
else:
self.activation = nn.Tanh()
self.fc1 = nn.Linear(d, m, bias=bias_status)
self.fc2 = nn.Linear(m, 1, bias=False)
output_weights = (1/m)*np.concatenate((-np.ones(int(m/2)), np.ones(int(m/2)))).reshape((1,m))
output_weights = torch.Tensor(output_weights)
if train_outer == False:
self.fc2.weight = nn.Parameter(output_weights, requires_grad=False)
else:
self.fc2.weight = nn.Parameter(output_weights, requires_grad=True)
def forward(self, x):
y = self.fc1(x)
z = self.activation(y)
y = self.fc2(z)
return y
# ----------- LOSS FUNCTIONS ----------------
class loss_regularized(nn.Module):
"""
Class for constructing regularized MSE objective.
"""
def __init__(self, loss_type, hidden_rtype, hidden_rweight, outer_rtype, outer_rweight):
super(loss_regularized, self).__init__()
self.loss_type = loss_type
self.hidden_rtype = hidden_rtype
self.outer_rtype = outer_rtype
self.hidden_rweight = hidden_rweight
self.outer_rweight = outer_rweight
def forward(self, predictions, target, W, a):
if self.loss_type == 'L2':
error = torch.mean(torch.square(predictions - target))
else:
relu = nn.ReLU()
error = torch.mean(relu(1 - predictions*target))
# error = torch.mean(torch.maximum(torch.zeros_like(target), 1 - predictions*target)) # default is hinge
if self.hidden_rtype == 'none':
hidden_reg = 0
else:
hidden_reg = torch.linalg.matrix_norm(W, ord=self.hidden_rtype)
if self.outer_rtype == 'none':
outer_reg = 0
else:
outer_reg = torch.linalg.vector_norm(a, ord=self.outer_rtype)
loss_value = error + self.hidden_rweight*hidden_reg + self.outer_rweight*outer_reg
return loss_value
# ----------- OLD ----------------
# class One_Hidden_Layer_Model_ReLU(nn.Module):
# """
# Class for dense feedforward network with 1 hidden ReLU layer of width m and a fixed output layer with sigmoid output
# non-linearity.
# """
# def __init__(self, d, m, bias_status):
# super(One_Hidden_Layer_Model, self).__init__()
# self.relu = nn.ReLU()
# self.sigmoid = nn.Sigmoid()
# self.fc1 = nn.Linear(d, m, bias=bias_status)
# self.fc2 = nn.Linear(m, 1, bias=False)
# a = (1/m)*np.concatenate((-np.ones(int(m/2)), np.ones(int(m/2)))).reshape((1,m))
# output_weights = torch.Tensor(a)
# self.fc2.weight = nn.Parameter(output_weights, requires_grad=False)
#
# def forward(self, x):
# y = self.fc1(x)
# z = self.relu(y)
# y = self.fc2(z)
# y = 2*self.sigmoid(y) - 1
# return y
#
#
# class One_Hidden_Layer_Model_Tanh(nn.Module):
# """
# Class for dense feedforward network with 1 hidden Tanh layer of width m, with fixed linear output layer.
# """
# def __init__(self, d, m, bias_status=False, train_outer=False):
# super(One_Hidden_Layer_Model_Tanh, self).__init__()
# self.tanh = nn.Tanh()
# self.fc1 = nn.Linear(d, m, bias=bias_status)
# self.fc2 = nn.Linear(m, 1, bias=False)
# output_weights = (1 / m) * torch.ones(m)
# if train_outer == False:
# self.fc2.weight = nn.Parameter(output_weights, requires_grad=False)
# else:
# self.fc2.weight = nn.Parameter(output_weights, requires_grad=True)
#
# def forward(self, x):
# y = self.fc1(x)
# z = self.tanh(y)
# y = self.fc2(z)
# return y
#
#
# class Two_Layer_Model_ReLU(nn.Module):
# """
# Class for dense feedforward network with 1 hidden ReLU layer of width m and univariate sigmoid output layer. Note both hidden
# and output layer are trainable.
# """
#
# def __init__(self, d, m):
# super(Two_Layer_Model, self).__init__()
# self.relu = nn.ReLU()
# self.sigmoid = nn.Sigmoid()
#
# self.fc1 = nn.Linear(d, m, bias=True)
# self.fc2 = nn.Linear(m, 1, bias=False)
#
# def forward(self, x):
# y = self.fc1(x)
# z = self.relu(y)
# y = self.fc2(z)
# y = 2*self.sigmoid(y) - 1
#
# return y