Repository navigation
Expand file tree
/
Copy path__init__.py
More file actions
executable file
·97 lines (77 loc) · 2.75 KB
/
Copy path__init__.py
File metadata and controls
executable file
·97 lines (77 loc) · 2.75 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
import torch
import torch.nn as nn
import torch.nn.functional as F
def update_learning_rate(P, optimizer, cur_epoch, n, n_total):
cur_epoch = cur_epoch - 1
lr = P.lr_init
if P.optimizer == 'sgd' or 'lars':
DECAY_RATIO = 0.1
elif P.optimizer == 'adam':
DECAY_RATIO = 0.3
else:
raise NotImplementedError()
if P.warmup > 0:
cur_iter = cur_epoch * n_total + n
if cur_iter <= P.warmup:
lr *= cur_iter / float(P.warmup)
if cur_epoch >= 0.5 * P.epochs:
lr *= DECAY_RATIO
if cur_epoch >= 0.75 * P.epochs:
lr *= DECAY_RATIO
for param_group in optimizer.param_groups:
param_group['lr'] = lr
return lr
def _cross_entropy(input, targets, reduction='mean'):
targets_prob = F.softmax(targets, dim=1)
xent = (-targets_prob * F.log_softmax(input, dim=1)).sum(1)
if reduction == 'sum':
return xent.sum()
elif reduction == 'mean':
return xent.mean()
elif reduction == 'none':
return xent
else:
raise NotImplementedError()
def _entropy(input, reduction='mean'):
return _cross_entropy(input, input, reduction)
def cross_entropy_soft(input, targets, reduction='mean'):
targets_prob = F.softmax(targets, dim=1)
xent = (-targets_prob * F.log_softmax(input, dim=1)).sum(1)
if reduction == 'sum':
return xent.sum()
elif reduction == 'mean':
return xent.mean()
elif reduction == 'none':
return xent
else:
raise NotImplementedError()
def kl_div(input, targets, reduction='batchmean'):
return F.kl_div(F.log_softmax(input, dim=1), F.softmax(targets, dim=1),
reduction=reduction)
def target_nll_loss(inputs, targets, reduction='none'):
inputs_t = -F.nll_loss(inputs, targets, reduction='none')
logit_diff = inputs - inputs_t.view(-1, 1)
logit_diff = logit_diff.scatter(1, targets.view(-1, 1), -1e8)
diff_max = logit_diff.max(1)[0]
if reduction == 'sum':
return diff_max.sum()
elif reduction == 'mean':
return diff_max.mean()
elif reduction == 'none':
return diff_max
else:
raise NotImplementedError()
def target_nll_c(inputs, targets, reduction='none'):
conf = torch.softmax(inputs, dim=1)
conf_t = -F.nll_loss(conf, targets, reduction='none')
conf_diff = conf - conf_t.view(-1, 1)
conf_diff = conf_diff.scatter(1, targets.view(-1, 1), -1)
diff_max = conf_diff.max(1)[0]
if reduction == 'sum':
return diff_max.sum()
elif reduction == 'mean':
return diff_max.mean()
elif reduction == 'none':
return diff_max
else:
raise NotImplementedError()