Loss calculation in pytorch

NLLLoss & CrossEntropyLoss

crossentropyloss = softmax + log + nllloss
$$L=-\frac{1}{N}\sum _i{p}_i\lg q_i$$

• $p_i$​ real label
• $q_i$ forecast lable probability

def test_NLLLoss():
    m = nn.LogSoftmax(dim=-1)
    loss = nn.NLLLoss()  # NLLLoss Internal only input Carry out negative operation , Often you need to be external to input Conduct softmax and log operation 
    loss1 = nn.CrossEntropyLoss()  # softmax + log + nllloss
    loss2 = nn.NLLLoss(reduction='none')
    # input is of size N x C = 3 x 5
    input = torch.randn(3, 5, requires_grad=True)
    # each element in target has to have 0 <= value < C
    # target = torch.empty(3, dtype=torch.long).random_(5)
    target = torch.tensor([1, 0, 4])
    output = loss(m(input), target)

    output_mine = -(torch.gather(m(input), index=target.unsqueeze(-1), dim=1)).sum() / target.size(0)

    print("NLLLoss")
    print("input:", input)
    print("log_softmax: ", m(input))
    print("target:", target)
    print("loss:", output)
    print("loss1:", loss1(input, target))
    print("none loss:", loss2(m(input), target))
    print("mine loss:", output_mine)
    
""" NLLLoss input: tensor([[-0.3630, -2.0179, -1.9629, -0.1293, 0.7163], [ 0.0691, -0.6838, 1.1787, -0.4356, -1.6239], [-1.1982, 0.4505, -0.8784, 1.1047, 0.9539]], requires_grad=True) log_softmax: tensor([[-1.7226, -3.3775, -3.3225, -1.4889, -0.6433], [-1.6661, -2.4191, -0.5566, -2.1709, -3.3592], [-3.2651, -1.6164, -2.9453, -0.9622, -1.1130]], grad_fn=<LogSoftmaxBackward>) target: tensor([1, 0, 4]) loss: tensor(2.0522, grad_fn=<NllLossBackward>) loss1: tensor(2.0522, grad_fn=<NllLossBackward>) none loss: tensor([3.3775, 1.6661, 1.1130], grad_fn=<NllLossBackward>) mine loss: tensor(2.0522, grad_fn=<DivBackward0>) """    

def test_CrossEntropyLoss():
    loss = nn.CrossEntropyLoss(ignore_index=1)  # CrossEntropyLoss It's going to be internal input Conduct softmax and log operation 
    loss1 = nn.CrossEntropyLoss(reduction='none', ignore_index=1)
    # input is of size N x C = 4 x 5
    input = torch.randn(4, 5, requires_grad=True)
    # each element in target has to have 0 <= value < C
    # target = torch.empty(4, dtype=torch.long).random_(5)
    target = torch.tensor([4, 2, 1, 1])
    output = loss(input, target)
    tar_mask = target.ne(1).float().detach()

    output_mine = (-torch.gather(F.log_softmax(input, dim=-1), index=target.unsqueeze(-1), dim=1) * tar_mask.unsqueeze(1)).sum() / tar_mask.sum()
    
    print("CrossEntropyLoss")
    print("input:", input)
    print("log_softmax: ", F.log_softmax(input, dim=-1))
    print("target:", target)
    print("loss:", output)
    print("none loss:", loss1(input, target))
    print("mine loss:", output_mine)
    
""" CrossEntropyLoss input: tensor([[ 0.5172, 0.4194, -1.2796, 1.0469, -1.2624], [-0.9485, -1.3813, -0.2962, -0.2169, -0.6744], [-1.9062, 0.0981, -2.3465, 0.1645, 0.5468], [ 0.6164, -0.7764, -0.3201, -0.6006, 0.9538]], requires_grad=True) log_softmax: tensor([[-1.3711, -1.4689, -3.1680, -0.8414, -3.1508], [-1.9405, -2.3733, -1.2882, -1.2089, -1.6664], [-3.3541, -1.3497, -3.7944, -1.2834, -0.9010], [-1.2053, -2.5981, -2.1418, -2.4223, -0.8679]], grad_fn=<LogSoftmaxBackward>) target: tensor([4, 2, 1, 1]) loss: tensor(2.2195, grad_fn=<NllLossBackward>) none loss: tensor([3.1508, 1.2882, 0.0000, 0.0000], grad_fn=<NllLossBackward>) mine loss: tensor(2.2195, grad_fn=<DivBackward0>) """

BCELoss

$$L=-\frac{1}{N}\sum _i\left(p_i\lg q_i+\left(1-p_i\right)\lg \left(1-q_i\right)\right)$$

def test_BCELoss():
    m = nn.Sigmoid()
    loss = nn.BCELoss()  # BCELoss It's going to be internal input Conduct log operation 
    loss1 = nn.BCEWithLogitsLoss()  # sigmoid + BCE
    loss2 = nn.BCELoss(reduction='none')
    input = torch.randn(5, requires_grad=True)
    # target = torch.empty(3).random_(2)
    target = torch.tensor([1, 0, 1, 0, 1]).float()
    output = loss(m(input), target)

    output_mine = -(target * m(input).log() + (1 - target) * (1 - m(input)).log()).sum() / input.size(0)

    print("\nBCELoss")
    print("input:", input)
    print("sigmoid input:", m(input))
    print("target:", target)
    print("loss:", output)
    print("loss1:", loss1(input, target))
    print("loss2:", loss2(m(input), target))
    print("mine loss:", output_mine)
""" BCELoss input: tensor([-1.5962, -0.2057, -0.2376, -0.1843, 0.3428], requires_grad=True) sigmoid input: tensor([0.1685, 0.4487, 0.4409, 0.4540, 0.5849], grad_fn=<SigmoidBackward>) target: tensor([1., 0., 1., 0., 1.]) loss: tensor(0.8674, grad_fn=<BinaryCrossEntropyBackward>) loss1: tensor(0.8674, grad_fn=<BinaryCrossEntropyWithLogitsBackward>) loss2: tensor([1.7807, 0.5956, 0.8190, 0.6052, 0.5364], grad_fn=<BinaryCrossEntropyBackward>) mine loss: tensor(0.8674, grad_fn=<DivBackward0>) """

KLDivLoss

$$L=-\sum _i\left(p_i\lg q_i-p_i\lg p_i\right)$$

def test_KLDivLoss():
    loss = nn.KLDivLoss(reduction='batchmean')  #  Just divide by batch_size
    loss1 = nn.KLDivLoss(reduction='none')  # KLDivLoss The inside won't be right input Conduct softmax and log operation 
    input = torch.log_softmax(torch.tensor([[-2., -6., -8.], [-7., -1., -2.]], requires_grad=True), dim=-1)
    target = torch.tensor([[0.8, 0.1, 0.1], [0.1, 0.7, 0.2]])
    output = loss(input, target)

    output_mine = torch.sum(torch.log(target)*target - target*input) / input.size(0)

    print("\nKLDivLoss")
    print("input:", input)
    print("target:", target)
    print("loss:", output)
    print("none loss:", loss1(input, target))
    print("mine loss:", output_mine)
""" KLDivLoss input: tensor([[-0.0206, -4.0206, -6.0206], [-6.3151, -0.3151, -1.3151]], grad_fn=<LogSoftmaxBackward>) target: tensor([[0.8000, 0.1000, 0.1000], [0.1000, 0.7000, 0.2000]]) loss: tensor(0.3474, grad_fn=<DivBackward0>) none loss: tensor([[-0.1620, 0.1718, 0.3718], [ 0.4012, -0.0291, -0.0589]], grad_fn=<KlDivBackward>) mine loss: tensor(0.3474, grad_fn=<DivBackward0>) """