Principle and implementation of cross entropy

principle

To be lazy , No hurry, no change

Realization

  #  After testing ,reduction  Add it or not, the result is the same , But for insurance , Or add 
  ce_loss = F.cross_entropy(logits, labels, reduction="none")
  print(ce_loss)

python Realization

softmax

import numpy as np
x=np.array([[0.3, 0.6, 0.9, 1], [0.6, 0.4, 0.9, 0.5], [0.6, 0.2, 0.9, 0.5]]) # input data 
W=np.random.randn(4,3) # Generate weights randomly 


def softmax(x,W):
    z=np.dot(x,W) # Get the vector of prediction probability 
    return np.exp(z) / np.sum(np.exp(z))
a=softmax(x,W)
print(a)
a0_result=np.argmax(a[0]) # On the sample a0, take a[0] The maximum probability in is taken as the classification result   Be careful a[0].sum=1
a1_result=np.argmax(a[1])
a2_result=np.argmax(a[2])
print(a0_result)

Cross entropy

stay softmax On the basis of ：

import torch
labels=np.array([1,0,2])
def cross_entropy(labels,log_pt):
    pt = torch.exp(log_pt)  #  adopt  softmax  Score after function 
    labels = labels.view(-1, 1)  #  Add one more dimension , For the use of  gather  Function to prepare 
    pt = pt.gather(1, labels)  #  Pick the one corresponding to the real value  softmax  Scoring , It can also be realized by using hot coding 
    #dim=1  Return by line 
    ce_loss = -torch.log(pt)
    return ce_loss

ce_loss=cross_entropy(torch.tensor(labels),torch.tensor(a))
print(ce_loss)