Self taught neural network series - 3. First knowledge of neural network

3.1 Construction of nonlinear function

• Any curve can be constructed with an activation function
• Any curve is the sum of some activation functions
• These activation functions Bias and weight are different
• Two ReLU Function to construct a step function or sigmoid function
• So in the same case ReLU The activation function of ( Neuron ) It needs to be doubled

#  Activation function 
#  Implementation of step function 
import numpy as np

def  step_func(x):
    if  x >0:
        return 1
    elif x<=0:
        return 0
#  test 
step_func(11)
x = np.array([11,11])
# x >0
# step_func([11,11])
# TypeError: '>' not supported between instances of 'list' and 'int'

#  utilize numpy Array improvements 
import numpy as np  
import matplotlib.pylab as plt

def step_function(x):
    ''' Implementation of step function '''
    y = x>0  #  The resulting Boolean array 
    return y.astype(np.int)   # convert to 0,1

x = np.arange(-5.0,5.0,0.1)
y = step_function(x)
#  visualization 
plt.plot(x,y)
plt.ylim(-0.1,1.1)
plt.show()

def sigmoid(x):
    return 1/(1+np.exp(-x))

x = np.arange(-5,5,0.1)
y = sigmoid(x)
plt.plot(x,y)
plt.ylim(-0.1,1.1)
plt.show()

# ReLU  Linear rectification function 
def ReLU(x):
    return np.maximum(0,x)

#  test 
ReLU(2)
x = np.arange(-5,5,0.1)
y = ReLU(x)
plt.plot(x,y)
plt.show()

3.2 Multidimensional arrays

import numpy as np

A= np.array([[[1,2],[3,4],[5,6]],[[2,2],[3,3],[4,4]]])
A.shape  #  Dimensions are read from outside to inside 

3.3 The realization of neural network

#  Matrix multiplication 
X = np.array([1,2])
W = np.array([[1,3,5],[2,4,6]])  #  The first is x1 The weight of , The second is x2 The weight of 
Y = np.dot(X,W)
print(Y)
#  Hidden layer 1 The implementation of the 
X = np.array([1.0,0.5])
W1 = np.array([[0.1,0.3,0.5],[0.2,0.4,0.6]])
B1 = np.array([0.1,0.2,0.3])
print(X.shape)
print(W1.shape)
print(B1.shape)
A1 = np.dot(X,W1)+B1
Z1 = sigmoid(A1)
print(A1)
print(Z1)
#  Hidden layer 2 The implementation of the 
W2 = np.array([[0.1,0.4],[0.2,0.5],[0.3,0.6]])
B2 =  np.array([0.1,0.2])
print(Z1.shape)
print(W2.shape)
print(B2.shape)
A2 = np.dot(Z1,W2) + B2
Z2 = sigmoid(A2)
#  Implementation of the output layer 
W3 = np.array([[0.1,0.3],[0.2,0.4]])
B3 = np.array([0.1,0.2])
A3 = np.dot(Z2,W3)+B3
#  Output layer activation function 
#  The regression problem uses the identity function , Classification questions use sigmoid function 、 Multi classification softmax function 
def identity_func(x):
    ''' Output layer activation function '''
    return x

def _softmax(x):
    exp_x = np.exp(x)
    sum_exp = np.sum(exp_x)
    y = exp_x /sum_exp
    return y

# softmax Need to be solved   An index , The data is too big 

def softmax(x):
    a = np.max(x)
    exp_x = np.exp(x-a)
    exp_sum = np.sum(exp_x)
    y = exp_x/exp_sum
    return y

Y = identity_func(A3) 
print(Y)

#  test 
x = np.array([1010,1000,990])
y1 = _softmax(x)
y2 = softmax(x)
print(y1)
print(y2)

 Object oriented form 

class network:

    def __init__(self,W1,b1,W2,b2,W3,b3):
        network = {
    }
        self.W1 = W1
        self.W2 = W2
        self.W3 = W3
        self.b1 = b1
        self.b2 = b2
        self.b3 = b3

    def forward(self,x):
        a1 = np.dot(x,self.W1)+self.b1
        z1 = sigmoid(a1)
        a2 = np.dot(z1,self.W2) + self.b2
        z2 = sigmoid(a2)
        a3 = np.dot(z2,self.W3) +self.b3
        y = identity_func(a3)
        return y

x = np.array([1.0,0.5])
W1 = np.array([[0.1,0.3,0.5],[0.2,0.4,0.6]])
b1 = np.array([0.1,0.2,0.3])
W2 = np.array([[0.1,0.4],[0.2,0.5],[0.3,0.6]])
b2 = np.array([0.1,0.2])
W3 = np.array([[0.1,0.3],[0.2,0.4]])
b3 = np.array([0.1,0.2])
network = network(W1,b1,W2,b2,W3,b3)
output = network.forward(x)
print(output)

3.4 Handwritten digit recognition

------------------------------- Supplementary information -----------------------------------------------------

# minst.py file 
try:
    import urllib.request
except ImportError:
    raise ImportError('You should use Python 3.x')
import os.path
import gzip
import pickle
import os
import numpy as np

url_base = 'http://yann.lecun.com/exdb/mnist/'
key_file = {
    
    'train_img': 'train-images-idx3-ubyte.gz',
    'train_label': 'train-labels-idx1-ubyte.gz',
    'test_img': 't10k-images-idx3-ubyte.gz',
    'test_label': 't10k-labels-idx1-ubyte.gz'
}

dataset_dir = os.path.dirname(os.path.abspath(__file__))
save_file = dataset_dir + "/mnist.pkl"

train_num = 60000
test_num = 10000
img_dim = (1, 28, 28)
img_size = 784


def _download(file_name):
    file_path = dataset_dir + "/" + file_name

    if os.path.exists(file_path):
        return

    print("Downloading " + file_name + " ... ")
    urllib.request.urlretrieve(url_base + file_name, file_path)
    print("Done")


def download_mnist():
    for v in key_file.values():
        _download(v)


def _load_label(file_name):
    file_path = dataset_dir + "/" + file_name

    print("Converting " + file_name + " to NumPy Array ...")
    with gzip.open(file_path, 'rb') as f:
        labels = np.frombuffer(f.read(), np.uint8, offset=8)
    print("Done")

    return labels


def _load_img(file_name):
    file_path = dataset_dir + "/" + file_name

    print("Converting " + file_name + " to NumPy Array ...")
    with gzip.open(file_path, 'rb') as f:
        data = np.frombuffer(f.read(), np.uint8, offset=16)
    data = data.reshape(-1, img_size)
    print("Done")

    return data


def _convert_numpy():
    dataset = {
    }
    dataset['train_img'] = _load_img(key_file['train_img'])
    dataset['train_label'] = _load_label(key_file['train_label'])
    dataset['test_img'] = _load_img(key_file['test_img'])
    dataset['test_label'] = _load_label(key_file['test_label'])

    return dataset


def init_mnist():
    download_mnist()
    dataset = _convert_numpy()
    print("Creating pickle file ...")
    with open(save_file, 'wb') as f:
        pickle.dump(dataset, f, -1)
    print("Done!")


def _change_one_hot_label(X):
    T = np.zeros((X.size, 10))
    for idx, row in enumerate(T):
        row[X[idx]] = 1

    return T


def load_mnist(normalize=True, flatten=True, one_hot_label=False):
    """ Read in MNIST Data sets  Parameters ---------- normalize :  Normalize the pixel value of the image to 0.0~1.0 one_hot_label : one_hot_label by True Under the circumstances , Label as one-hot Array return  one-hot Array refers to [0,0,1,0,0,0,0,0,0,0] Such an array  flatten :  Whether to expand the image into a one-dimensional array  Returns ------- ( Training images ,  Training tags ), ( Test image ,  Test label ) """
    if not os.path.exists(save_file):
        init_mnist()

    with open(save_file, 'rb') as f:
        dataset = pickle.load(f)

    if normalize:
        for key in ('train_img', 'test_img'):
            dataset[key] = dataset[key].astype(np.float32)
            dataset[key] /= 255.0

    if one_hot_label:
        dataset['train_label'] = _change_one_hot_label(dataset['train_label'])
        dataset['test_label'] = _change_one_hot_label(dataset['test_label'])

    if not flatten:
        for key in ('train_img', 'test_img'):
            dataset[key] = dataset[key].reshape(-1, 1, 28, 28)

    return (dataset['train_img'],
            dataset['train_label']), (dataset['test_img'],
                                      dataset['test_label'])


if __name__ == '__main__':
    init_mnist()

----------------------------------------------- Text --------------------------------------------

import sys,os
sys.path.append(os.pardir)
from res.mnist import load_mnist
from PIL import Image

# (x_train, t_train),(x_test,t_test) = load_mnist(flatten=True,normalize=False)
# print(x_train.shape)
# print(t_train.shape)
# print(x_test.shape)
# print(t_test.shape)

#  Show the first photo 
def img_show(img):
    pil_img = Image.fromarray(np.uint8(img))
    pil_img.show()

(x_train, t_train),(x_test,t_test) = load_mnist(flatten=True,normalize=False)
# pickle The function can save the running object as a file , You can use it directly next time 
#  Used in function pickle function 
img = x_train[0]
label = t_train[0]
print(label)
print(img.shape)
img = img.reshape(28,28)
print(img.shape)
img_show(img)

#  Neural networks predict 
# 1*28*28 The picture of the dimension corresponds to 784 Neurons , stay 255*255*255 The selected part 
#  Input 784 Neurons , Output 0-9 Of 10 Neurons 
#  Suppose the hidden layer 1 Yes 50 Neurons , Hidden layer 2 Yes 100 Neurons 

'''1、 Get input data  2、 Define neural networks  3、 Using neural network to predict  '''


def  get_data():
        (xtrain,t_train),(x_test,t_test) = load_mnist(
            normalize=True,flatten=True,one_hot_label=False )
        return x_test,t_test
    
def  _network():
        import pickle
        with open('sample_weight.pkl', 'rb') as f:
            network = pickle.load(f)
        return network
    
def predict(network,x):
        W1,W2,W3  = network['W1'], network['W2'], network['W3']
        b1,b2,b3 = network['b1'],network['b2'],network['b3']
        a1 = np.dot(x,W1)+b1
        z1 = sigmoid(a1)
        a2 = np.dot(z1,W2) + b2
        z2 = sigmoid(a2)
        a3  = np.dot(z2,W3) + b3
        y =softmax(a3)

        return y
x,t = get_data()
network = _network()
accuracy_cnt = 0
for i in range(len(x)):
    y = predict(network,x[i])
    p = np.argmax(y) 
    if p ==t[i]:
        accuracy_cnt +=1
print("Accuracy:" + str(float(accuracy_cnt)/len(x)))

#  The parameter characteristics of the neural network used 
print(x.shape)
print(x[0].shape)
print(W1.shape)
W1,W2,W3  = network['W1'], network['W2'], network['W3']
print(W1.shape)
print(W2.shape)
print(W3.shape) 

#  Because neural networks only limit columns , So the input 100 Pixels and 1 There is no difference in the pixels of the photos 
#  Use batch Can improve computing performance 
x,t = get_data()
network = _network()
batch_size = 100
accuracy_cnt = 0

for i in range(0,len(x),batch_size): #  take 0 100 200  Read data faster 
    x_batch = x[i:i+batch_size]      #  Take... Again in the loop  1 2 101 102  It is equivalent to two cycles  
    y_batch = predict(network,x_batch)
    p = np.argmax(y_batch,axis=1)
    accuracy_cnt += np.sum(p==t[i:i+batch_size])

print("Accuracy:" + str(float(accuracy_cnt/len(x))))