Principle of logistic regression

1.sigmoid function

         The formula ： 

          The argument value is any real number , range [0,1]

         explain ： Mapping arbitrary input to [0,1] Section , We can get a prediction in linear regression , Then map the value to Sigmoid This completes the conversion from value to probability , That is to say, classified tasks

         Prediction function ：

          among ,

          Classification task ：

          Integrate ：

         explain ： For binary tasks （0,1）, After integration y take 0 Only keep  ,y take 1 Only keep

          Likelihood function ：

          Log likelihood ：

          In this case, the gradient rise is applied to find the maximum value , introduce

          Convert to gradient descent task

         Derivation process ：

Parameters are updated ： 

Multi category softmax:  

 2. Code implementation

       

import numpy as np
from utils.features import prepare_for_training
from scipy.optimize import minimize
from utils.hypothesis import sigmoid

class LogisticRegression:
    def __init__(self, data, labels, polynomial_degree=0, sinusoid_degree=0, normalize_data=False):

        (data_processed, features_mean,
         features_deviation) = prepare_for_training(data, polynomial_degree, sinusoid_degree,
                                                    normalize_data)
        self.data = data_processed
        self.labels = labels
        self.unique_labels = np.unique(labels)
        self.features_mean = features_mean
        self.features_deviation = features_deviation
        self.polynomial_degree = polynomial_degree
        self.sinusoid_degree = sinusoid_degree
        self.normalize_data = normalize_data
        # Data preprocessing
        num_unique_labels = len(np.unique(labels))
        num_features = self.data.shape[1]
        self.theta = np.zeros((num_unique_labels, num_features))

    def train(self, n_iterations=500):
        cost_histories = []
        num_features = self.data.shape[1]
        for label_index, unique_label in enumerate(self.unique_labels):
            current_initial_theta = np.copy(self.theta[label_index].reshape(num_features, 1))
            current_lables = (self.labels == unique_label).astype(float)
            (theta, cost_history) = LogisticRegression.gradient_descent(self.data, current_initial_theta,
                                                                          current_lables, n_iterations)
            self.theta[label_index]=theta.T
            cost_histories.append(cost_history)
        return self.theta,cost_histories

    @staticmethod
    def gradient_descent(data, current_initial_theta, current_lables, n_iterations):
        cost_history = []
        num_fratures = data.shape[1]
        result = minimize(
            # The goal of optimization
            lambda x: LogisticRegression.cost_function(data, current_lables, x.reshape(num_fratures,1)),
            # Initialized weight parameters
            current_initial_theta,
            # Choose optimization strategy
            method='CG',
            # Gradient descent iterative calculation formula
            jac=lambda x: LogisticRegression.gradient_step(data, current_lables, x.reshape(num_fratures,1)),
            callback=lambda x: cost_history.append(
                LogisticRegression.cost_function(data, current_lables, x.reshape(num_fratures,1))),
            options={
                "maxiter": n_iterations
            }
        )

        if not result.success:
            raise ArithmeticError('Can not minimize cost function' + result.message)
        theta = result.x.reshape(num_fratures,1)
        return theta,cost_history

    @staticmethod
    def cost_function(data, label, theta):
        num_examples = data.shape[0]
        prediction = LogisticRegression.hypothesis(data, theta)
        y_true_cost = np.dot(label[label == 1].T, np.log(prediction[label == 1]))
        y_false_cost = np.dot(1 - label[label == 0].T, np.log(1 - prediction[label == 0]))
        cost = (-1 / num_examples) * (y_false_cost + y_true_cost)
        return cost

    @staticmethod
    def hypothesis(data, theta):
        predictions = sigmoid(np.dot(data, theta))
        return predictions

    @staticmethod
    def gradient_step(data, label, theta):
        num_examples = data.shape[0]
        prediction = LogisticRegression.hypothesis(data, theta)
        label_diff = prediction - label
        gradients = (1 / num_examples) * np.dot(data.T, label_diff)
        return gradients.T.flatten()

    def predict(self,data):
        num_examples = data.shape[0]
        data_processed = prepare_for_training(data, self.polynomial_degree, self.sinusoid_degree,
                                              self.normalize_data)[0]
        prediction = LogisticRegression.hypothesis(data_processed, self.theta.T)
        arg = np.argmax(prediction,axis=1)
        class_prediction = np.empty(arg.shape,dtype=object)
        for index,unique_label in enumerate(self.unique_labels):
            class_prediction[arg == index] = unique_label
        return class_prediction.reshape((num_examples,1))