Logistic regression

import numpy as np

def sigmoid(z):
    """
    Apply sigmoid function.
    """
    return 1 / (1 + np.exp(-z))

def cost_function(X, y, weights):
    """
    Compute the cost function for all the training samples.
    """
    m = X.shape[0]
    h = sigmoid(X @ weights)
    epsilon = 1e-5  # to avoid log(0)
    cost = (-1/m) * ((y.T @ np.log(h + epsilon)) + ((1 - y).T @ np.log(1 - h + epsilon)))
    return cost

def gradient_descent(X, y, weights, alpha, iterations):
    """
    Perform gradient descent to learn weight parameters.
    """
    m = X.shape[0]
    cost_history = []

    for i in range(iterations):
        # Update weights based on output from sigmoid
        weights = weights - (alpha/m) * X.T @ (sigmoid(X @ weights) - y)
        # Calculate cost so we can see the change, here i am using binary cross entrupy
        cost = cost_function(X, y, weights)
        # Save the cost from each iteration
        cost_history.append(cost)

    return weights, cost_history

def predict(X, weights):
    """
    Predict whether the label is 0 or 1 using learned logistic regression parameters.
    """
    predictions = sigmoid(X @ weights)
    return predictions.round()

# Example usage:
if __name__ == "__main__":
    # Example data
    X = np.array([
        [1, 2],
        [1, 3],
        [1, 4],
        [1, 5]
    ])
    y = np.array([0, 0, 1, 1])

    # Add intercept term to X
    X = np.hstack((np.ones((X.shape[0], 1)), X))

    # Initial weights (parameters)
    initial_weights = np.zeros(X.shape[1])

    # Gradient descent settings
    iterations = 1000
    alpha = 0.01

    weights, cost_history = gradient_descent(X, y, initial_weights, alpha, iterations)

    print("Learned weights:", weights)
    # Predictions
    predictions = predict(X, weights)
    print("Predictions:", predictions)

---

Last update: March 21, 2024