From 3312167e17a461937aee78cc3df29b3b08d78741 Mon Sep 17 00:00:00 2001 From: Mahadev-05 Date: Tue, 16 Dec 2025 01:38:04 +0530 Subject: [PATCH 1/6] machine_learning: add RidgeRegression with tests and demo --- machine_learning/__init__.py | 3 + machine_learning/ridge_regression.py | 169 ++++++++++++++++++ machine_learning/tests/conftest.py | 9 + .../tests/test_ridge_regression.py | 51 ++++++ 4 files changed, 232 insertions(+) create mode 100644 machine_learning/ridge_regression.py create mode 100644 machine_learning/tests/conftest.py create mode 100644 machine_learning/tests/test_ridge_regression.py diff --git a/machine_learning/__init__.py b/machine_learning/__init__.py index e69de29bb2d1..1ad13b27854e 100644 --- a/machine_learning/__init__.py +++ b/machine_learning/__init__.py @@ -0,0 +1,3 @@ +from .ridge_regression import RidgeRegression, mean_absolute_error + +__all__ = ["RidgeRegression", "mean_absolute_error"] diff --git a/machine_learning/ridge_regression.py b/machine_learning/ridge_regression.py new file mode 100644 index 000000000000..a637624eed4a --- /dev/null +++ b/machine_learning/ridge_regression.py @@ -0,0 +1,169 @@ +"""Ridge Regression (L2 regularization) implemented with batch gradient descent. + +This module provides a small, well-tested `RidgeRegression` class that is +compatible with the existing `linear_regression` demo dataset (ADR vs Rating). + +Features: +- Bias (intercept) handled automatically unless the caller provides an + already-augmented feature matrix. +- L2 regularization that excludes the bias term. +- `mean_absolute_error` utility and a small `main()` demo that fetches the + CSGO ADR vs Rating CSV used elsewhere in the repository. + +Examples +-------- +>>> import numpy as np +>>> X = np.array([[1.0], [2.0], [3.0]]) +>>> y = np.array([2.0, 4.0, 6.0]) +>>> model = RidgeRegression(learning_rate=0.1, lambda_=0.0, epochs=2000) +>>> model.fit(X, y) +>>> np.allclose(model.weights, [0.0, 2.0], atol=1e-2) +True +>>> model.predict(np.array([[4.0], [5.0]])) +array([ 8., 10.]) +""" + +from __future__ import annotations + +from dataclasses import dataclass +import httpx +import numpy as np +from typing import Optional + + +@dataclass +class RidgeRegression: + """Ridge Regression using batch gradient descent. + + Parameters + ---------- + learning_rate: float + Step size for gradient descent (must be > 0). + lambda_: float + L2 regularization strength (must be >= 0). Regularization is NOT + applied to the bias (intercept) term. + epochs: int + Number of gradient descent iterations (must be > 0). + """ + + learning_rate: float = 0.01 + lambda_: float = 0.1 + epochs: int = 1000 + weights: Optional[np.ndarray] = None + + def __post_init__(self) -> None: + if self.learning_rate <= 0: + raise ValueError("learning_rate must be positive") + if self.lambda_ < 0: + raise ValueError("lambda_ must be non-negative") + if self.epochs <= 0: + raise ValueError("epochs must be positive") + + @staticmethod + def _add_intercept(features: np.ndarray) -> np.ndarray: + if features.ndim != 2: + raise ValueError("features must be a 2D array") + n_samples = features.shape[0] + return np.c_[np.ones(n_samples), features] + + def fit(self, features: np.ndarray, target: np.ndarray, add_intercept: bool = True) -> None: + """Train the ridge regression model. + + Parameters + ---------- + features: np.ndarray + 2D array (n_samples, n_features) + target: np.ndarray + 1D array (n_samples,) + add_intercept: bool + If True the model will add a bias column of ones to `features`. + """ + if features.ndim != 2: + raise ValueError("features must be a 2D array") + if target.ndim != 1: + raise ValueError("target must be a 1D array") + if features.shape[0] != target.shape[0]: + raise ValueError("Number of samples must match") + + X = features if not add_intercept else self._add_intercept(features) + n_samples, n_features = X.shape + + # initialize weights (including bias as weights[0]) + self.weights = np.zeros(n_features) + + for _ in range(self.epochs): + preds = X @ self.weights + errors = preds - target + + # gradient without regularization + grad = (X.T @ errors) / n_samples + + # add L2 regularization term (do not regularize bias term) + reg = np.concatenate(([0.0], 2 * self.lambda_ * self.weights[1:])) + grad += reg + + self.weights -= self.learning_rate * grad + + def predict(self, features: np.ndarray, add_intercept: bool = True) -> np.ndarray: + """Predict target values for `features`. + + Parameters + ---------- + features: np.ndarray + 2D array (n_samples, n_features) + add_intercept: bool + If True, add bias column to features before prediction. + """ + if self.weights is None: + raise ValueError("Model is not trained") + X = features if not add_intercept else self._add_intercept(features) + return X @ self.weights + + +def mean_absolute_error(predicted: np.ndarray, actual: np.ndarray) -> float: + """Return mean absolute error between two 1D arrays.""" + predicted = np.asarray(predicted) + actual = np.asarray(actual) + if predicted.shape != actual.shape: + raise ValueError("predicted and actual must have the same shape") + return float(np.mean(np.abs(predicted - actual))) + + +def collect_dataset() -> np.matrix: + """Fetch the ADR vs Rating CSV used in the repo's linear regression demo.""" + response = httpx.get( + "https://raw.githubusercontent.com/yashLadha/The_Math_of_Intelligence/" + "master/Week1/ADRvsRating.csv", + timeout=10, + ) + lines = response.text.splitlines() + data = [line.split(",") for line in lines] + data.pop(0) + return np.matrix(data) + + +def main() -> None: + data = collect_dataset() + n = data.shape[0] + + # features and target (same layout as linear_regression.py) + X = np.c_[data[:, 0].astype(float)] + y = np.ravel(data[:, 1].astype(float)) + + model = RidgeRegression(learning_rate=0.0002, lambda_=0.01, epochs=50000) + model.fit(X, y) + + preds = model.predict(X) + mae = mean_absolute_error(preds, y) + + print("Learned weights:") + for i, w in enumerate(model.weights): + print(f"w[{i}] = {w:.6f}") + print(f"MAE on training data: {mae:.6f}") + + +if __name__ == "__main__": + import doctest + + doctest.testmod() + main() diff --git a/machine_learning/tests/conftest.py b/machine_learning/tests/conftest.py new file mode 100644 index 000000000000..7a69a777c73d --- /dev/null +++ b/machine_learning/tests/conftest.py @@ -0,0 +1,9 @@ +import os +import sys + +# Ensure project root (the parent of `machine_learning`) is on sys.path so +# tests can import `machine_learning` when pytest runs tests from inside +# subdirectories. +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if ROOT not in sys.path: + sys.path.insert(0, ROOT) diff --git a/machine_learning/tests/test_ridge_regression.py b/machine_learning/tests/test_ridge_regression.py new file mode 100644 index 000000000000..1fb9369b39a2 --- /dev/null +++ b/machine_learning/tests/test_ridge_regression.py @@ -0,0 +1,51 @@ +import numpy as np +import pytest + +from machine_learning import RidgeRegression, mean_absolute_error + + +def test_fit_perfect_linear_no_regularization(): + X = np.array([[1.0], [2.0], [3.0]]) + y = np.array([2.0, 4.0, 6.0]) + + model = RidgeRegression(learning_rate=0.1, lambda_=0.0, epochs=2000) + model.fit(X, y) + + # bias ~ 0, slope ~ 2 + assert pytest.approx(0.0, abs=1e-2) == model.weights[0] + assert pytest.approx(2.0, abs=1e-2) == model.weights[1] + + +def test_regularization_reduces_weight_norm(): + rng = np.random.default_rng(0) + X = rng.normal(size=(200, 2)) + true_w = np.array([0.0, 5.0, -3.0]) + y = X @ true_w[1:] + true_w[0] + rng.normal(scale=0.1, size=200) + + no_reg = RidgeRegression(learning_rate=0.01, lambda_=0.0, epochs=5000) + no_reg.fit(X, y) + + strong_reg = RidgeRegression(learning_rate=0.01, lambda_=10.0, epochs=5000) + strong_reg.fit(X, y) + + norm_no_reg = np.linalg.norm(no_reg.weights[1:]) + norm_strong_reg = np.linalg.norm(strong_reg.weights[1:]) + + assert norm_strong_reg < norm_no_reg + + +def test_predict_and_mae(): + X = np.array([[1.0], [2.0]]) + y = np.array([3.0, 5.0]) + model = RidgeRegression(learning_rate=0.1, lambda_=0.0, epochs=1000) + model.fit(X, y) + + preds = model.predict(X) + assert preds.shape == (2,) + assert mean_absolute_error(preds, y) < 1e-2 + + +def test_input_validation(): + model = RidgeRegression() + with pytest.raises(ValueError): + model.fit(np.array([1, 2, 3]), np.array([1, 2, 3])) From 51d9176f58c539a63abe3f0477302b59d7775f57 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:21:00 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- machine_learning/ridge_regression.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/machine_learning/ridge_regression.py b/machine_learning/ridge_regression.py index a637624eed4a..3808aa7eb50b 100644 --- a/machine_learning/ridge_regression.py +++ b/machine_learning/ridge_regression.py @@ -66,7 +66,9 @@ def _add_intercept(features: np.ndarray) -> np.ndarray: n_samples = features.shape[0] return np.c_[np.ones(n_samples), features] - def fit(self, features: np.ndarray, target: np.ndarray, add_intercept: bool = True) -> None: + def fit( + self, features: np.ndarray, target: np.ndarray, add_intercept: bool = True + ) -> None: """Train the ridge regression model. Parameters From a0db9d6033693f8cc7f8c07d4c3bdea817019e4d Mon Sep 17 00:00:00 2001 From: Mahadev-05 Date: Tue, 16 Dec 2025 01:55:59 +0530 Subject: [PATCH 3/6] machine_learning: fix ruff issues (imports, annotations, naming, tests package) --- machine_learning/ridge_regression.py | 24 +++++++++---------- machine_learning/tests/__init__.py | 1 + .../tests/test_ridge_regression.py | 18 +++++++------- 3 files changed, 22 insertions(+), 21 deletions(-) create mode 100644 machine_learning/tests/__init__.py diff --git a/machine_learning/ridge_regression.py b/machine_learning/ridge_regression.py index 3808aa7eb50b..b757b1a9bd19 100644 --- a/machine_learning/ridge_regression.py +++ b/machine_learning/ridge_regression.py @@ -26,9 +26,10 @@ from __future__ import annotations from dataclasses import dataclass +from typing import Optional + import httpx import numpy as np -from typing import Optional @dataclass @@ -49,7 +50,7 @@ class RidgeRegression: learning_rate: float = 0.01 lambda_: float = 0.1 epochs: int = 1000 - weights: Optional[np.ndarray] = None + weights: np.ndarray | None = None def __post_init__(self) -> None: if self.learning_rate <= 0: @@ -87,18 +88,18 @@ def fit( if features.shape[0] != target.shape[0]: raise ValueError("Number of samples must match") - X = features if not add_intercept else self._add_intercept(features) - n_samples, n_features = X.shape + x = features if not add_intercept else self._add_intercept(features) + n_samples, n_features = x.shape # initialize weights (including bias as weights[0]) self.weights = np.zeros(n_features) for _ in range(self.epochs): - preds = X @ self.weights + preds = x @ self.weights errors = preds - target # gradient without regularization - grad = (X.T @ errors) / n_samples + grad = (x.T @ errors) / n_samples # add L2 regularization term (do not regularize bias term) reg = np.concatenate(([0.0], 2 * self.lambda_ * self.weights[1:])) @@ -118,8 +119,8 @@ def predict(self, features: np.ndarray, add_intercept: bool = True) -> np.ndarra """ if self.weights is None: raise ValueError("Model is not trained") - X = features if not add_intercept else self._add_intercept(features) - return X @ self.weights + x = features if not add_intercept else self._add_intercept(features) + return x @ self.weights def mean_absolute_error(predicted: np.ndarray, actual: np.ndarray) -> float: @@ -146,16 +147,15 @@ def collect_dataset() -> np.matrix: def main() -> None: data = collect_dataset() - n = data.shape[0] # features and target (same layout as linear_regression.py) - X = np.c_[data[:, 0].astype(float)] + x = np.c_[data[:, 0].astype(float)] y = np.ravel(data[:, 1].astype(float)) model = RidgeRegression(learning_rate=0.0002, lambda_=0.01, epochs=50000) - model.fit(X, y) + model.fit(x, y) - preds = model.predict(X) + preds = model.predict(x) mae = mean_absolute_error(preds, y) print("Learned weights:") diff --git a/machine_learning/tests/__init__.py b/machine_learning/tests/__init__.py new file mode 100644 index 000000000000..a174c30273b8 --- /dev/null +++ b/machine_learning/tests/__init__.py @@ -0,0 +1 @@ +# Package for machine_learning tests diff --git a/machine_learning/tests/test_ridge_regression.py b/machine_learning/tests/test_ridge_regression.py index 1fb9369b39a2..c15f5d951019 100644 --- a/machine_learning/tests/test_ridge_regression.py +++ b/machine_learning/tests/test_ridge_regression.py @@ -5,11 +5,11 @@ def test_fit_perfect_linear_no_regularization(): - X = np.array([[1.0], [2.0], [3.0]]) + x = np.array([[1.0], [2.0], [3.0]]) y = np.array([2.0, 4.0, 6.0]) model = RidgeRegression(learning_rate=0.1, lambda_=0.0, epochs=2000) - model.fit(X, y) + model.fit(x, y) # bias ~ 0, slope ~ 2 assert pytest.approx(0.0, abs=1e-2) == model.weights[0] @@ -18,15 +18,15 @@ def test_fit_perfect_linear_no_regularization(): def test_regularization_reduces_weight_norm(): rng = np.random.default_rng(0) - X = rng.normal(size=(200, 2)) + x = rng.normal(size=(200, 2)) true_w = np.array([0.0, 5.0, -3.0]) - y = X @ true_w[1:] + true_w[0] + rng.normal(scale=0.1, size=200) + y = x @ true_w[1:] + true_w[0] + rng.normal(scale=0.1, size=200) no_reg = RidgeRegression(learning_rate=0.01, lambda_=0.0, epochs=5000) - no_reg.fit(X, y) + no_reg.fit(x, y) strong_reg = RidgeRegression(learning_rate=0.01, lambda_=10.0, epochs=5000) - strong_reg.fit(X, y) + strong_reg.fit(x, y) norm_no_reg = np.linalg.norm(no_reg.weights[1:]) norm_strong_reg = np.linalg.norm(strong_reg.weights[1:]) @@ -35,12 +35,12 @@ def test_regularization_reduces_weight_norm(): def test_predict_and_mae(): - X = np.array([[1.0], [2.0]]) + x = np.array([[1.0], [2.0]]) y = np.array([3.0, 5.0]) model = RidgeRegression(learning_rate=0.1, lambda_=0.0, epochs=1000) - model.fit(X, y) + model.fit(x, y) - preds = model.predict(X) + preds = model.predict(x) assert preds.shape == (2,) assert mean_absolute_error(preds, y) < 1e-2 From ddb4638d9d1c867d6c6e298a9fc050d3a8cdc799 Mon Sep 17 00:00:00 2001 From: Mahadev-05 Date: Tue, 16 Dec 2025 02:05:37 +0530 Subject: [PATCH 4/6] machine_learning: accept numpy.matrix inputs and add test --- machine_learning/ridge_regression.py | 14 ++++++++++---- machine_learning/tests/test_ridge_regression.py | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/machine_learning/ridge_regression.py b/machine_learning/ridge_regression.py index b757b1a9bd19..7439c735ee26 100644 --- a/machine_learning/ridge_regression.py +++ b/machine_learning/ridge_regression.py @@ -26,7 +26,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Optional + import httpx import numpy as np @@ -62,10 +62,11 @@ def __post_init__(self) -> None: @staticmethod def _add_intercept(features: np.ndarray) -> np.ndarray: - if features.ndim != 2: + arr = np.asarray(features, dtype=float) + if arr.ndim != 2: raise ValueError("features must be a 2D array") - n_samples = features.shape[0] - return np.c_[np.ones(n_samples), features] + n_samples = arr.shape[0] + return np.c_[np.ones(n_samples), arr] def fit( self, features: np.ndarray, target: np.ndarray, add_intercept: bool = True @@ -81,6 +82,9 @@ def fit( add_intercept: bool If True the model will add a bias column of ones to `features`. """ + features = np.asarray(features, dtype=float) + target = np.asarray(target, dtype=float) + if features.ndim != 2: raise ValueError("features must be a 2D array") if target.ndim != 1: @@ -119,6 +123,8 @@ def predict(self, features: np.ndarray, add_intercept: bool = True) -> np.ndarra """ if self.weights is None: raise ValueError("Model is not trained") + + features = np.asarray(features, dtype=float) x = features if not add_intercept else self._add_intercept(features) return x @ self.weights diff --git a/machine_learning/tests/test_ridge_regression.py b/machine_learning/tests/test_ridge_regression.py index c15f5d951019..a8e374a1f71d 100644 --- a/machine_learning/tests/test_ridge_regression.py +++ b/machine_learning/tests/test_ridge_regression.py @@ -49,3 +49,17 @@ def test_input_validation(): model = RidgeRegression() with pytest.raises(ValueError): model.fit(np.array([1, 2, 3]), np.array([1, 2, 3])) + + +def test_accepts_numpy_matrix(): + from machine_learning.ridge_regression import collect_dataset + + data = collect_dataset() + X = np.c_[data[:, 0].astype(float)] # numpy.matrix + y = np.ravel(data[:, 1].astype(float)) + + model = RidgeRegression(learning_rate=0.0002, lambda_=0.01, epochs=500) + model.fit(X, y) + preds = model.predict(X) + assert preds.shape == (y.shape[0],) + assert mean_absolute_error(preds, y) >= 0.0 From f61c33fb4598b345212d4ad0cd00f0f515712500 Mon Sep 17 00:00:00 2001 From: Mahadev-05 Date: Tue, 16 Dec 2025 02:14:11 +0530 Subject: [PATCH 5/6] docs: add Ridge Regression entry to DIRECTORY.md --- DIRECTORY.md | 1 + 1 file changed, 1 insertion(+) diff --git a/DIRECTORY.md b/DIRECTORY.md index 0f9859577493..e4e80a7dfd2c 100644 --- a/DIRECTORY.md +++ b/DIRECTORY.md @@ -609,6 +609,7 @@ * [K Nearest Neighbours](machine_learning/k_nearest_neighbours.py) * [Linear Discriminant Analysis](machine_learning/linear_discriminant_analysis.py) * [Linear Regression](machine_learning/linear_regression.py) + * [Ridge Regression](machine_learning/ridge_regression.py) * Local Weighted Learning * [Local Weighted Learning](machine_learning/local_weighted_learning/local_weighted_learning.py) * [Logistic Regression](machine_learning/logistic_regression.py) From 1fe676ac555f15660bbe23c205bd759d84c091c0 Mon Sep 17 00:00:00 2001 From: Mahadev-05 Date: Tue, 16 Dec 2025 02:18:15 +0530 Subject: [PATCH 6/6] =?UTF-8?q?fix(ci):=20ruff/mypy=20issues=20=E2=80=94?= =?UTF-8?q?=20lower-case=20doctest=20vars,=20assert=20weights=20before=20e?= =?UTF-8?q?numerate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- machine_learning/ridge_regression.py | 6 +++--- machine_learning/tests/test_ridge_regression.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/machine_learning/ridge_regression.py b/machine_learning/ridge_regression.py index 7439c735ee26..68e4cb3caf65 100644 --- a/machine_learning/ridge_regression.py +++ b/machine_learning/ridge_regression.py @@ -13,10 +13,10 @@ Examples -------- >>> import numpy as np ->>> X = np.array([[1.0], [2.0], [3.0]]) +>>> x = np.array([[1.0], [2.0], [3.0]]) >>> y = np.array([2.0, 4.0, 6.0]) >>> model = RidgeRegression(learning_rate=0.1, lambda_=0.0, epochs=2000) ->>> model.fit(X, y) +>>> model.fit(x, y) >>> np.allclose(model.weights, [0.0, 2.0], atol=1e-2) True >>> model.predict(np.array([[4.0], [5.0]])) @@ -27,7 +27,6 @@ from dataclasses import dataclass - import httpx import numpy as np @@ -165,6 +164,7 @@ def main() -> None: mae = mean_absolute_error(preds, y) print("Learned weights:") + assert model.weights is not None for i, w in enumerate(model.weights): print(f"w[{i}] = {w:.6f}") print(f"MAE on training data: {mae:.6f}") diff --git a/machine_learning/tests/test_ridge_regression.py b/machine_learning/tests/test_ridge_regression.py index a8e374a1f71d..c05a16ca98a4 100644 --- a/machine_learning/tests/test_ridge_regression.py +++ b/machine_learning/tests/test_ridge_regression.py @@ -55,11 +55,11 @@ def test_accepts_numpy_matrix(): from machine_learning.ridge_regression import collect_dataset data = collect_dataset() - X = np.c_[data[:, 0].astype(float)] # numpy.matrix + x = np.c_[data[:, 0].astype(float)] # numpy.matrix y = np.ravel(data[:, 1].astype(float)) model = RidgeRegression(learning_rate=0.0002, lambda_=0.01, epochs=500) - model.fit(X, y) - preds = model.predict(X) + model.fit(x, y) + preds = model.predict(x) assert preds.shape == (y.shape[0],) assert mean_absolute_error(preds, y) >= 0.0