Lecture 06: Linear Regression (1) – CS 189, Fall 2025
In this lecture we will explore the formulation of linear regression, basis functions, and geometry of linear regression.
import matplotlib.pyplot as plt
import numpy as np
import plotly.graph_objects as go
from sklearn.linear_model import LinearRegression
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF
from sklearn.linear_model import LinearRegression
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import PolynomialFeatures
import warnings
warnings.filterwarnings("ignore", category=RuntimeWarning)
np.random.seed(42)
import plotly.io as pio
pio.renderers.default = "notebook"
This Jupyter Notebook is designed to provide an interactive exploration of linear regression.
Key Learning Objectives:¶
- Understand the Basics of Linear Regression:
- Explore the concept of fitting a straight line to data points.
- Learn about the parameters of a linear model: intercept and slope.
- Explore Basis Functions:
- Learn how different basis functions (e.g., polynomial, Gaussian, Fourier) can transform the input space.
- Understand how these transformations enable linear regression to model non-linear relationships.
- Compare Linear and Polynomial Regression:
- Fit and compare linear and polynomial regression models.
- Understand the trade-offs between underfitting and overfitting.
- Visualize Linear Regression Geometry:
- Understand the geometric interpretation of linear regression in 2D and 3D spaces.
- Explore the concepts of residuals, hyperplanes, and the span of input features.
- Analyze Residuals:
- Learn how to interpret residual plots to assess model fit.
- Understand the concept of heteroscedasticity and its implications.
- Apply Linear Regression to Real-World Data:
- Use the California Housing dataset to fit a simple linear regression model.
- Evaluate the model using metrics like Mean Squared Error (MSE) and R-squared.
1. Understand the Basics of Linear Regression:¶
A basic linear regression model predicts the output $y$ as a linear combination of input features $x$ and model parameters $w$. The general form of the model is:
$$ y(x, w) = x^T w $$
Here, $x$ is the input vector, $w$ is the parameter vector, and $y(x, w)$ represents the predicted output. For a dataset with $D$ features, the model can be expressed as:
$$ y(x, w) = w_0 + w_1 x_1 + w_2 x_2 + \dots + w_D x_D $$
where:
- $w_0$ is the intercept (bias term),
- $w_1, w_2, \dots, w_D$ are the weights (coefficients) for the input features,
- $x = (x_1, x_2, \dots, x_D)^T \in \mathbb{R}^D$ is the input vector.
The goal of linear regression is to learn the parameters $w$ from the training data such that the predictions $y(x, w)$ closely match the true target values $t$.
N = 50
x = np.random.rand(N) * 10
t = 2 * x + 1 + np.random.randn(N) * 2
x_pred = np.linspace(0, 10, 100)
y_conceptual = 2 * x_pred + 1
plt.figure(figsize=(8, 6))
plt.scatter(x, t, alpha=0.7, label='Dataset $(x_n, t_n)$', s=50)
plt.plot(x_pred, y_conceptual, color='red', linestyle='--', label='Prediction y(x)', linewidth=4.0)
plt.xlabel('x (input)', fontsize=20)
plt.ylabel('Output (t or y(x))', fontsize=20)
plt.title('Visualizing Dataset and a Linear Regression Function', fontsize=20)
plt.legend(fontsize=15)
plt.grid(True)
# plt.savefig('conceptual_prediction.png')
plt.show()
example_w0 = 1.0
example_w1 = 2.0
x_pred = np.linspace(0, 10, 100)
y_example_linear = example_w0 + example_w1 * x_pred
plt.figure(figsize=(8, 6))
plt.scatter(x, t, alpha=0.7, label='Dataset $(x_n, t_n)$', s=100)
plt.plot(x_pred, y_example_linear, color='red', linestyle='--', label=f'Linear Prediction $y(x) = {example_w0} + {example_w1}x$', linewidth=4.0)
plt.axhline(0, color='grey', lw=0.5)
plt.axvline(0, color='grey', lw=0.5)
plt.plot(0, example_w0, 'go', markersize=12, label=f'Intercept ($w_0={example_w0}$)')
x_point_1 = 2
x_point_2 = 4
y_point_1 = example_w0 + example_w1 * x_point_1
y_point_2 = example_w0 + example_w1 * x_point_2
plt.plot([x_point_1, x_point_2], [y_point_1, y_point_1], 'k--', linewidth=4.0)
plt.plot([x_point_2, x_point_2], [y_point_1, y_point_2], 'k--', linewidth=4.0)
plt.text(x_point_1 + (x_point_2 - x_point_1)/2, y_point_1 - 2, f'$\Delta x = {x_point_2 - x_point_1}$', horizontalalignment='center', verticalalignment='top', fontsize=15)
plt.text(x_point_2 + 0.3, y_point_1 + (y_point_2 - y_point_1)/2, f'$\Delta y = {y_point_2 - y_point_1}$', horizontalalignment='left', verticalalignment='center', fontsize=15)
plt.text(8, 2, f'Slope $w_1 = \\frac{{\Delta y}}{{\Delta x}} = {example_w1}$', horizontalalignment='right', verticalalignment='bottom', fontsize=15)
plt.xlabel('x (input)', fontsize=20)
plt.ylabel('Output (t or y(x))', fontsize=20)
plt.title('Intercept and Slope in a Linear Regression Function', fontsize=20)
plt.legend(fontsize=15)
plt.grid(True)
# plt.savefig('intercept_slope_illustration.png')
plt.show()
In higher dimensions, the predicted values from a linear regression model form a hyperplane in $ℝ^{𝐷+1}$, where $𝐷$ is the number of input features.
This hyperplane represents the relationship between the input features and the predicted output. Each point on the hyperplane corresponds to a predicted value for a specific combination of input features. The geometry of this hyperplane is determined by the model's coefficients (slopes) and the intercept (bias term).
N = 50
x1 = np.random.rand(N) * 10
x2 = np.random.rand(N) * 10
t = 2 * x1 + 3 * x2 + 5 + np.random.randn(N) * 5
X = np.vstack((x1, x2)).T
model = LinearRegression()
model.fit(X, t)
# Get the coefficients (w1, w2) and the intercept (w0)
w1, w2 = model.coef_
w0 = model.intercept_
print(f"Fitted model: t = {w0:.2f} + {w1:.2f}*x1 + {w2:.2f}*x2")
# Create a grid of x1 and x2 values for the hyperplane
x1_surf, x2_surf = np.meshgrid(np.linspace(x1.min(), x1.max(), 12),
np.linspace(x2.min(), x2.max(), 12))
t_surf = w0 + w1 * x1_surf + w2 * x2_surf
fig = go.Figure()
fig.add_trace(go.Scatter3d(
x=x1, y=x2, z=t,
mode='markers',
marker=dict(
size=5,
color='blue',
opacity=0.8
),
name='Dataset (x₁, x₂, t)'
))
fig.add_trace(go.Surface(
x=x1_surf, y=x2_surf, z=t_surf,
colorscale=[[0, 'red'], [1, 'red']],
opacity=0.5,
showscale=False,
name='Fitted Hyperplane'
))
fig.update_layout(
title='3D Visualization of Linear Regression and Fitted Hyperplane',
scene=dict(
xaxis=dict(
title='x₁',
backgroundcolor='white',
gridcolor='lightgray'
),
yaxis=dict(
title='x₂',
backgroundcolor='white',
gridcolor='lightgray'
),
zaxis=dict(
title='Output (t or y(x))',
backgroundcolor='white',
gridcolor='lightgray',
),
bgcolor='white'
),
margin=dict(l=0, r=0, b=0, t=40),
font=dict(size=14),
legend=dict(font=dict(size=12)),
)
# fig.to_image(format="png", width=800, height=400)
# fig.write_html("linear_regression_3d.html")
fig.show()
Fitted model: t = 6.79 + 2.24*x1 + 2.64*x2
x_points = np.linspace(0, 10, 5)
t_points = 2 * x_points + 1 + np.random.randn(5) * 5
def plot_polynomial_with_residuals(x_data, t_data, degree=1, color='red', label='y(x, w)'):
w = np.polyfit(x_data, t_data, degree)
polynomial = np.poly1d(w)
x_plot = np.linspace(x_data.min(), x_data.max(), 100)
y_plot = polynomial(x_plot)
plt.figure(figsize=(10, 7))
plt.plot(x_plot, y_plot, color=color, label=label, linewidth=4)
plt.scatter(x_data, t_data, color='blue', label='Data Points ($t_n$)', s=150)
# Predict y values for all points
y_predicted_points = polynomial(x_data)
# Calculate residuals
residuals = t_data - y_predicted_points
# Show the distances (residuals) with green lines and indicate positive/negative
for i in range(len(x_data)):
x_i = x_data[i]
t_i = t_data[i]
y_pred_i = y_predicted_points[i]
residual_i = residuals[i]
plt.plot([x_i, x_i], [t_i, y_pred_i], color='green', linestyle='--', linewidth=4, alpha=0.7)
text_x_pos = x_i + (x_data.max() - x_data.min()) * 0.01
text_y_pos = (t_i + y_pred_i) / 2
sign = '+' if residual_i >= 0 else '-'
plt.text(text_x_pos, text_y_pos, f'{residual_i:.2f} ({sign})',
fontsize=14, color='green', ha='left', va='center')
plt.xlabel('x', fontsize=16)
plt.ylabel('t / y(x, w)', fontsize=16)
plt.title(f'Linear Regression plot with Residuals', fontsize=18)
plt.legend(fontsize=14)
plt.grid(True)
# plt.savefig('linear_regression_with_residuals.png')
plt.show()
plot_polynomial_with_residuals(x_points, t_points, degree=1, color='red', label='y(x, w)')