loclm() implements local linear regression, using
high-level functions. It can handle both numerical and non-numerical
variables. Non-numerical variables are converted to factors.
library(loclm)
show(packageVersion("loclm"))
#> [1] '1.1.0'
show(date())
#> [1] "Tue Nov 17 16:49:02 2026"Here I’m just setting up a nicer plotting frame.
Let’s use the multivariate Banana function variant B from
S. Kok and C. Sandrock, 2009, Locating and Characterizing the Stationary Points of the Extended Rosenbrock Function, Evolutionary Computing, 17(3), pages 437-453.
As the paper explains, variant B is pathological, especially in higher dimensions.
#### d-dimensional Banana function
## (d >= 2 is implied by x)
banana <- function(x) {
stopifnot(is.vector(x, mode = "numeric"))
d <- length(x)
if (d == 1) {
x <- c(x, 1)
d <- 2L
}
robj <- 0
for (j in seq_len(d - 1)) {
robj <- robj + 100 * (x[j]^2 - x[j+1])^2 + (x[j] - 1)^2
}
robj
}Here is a dataset of 101 runs from a 5D banana function, plus a
little noise. Feel free to play with d and
sigma.
## make a dataset
set.seed(1234) # for reproducibility, feel free to change
d <- 3
sigma <- 1
X <- matrix(runif(101 * d), ncol = d)
y <- apply(X, 1, banana) + rnorm(nrow(X), sd = sigma)
## here's where we'll predict
newX <- matrix(runif(23 * d), ncol = d)
newy <- apply(newX, 1, banana) # leave off the noiseI have left the column-names of X and newX
blank, and they will be assigned default values of V1,
V2, and V3 inside loclm() by
as.data.frame(). It would be be safer, though, to assign
meaningful column-names.
## to assign column names
# nms <- paste0("x", 1:ncol(X)) # or something else
# colnames(X) <- nms
# colnames(newX) <- nmsFirst, fit the model with the default span.
## default span
fit <- loclm(X, y)
pp <- predict(fit, newX)
cex <- 0.6
col <- c("darkblue", "forestgreen")
par(op)
matplot(newX[, 2], cbind(newy, pp), pch = c(1, 2), cex = cex,
col = col, xlab = "x[2]", ylab = "Actual/predicted",
main = "Default span")
legend("topleft", c("actual", "predicted"), pch = 1:2, col = col,
cex = par("cex.axis"), bty = "n")Not great.
Now use Leave-One-Out (LOO) cross-validation to set the span. There
is a bespoke function LOO_loclm(). This always executes
sequentially, so that it can be embedded inside a parallel call over
multiple values of span.
## try different spans, can be executed in parallel
mc <- options("mc.cores")
options(mc.cores = 2L) # or more
span_vals <- c(seq(0.1, 0.2, 0.02), 0.75)
rmse <- parallel::mclapply(span_vals, \(span) {
loo_resid <- LOO_loclm(fit, span = span)
sqrt(mean(loo_resid^2))
}) |> unlist()
show(cbind(span = span_vals, rmse = rmse))
#> span rmse
#> [1,] 0.10 8.568039
#> [2,] 0.12 8.079310
#> [3,] 0.14 7.883803
#> [4,] 0.16 7.232624
#> [5,] 0.18 7.401626
#> [6,] 0.20 7.449551
#> [7,] 0.75 14.292123
spanhat <- span_vals[which.min(rmse)]
show(c(span = spanhat))
#> span
#> 0.16The LOO RMSE has dropped from 14.3 to 7.23. Try predicting with the optimal span.
fit <- loclm(X, y, span = spanhat)
pp <- predict(fit, newX)
par(op)
matplot(newX[, 2], cbind(newy, pp), pch = c(1, 2), cex = cex,
col = col, xlab = "x[2]", ylab = "Actual/predicted",
main = sprintf("span = %s (optimal)", format(spanhat)))
legend("topleft", c("actual", "predicted"), pch = 1:2, col = col,
cex = par("cex.axis"), bty = "n")That’s an improvement.
Let’s make the first input into a factor.
## first input is a factor
breaks <- seq(0, 1, 0.2)
X <- as.data.frame(X) # into a data.frame
X[[1]] <- factor(LETTERS[cut(X[[1]], breaks)]) # five levels
show(head(X, 10))
#> V1 V2 V3
#> 1 A 0.56507611 0.3174938
#> 2 D 0.28025778 0.7678555
#> 3 D 0.20419632 0.5263085
#> 4 D 0.13373890 0.7323019
#> 5 E 0.32568192 0.3076657
#> 6 D 0.15506197 0.4041733
#> 7 A 0.12996214 0.2044024
#> 8 B 0.43553106 0.9856331
#> 9 D 0.03864265 0.5663108
#> 10 C 0.71330156 0.2803751
newX <- as.data.frame(newX)
newX[[1]] <- factor(LETTERS[cut(newX[[1]], breaks)])For fitting I should really retune the span value, but
for speed let’s just crack on.
## fit, don't bother to retune
fit <- loclm(X, y, span = spanhat)
pp <- predict(fit, newX)
par(op)
matplot(newX[, 2], cbind(newy, pp), pch = c(1, 2), cex = cex,
col = col, xlab = "x[2]", ylab = "Actual/predicted",
main = sprintf("First input is a factor"))
legend("topleft", c("actual", "predicted"), pch = 1:2, col = col,
cex = par("cex.axis"), bty = "n")Not a disaster.
predict.loclm() is currently using
future.apply::future_vapply() under the hood. The default
is sequential (one core). If newdata has many rows it will
be faster to use multiple cores. The following code is not run, but the
rough timings are written in the comments.
## parallel processing -- NOT RUN
if (requireNamespace("future", quietly = TRUE)) {
big_newX <- do.call("rbind", rep(list(newX), 1E4))
show(dim(big_newX))
t1 <- system.time(pp1 <- predict(fit, big_newX))
show(t1) # 22 sec
future::plan(future::multisession, workers = 2) # two cores
t2 <- system.time(pp2 <- predict(fit, big_newX))
show(t2) # 11 sec
all.equal(pp1, pp2) # TRUE
future::plan(future::sequential) # single core
}See future::plan() for more details.
In R the generic function hatvalues() accesses the
diagonal of the hat matrix, and therefore
sum(hatvalues(model)) is the effective number of
parameters. In local linear regression this depends on the
span, with span = Inf being linear regression,
when the number of parameters is the number of columns in the model
matrix, which is one plus the number of variables if all variables are
numeric.
## effective number of params
npeff <- sum(hatvalues(fit))
show(npeff)
#> [1] 57.29056
fit_inf <- loclm(X, y, span = Inf)
np_inf <- sum(hatvalues(fit_inf))
show(np_inf) # 7 (hopefully!)
#> [1] 7The banana function is very non-linear, by construction. For linear
regression, np_inf should be seven, because X
has a constant, a five-level factor (four dummy columns), and two
numeric variables.`