diff --git a/NEWS.md b/NEWS.md
index 9e8d26f28b..f2879bd341 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -69,6 +69,7 @@
 * A stacking bug in `stat_align()` was fixed (@teunbrand, #5176).
 * `stat_contour()` and `stat_contour_filled()` now warn about and remove
   duplicated coordinates (@teunbrand, #5215).
+* Improve performance of layers without positional scales (@zeehio, #4990)
 
 # ggplot2 3.4.2
 This is a hotfix release anticipating changes in r-devel, but folds in upkeep
diff --git a/R/facet-.R b/R/facet-.R
index 2f92697d25..46e5a1c61f 100644
--- a/R/facet-.R
+++ b/R/facet-.R
@@ -101,20 +101,27 @@ Facet <- ggproto("Facet", NULL,
   train_scales = function(x_scales, y_scales, layout, data, params) {
     # loop over each layer, training x and y scales in turn
     for (layer_data in data) {
-      match_id <- match(layer_data$PANEL, layout$PANEL)
+      match_id <- NULL
 
       if (!is.null(x_scales)) {
         x_vars <- intersect(x_scales[[1]]$aesthetics, names(layer_data))
-        SCALE_X <- layout$SCALE_X[match_id]
-
-        scale_apply(layer_data, x_vars, "train", SCALE_X, x_scales)
+        if (length(x_vars) > 0) {
+          match_id <- match(layer_data$PANEL, layout$PANEL)
+          SCALE_X <- layout$SCALE_X[match_id]
+          scale_apply(layer_data, x_vars, "train", SCALE_X, x_scales)
+        }
       }
 
       if (!is.null(y_scales)) {
         y_vars <- intersect(y_scales[[1]]$aesthetics, names(layer_data))
-        SCALE_Y <- layout$SCALE_Y[match_id]
-
-        scale_apply(layer_data, y_vars, "train", SCALE_Y, y_scales)
+        if (length(y_vars) > 0) {
+          if (is.null(match_id)) {
+            match_id <- match(layer_data$PANEL, layout$PANEL)
+          }
+          SCALE_Y <- layout$SCALE_Y[match_id]
+
+          scale_apply(layer_data, y_vars, "train", SCALE_Y, y_scales)
+        }
       }
     }
   },
diff --git a/R/layout.R b/R/layout.R
index 56841b647b..6e2124e8be 100644
--- a/R/layout.R
+++ b/R/layout.R
@@ -151,21 +151,29 @@ Layout <- ggproto("Layout", NULL,
     layout <- self$layout
 
     lapply(data, function(layer_data) {
-      match_id <- match(layer_data$PANEL, layout$PANEL)
+      match_id <- NULL
 
       # Loop through each variable, mapping across each scale, then joining
       # back together
       x_vars <- intersect(self$panel_scales_x[[1]]$aesthetics, names(layer_data))
-      names(x_vars) <- x_vars
-      SCALE_X <- layout$SCALE_X[match_id]
-      new_x <- scale_apply(layer_data, x_vars, "map", SCALE_X, self$panel_scales_x)
-      layer_data[, x_vars] <- new_x
+      if (length(x_vars) > 0) {
+        match_id <- match(layer_data$PANEL, layout$PANEL)
+        names(x_vars) <- x_vars
+        SCALE_X <- layout$SCALE_X[match_id]
+        new_x <- scale_apply(layer_data, x_vars, "map", SCALE_X, self$panel_scales_x)
+        layer_data[, x_vars] <- new_x
+      }
 
       y_vars <- intersect(self$panel_scales_y[[1]]$aesthetics, names(layer_data))
-      names(y_vars) <- y_vars
-      SCALE_Y <- layout$SCALE_Y[match_id]
-      new_y <- scale_apply(layer_data, y_vars, "map", SCALE_Y, self$panel_scales_y)
-      layer_data[, y_vars] <- new_y
+      if (length(y_vars) > 0) {
+        if (is.null(match_id)) {
+          match_id <- match(layer_data$PANEL, layout$PANEL)
+        }
+        names(y_vars) <- y_vars
+        SCALE_Y <- layout$SCALE_Y[match_id]
+        new_y <- scale_apply(layer_data, y_vars, "map", SCALE_Y, self$panel_scales_y)
+        layer_data[, y_vars] <- new_y
+      }
 
       layer_data
     })