diff --git a/R/position-stack.R b/R/position-stack.R
index 6d77269a1f..de23456625 100644
--- a/R/position-stack.R
+++ b/R/position-stack.R
@@ -153,8 +153,14 @@ PositionStack <- ggproto("PositionStack", Position,
   setup_params = function(self, data) {
     flipped_aes <- has_flipped_aes(data)
     data <- flip_data(data, flipped_aes)
+    var <- self$var %||% stack_var(data)
+    if (!vec_duplicate_any(data$x)) {
+      # We skip stacking when all data have different x positions so that
+      # there is nothing to stack
+      var <- NULL
+    }
     list(
-      var = self$var %||% stack_var(data),
+      var = var,
       fill = self$fill,
       vjust = self$vjust,
       reverse = self$reverse,
@@ -185,10 +191,6 @@ PositionStack <- ggproto("PositionStack", Position,
     if (is.null(params$var)) {
       return(data)
     }
-    if (!vec_duplicate_any(data$x)) {
-      # Every x is unique, nothing to stack here
-      return(flip_data(data, params$flipped_aes))
-    }
 
     negative <- data$ymax < 0
     negative[is.na(negative)] <- FALSE
diff --git a/tests/testthat/test-position-stack.R b/tests/testthat/test-position-stack.R
index 6a9353a300..54c6b858e8 100644
--- a/tests/testthat/test-position-stack.R
+++ b/tests/testthat/test-position-stack.R
@@ -25,6 +25,12 @@ test_that("negative and positive values are handled separately", {
 
   expect_equal(dat$ymin[dat$x == 2], c(0, -3))
   expect_equal(dat$ymax[dat$x == 2], c(2, 0))
+
+  # Test only negatives #6088
+  df <- data_frame0(x = c(1, 1, 2, 2), y = c(-1, -1, -1, -1), g = LETTERS[1:4])
+  dat <- get_layer_data(ggplot(df, aes(x, y, fill = factor(g))) + geom_col())
+  expect_equal(dat$ymax, dat$ymin + 1)
+  expect_equal(dat$ymin, c(-2, -1, -2, -1))
 })
 
 test_that("can request reverse stacking", {