From 06927206004716f26686039be6584ccafc5d68f3 Mon Sep 17 00:00:00 2001 From: brijrajk <22271048+brijrajk@users.noreply.github.com> Date: Fri, 26 Jun 2026 20:07:55 +0530 Subject: [PATCH] [GLUTEN-11539][VL] Improve error message when spark.io.compression.codec=none Emit a unified error for all unsupported codecs that names the offending codec, hints at spark.shuffle.compress=false to disable compression entirely, and lists the valid codec choices. Co-Authored-By: Claude Sonnet 4.6 --- .../org/apache/gluten/execution/MiscOperatorSuite.scala | 7 ++++--- .../org/apache/spark/shuffle/GlutenShuffleUtils.scala | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala index fcf1fbbbf77..51cb918aa61 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala @@ -2257,16 +2257,17 @@ class MiscOperatorSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa val ex = intercept[IllegalArgumentException] { GlutenShuffleUtils.getCompressionCodec(conf) } - assert(ex.getMessage.contains("snappy is not supported")) + assert(ex.getMessage.contains("does not support codec 'snappy'")) + assert(ex.getMessage.contains("spark.shuffle.compress=false")) assert(ex.getMessage.contains(GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key)) } - test("GLUTEN-11539: spark.io.compression.codec=none throws with actionable message") { + test("GLUTEN-11539: spark.io.compression.codec=none throws pointing to spark.shuffle.compress") { val conf = spark.sparkContext.getConf.clone().set("spark.io.compression.codec", "none") val ex = intercept[IllegalArgumentException] { GlutenShuffleUtils.getCompressionCodec(conf) } - assert(ex.getMessage.contains("none is not supported")) + assert(ex.getMessage.contains("spark.shuffle.compress=false")) assert(ex.getMessage.contains(GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key)) } diff --git a/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala b/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala index 4bd89582c18..d7d8d4cbd9c 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala @@ -77,9 +77,9 @@ object GlutenShuffleUtils { val supportedCodecs = BackendsApiManager.getSettings.shuffleSupportedCodec() if (!supportedCodecs.contains(codec)) { throw new IllegalArgumentException( - s"Gluten shuffle only supports ${supportedCodecs.mkString(", ")}. " + - s"$codec is not supported. " + - s"You may configure ${GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key} " + + s"Gluten shuffle does not support codec '$codec'. " + + s"To disable shuffle compression, set spark.shuffle.compress=false. " + + s"To use a supported codec, set ${GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key} " + s"to ${supportedCodecs.mkString(" or ")}.") } codec