diff --git a/core/src/main/java/org/apache/calcite/sql/validate/implicit/AbstractTypeCoercion.java b/core/src/main/java/org/apache/calcite/sql/validate/implicit/AbstractTypeCoercion.java
index 7e442fc9f55..167d769f85c 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/implicit/AbstractTypeCoercion.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/implicit/AbstractTypeCoercion.java
@@ -289,6 +289,16 @@ protected boolean needToCast(SqlValidatorScope scope, SqlNode node,
       return false;
     }
 
+    // No casts to binary except from strings
+    if (SqlTypeUtil.isBinary(fromType) && !SqlTypeUtil.isString(toType)) {
+      return false;
+    }
+
+    // No casts from binary except to strings
+    if (SqlTypeUtil.isBinary(toType) && !SqlTypeUtil.isString(fromType)) {
+      return false;
+    }
+
     // Implicit type coercion does not handle nullability.
     if (SqlTypeUtil.equalSansNullability(factory, fromType, toType)) {
       return false;
diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
index 9f4a2da5636..f17ceb2a7bd 100644
--- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
+++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
@@ -1465,6 +1465,26 @@ void testCastToBoolean(CastType castType, SqlOperatorFixture f) {
     f.checkNull("cast(null as row(f0 varchar, f1 varchar))");
   }
 
+  /** Test case for
+   * <a href="https://issues.apache.org/jira/projects/CALCITE/issues/CALCITE-6095">
+   * [CALCITE-6095] Arithmetic expression with VARBINARY value causes AssertionFailure</a>.
+   */
+  @Test public void testVarbitArithmetic() {
+    SqlOperatorFixture f = fixture();
+    String error = "Cannot apply '\\+' to arguments of type .*\\."
+        + " Supported form\\(s\\): '<NUMERIC> \\+ <NUMERIC>'\n"
+        + "'<DATETIME_INTERVAL> \\+ <DATETIME_INTERVAL>'\n"
+        + "'<DATETIME> \\+ <DATETIME_INTERVAL>'\n"
+        + "'<DATETIME_INTERVAL> \\+ <DATETIME>'";
+    f.checkFails("SELECT ^x'31' + 0^", error, false);
+    f.checkFails("SELECT ^x'31' + x'31'^", error, false);
+    f.checkFails("SELECT ^0 + x'31'^", error, false);
+    f.checkFails("SELECT ^'a' + x'31'^", error, false);
+    f.checkFails("SELECT ^0.0 + x'31'^", error, false);
+    f.checkFails("SELECT ^0e0 + x'31'^", error, false);
+    f.checkFails("SELECT ^DATE '2000-01-01' + x'31'^", error, false);
+  }
+
   /** Test case for
    * <a href="https://issues.apache.org/jira/browse/CALCITE-4861">[CALCITE-4861]
    * Optimization of chained CAST calls leads to unexpected behavior</a>. */