[bug] java.lang.ClassCastException: class org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema cannot be cast to class org.apache.avro.generic.IndexedRecord

[lev112]

I got this error after upgrading from 3.0.0 to 3.1.1.
the error happens while writing into a snowflake table with a DataframeWriter

driver stack trace:

org.apache.spark.rdd.RDD.collect(RDD.scala:1048)
net.snowflake.spark.snowflake.io.CloudStorage.uploadRDD(CloudStorageOperations.scala:801)
net.snowflake.spark.snowflake.io.CloudStorage.uploadRDD$(CloudStorageOperations.scala:751)
net.snowflake.spark.snowflake.io.InternalS3Storage.uploadRDD(CloudStorageOperations.scala:1305)
net.snowflake.spark.snowflake.io.CloudStorage.upload(CloudStorageOperations.scala:531)
net.snowflake.spark.snowflake.io.CloudStorage.upload$(CloudStorageOperations.scala:526)
net.snowflake.spark.snowflake.io.InternalS3Storage.upload(CloudStorageOperations.scala:1305)
net.snowflake.spark.snowflake.io.StageWriter$.writeToStage(StageWriter.scala:219)
net.snowflake.spark.snowflake.io.package$.writeRDD(package.scala:50)
net.snowflake.spark.snowflake.SnowflakeWriter.save(SnowflakeWriter.scala:110)
net.snowflake.spark.snowflake.DefaultSource.createRelation(DefaultSource.scala:113)
org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:48)
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:75)
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:73)
org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:84)
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:107)
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$6(SQLExecution.scala:125)
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:201)
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:108)
org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:900)

executor stack trace:

Job aborted due to stage failure: Task 11 in stage 257.0 failed 1 times, most recent failure: Lost task 11.0 in stage 257.0 (TID 11297) (ip-xxxxxx.eu-west-1.compute.internal executor driver): java.lang.ClassCastException: class org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema cannot be cast to class org.apache.avro.generic.IndexedRecord (org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema and org.apache.avro.generic.IndexedRecord are in unnamed module of loader 'app')
	at org.apache.avro.generic.GenericData.getField(GenericData.java:858)
	at org.apache.parquet.avro.AvroWriteSupport.writeRecordFields(AvroWriteSupport.java:201)
	at org.apache.parquet.avro.AvroWriteSupport.writeRecord(AvroWriteSupport.java:186)
	at org.apache.parquet.avro.AvroWriteSupport.writeValueWithoutConversion(AvroWriteSupport.java:366)
	at org.apache.parquet.avro.AvroWriteSupport.writeValue(AvroWriteSupport.java:291)
	at org.apache.parquet.avro.AvroWriteSupport.access$400(AvroWriteSupport.java:54)
	at org.apache.parquet.avro.AvroWriteSupport$ThreeLevelListWriter.writeObjectArray(AvroWriteSupport.java:685)
	at org.apache.parquet.avro.AvroWriteSupport$ListWriter.writeJavaArray(AvroWriteSupport.java:459)
	at org.apache.parquet.avro.AvroWriteSupport$ListWriter.writeList(AvroWriteSupport.java:449)
	at org.apache.parquet.avro.AvroWriteSupport.writeValueWithoutConversion(AvroWriteSupport.java:372)
	at org.apache.parquet.avro.AvroWriteSupport.writeValue(AvroWriteSupport.java:291)
	at org.apache.parquet.avro.AvroWriteSupport.writeRecordFields(AvroWriteSupport.java:204)
	at org.apache.parquet.avro.AvroWriteSupport.write(AvroWriteSupport.java:178)
	at org.apache.parquet.hadoop.InternalParquetRecordWriter.write(InternalParquetRecordWriter.java:138)
	at org.apache.parquet.hadoop.ParquetWriter.write(ParquetWriter.java:310)
	at net.snowflake.spark.snowflake.io.CloudStorage.$anonfun$doUploadPartition$2(CloudStorageOperations.scala:681)
	at net.snowflake.spark.snowflake.io.CloudStorage.$anonfun$doUploadPartition$2$adapted(CloudStorageOperations.scala:679)
	at scala.collection.Iterator.foreach(Iterator.scala:943)
	at scala.collection.Iterator.foreach$(Iterator.scala:943)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
	at net.snowflake.spark.snowflake.io.CloudStorage.doUploadPartition(CloudStorageOperations.scala:679)
	at net.snowflake.spark.snowflake.io.CloudStorage.uploadPartition(CloudStorageOperations.scala:577)
	at net.snowflake.spark.snowflake.io.CloudStorage.uploadPartition$(CloudStorageOperations.scala:559)
	at net.snowflake.spark.snowflake.io.InternalS3Storage.uploadPartition(CloudStorageOperations.scala:1305)
	at net.snowflake.spark.snowflake.io.CloudStorage.$anonfun$uploadRDD$2(CloudStorageOperations.scala:794)
	at net.snowflake.spark.snowflake.io.CloudStorage.$anonfun$uploadRDD$2$adapted(CloudStorageOperations.scala:784)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsWithIndex$2(RDD.scala:910)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsWithIndex$2$adapted(RDD.scala:910)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
	at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
	at org.apache.spark.scheduler.Task.run(Task.scala:141)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:620)
	at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
	at org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:623)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
	at java.base/java.lang.Thread.run(Thread.java:829)