Iceberg connector can create tables having difference case fields in row types, but can't read it

[ebyhr]

It would be better to deny when creating tables.

trino:tpch> CREATE TABLE test AS SELECT cast(row(1,2) AS row("X" int, x int)) col;
CREATE TABLE: 1 row

-- ORC
trino:tpch> TABLE test;
io.trino.spi.TrinoException: Error opening Iceberg split /var/folders/8s/dkvf18z55lj_9yxhy1n54sph0000gn/T/TrinoTest14995291286640599612/iceberg_data/tpch/test-71cbb7a701ab43fa9ef9bffe5fc77129/data/20230227_062327_00025_dg69i-beb13022-c1e7-4db3-b9e1-60190e22ceba.orc (offset=0, length=421): Multiple entries with same key: x=3 and x=2
	at io.trino.plugin.iceberg.IcebergPageSourceProvider.createOrcPageSource(IcebergPageSourceProvider.java:699)
	at io.trino.plugin.iceberg.IcebergPageSourceProvider.createDataPageSource(IcebergPageSourceProvider.java:480)
	at io.trino.plugin.iceberg.IcebergPageSourceProvider.createPageSource(IcebergPageSourceProvider.java:296)
	at io.trino.plugin.base.classloader.ClassLoaderSafeConnectorPageSourceProvider.createPageSource(ClassLoaderSafeConnectorPageSourceProvider.java:49)
	at io.trino.split.PageSourceManager.createPageSource(PageSourceManager.java:62)
	at io.trino.operator.TableScanOperator.getOutput(TableScanOperator.java:298)
	at io.trino.operator.Driver.processInternal(Driver.java:394)
	at io.trino.operator.Driver.lambda$process$8(Driver.java:297)
	at io.trino.operator.Driver.tryWithLock(Driver.java:689)
	at io.trino.operator.Driver.process(Driver.java:289)
	at io.trino.operator.Driver.processForDuration(Driver.java:260)
	at io.trino.execution.SqlTaskExecution$DriverSplitRunner.processFor(SqlTaskExecution.java:773)
	at io.trino.execution.executor.PrioritizedSplitRunner.process(PrioritizedSplitRunner.java:165)
	at io.trino.execution.executor.TaskExecutor$TaskRunner.run(TaskExecutor.java:523)
	at io.trino.$gen.Trino_testversion____20230227_062239_36.run(Unknown Source)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.IllegalArgumentException: Multiple entries with same key: x=3 and x=2
	at com.google.common.collect.ImmutableMap.conflictException(ImmutableMap.java:377)
	at com.google.common.collect.ImmutableMap.checkNoConflict(ImmutableMap.java:371)
	at com.google.common.collect.RegularImmutableMap.checkNoConflictInKeyBucket(RegularImmutableMap.java:241)
	at com.google.common.collect.RegularImmutableMap.fromEntryArrayCheckingBucketOverflow(RegularImmutableMap.java:132)
	at com.google.common.collect.RegularImmutableMap.fromEntryArray(RegularImmutableMap.java:94)
	at com.google.common.collect.ImmutableMap$Builder.build(ImmutableMap.java:573)
	at com.google.common.collect.ImmutableMap$Builder.buildOrThrow(ImmutableMap.java:601)
	at com.google.common.collect.ImmutableMap$Builder.build(ImmutableMap.java:588)
	at java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:686)
	at io.trino.plugin.iceberg.IcebergPageSourceProvider$IdBasedFieldMapperFactory.populateMapping(IcebergPageSourceProvider.java:849)
	at io.trino.plugin.iceberg.IcebergPageSourceProvider$IdBasedFieldMapperFactory.<init>(IcebergPageSourceProvider.java:823)
	at io.trino.plugin.iceberg.IcebergPageSourceProvider.createOrcPageSource(IcebergPageSourceProvider.java:657)

-- Parquet
io.trino.spi.TrinoException: Error opening Iceberg split /var/folders/8s/dkvf18z55lj_9yxhy1n54sph0000gn/T/TrinoTest14995291286640599612/iceberg_data/tpch/test-f1021397fddd4b5798961effbe0681fe/data/20230227_062436_00032_dg69i-d12244b7-cd3a-4fef-9136-da9f49d614f0.parquet (offset=0, length=337): Multiple entries with same key: [col, x] optional int32 x = 3=min: 2, max: 2, num_nulls: 0 and [col, x] optional int32 x = 3=min: 1, max: 1, num_nulls: 0
	at io.trino.plugin.iceberg.IcebergPageSourceProvider.createParquetPageSource(IcebergPageSourceProvider.java:1050)
	at io.trino.plugin.iceberg.IcebergPageSourceProvider.createDataPageSource(IcebergPageSourceProvider.java:503)
	at io.trino.plugin.iceberg.IcebergPageSourceProvider.createPageSource(IcebergPageSourceProvider.java:296)
	at io.trino.plugin.base.classloader.ClassLoaderSafeConnectorPageSourceProvider.createPageSource(ClassLoaderSafeConnectorPageSourceProvider.java:49)
	at io.trino.split.PageSourceManager.createPageSource(PageSourceManager.java:62)
	at io.trino.operator.TableScanOperator.getOutput(TableScanOperator.java:298)
	at io.trino.operator.Driver.processInternal(Driver.java:394)
	at io.trino.operator.Driver.lambda$process$8(Driver.java:297)
	at io.trino.operator.Driver.tryWithLock(Driver.java:689)
	at io.trino.operator.Driver.process(Driver.java:289)
	at io.trino.operator.Driver.processForDuration(Driver.java:260)
	at io.trino.execution.SqlTaskExecution$DriverSplitRunner.processFor(SqlTaskExecution.java:773)
	at io.trino.execution.executor.PrioritizedSplitRunner.process(PrioritizedSplitRunner.java:165)
	at io.trino.execution.executor.TaskExecutor$TaskRunner.run(TaskExecutor.java:523)
	at io.trino.$gen.Trino_testversion____20230227_062239_36.run(Unknown Source)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.IllegalArgumentException: Multiple entries with same key: [col, x] optional int32 x = 3=min: 2, max: 2, num_nulls: 0 and [col, x] optional int32 x = 3=min: 1, max: 1, num_nulls: 0
	at com.google.common.collect.ImmutableMap.conflictException(ImmutableMap.java:377)
	at com.google.common.collect.ImmutableMap.checkNoConflict(ImmutableMap.java:371)
	at com.google.common.collect.RegularImmutableMap.checkNoConflictInKeyBucket(RegularImmutableMap.java:241)
	at com.google.common.collect.RegularImmutableMap.fromEntryArrayCheckingBucketOverflow(RegularImmutableMap.java:132)
	at com.google.common.collect.RegularImmutableMap.fromEntryArray(RegularImmutableMap.java:94)
	at com.google.common.collect.ImmutableMap$Builder.build(ImmutableMap.java:573)
	at com.google.common.collect.ImmutableMap$Builder.buildOrThrow(ImmutableMap.java:601)
	at io.trino.parquet.predicate.PredicateUtils.getStatistics(PredicateUtils.java:186)
	at io.trino.parquet.predicate.PredicateUtils.predicateMatches(PredicateUtils.java:142)
	at io.trino.plugin.iceberg.IcebergPageSourceProvider.createParquetPageSource(IcebergPageSourceProvider.java:933)
	... 17 more

-- Avro
    col
------------
 {X=1, x=2}

[findepi]

Does Iceberg with Spark allow creation of such tables?

[ebyhr]

Spark (catalyst) disallows creating such a table:

spark-sql (default)> create table test_iceberg using iceberg as select named_struct('x', 1, 'X', 'Field2') a;
[COLUMN_ALREADY_EXISTS] The column `a`.`x` already exists. Consider to choose another name or rename the existing column.

https://github.com/apache/spark/blob/f0b7cfa56cb90ef70132d9656299956cbde00b53/sql/catalyst/src/main/scala/org/apache/spark/sql/util/SchemaUtils.scala#L107-L117