Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update Iceberg version to 1.8.1 #25094

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Update Iceberg version to 1.8.1 #25094

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ public static IcebergFileFormat fromIceberg(FileFormat format)
case AVRO -> AVRO;
// Not used as a data file format
case METADATA -> throw new IllegalArgumentException("Unexpected METADATA file format");
case PUFFIN -> throw new IllegalArgumentException("Unexpected PUFFIN file format");
};
}

Expand Down
6 changes: 3 additions & 3 deletions plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1484,7 +1484,7 @@ else if (!computedStatistics.isEmpty()) {
INCREMENTAL_UPDATE,
collectedStatistics);
transaction.updateStatistics()
.setStatistics(newSnapshotId, statisticsFile)
.setStatistics(statisticsFile)
.commit();

commitTransaction(transaction, "update statistics on insert");
Expand Down Expand Up @@ -1996,7 +1996,7 @@ private void finishOptimize(ConnectorSession session, IcebergTableExecuteHandle
StatisticsFile newStatsFile = tableStatisticsWriter.rewriteStatisticsFile(session, reloadedTable, newSnapshotId);

transaction.updateStatistics()
.setStatistics(newSnapshotId, newStatsFile)
.setStatistics(newStatsFile)
.commit();
commitTransaction(transaction, "update statistics after optimize");
}
Expand Down Expand Up @@ -2944,7 +2944,7 @@ public void finishStatisticsCollection(ConnectorSession session, ConnectorTableH
REPLACE,
collectedStatistics);
transaction.updateStatistics()
.setStatistics(snapshotId, statisticsFile)
.setStatistics(statisticsFile)
.commit();

commitTransaction(transaction, "statistics collection");
Expand Down
3 changes: 3 additions & 0 deletions plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/PartitionData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,9 @@ public static Object getValue(JsonNode partitionValue, Type type)
createDecimalType(decimalType.precision(), decimalType.scale()));
// TODO https://github.com/trinodb/trino/issues/19753 Support Iceberg timestamp types with nanosecond precision
case TIMESTAMP_NANO:
// TODO https://github.com/trinodb/trino/issues/24538 Support variant type
case VARIANT:
case UNKNOWN:
case LIST:
case MAP:
case STRUCT:
Expand Down
5 changes: 5 additions & 0 deletions plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TypeConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,11 @@ public static Type toTrinoType(org.apache.iceberg.types.Type type, TypeManager t
return RowType.from(fields.stream()
.map(field -> new RowType.Field(Optional.of(field.name()), toTrinoType(field.type(), typeManager)))
.collect(toImmutableList()));
case VARIANT:
// TODO https://github.com/trinodb/trino/issues/24538 Support variant type
break;
case UNKNOWN:
break;
}
throw new UnsupportedOperationException(format("Cannot convert from Iceberg type '%s' (%s) to Trino type", type, type.typeId()));
}
Expand Down
3 changes: 3 additions & 0 deletions plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/HiveSchemaUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@ private static String convertToTypeString(Type type)
case TIMESTAMP_NANO -> throw new TrinoException(NOT_SUPPORTED, "Unsupported Iceberg type: TIMESTAMP_NANO");
case FIXED, BINARY -> "binary";
case DECIMAL -> "decimal(%s,%s)".formatted(((DecimalType) type).precision(), ((DecimalType) type).scale());
case UNKNOWN -> throw new TrinoException(NOT_SUPPORTED, "Unsupported Iceberg type: UNKNOWN");
// TODO https://github.com/trinodb/trino/issues/24538 Support variant type
case VARIANT -> throw new TrinoException(NOT_SUPPORTED, "Unsupported Iceberg type: VARIANT");
case LIST -> "array<%s>".formatted(convert(type.asListType().elementType()));
case MAP -> "map<%s,%s>".formatted(convert(type.asMapType().keyType()), convert(type.asMapType().valueType()));
case STRUCT -> "struct<%s>".formatted(type.asStructType().fields().stream()
Expand Down
2 changes: 2 additions & 0 deletions plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/OrcTypeConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,8 @@ private static List<OrcType> toOrcType(int nextFieldTypeIndex, Type type, Map<St
.buildOrThrow();
yield ImmutableList.of(new OrcType(OrcTypeKind.BINARY, ImmutableList.of(), ImmutableList.of(), Optional.empty(), Optional.empty(), Optional.empty(), attributes));
}
case VARIANT -> throw new TrinoException(NOT_SUPPORTED, "Unsupported Iceberg type: VARIANT");
case UNKNOWN -> throw new TrinoException(NOT_SUPPORTED, "Unsupported Iceberg type: UNKNOWN");
case STRUCT -> toOrcStructType(nextFieldTypeIndex, (StructType) type, attributes);
case LIST -> toOrcListType(nextFieldTypeIndex, (ListType) type, attributes);
case MAP -> toOrcMapType(nextFieldTypeIndex, (MapType) type, attributes);
Expand Down
2 changes: 1 addition & 1 deletion plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergStatistics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ public void testAnalyzeWithSchemaEvolution()
double infoDataSize = (double) computeActual("SHOW STATS FOR " + tableName).getMaterializedRows().stream()
.filter(row -> "info".equals(row.getField(0)))
.collect(onlyElement()).getField(1);
assertThat(infoDataSize).isBetween(2000.0, 4000.0);
assertThat(infoDataSize).isBetween(2000.0, 5000.0);
assertQuery(
"SHOW STATS FOR " + tableName,
"""
Expand Down
18 changes: 11 additions & 7 deletions plugin/trino-iceberg/src/test/java/org/apache/iceberg/rest/RestCatalogServlet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import org.apache.iceberg.exceptions.RESTException;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.io.CharStreams;
import org.apache.iceberg.rest.RESTCatalogAdapter.HTTPMethod;
import org.apache.iceberg.rest.HTTPRequest.HTTPMethod;
import org.apache.iceberg.rest.RESTCatalogAdapter.Route;
import org.apache.iceberg.rest.responses.ErrorResponse;
import org.apache.iceberg.util.Pair;
Expand Down Expand Up @@ -98,15 +98,19 @@ protected void execute(ServletRequestContext context, HttpServletResponse respon
return;
}

HTTPRequest request = restCatalogAdapter.buildRequest(
context.method(),
context.path(),
context.queryParams(),
context.headers(),
context.body());

try {
Object responseBody = restCatalogAdapter.execute(
context.method(),
context.path(),
context.queryParams(),
context.body(),
request,
context.route().responseClass(),
context.headers(),
handle(response));
handle(response),
_ -> {});

if (responseBody != null) {
RESTObjectMapper.mapper().writeValue(response.getWriter(), responseBody);
Expand Down
10 changes: 9 additions & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@
<dep.gib.version>4.5.4</dep.gib.version>
<dep.google.http.client.version>1.46.1</dep.google.http.client.version>
<dep.httpcore5.version>5.3.3</dep.httpcore5.version>
<dep.iceberg.version>1.7.1</dep.iceberg.version>
<dep.iceberg.version>1.8.1</dep.iceberg.version>
<dep.jna.version>5.16.0</dep.jna.version>
<dep.joda.version>2.13.1</dep.joda.version>
<dep.jsonwebtoken.version>0.12.6</dep.jsonwebtoken.version>
Expand Down Expand Up @@ -2380,6 +2380,14 @@
</dependencies>
</dependencyManagement>

<repositories>
<repository>
<id>iceberg-release-candidate</id>
<name>Iceberg Release Candidate</name>
<url>https://repository.apache.org/content/repositories/orgapacheiceberg-1184/</url>
</repository>
</repositories>

Comment on lines +2383 to +2390
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will revert this change before merging this PR.

<build>
<pluginManagement>
<plugins>
Expand Down
2 changes: 1 addition & 1 deletion ...tainers/src/main/java/io/trino/testing/containers/IcebergRestCatalogBackendContainer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ public IcebergRestCatalogBackendContainer(
String minioSessionToken)
{
super(
"tabulario/iceberg-rest:1.5.0",
"tabulario/iceberg-rest:1.6.0",
"iceberg-rest",
ImmutableSet.of(8181),
ImmutableMap.of(),
Expand Down