[fix][dingo-calcite] Support operations on tables without primary keys

guojn1 · guojn1 · commit 0e04da29402b · 2025-09-18T09:52:46.000+08:00
diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/DingoRelOptTable.java b/dingo-calcite/src/main/java/io/dingodb/calcite/DingoRelOptTable.java
@@ -92,23 +92,6 @@ public double getRowCount() {
 
     @Override
     public RelDataType getRowType() {
-        StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
-        StackTraceElement stackTraceElement = null;
-        if (stackTrace.length >= 2) {
-            stackTraceElement = stackTrace[2];
-        }
-        DingoTable table = relOptTable.unwrap(DingoTable.class);
-        if (table == null) {
-            throw new RuntimeException("getRowType exception, table is null");
-        }
-        boolean withoutPri =
-            table.getTable().getColumns().stream().anyMatch(col -> col.getState() == 2);
-        if (!withoutPri) {
-            return relOptTable.getRowType();
-        }
-        if (withOutPrimaryRowType(stackTraceElement)) {
-            return getWithoutPriRowType();
-        }
         return relOptTable.getRowType();
     }
 
diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/DingoSqlValidator.java b/dingo-calcite/src/main/java/io/dingodb/calcite/DingoSqlValidator.java
@@ -70,6 +70,7 @@
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
+import static io.dingodb.common.environment.ExecutionEnvironment.IMPLICT_NAME;
 import static org.apache.calcite.util.Static.RESOURCE;
 
 public class DingoSqlValidator extends SqlValidatorImpl {
@@ -338,7 +339,11 @@ public void checkTypeAssignment(
 
         // Fall back to default behavior: compare the type families.
         for (int i = 0; i < sourceCount; ++i) {
-            RelDataType sourceType = sourceFields.get(i).getType();
+            RelDataTypeField sourceTypeField = sourceFields.get(i);
+            if (IMPLICT_NAME.equals(sourceTypeField.getName())) {
+                continue;
+            }
+            RelDataType sourceType = sourceTypeField.getType();
             RelDataType targetType = targetFields.get(i).getType();
             if (!SqlTypeUtil.canAssignFrom(targetType, sourceType)) {
                 SqlNode node = getNthExpr(query, i, sourceCount);
diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/executor/LoadDataExecutor.java b/dingo-calcite/src/main/java/io/dingodb/calcite/executor/LoadDataExecutor.java
@@ -107,7 +107,6 @@ public class LoadDataExecutor implements DmlExecutor {
     private volatile String errMessage;
     private Table table;
     private final KeyValueCodec codec;
-    private NavigableMap<ByteArrayUtils.ComparableByteArray, RangeDistribution> distributions;
     private final DingoType schema;
 
     MetaService metaService;
@@ -183,7 +182,6 @@ public LoadDataExecutor(
         codec = CodecService.getDefault().createKeyValueCodec(
             table.getCodecVersion(), table.version, table.tupleType(), table.keyMapping()
         );
-        distributions = metaService.getRangeDistribution(table.tableId);
         schema = table.tupleType();
         this.isTxn = checkEngine();
         this.statementId = UUID.randomUUID().toString();
@@ -445,9 +443,7 @@ private void insertTuples(Object[] tuples) {
 
     public void insertWithoutTxn(Object[] tuples, boolean retry) {
         try {
-            if (retry) {
-                distributions = metaService.getRangeDistribution(table.tableId);
-            }
+            NavigableMap<ByteArrayUtils.ComparableByteArray, RangeDistribution> distributions = metaService.getRangeDistribution(table.tableId);
             CommonId partId = PartitionService.getService(
                     Optional.ofNullable(table.getPartitionStrategy())
                         .orElse(DingoPartitionServiceProvider.RANGE_FUNC_NAME))
diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/schema/SubCalciteSchema.java b/dingo-calcite/src/main/java/io/dingodb/calcite/schema/SubCalciteSchema.java
@@ -68,19 +68,9 @@ protected SubCalciteSchema(
 
     @Override
     public @Nullable TableEntry getImplicitTable(String tableName, boolean caseSensitive) {
-        StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
-        boolean forValidate = false;
-        if (stackTrace.length > 19) {
-            forValidate = stackTrace[19].getMethodName().equalsIgnoreCase("validate");
-        }
         SubSnapshotSchema subSnapshotSchema = (SubSnapshotSchema) schema;
         boolean inTxn = subSnapshotSchema.inTransaction();
-        Table table;
-        if (forValidate) {
-            table = ((SubSnapshotSchema) schema).getValidateTable(tableName);
-        } else {
-            table = schema.getTable(tableName);
-        }
+        Table table = schema.getTable(tableName);
         if (table != null && inTxn) {
             DingoTable dingoTable = (DingoTable) table;
             rootCalciteSchema.putRelatedTable(dingoTable.getTableId().seq, ((SubSnapshotSchema) schema).getSchemaVer());
diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/utils/RuleUtils.java b/dingo-calcite/src/main/java/io/dingodb/calcite/utils/RuleUtils.java
@@ -18,7 +18,6 @@
 
 import io.dingodb.calcite.type.DingoSqlTypeFactory;
 import io.dingodb.expr.runtime.utils.DateTimeUtils;
-import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexCall;
 import org.apache.calcite.rex.RexInputRef;
@@ -34,10 +33,9 @@
 import org.checkerframework.checker.nullness.qual.Nullable;
 
 import java.sql.Date;
-import java.sql.Time;
 import java.sql.Timestamp;
 import java.util.Calendar;
-import java.util.Objects;
+import java.util.TimeZone;
 
 public class RuleUtils {
 
@@ -114,6 +112,7 @@ private static boolean checkConditionOp(@NonNull RexNode op0, RexNode op1, Condi
                         }
                         Calendar calendar = Calendar.getInstance();
                         calendar.setTimeInMillis(timestamp.getTime());
+                        calendar.setTimeZone(TimeZone.getTimeZone("UTC"));
                         TimestampString timestampString = TimestampString.fromCalendarFields(calendar);
                         info.index = ((RexInputRef) op0).getIndex();
                         info.value = rexBuilder.makeTimestampLiteral(timestampString, 19);
diff --git a/dingo-calcite/src/main/java/org/apache/calcite/sql/validate/implicit/DingoTypeCoercionImpl.java b/dingo-calcite/src/main/java/org/apache/calcite/sql/validate/implicit/DingoTypeCoercionImpl.java
@@ -45,6 +45,7 @@
 import java.util.List;
 
 import static io.dingodb.calcite.DingoTypeMapper.getAggregateResultType;
+import static io.dingodb.common.environment.ExecutionEnvironment.IMPLICT_NAME;
 import static org.apache.calcite.linq4j.Nullness.castNonNull;
 
 @Slf4j
@@ -59,7 +60,11 @@ public DingoTypeCoercionImpl(RelDataTypeFactory typeFactory, SqlValidator valida
         final List<RelDataTypeField> targetFields = targetRowType.getFieldList();
         final int sourceCount = sourceFields.size();
         for (int i = 0; i < sourceCount; i++) {
-            RelDataType sourceType = sourceFields.get(i).getType();
+            RelDataTypeField sourceTypeField = sourceFields.get(i);
+            if (IMPLICT_NAME.equals(sourceTypeField.getName())) {
+                continue;
+            }
+            RelDataType sourceType = sourceTypeField.getType();
             RelDataType targetType = targetFields.get(i).getType();
             if (!SqlTypeUtil.equalSansNullability(validator.getTypeFactory(), sourceType, targetType)
                 && !SqlTypeUtil.canCastFrom(targetType, sourceType, true)) {
@@ -83,6 +88,10 @@ public DingoTypeCoercionImpl(RelDataTypeFactory typeFactory, SqlValidator valida
         }
         boolean coerced = false;
         for (int i = 0; i < sourceFields.size(); i++) {
+            RelDataTypeField sourceTypeField = sourceFields.get(i);
+            if (IMPLICT_NAME.equals(sourceTypeField.getName())) {
+                continue;
+            }
             RelDataType targetType = targetFields.get(i).getType();
             coerced = coerceSourceRowType1(scope, query, i, targetType) || coerced;
         }
diff --git a/dingo-codec-api/src/main/java/io/dingodb/codec/KeyValueCodec.java b/dingo-codec-api/src/main/java/io/dingodb/codec/KeyValueCodec.java
@@ -17,6 +17,7 @@
 package io.dingodb.codec;
 
 import io.dingodb.common.store.KeyValue;
+import io.dingodb.common.type.DingoType;
 import org.checkerframework.checker.nullness.qual.NonNull;
 
 public interface KeyValueCodec {
@@ -72,4 +73,6 @@ public interface KeyValueCodec {
     default Object[] mapKeyAndDecodeValue(Object[] keys, byte[] bytes) {
         throw new UnsupportedOperationException();
     }
+
+    default DingoType getDingoType() { return null; }
 }
diff --git a/dingo-common/src/main/java/io/dingodb/common/environment/ExecutionEnvironment.java b/dingo-common/src/main/java/io/dingodb/common/environment/ExecutionEnvironment.java
@@ -46,6 +46,7 @@ public class ExecutionEnvironment {
     public MdlCheckTableInfo mdlCheckTableInfo = new MdlCheckTableInfo();
 
     public ClientIdentity clientIdentity = new ClientIdentity();
+    public static final String IMPLICT_NAME = "_ROWID";
 
     public volatile long safePointTs;
 
diff --git a/dingo-common/src/main/java/io/dingodb/common/type/DingoType.java b/dingo-common/src/main/java/io/dingodb/common/type/DingoType.java
@@ -119,4 +119,8 @@ public interface DingoType extends CompileContext {
     default void setHidden(boolean hidden){};
 
     default boolean isHidden() { return false; };
+
+    default void setCheckFieldCount(boolean check) {};
+
+    default boolean checkFieldCount() { return true; }
 }
diff --git a/dingo-common/src/main/java/io/dingodb/common/type/TupleType.java b/dingo-common/src/main/java/io/dingodb/common/type/TupleType.java
@@ -44,6 +44,18 @@ public class TupleType extends AbstractDingoType {
     @Getter
     private final Type type;
 
+    // create table b(id int,age int);
+    // When executing insert into tab select * from b on a table without a primary key,
+    // the hidden primary key is found but the target table does not have the relevant schema.
+    // In this case, the fieldCount and tuple. length are inconsistent, No need to check
+    // example: TxnPartInsertOperator,TxnPartInsertIgnoreOperator
+    public boolean checkFieldCount;
+
+    @Override
+    public void setCheckFieldCount(boolean checkFieldCount) {
+        this.checkFieldCount = checkFieldCount;
+    }
+
     @JsonCreator
     public TupleType(
         @JsonProperty("fields") DingoType[] fields
@@ -65,6 +77,7 @@ public TupleType(
                 }
             }
         ).toArray(Type[]::new));
+        this.checkFieldCount = true;
     }
 
     private void setElementIds() {
@@ -96,7 +109,15 @@ public int fieldCount() {
 
     @Override
     public DingoType getChild(@NonNull Object index) {
-        return fields[(int) index];
+        int idx = (int) index;
+        if (idx < fieldCount()) {
+            return fields[(int) index];
+        } else {
+            // When executing insert into tab select * from without PrimaryTab on a table without a primary key,
+            // the hidden primary key is found but the target table does not have the relevant schema
+            // set type = null
+            return null;
+        }
     }
 
     @Override
@@ -169,11 +190,16 @@ public String toString() {
     }
 
     private Object @NonNull [] checkFieldCount(Object @NonNull [] tuple) {
-        if (tuple.length == fieldCount()) {
+        if (checkFieldCount) {
+            if (tuple.length == fieldCount()) {
+                return tuple;
+            } else {
+                throw new IllegalArgumentException(
+                    "Required " + fieldCount() + " elements, but " + tuple.length + " provided."
+                );
+            }
+        } else {
             return tuple;
         }
-        throw new IllegalArgumentException(
-            "Required " + fieldCount() + " elements, but " + tuple.length + " provided."
-        );
     }
 }
diff --git a/dingo-common/src/main/java/io/dingodb/common/type/converter/DataConverter.java b/dingo-common/src/main/java/io/dingodb/common/type/converter/DataConverter.java
@@ -278,7 +278,13 @@ default Object convertObjectFrom(@NonNull Object value) {
     default Object[] convertTupleFrom(@NonNull Object value, @NonNull DingoType type) {
         Object[] tuple = (Object[]) value;
         return IntStream.range(0, tuple.length)
-            .mapToObj(i -> Objects.requireNonNull(type.getChild(i)).convertFrom(tuple[i], this))
+            .mapToObj(i -> {
+                DingoType dingoType = type.getChild(i);
+                if (dingoType == null) {
+                    return null;
+                }
+                return Objects.requireNonNull(dingoType).convertFrom(tuple[i], this);
+            })
             .toArray(Object[]::new);
     }
 
diff --git a/dingo-driver/host/src/main/java/io/dingodb/driver/DingoDriverParser.java b/dingo-driver/host/src/main/java/io/dingodb/driver/DingoDriverParser.java
@@ -131,6 +131,7 @@
 import javax.annotation.Nonnull;
 
 import static io.dingodb.common.SimpleExecuteVariablesFactory.variablesFactory;
+import static io.dingodb.common.environment.ExecutionEnvironment.IMPLICT_NAME;
 import static io.dingodb.common.mysql.error.ErrorCode.ErrUnknown;
 import static io.dingodb.common.util.NameCaseUtils.convertName;
 import static io.dingodb.exec.transaction.base.TransactionType.NONE;
@@ -179,6 +180,9 @@ private List<ColumnMetaData> getColumnMetaDataList(
             RelDataTypeField field = fieldList.get(i);
             List<String> colList = originList.get(i);
             boolean hidden = SchemaStateUtils.columnHidden(connection, colList);
+            if (!hidden) {
+                hidden = IMPLICT_NAME.equals(field.getName());
+            }
             //continue;
             columns.add(metaData(
                 typeFactory,
@@ -473,6 +477,7 @@ public Meta.Signature parseQuery(
             LogUtils.error(log, "Parse and validate error, sql: <[{}]>.", sql, e);
             throw ExceptionUtils.toRuntime(e);
         } catch (RuntimeException e) {
+            LogUtils.error(log, "Parse and validate error, sql: <[{}]>.", sql, e);
             throw DingoErrUtil.newStdErr(e.getMessage());
         }
         planProfile.endValidator();
diff --git a/dingo-exec/src/main/java/io/dingodb/exec/operator/ScanOperator.java b/dingo-exec/src/main/java/io/dingodb/exec/operator/ScanOperator.java
@@ -24,6 +24,8 @@
 import io.dingodb.exec.operator.data.Context;
 import io.dingodb.exec.operator.params.ScanParam;
 import io.dingodb.exec.utils.RelOpUtils;
+import io.dingodb.meta.DdlService;
+import io.dingodb.meta.entity.Table;
 import io.dingodb.store.api.StoreInstance;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -47,6 +49,14 @@ public final class ScanOperator extends ScanOperatorBase {
         byte[] endKey = rd.getEndKey();
         boolean includeStart = rd.isWithStart();
         boolean includeEnd = rd.isWithEnd();
+        Table table = DdlService.root().getTable(param.tableId);
+        if (table != null) {
+            boolean withoutPrimary = table.getColumns()
+                .stream().anyMatch(column -> column.isPrimary() && column.getState() == 2);
+            if (withoutPrimary) {
+                context.setWithoutPrimary(true);
+            }
+        }
         SourceProfile profile = param.getSourceProfile("scanBase");
         long start = System.currentTimeMillis();
         StoreInstance storeInstance = Services.KV_STORE.getInstance(param.getTableId(), rd.getId());
diff --git a/dingo-exec/src/main/java/io/dingodb/exec/operator/ScanWithRelOpOperatorBase.java b/dingo-exec/src/main/java/io/dingodb/exec/operator/ScanWithRelOpOperatorBase.java
@@ -24,6 +24,8 @@
 import io.dingodb.exec.dag.Vertex;
 import io.dingodb.exec.operator.data.Context;
 import io.dingodb.exec.operator.params.ScanWithRelOpParam;
+import io.dingodb.meta.DdlService;
+import io.dingodb.meta.entity.Table;
 import io.dingodb.store.api.StoreInstance;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -44,6 +46,14 @@ public abstract class ScanWithRelOpOperatorBase extends ScanOperatorBase {
     ) {
         ScanWithRelOpParam param = vertex.getParam();
         SourceProfile profile = param.getSourceProfile("scanBase");
+        Table table = DdlService.root().getTable(param.tableId);
+        if (table != null) {
+            boolean withoutPrimary = table.getColumns()
+                .stream().anyMatch(column -> column.isPrimary() && column.getState() == 2);
+            if (withoutPrimary) {
+                context.setWithoutPrimary(true);
+            }
+        }
         long start = System.currentTimeMillis();
         RangeDistribution rd = context.getDistribution();
         byte[] startKey = rd.getStartKey();
diff --git a/dingo-exec/src/main/java/io/dingodb/exec/operator/TxnPartInsertIgnoreOperator.java b/dingo-exec/src/main/java/io/dingodb/exec/operator/TxnPartInsertIgnoreOperator.java
@@ -168,6 +168,13 @@ protected boolean pushTuple(Context context, Object[] tuple, Vertex vertex) {
                 indexTable.getCodecVersion(), indexTable.version, indexTable.tupleType(), indexTable.keyMapping()
             );
         }
+        if (context.isWithoutPrimary()) {
+            schema.setCheckFieldCount(false);
+            DingoType dingoType = codec.getDingoType();
+            if (dingoType != null) {
+                dingoType.setCheckFieldCount(false);
+            }
+        }
         Object[] newTuple = (Object[]) schema.convertFrom(tuple, ValueConverter.INSTANCE);
         assert newTuple != null;
         KeyValue keyValue = wrap(codec::encode).apply(newTuple);
diff --git a/dingo-exec/src/main/java/io/dingodb/exec/operator/TxnPartInsertOperator.java b/dingo-exec/src/main/java/io/dingodb/exec/operator/TxnPartInsertOperator.java
@@ -265,6 +265,13 @@ protected boolean pushTuple(Context context, Object[] tuple, Vertex vertex) {
                 }
             }
         }
+        if (context.isWithoutPrimary()) {
+            schema.setCheckFieldCount(false);
+            DingoType dingoType = codec.getDingoType();
+            if (dingoType != null) {
+                dingoType.setCheckFieldCount(false);
+            }
+        }
         Object[] newTuple = (Object[]) schema.convertFrom(tuple, ValueConverter.INSTANCE);
         KeyValue keyValue = wrap(codec::encode).apply(newTuple);
         CodecService.getDefault().setId(keyValue.getKey(), partId.domain);
diff --git a/dingo-exec/src/main/java/io/dingodb/exec/operator/TxnScanOperator.java b/dingo-exec/src/main/java/io/dingodb/exec/operator/TxnScanOperator.java
@@ -24,6 +24,8 @@
 import io.dingodb.exec.operator.data.Context;
 import io.dingodb.exec.operator.params.TxnScanParam;
 import io.dingodb.exec.utils.RelOpUtils;
+import io.dingodb.meta.DdlService;
+import io.dingodb.meta.entity.Table;
 import io.dingodb.store.api.transaction.DingoTransformedIterator;
 import lombok.extern.slf4j.Slf4j;
 import org.checkerframework.checker.nullness.qual.NonNull;
@@ -42,6 +44,14 @@ public final class TxnScanOperator extends TxnScanOperatorBase {
         SourceProfile profile = param.getSourceProfile("scanBase");
         long start = System.currentTimeMillis();
         CommonId tableId = param.getTableId();
+        Table table = DdlService.root().getTable(tableId);
+        if (table != null) {
+            boolean withoutPrimary = table.getColumns()
+                .stream().anyMatch(column -> column.isPrimary() && column.getState() == 2);
+            if (withoutPrimary) {
+                context.setWithoutPrimary(true);
+            }
+        }
         CommonId txnId = vertex.getTask().getTxnId();
         RangeDistribution distribution = context.getDistribution();
         if (param.isAutoCommit()) {
diff --git a/dingo-exec/src/main/java/io/dingodb/exec/operator/TxnScanWithRelOpOperatorBase.java b/dingo-exec/src/main/java/io/dingodb/exec/operator/TxnScanWithRelOpOperatorBase.java
diff --git a/dingo-exec/src/main/java/io/dingodb/exec/operator/data/Context.java b/dingo-exec/src/main/java/io/dingodb/exec/operator/data/Context.java
diff --git a/dingo-store-proxy/src/main/java/io/dingodb/store/proxy/ddl/DdlService.java b/dingo-store-proxy/src/main/java/io/dingodb/store/proxy/ddl/DdlService.java
diff --git a/dingo-store-proxy/src/main/java/io/dingodb/store/proxy/service/CodecService.java b/dingo-store-proxy/src/main/java/io/dingodb/store/proxy/service/CodecService.java

Original file line number	Diff line number	Diff line change
`@@ -17,6 +17,7 @@`
`17`	`17`	`package io.dingodb.codec;`
`18`	`18`
`19`	`19`	`import io.dingodb.common.store.KeyValue;`
	`20`	`+import io.dingodb.common.type.DingoType;`
`20`	`21`	`import org.checkerframework.checker.nullness.qual.NonNull;`
`21`	`22`
`22`	`23`	`public interface KeyValueCodec {`
`@@ -72,4 +73,6 @@ public interface KeyValueCodec {`
`72`	`73`	`default Object[] mapKeyAndDecodeValue(Object[] keys, byte[] bytes) {`
`73`	`74`	`throw new UnsupportedOperationException();`
`74`	`75`	`}`
	`76`	`+`
	`77`	`+ default DingoType getDingoType() { return null; }`
`75`	`78`	`}`
Original file line number	Diff line number	Diff line change
`@@ -119,4 +119,8 @@ public interface DingoType extends CompileContext {`
`119`	`119`	`default void setHidden(boolean hidden){};`
`120`	`120`
`121`	`121`	`default boolean isHidden() { return false; };`
	`122`	`+`
	`123`	`+ default void setCheckFieldCount(boolean check) {};`
	`124`	`+`
	`125`	`+ default boolean checkFieldCount() { return true; }`
`122`	`126`	`}`