diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java index f530632283bc..286a755756b2 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java @@ -49,6 +49,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.utils.Binary; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,6 +57,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintWriter; +import java.time.LocalDate; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; @@ -176,7 +178,6 @@ public abstract class AbstractDataTool { protected static Boolean aligned; protected static Session session; protected static final LongAdder loadFileSuccessfulNum = new LongAdder(); - protected static final Map TYPE_INFER_KEY_DICT = new HashMap<>(); protected static final String DATATYPE_BOOLEAN = "boolean"; protected static final String DATATYPE_INT = "int"; @@ -188,10 +189,13 @@ public abstract class AbstractDataTool { protected static final String DATATYPE_BLOB = "blob"; protected static final String DATATYPE_NAN = "NaN"; protected static final String DATATYPE_TEXT = "text"; + protected static final String DATATYPE_STRING = "string"; protected static final String DATATYPE_NULL = "null"; protected static int batchPointSize = 100_000; + protected static final Map TYPE_INFER_KEY_DICT = new HashMap<>(); + static { TYPE_INFER_KEY_DICT.put(DATATYPE_BOOLEAN, TSDataType.BOOLEAN); TYPE_INFER_KEY_DICT.put(DATATYPE_INT, TSDataType.FLOAT); @@ -200,8 +204,9 @@ public abstract class AbstractDataTool { TYPE_INFER_KEY_DICT.put(DATATYPE_DOUBLE, TSDataType.DOUBLE); TYPE_INFER_KEY_DICT.put(DATATYPE_TIMESTAMP, TSDataType.TIMESTAMP); TYPE_INFER_KEY_DICT.put(DATATYPE_DATE, TSDataType.TIMESTAMP); - TYPE_INFER_KEY_DICT.put(DATATYPE_BLOB, TSDataType.TEXT); + TYPE_INFER_KEY_DICT.put(DATATYPE_BLOB, TSDataType.BLOB); TYPE_INFER_KEY_DICT.put(DATATYPE_NAN, TSDataType.DOUBLE); + TYPE_INFER_KEY_DICT.put(DATATYPE_STRING, TSDataType.STRING); } protected static final Map TYPE_INFER_VALUE_DICT = new HashMap<>(); @@ -213,9 +218,10 @@ public abstract class AbstractDataTool { TYPE_INFER_VALUE_DICT.put(DATATYPE_FLOAT, TSDataType.FLOAT); TYPE_INFER_VALUE_DICT.put(DATATYPE_DOUBLE, TSDataType.DOUBLE); TYPE_INFER_VALUE_DICT.put(DATATYPE_TIMESTAMP, TSDataType.TIMESTAMP); - TYPE_INFER_VALUE_DICT.put(DATATYPE_DATE, TSDataType.TIMESTAMP); - TYPE_INFER_VALUE_DICT.put(DATATYPE_BLOB, TSDataType.TEXT); + TYPE_INFER_VALUE_DICT.put(DATATYPE_DATE, TSDataType.DATE); + TYPE_INFER_VALUE_DICT.put(DATATYPE_BLOB, TSDataType.BLOB); TYPE_INFER_VALUE_DICT.put(DATATYPE_TEXT, TSDataType.TEXT); + TYPE_INFER_VALUE_DICT.put(DATATYPE_STRING, TSDataType.STRING); } private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); @@ -689,6 +695,8 @@ private static TSDataType typeInfer(String strValue) { // "NaN" is returned if the NaN Literal is given in Parser } else if (DATATYPE_NAN.equals(strValue)) { return TYPE_INFER_KEY_DICT.get(DATATYPE_NAN); + } else if (isBlob(strValue)) { + return TYPE_INFER_KEY_DICT.get(DATATYPE_BLOB); } else if (strValue.length() <= 512) { return STRING; } else { @@ -708,6 +716,10 @@ static boolean isNumber(String s) { return true; } + private static boolean isBlob(String s) { + return s.length() >= 3 && s.startsWith("X'") && s.endsWith("'"); + } + private static boolean isBoolean(String s) { return s.equalsIgnoreCase(SqlConstant.BOOLEAN_TRUE) || s.equalsIgnoreCase(SqlConstant.BOOLEAN_FALSE); @@ -745,8 +757,11 @@ private static Object typeTrans(String value, TSDataType type) { case DOUBLE: return Double.parseDouble(value); case TIMESTAMP: + return Long.parseLong(value); case DATE: + return LocalDate.parse(value); case BLOB: + return new Binary(parseHexStringToByteArray(value.replaceFirst("0x", ""))); default: return null; } @@ -755,6 +770,15 @@ private static Object typeTrans(String value, TSDataType type) { } } + private static byte[] parseHexStringToByteArray(String hexString) { + byte[] bytes = new byte[hexString.length() / 2]; + for (int i = 0; i < hexString.length(); i += 2) { + int value = Integer.parseInt(hexString.substring(i, i + 2), 16); + bytes[i / 2] = (byte) value; + } + return bytes; + } + private static long parseTimestamp(String str) { long timestamp; try { diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportData.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportData.java index 13a43931cca8..0f60c38659c6 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportData.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportData.java @@ -46,6 +46,7 @@ import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.Path; import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.utils.BytesUtils; import org.jline.reader.LineReader; import java.io.BufferedReader; @@ -54,6 +55,7 @@ import java.io.FileWriter; import java.io.IOException; import java.time.Instant; +import java.time.LocalDate; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; @@ -651,7 +653,7 @@ public static void writeCsvFile( while (i++ < linesPerFile) { if (sessionDataSet.hasNext()) { RowRecord rowRecord = sessionDataSet.next(); - if (rowRecord.getTimestamp() != 0) { + if ("Time".equals(headers.get(0))) { csvPrinterWrapper.print(timeTrans(rowRecord.getTimestamp())); } rowRecord @@ -660,10 +662,24 @@ public static void writeCsvFile( field -> { String fieldStringValue = field.getStringValue(); if (!"null".equals(field.getStringValue())) { - if ((field.getDataType() == TSDataType.TEXT - || field.getDataType() == TSDataType.STRING) + final TSDataType dataType = field.getDataType(); + if ((dataType == TSDataType.TEXT || dataType == TSDataType.STRING) && !fieldStringValue.startsWith("root.")) { fieldStringValue = "\"" + fieldStringValue + "\""; + } else if (dataType == TSDataType.BLOB) { + final byte[] v = field.getBinaryV().getValues(); + if (v == null) { + fieldStringValue = null; + } else { + fieldStringValue = BytesUtils.parseBlobByteArrayToString(v); + } + } else if (dataType == TSDataType.DATE) { + final LocalDate dateV = field.getDateV(); + if (dateV == null) { + fieldStringValue = null; + } else { + fieldStringValue = dateV.toString(); + } } csvPrinterWrapper.print(fieldStringValue); } else { @@ -757,8 +773,25 @@ public static void writeSqlFile( headersTemp.remove(seriesList.get(index)); continue; } - if ("TEXT".equalsIgnoreCase(timeseriesList.get(3).getStringValue())) { - values.add("\"" + value + "\""); + final String dataType = timeseriesList.get(3).getStringValue(); + if (TSDataType.TEXT.name().equalsIgnoreCase(dataType) + || TSDataType.STRING.name().equalsIgnoreCase(dataType)) { + values.add("\'" + value + "\'"); + } else if (TSDataType.BLOB.name().equalsIgnoreCase(dataType)) { + final byte[] v = fields.get(index).getBinaryV().getValues(); + if (v == null) { + values.add(null); + } else { + values.add( + BytesUtils.parseBlobByteArrayToString(v).replaceFirst("0x", "X'") + "'"); + } + } else if (TSDataType.DATE.name().equalsIgnoreCase(dataType)) { + final LocalDate dateV = fields.get(index).getDateV(); + if (dateV == null) { + values.add(null); + } else { + values.add("'" + dateV.toString() + "'"); + } } else { values.add(value); } diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java index 76d9417777d0..753713b9d1f6 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java @@ -402,13 +402,33 @@ private static void applyTypeInferArgs(String key, String value) throws ArgsErro if (key.equals(DATATYPE_NAN) && !(value.equals(DATATYPE_FLOAT) || value.equals(DATATYPE_DOUBLE) - || value.equals(DATATYPE_TEXT))) { + || value.equals(DATATYPE_TEXT) + || value.equals(DATATYPE_STRING))) { throw new ArgsErrorException("NaN can not convert to " + value); } if (key.equals(DATATYPE_BOOLEAN) - && !(value.equals(DATATYPE_BOOLEAN) || value.equals(DATATYPE_TEXT))) { + && !(value.equals(DATATYPE_BOOLEAN) + || value.equals(DATATYPE_TEXT) + || value.equals(DATATYPE_STRING))) { throw new ArgsErrorException("Boolean can not convert to " + value); } + if (key.equals(DATATYPE_DATE) + && !(value.equals(DATATYPE_DATE) + || value.equals(DATATYPE_TEXT) + || value.equals(DATATYPE_STRING))) { + throw new ArgsErrorException("Date can not convert to " + value); + } + if (key.equals(DATATYPE_TIMESTAMP) + && !(value.equals(DATATYPE_TIMESTAMP) + || value.equals(DATATYPE_TEXT) + || value.equals(DATATYPE_STRING) + || value.equals(DATATYPE_DOUBLE) + || value.equals(DATATYPE_LONG))) { + throw new ArgsErrorException("Timestamp can not convert to " + value); + } + if (key.equals(DATATYPE_BLOB) && !(value.equals(DATATYPE_BLOB))) { + throw new ArgsErrorException("Blob can not convert to " + value); + } final TSDataType srcType = TYPE_INFER_VALUE_DICT.get(key); final TSDataType dstType = TYPE_INFER_VALUE_DICT.get(value); if (dstType.getType() < srcType.getType()) {