From 364b0cf9131fc177a05a560efb1d95db4965c498 Mon Sep 17 00:00:00 2001 From: xufei Date: Tue, 14 Apr 2020 21:36:02 +0800 Subject: [PATCH] support fromUnixTime and dateFormat (#629) (#634) --- dbms/src/Common/MyTime.cpp | 467 ++++++++++++++---- dbms/src/Common/MyTime.h | 25 +- dbms/src/Debug/dbgFuncCoprocessor.cpp | 22 + dbms/src/Flash/Coprocessor/DAGDriver.cpp | 1 + .../Coprocessor/DAGExpressionAnalyzer.cpp | 36 +- .../Flash/Coprocessor/DAGExpressionAnalyzer.h | 5 +- dbms/src/Flash/Coprocessor/DAGUtils.cpp | 5 +- dbms/src/Flash/Coprocessor/InterpreterDAG.cpp | 4 +- dbms/src/Functions/FunctionsConversion.cpp | 3 + dbms/src/Functions/FunctionsConversion.h | 259 ++++++++++ dbms/src/Functions/FunctionsDateTime.h | 18 - dbms/src/IO/WriteHelpers.h | 6 + dbms/src/Interpreters/Context.cpp | 1 + dbms/src/Interpreters/Context.h | 6 + dbms/src/Interpreters/TimezoneInfo.h | 60 +++ tests/mutable-test/expr/date_format.test | 30 ++ tests/mutable-test/expr/from_unixtime.test | 42 ++ 17 files changed, 853 insertions(+), 137 deletions(-) create mode 100644 dbms/src/Interpreters/TimezoneInfo.h create mode 100644 tests/mutable-test/expr/date_format.test create mode 100644 tests/mutable-test/expr/from_unixtime.test diff --git a/dbms/src/Common/MyTime.cpp b/dbms/src/Common/MyTime.cpp index 898885a28f3..494c462eeb3 100644 --- a/dbms/src/Common/MyTime.cpp +++ b/dbms/src/Common/MyTime.cpp @@ -1,11 +1,10 @@ #include +#include #include #include #include -#include - namespace DB { @@ -115,6 +114,7 @@ UInt64 MyTimeBase::toPackedUInt() const return (ymd << 17 | hms) << 24 | micro_second; } +// the implementation is the same as TiDB String MyTimeBase::dateFormat(const String & layout) const { String result; @@ -137,29 +137,210 @@ String MyTimeBase::dateFormat(const String & layout) const return result; } +// the implementation is the same as TiDB +int MyTimeBase::yearDay() const +{ + if (month == 0 || day == 0) + { + return 0; + } + return calcDayNum(year, month, day) - calcDayNum(year, 1, 1) + 1; +} + +UInt32 adjustWeekMode(UInt32 mode) +{ + mode &= 7u; + if (!(mode & MyTimeBase::WEEK_BEHAVIOR_MONDAY_FIRST)) + mode ^= MyTimeBase::WEEK_BEHAVIOR_FIRST_WEEKDAY; + return mode; +} + +// the implementation is the same as TiDB +int MyTimeBase::week(UInt32 mode) const +{ + if (month == 0 || day == 0) + { + return 0; + } + auto [year, week] = calcWeek(adjustWeekMode(mode)); + std::ignore = year; + return week; +} + +// calcWeekday calculates weekday from daynr, returns 0 for Monday, 1 for Tuesday ... +// the implementation is the same as TiDB +int calcWeekday(int day_num, bool sunday_first_day_of_week) +{ + day_num += 5; + if (sunday_first_day_of_week) + day_num++; + return day_num % 7; +} + +// the implementation is the same as TiDB +int calcDaysInYear(int year) +{ + if ((year & 3u) == 0 && (year % 100 != 0 || (year % 400 == 0 && (year != 0)))) + return 366; + return 365; +} + +// the implementation is the same as TiDB +std::tuple MyTimeBase::calcWeek(UInt32 mode) const +{ + int days, ret_year, ret_week; + int ty = year, tm = month, td = day; + int day_num = calcDayNum(ty, tm, td); + int first_day_num = calcDayNum(ty, 1, 1); + bool monday_first = mode & WEEK_BEHAVIOR_MONDAY_FIRST; + bool week_year = mode & WEEK_BEHAVIOR_YEAR; + bool first_week_day = mode & WEEK_BEHAVIOR_FIRST_WEEKDAY; + + int week_day = calcWeekday(first_day_num, !monday_first); + + ret_year = ty; + + if (tm == 1 && td <= 7 - week_day) + { + if (!week_year && ((first_week_day && week_day != 0) || (!first_week_day && week_day >= 4))) + { + ret_week = 0; + return std::make_tuple(ret_year, ret_week); + } + week_year = true; + ret_year--; + days = calcDaysInYear(ret_year); + first_day_num -= days; + week_day = (week_day + 53 * 7 - days) % 7; + } + + if ((first_week_day && week_day != 0) || (!first_week_day && week_day >= 4)) + { + days = day_num - (first_day_num + 7 - week_day); + } + else + { + days = day_num - (first_day_num - week_day); + } + + if (week_year && days >= 52 * 7) + { + week_day = (week_day + calcDaysInYear(year)) % 7; + if ((!first_week_day && week_day < 4) || (first_week_day && week_day == 0)) + { + ret_year++; + ret_week = 1; + return std::make_tuple(ret_year, ret_week); + } + } + ret_week = days / 7 + 1; + return std::make_tuple(ret_year, ret_week); +} + +static const String month_names[] = { + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +}; + +static const String abbrev_month_names[] = { + "Jan", + "Feb", + "Mar", + "Apr", + "May", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec", +}; + +static const String abbrev_weekday_names[] = { + "Sun", + "Mon", + "Tue", + "Wed", + "Thu", + "Fri", + "Sat", +}; + +static const String weekday_names[] = { + "Sunday", + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday", + "Saturday", +}; + +String abbrDayOfMonth(int day) +{ + switch (day) + { + case 1: + case 21: + case 31: + return "st"; + case 2: + case 22: + return "nd"; + case 3: + case 23: + return "rd"; + default: + return "th"; + } +} + +int MyTimeBase::weekDay() const +{ + int current_abs_day_num = calcDayNum(year, month, day); + // 1986-01-05 is sunday + int reference_abs_day_num = calcDayNum(1986, 1, 5); + int diff = current_abs_day_num - reference_abs_day_num; + if (diff < 0) + diff += (-diff / 7 + 1) * 7; + diff = diff % 7; + return diff; +} + +// the implementation is the same as TiDB void MyTimeBase::convertDateFormat(char c, String & result) const { - // TODO:: Implement other formats. switch (c) { - //case 'b': - //{ - // if (month == 0 || month > 12) - // { - // throw Exception("invalid time format"); - // } - // result.append(String(MonthNames[month-1], 3)); - // break; - //} - //case 'M': - //{ - // if (month == 0 || month > 12) - // { - // throw Exception("invalid time format"); - // } - // result.append(String(MonthNames[month-1], 3)); - // break; - //} + case 'b': + { + if (month == 0 || month > 12) + { + throw Exception("invalid time format"); + } + result.append(abbrev_month_names[month - 1]); + break; + } + case 'M': + { + if (month == 0 || month > 12) + { + throw Exception("invalid time format"); + } + result.append(month_names[month - 1]); + break; + } case 'm': { char buf[16]; @@ -174,14 +355,14 @@ void MyTimeBase::convertDateFormat(char c, String & result) const result.append(String(buf)); break; } - //case 'D': - //{ - // char buf[16]; - // sprintf(buf, "%d", month); - // result.append(String(buf)); - // result.append(abbrDayOfMonth(day)); - // break; - //} + case 'D': + { + char buf[16]; + sprintf(buf, "%d", day); + result.append(String(buf)); + result.append(abbrDayOfMonth(day)); + break; + } case 'd': { char buf[16]; @@ -196,13 +377,13 @@ void MyTimeBase::convertDateFormat(char c, String & result) const result.append(String(buf)); break; } - //case 'j': - //{ - // char buf[16]; - // sprintf(buf, "%03d", yearDay()); - // result.append(String(buf)); - // break; - //} + case 'j': + { + char buf[16]; + sprintf(buf, "%03d", yearDay()); + result.append(String(buf)); + break; + } case 'H': { char buf[16]; @@ -290,50 +471,75 @@ void MyTimeBase::convertDateFormat(char c, String & result) const result.append(String(buf)); break; } - //case 'U': - //{ - // char buf[16]; - // auto w = week(0); - // sprintf(buf, "%02d", w); - // result.append(String(buf)); - // break; - //} - //case 'u': - //{ - // char buf[16]; - // auto w = week(1); - // sprintf(buf, "%02d", w); - // result.append(String(buf)); - // break; - //} - //case 'V': - //{ - // char buf[16]; - // auto w = week(2); - // sprintf(buf, "%02d", w); - // result.append(String(buf)); - // break; - //} - //case 'v': - //{ - // char buf[16]; - // auto w = yearWeek(2); - // sprintf(buf, "%02d", w); - // result.append(String(buf)); - // break; - //} - //case 'a': - //{ - // auto weekDay = weekDay(); - // result.append(String(abbrevWeekdayName[weekDay])); - // break; - //} - //case 'w': - //{ - // auto weekDay = weekDay(); - // result.append(std::to_string(weekDay)); - // break; - //} + case 'U': + { + char buf[16]; + auto w = week(0); + sprintf(buf, "%02d", w); + result.append(String(buf)); + break; + } + case 'u': + { + char buf[16]; + auto w = week(1); + sprintf(buf, "%02d", w); + result.append(String(buf)); + break; + } + case 'V': + { + char buf[16]; + auto w = week(2); + sprintf(buf, "%02d", w); + result.append(String(buf)); + break; + } + case 'v': + { + char buf[16]; + auto [year, week] = calcWeek(3); + std::ignore = year; + sprintf(buf, "%02d", week); + result.append(String(buf)); + break; + } + case 'a': + { + auto week_day = weekDay(); + result.append(abbrev_weekday_names[week_day]); + break; + } + case 'W': + { + auto week_day = weekDay(); + result.append(weekday_names[week_day]); + break; + } + case 'w': + { + auto week_day = weekDay(); + result.append(std::to_string(week_day)); + break; + } + case 'X': + { + char buf[16]; + auto [year, week] = calcWeek(6); + std::ignore = week; + sprintf(buf, "%04d", year); + result.append(String(buf)); + break; + } + case 'x': + { + char buf[16]; + auto [year, week] = calcWeek(3); + std::ignore = week; + sprintf(buf, "%04d", year); + result.append(String(buf)); + break; + } case 'Y': { char buf[16]; @@ -463,10 +669,7 @@ String MyDateTime::toString(int fsp) const return result; } -bool isZeroDate(UInt64 time) -{ - return time == 0; -} +bool isZeroDate(UInt64 time) { return time == 0; } void convertTimeZone(UInt64 from_time, UInt64 & to_time, const DateLUTImpl & time_zone_from, const DateLUTImpl & time_zone_to) { @@ -494,9 +697,101 @@ void convertTimeZoneByOffset(UInt64 from_time, UInt64 & to_time, Int64 offset, c time_t epoch = time_zone.makeDateTime( from_my_time.year, from_my_time.month, from_my_time.day, from_my_time.hour, from_my_time.minute, from_my_time.second); epoch += offset; - MyDateTime to_my_time(time_zone.toYear(epoch), time_zone.toMonth(epoch), time_zone.toDayOfMonth(epoch), - time_zone.toHour(epoch), time_zone.toMinute(epoch), time_zone.toSecond(epoch), from_my_time.micro_second); + MyDateTime to_my_time(time_zone.toYear(epoch), time_zone.toMonth(epoch), time_zone.toDayOfMonth(epoch), time_zone.toHour(epoch), + time_zone.toMinute(epoch), time_zone.toSecond(epoch), from_my_time.micro_second); to_time = to_my_time.toPackedUInt(); } +// the implementation is the same as TiDB +int calcDayNum(int year, int month, int day) +{ + if (year == 0 || month == 0) + return 0; + int delsum = 365 * year + 31 * (month - 1) + day; + if (month <= 2) + { + year--; + } + else + { + delsum -= (month * 4 + 23) / 10; + } + int temp = ((year / 100 + 1) * 3) / 4; + return delsum + year / 4 - temp; +} + +size_t maxFormattedDateTimeStringLength(const String & format) +{ + size_t result = 0; + bool in_pattern_match = false; + for (size_t i = 0; i < format.size(); i++) + { + char x = format[i]; + if (in_pattern_match) + { + switch (x) + { + case 'b': + case 'j': + case 'a': + result += 3; + break; + case 'M': + case 'W': + result += 9; + break; + case 'm': + case 'c': + case 'd': + case 'e': + case 'H': + case 'k': + case 'h': + case 'I': + case 'l': + case 'i': + case 'p': + case 'S': + case 's': + case 'U': + case 'u': + case 'V': + case 'v': + case 'y': + result += 2; + break; + case 'D': + case 'X': + case 'x': + case 'Y': + result += 4; + break; + case 'r': + result += 11; + break; + case 'T': + result += 8; + break; + case 'f': + result += 6; + break; + case 'w': + result += 1; + break; + default: + result += 1; + break; + } + in_pattern_match = false; + continue; + } + + if (x == '%') + in_pattern_match = true; + else + result++; + } + return std::max(result, 1); +} + } // namespace DB diff --git a/dbms/src/Common/MyTime.h b/dbms/src/Common/MyTime.h index b54bb35f0d4..5b60df1e19f 100644 --- a/dbms/src/Common/MyTime.h +++ b/dbms/src/Common/MyTime.h @@ -9,7 +9,16 @@ namespace DB struct MyTimeBase { - static const UInt64 YMD_MASK = ~((1ull << 41) -1); + static const UInt64 YMD_MASK = ~((1ull << 41) - 1); + + // weekBehaviourMondayFirst set Monday as first day of week; otherwise Sunday is first day of week + static const UInt32 WEEK_BEHAVIOR_MONDAY_FIRST = 1; + // If set, Week is in range 1-53, otherwise Week is in range 0-53. + // Note that this flag is only relevant if WEEK_JANUARY is not set + static const UInt32 WEEK_BEHAVIOR_YEAR = 2; + // If not set, Weeks are numbered according to ISO 8601:1988. + // If set, the week that contains the first 'first-day-of-week' is week 1. + static const UInt32 WEEK_BEHAVIOR_FIRST_WEEKDAY = 4; enum MyTimeType : UInt8 { @@ -39,6 +48,15 @@ struct MyTimeBase // See http://dev.mysql.com/doc/refman/5.7/en/date-and-time-functions.html#function_date-format String dateFormat(const String & layout) const; + // returns the week day of current date(0 as sunday) + int weekDay() const; + // the following methods are port from TiDB + int yearDay() const; + + int week(UInt32 mode) const; + + std::tuple calcWeek(UInt32 mode) const; + protected: void convertDateFormat(char c, String & result) const; }; @@ -69,4 +87,9 @@ void convertTimeZone(UInt64 from_time, UInt64 & to_time, const DateLUTImpl & tim void convertTimeZoneByOffset(UInt64 from_time, UInt64 & to_time, Int64 offset, const DateLUTImpl & time_zone); +int calcDayNum(int year, int month, int day); + +size_t maxFormattedDateTimeStringLength(const String & format); + + } // namespace DB diff --git a/dbms/src/Debug/dbgFuncCoprocessor.cpp b/dbms/src/Debug/dbgFuncCoprocessor.cpp index 9f9dc65ec3c..8d5e52a804d 100644 --- a/dbms/src/Debug/dbgFuncCoprocessor.cpp +++ b/dbms/src/Debug/dbgFuncCoprocessor.cpp @@ -265,6 +265,28 @@ void compileExpr(const DAGSchema & input, ASTPtr ast, tipb::Expr * expr, std::un } return; } + else if (func_name_lowercase == "from_unixtime") + { + if (func->arguments->children.size() == 1) + { + expr->set_sig(tipb::ScalarFuncSig::FromUnixTime1Arg); + auto * ft = expr->mutable_field_type(); + ft->set_tp(TiDB::TypeDatetime); + ft->set_decimal(6); + } + else + { + expr->set_sig(tipb::ScalarFuncSig::FromUnixTime2Arg); + auto * ft = expr->mutable_field_type(); + ft->set_tp(TiDB::TypeString); + } + } + else if (func_name_lowercase == "date_format") + { + expr->set_sig(tipb::ScalarFuncSig::DateFormatSig); + auto * ft = expr->mutable_field_type(); + ft->set_tp(TiDB::TypeString); + } else { throw Exception("Unsupported function: " + func_name_lowercase, ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Flash/Coprocessor/DAGDriver.cpp b/dbms/src/Flash/Coprocessor/DAGDriver.cpp index ee5ea3ccb41..77a1d2959d5 100644 --- a/dbms/src/Flash/Coprocessor/DAGDriver.cpp +++ b/dbms/src/Flash/Coprocessor/DAGDriver.cpp @@ -38,6 +38,7 @@ DAGDriver::DAGDriver(Context & context_, const tipb::DAGRequest & dag_request_, if (schema_ver) // schema_ver being 0 means TiDB/TiSpark hasn't specified schema version. context.setSetting("schema_version", schema_ver); + context.getTimezoneInfo().resetByDAGRequest(dag_request); } void DAGDriver::execute() diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp index 4a16aa90512..39dec9a7342 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp @@ -379,21 +379,12 @@ void DAGExpressionAnalyzer::appendFinalProject(ExpressionActionsChain & chain, c } } -void constructTZExpr(tipb::Expr & tz_expr, const tipb::DAGRequest & rqst, bool from_utc) +void constructTZExpr(tipb::Expr & tz_expr, const TimezoneInfo & dag_timezone_info, bool from_utc) { - if (rqst.has_time_zone_name() && rqst.time_zone_name().length() > 0) - constructStringLiteralTiExpr(tz_expr, rqst.time_zone_name()); + if (dag_timezone_info.is_name_based) + constructStringLiteralTiExpr(tz_expr, dag_timezone_info.timezone_name); else - constructInt64LiteralTiExpr(tz_expr, from_utc ? rqst.time_zone_offset() : -rqst.time_zone_offset()); -} - -bool hasMeaningfulTZInfo(const tipb::DAGRequest & rqst) -{ - if (rqst.has_time_zone_name() && rqst.time_zone_name().length() > 0) - return rqst.time_zone_name() != "UTC"; - if (rqst.has_time_zone_offset()) - return rqst.has_time_zone_offset() != 0; - return false; + constructInt64LiteralTiExpr(tz_expr, from_utc ? dag_timezone_info.timezone_offset : -dag_timezone_info.timezone_offset); } String DAGExpressionAnalyzer::appendTimeZoneCast( @@ -414,20 +405,18 @@ String DAGExpressionAnalyzer::appendTimeZoneCast( // useless casts to all the timestamp columns, in order to avoid redundant cast, when cast the ts // column to the columns with session-level timezone info, the original ts columns with UTC timezone // are still kept, and the InterpreterDAG will choose the correct column based on encode type -bool DAGExpressionAnalyzer::appendTimeZoneCastsAfterTS( - ExpressionActionsChain & chain, std::vector is_ts_column, const tipb::DAGRequest & rqst) +bool DAGExpressionAnalyzer::appendTimeZoneCastsAfterTS(ExpressionActionsChain & chain, std::vector is_ts_column) { - if (!hasMeaningfulTZInfo(rqst)) + if (context.getTimezoneInfo().is_utc_timezone) return false; bool ret = false; initChain(chain, getCurrentInputColumns()); ExpressionActionsPtr actions = chain.getLastActions(); tipb::Expr tz_expr; - constructTZExpr(tz_expr, rqst, true); + constructTZExpr(tz_expr, context.getTimezoneInfo(), true); String tz_col; - String func_name - = rqst.has_time_zone_name() && rqst.time_zone_name().length() > 0 ? "ConvertTimeZoneFromUTC" : "ConvertTimeZoneByOffset"; + String func_name = context.getTimezoneInfo().is_name_based ? "ConvertTimeZoneFromUTC" : "ConvertTimeZoneByOffset"; for (size_t i = 0; i < is_ts_column.size(); i++) { if (is_ts_column[i]) @@ -444,19 +433,18 @@ bool DAGExpressionAnalyzer::appendTimeZoneCastsAfterTS( } void DAGExpressionAnalyzer::appendAggSelect( - ExpressionActionsChain & chain, const tipb::Aggregation & aggregation, const tipb::DAGRequest & rqst, bool keep_session_timezone_info) + ExpressionActionsChain & chain, const tipb::Aggregation & aggregation, bool keep_session_timezone_info) { initChain(chain, getCurrentInputColumns()); bool need_update_aggregated_columns = false; std::vector updated_aggregated_columns; ExpressionActionsChain::Step step = chain.steps.back(); - bool need_append_timezone_cast = !keep_session_timezone_info && hasMeaningfulTZInfo(rqst); + bool need_append_timezone_cast = !keep_session_timezone_info && !context.getTimezoneInfo().is_utc_timezone; tipb::Expr tz_expr; if (need_append_timezone_cast) - constructTZExpr(tz_expr, rqst, false); + constructTZExpr(tz_expr, context.getTimezoneInfo(), false); String tz_col; - String tz_cast_func_name - = rqst.has_time_zone_name() && rqst.time_zone_name().length() > 0 ? "ConvertTimeZoneToUTC" : "ConvertTimeZoneByOffset"; + String tz_cast_func_name = context.getTimezoneInfo().is_name_based ? "ConvertTimeZoneToUTC" : "ConvertTimeZoneByOffset"; for (Int32 i = 0; i < aggregation.agg_func_size(); i++) { String & name = aggregated_columns[i].name; diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.h b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.h index 0e7063c778c..a88f8815150 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.h +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.h @@ -42,8 +42,7 @@ class DAGExpressionAnalyzer : private boost::noncopyable void appendOrderBy(ExpressionActionsChain & chain, const tipb::TopN & topN, Strings & order_column_names); void appendAggregation(ExpressionActionsChain & chain, const tipb::Aggregation & agg, Names & aggregate_keys, AggregateDescriptions & aggregate_descriptions); - void appendAggSelect( - ExpressionActionsChain & chain, const tipb::Aggregation & agg, const tipb::DAGRequest & rqst, bool keep_session_timezone_info); + void appendAggSelect(ExpressionActionsChain & chain, const tipb::Aggregation & agg, bool keep_session_timezone_info); String appendCastIfNeeded(const tipb::Expr & expr, ExpressionActionsPtr & actions, const String & expr_name, bool explicit_cast); String alignReturnType(const tipb::Expr & expr, ExpressionActionsPtr & actions, const String & expr_name, bool force_uint8); void initChain(ExpressionActionsChain & chain, const std::vector & columns) const @@ -66,7 +65,7 @@ class DAGExpressionAnalyzer : private boost::noncopyable void makeExplicitSetForIndex(const tipb::Expr & expr, const ManageableStoragePtr & storage); String applyFunction(const String & func_name, const Names & arg_names, ExpressionActionsPtr & actions); Int32 getImplicitCastCount() { return implicit_cast_count; }; - bool appendTimeZoneCastsAfterTS(ExpressionActionsChain & chain, std::vector is_ts_column, const tipb::DAGRequest & rqst); + bool appendTimeZoneCastsAfterTS(ExpressionActionsChain & chain, std::vector is_ts_column); String appendTimeZoneCast(const String & tz_col, const String & ts_col, const String & func_name, ExpressionActionsPtr & actions); DAGPreparedSets & getPreparedSets() { return prepared_sets; } String convertToUInt8(ExpressionActionsPtr & actions, const String & column_name); diff --git a/dbms/src/Flash/Coprocessor/DAGUtils.cpp b/dbms/src/Flash/Coprocessor/DAGUtils.cpp index bc2b70d4ee0..dcdea010203 100644 --- a/dbms/src/Flash/Coprocessor/DAGUtils.cpp +++ b/dbms/src/Flash/Coprocessor/DAGUtils.cpp @@ -674,7 +674,7 @@ std::unordered_map scalar_func_map({ //{tipb::ScalarFuncSig::JsonKeys2ArgsSig, "cast"}, //{tipb::ScalarFuncSig::JsonValidStringSig, "cast"}, - //{tipb::ScalarFuncSig::DateFormatSig, "cast"}, + {tipb::ScalarFuncSig::DateFormatSig, "dateFormat"}, //{tipb::ScalarFuncSig::DateLiteral, "cast"}, //{tipb::ScalarFuncSig::DateDiff, "cast"}, //{tipb::ScalarFuncSig::NullTimeDiff, "cast"}, @@ -791,8 +791,7 @@ std::unordered_map scalar_func_map({ //{tipb::ScalarFuncSig::StrToDateDate, "cast"}, //{tipb::ScalarFuncSig::StrToDateDatetime, "cast"}, //{tipb::ScalarFuncSig::StrToDateDuration, "cast"}, - //{tipb::ScalarFuncSig::FromUnixTime1Arg, "cast"}, - //{tipb::ScalarFuncSig::FromUnixTime2Arg, "cast"}, + {tipb::ScalarFuncSig::FromUnixTime1Arg, "fromUnixTime"}, {tipb::ScalarFuncSig::FromUnixTime2Arg, "fromUnixTime"}, //{tipb::ScalarFuncSig::ExtractDatetime, "cast"}, //{tipb::ScalarFuncSig::ExtractDuration, "cast"}, diff --git a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp index 39a98c694c8..ed3ea745e61 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp +++ b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp @@ -505,7 +505,7 @@ bool InterpreterDAG::addTimeZoneCastAfterTS(std::vector & is_ts_column, Pi return false; ExpressionActionsChain chain; - if (analyzer->appendTimeZoneCastsAfterTS(chain, is_ts_column, dag.getDAGRequest())) + if (analyzer->appendTimeZoneCastsAfterTS(chain, is_ts_column)) { pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, chain.getLastActions()); }); return true; @@ -536,7 +536,7 @@ InterpreterDAG::AnalysisResult InterpreterDAG::analyzeExpressions() chain.clear(); // add cast if type is not match - analyzer->appendAggSelect(chain, dag.getAggregation(), dag.getDAGRequest(), keep_session_timezone_info); + analyzer->appendAggSelect(chain, dag.getAggregation(), keep_session_timezone_info); //todo use output_offset to reconstruct the final project columns for (auto & element : analyzer->getCurrentInputColumns()) { diff --git a/dbms/src/Functions/FunctionsConversion.cpp b/dbms/src/Functions/FunctionsConversion.cpp index 146efd3e355..0ef4369d4ac 100644 --- a/dbms/src/Functions/FunctionsConversion.cpp +++ b/dbms/src/Functions/FunctionsConversion.cpp @@ -88,6 +88,9 @@ void registerFunctionsConversion(FunctionFactory & factory) factory.registerFunction>(); factory.registerFunction>(); factory.registerFunction>(); + + factory.registerFunction(); + factory.registerFunction(); } } diff --git a/dbms/src/Functions/FunctionsConversion.h b/dbms/src/Functions/FunctionsConversion.h index a460ef28b4b..09bf58bb338 100644 --- a/dbms/src/Functions/FunctionsConversion.h +++ b/dbms/src/Functions/FunctionsConversion.h @@ -37,6 +37,8 @@ #include #include #include +#include +#include namespace DB @@ -1279,6 +1281,263 @@ class FunctionToFixedString : public IFunction } }; +class FunctionFromUnixTime : public IFunction +{ +public: + static constexpr auto name = "fromUnixTime"; + static FunctionPtr create(const Context & context_) { return std::make_shared(context_); }; + FunctionFromUnixTime(const Context & context_) : context(context_) {}; + + String getName() const override + { + return name; + } + + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + bool isInjective(const Block &) override { return true; } + bool useDefaultImplementationForNulls() const override { return false; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (arguments.size() != 1 && arguments.size() != 2) + throw Exception("Function " + getName() + " only accept 1 or 2 arguments"); + if (!removeNullable(arguments[0].type)->isDecimal()) + throw Exception("First argument for function " + getName() + " must be decimal type", ErrorCodes::ILLEGAL_COLUMN); + if (arguments.size() == 2 && (!removeNullable(arguments[1].type)->isString())) + throw Exception("Second argument of function " + getName() + " must be string constant", ErrorCodes::ILLEGAL_COLUMN); + + auto scale = std::min(6, getDecimalScale(*removeNullable(arguments[0].type), 6)); + if (arguments.size() == 1) + return makeNullable(std::make_shared(scale)); + return makeNullable(std::make_shared()); + } + + /// scale_multiplier is used to compensate the fsp part if the scale is less than 6 + /// for example, for decimal xxxx.23, the decimal part is 23, and it should be + /// converted to 230000 + static constexpr int scale_multiplier[] = {1000000,100000,10000,1000,100,10,1}; + + template + void decimalToMyDatetime(const ColumnPtr & input_col, ColumnUInt64::Container & datetime_res, ColumnUInt8::Container & null_res, + UInt32 scale, Int256 & scale_divisor, Int256 & scale_round_divisor) + { + const auto & timezone_info = context.getTimezoneInfo(); + const auto * datelut = timezone_info.timezone; + const auto decimal_col = checkAndGetColumn>(input_col.get()); + const typename ColumnDecimal::Container & vec_from = decimal_col->getData(); + Int64 scale_divisor_64 = 1; + Int64 scale_round_divisor_64 = 1; + bool scale_divisor_fit_64 = false; + if (scale_divisor < std::numeric_limits::max()) + { + scale_divisor_64 = static_cast(scale_divisor); + scale_round_divisor_64 = static_cast(scale_round_divisor); + scale_divisor_fit_64 = true; + } + for (size_t i = 0; i < decimal_col->size(); i++) + { + const auto & decimal = vec_from[i]; + if (decimal.value < 0) + { + null_res[i] = 1; + datetime_res[i] = 0; + continue; + } + Int64 integer_part; + Int64 fsp_part = 0; + if (likely(scale_divisor_fit_64 && decimal.value < std::numeric_limits::max())) + { + auto value = static_cast(decimal.value); + integer_part = value / scale_divisor_64; + fsp_part = value % scale_divisor_64; + if (scale <= 6) + fsp_part *= scale_multiplier[scale]; + else + { + /// according to TiDB impl, should use half even round mode + if (fsp_part % scale_round_divisor_64 >= scale_round_divisor_64 / 2) + fsp_part = (fsp_part / scale_round_divisor_64) + 1; + else + fsp_part = fsp_part / scale_round_divisor_64; + } + } + else + { + Int256 value = decimal.value; + Int256 integer_part_256 = value / scale_divisor; + Int256 fsp_part_256 = value % scale_divisor; + if (integer_part_256 > std::numeric_limits::max()) + integer_part = -1; + else + { + integer_part = static_cast(integer_part_256); + if (fsp_part_256 % scale_round_divisor >= scale_round_divisor / 2) + fsp_part_256 = (fsp_part_256 / scale_round_divisor) + 1; + else + fsp_part_256 = fsp_part_256 / scale_round_divisor; + fsp_part = static_cast(fsp_part_256); + } + } + if (integer_part > std::numeric_limits::max() || integer_part < 0) + { + null_res[i] = 1; + datetime_res[i] = 0; + continue; + } + + if (timezone_info.timezone_offset != 0) + integer_part += timezone_info.timezone_offset; + MyDateTime result(datelut->toYear(integer_part), datelut->toMonth(integer_part), datelut->toDayOfMonth(integer_part), + datelut->toHour(integer_part), datelut->toMinute(integer_part), datelut->toSecond(integer_part), fsp_part); + null_res[i] = 0; + datetime_res[i] = result.toPackedUInt(); + } + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, const size_t result) override + { + const auto & input_column = block.getByPosition(arguments[0]).column; + const auto & decimal_column = input_column->isColumnNullable() ? dynamic_cast(*input_column).getNestedColumnPtr() : input_column; + size_t rows = decimal_column->size(); + + + auto scale = getDecimalScale(*removeNullable(block.getByPosition(arguments[0]).type), 6); + /// scale_divisor is used to extract the integer/decimal part from the origin decimal + Int256 scale_divisor = 1; + for (size_t i = 0; i < scale; i++) + scale_divisor *= 10; + /// scale_round_divisor is used to round the decimal part if the scale is bigger than 6(which is the max fsp for datetime type) + Int256 scale_round_divisor = 1; + for (size_t i = 6; i < scale; i++) + scale_round_divisor *= 10; + + auto datetime_column = ColumnVector::create(); + auto null_column = ColumnUInt8::create(); + + auto & datetime_res = datetime_column->getData(); + datetime_res.assign(rows, (UInt64)0); + auto & null_res = null_column->getData(); + null_res.assign(rows, (UInt8)0); + if (input_column->isColumnNullable()) + { + for (size_t i = 0; i < rows; i++) + null_res[i] = input_column->isNullAt(i); + } + if (checkDataType(removeNullable(block.getByPosition(arguments[0]).type).get())) + { + decimalToMyDatetime(decimal_column, datetime_res, null_res, scale, scale_divisor, scale_round_divisor); + } + else if (checkDataType(removeNullable(block.getByPosition(arguments[0]).type).get())) + { + decimalToMyDatetime(decimal_column, datetime_res, null_res, scale, scale_divisor, scale_round_divisor); + } + else if (checkDataType(removeNullable(block.getByPosition(arguments[0]).type).get())) + { + decimalToMyDatetime(decimal_column, datetime_res, null_res, scale, scale_divisor, scale_round_divisor); + } + else if (checkDataType(removeNullable(block.getByPosition(arguments[0]).type).get())) + { + decimalToMyDatetime(decimal_column, datetime_res, null_res, scale, scale_divisor, scale_round_divisor); + } + else + { + throw Exception("The first argument of " + getName() + " must be decimal type", ErrorCodes::ILLEGAL_COLUMN); + } + + if (arguments.size() == 1) + block.getByPosition(result).column = ColumnNullable::create(std::move(datetime_column), std::move(null_column)); + else + { + // need append date format + Block temporary_block + { + { + ColumnNullable::create(std::move(datetime_column), std::move(null_column)), + makeNullable(std::make_shared(std::min(scale, 6))), + "" + }, + block.getByPosition(arguments[1]), + { + nullptr, + makeNullable(std::make_shared()), + "" + } + }; + FunctionBuilderPtr func_builder_date_format = FunctionFactory::instance().get("dateFormat", context); + ColumnsWithTypeAndName args{ temporary_block.getByPosition(0), temporary_block.getByPosition(1) }; + auto func_date_format = func_builder_date_format->build(args); + + func_date_format->execute(temporary_block, {0, 1}, 2); + block.getByPosition(result).column = std::move(temporary_block.getByPosition(2).column); + } + } + +private: + const Context & context; + +}; + +class FunctionDateFormat : public IFunction +{ +public: + static constexpr auto name = "dateFormat"; + static FunctionPtr create(const Context &) { return std::make_shared(); }; + + String getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 2; } + bool isInjective(const Block &) override { return false; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (!arguments[0].type->isDateOrDateTime()) + throw Exception("First argument for function " + getName() + " must be date or datetime type", ErrorCodes::ILLEGAL_COLUMN); + if (!arguments[1].type->isString() || !arguments[1].column) + throw Exception("Second argument for function " + getName() + " must be String constant", ErrorCodes::ILLEGAL_COLUMN); + + return std::make_shared(); + } + + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + + void executeImpl(Block & block, const ColumnNumbers & arguments, const size_t result) override + { + const auto * col_from = checkAndGetColumn>(block.getByPosition(arguments[0]).column.get()); + const auto & vec_from = col_from->getData(); + auto col_to = ColumnString::create(); + size_t size = col_from->size(); + const auto & format_col = block.getByPosition(arguments[1]).column; + if (format_col->isColumnConst()) + { + const auto & col_const = checkAndGetColumnConst(format_col.get()); + auto format = col_const->getValue(); + ColumnString::Chars_t & data_to = col_to->getChars(); + ColumnString::Offsets & offsets_to = col_to->getOffsets(); + data_to.resize(size * maxFormattedDateTimeStringLength(format)); + offsets_to.resize(size); + WriteBufferFromVector write_buffer(data_to); + for (size_t i = 0; i < size; i++) + { + writeMyDateTimeTextWithFormat(vec_from[i], write_buffer, format); + writeChar(0, write_buffer); + offsets_to[i] = write_buffer.count(); + } + data_to.resize(write_buffer.count()); + block.getByPosition(result).column = std::move(col_to); + } + else + { + throw Exception("Second argument for function " + getName() + " must be String constant", ErrorCodes::ILLEGAL_COLUMN); + } + } + +}; + /// Monotonicity. diff --git a/dbms/src/Functions/FunctionsDateTime.h b/dbms/src/Functions/FunctionsDateTime.h index 404b0025d02..0c99a1241cf 100644 --- a/dbms/src/Functions/FunctionsDateTime.h +++ b/dbms/src/Functions/FunctionsDateTime.h @@ -41,24 +41,6 @@ namespace ErrorCodes static const Int64 SECOND_IN_ONE_DAY = 86400; static const Int64 E6 = 1000000; -//calculates days since 0000-00-00(may 0000-01-01??) -inline int calcDayNum(int year, int month, int day) -{ - if(year == 0 || month == 0) - return 0; - int delsum = 365*year + 31*(month-1) + day; - if (month <= 2) - { - year--; - } - else - { - delsum -= (month*4 + 23) / 10; - } - int temp = ((year/100 + 1) * 3) / 4; - return delsum + year/4 - temp; -} - // day number per 400 years, from the year that year % 400 = 1 static const int DAY_NUM_PER_400_YEARS = 365 * 400 + 97; // day number per 100 years in every 400 years, from the year that year % 100 = 1 diff --git a/dbms/src/IO/WriteHelpers.h b/dbms/src/IO/WriteHelpers.h index f005087f976..bb61ccef585 100644 --- a/dbms/src/IO/WriteHelpers.h +++ b/dbms/src/IO/WriteHelpers.h @@ -624,6 +624,12 @@ inline void writeDateTimeText(const LocalDateTime & datetime, WriteBuffer & buf) } } +inline void writeMyDateTimeTextWithFormat(UInt64 packed, WriteBuffer & buf, String & format) +{ + const String & str = MyDateTime(packed).dateFormat(format); + buf.write(str.c_str(), str.size()); +} + inline void writeMyDateTimeText(UInt64 packed, int fsp, WriteBuffer & buf) { const String & str = MyDateTime(packed).toString(fsp); diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 0f89de72d11..5792522f31d 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -283,6 +283,7 @@ Context Context::createGlobal(std::shared_ptr runtime res.runtime_components_factory = runtime_components_factory; res.shared = std::make_shared(runtime_components_factory); res.quota = std::make_shared(); + res.timezone_info.init(); return res; } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 4889bba90c6..4ab8ca6f314 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -139,6 +140,8 @@ class Context bool use_l0_opt = true; + TimezoneInfo timezone_info; + using DatabasePtr = std::shared_ptr; using Databases = std::map>; @@ -448,6 +451,9 @@ class Context SharedQueriesPtr getSharedQueries(); + const TimezoneInfo & getTimezoneInfo() const { return timezone_info; }; + TimezoneInfo & getTimezoneInfo() { return timezone_info; }; + /// User name and session identifier. Named sessions are local to users. using SessionKey = std::pair; diff --git a/dbms/src/Interpreters/TimezoneInfo.h b/dbms/src/Interpreters/TimezoneInfo.h new file mode 100644 index 00000000000..35d1e3477aa --- /dev/null +++ b/dbms/src/Interpreters/TimezoneInfo.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +/// A class used to store timezone info, currently only used when handling coprocessor request +struct TimezoneInfo +{ + String timezone_name; + Int64 timezone_offset; + bool is_utc_timezone; + bool is_name_based; + const DateLUTImpl * timezone; + void init() + { + is_name_based = true; + timezone_offset = 0; + timezone = &DateLUT::instance(); + timezone_name = timezone->getTimeZone(); + is_utc_timezone = timezone_name == "UTC"; + } + void resetByDAGRequest(const tipb::DAGRequest & rqst) + { + if (rqst.has_time_zone_name() && !rqst.time_zone_name().empty()) + { + // dag request use name based timezone info + is_name_based = true; + timezone_offset = 0; + timezone = &DateLUT::instance(rqst.time_zone_name()); + timezone_name = timezone->getTimeZone(); + is_utc_timezone = timezone_name == "UTC"; + } + else if (rqst.has_time_zone_offset()) + { + // dag request use offset based timezone info + is_name_based = false; + timezone_offset = rqst.time_zone_offset(); + timezone = &DateLUT::instance("UTC"); + timezone_name = ""; + is_utc_timezone = timezone_offset == 0; + } + else + { + // dag request does not have timezone info + is_name_based = false; + timezone_offset = 0; + // set the default timezone to UTC because TiDB assumes + // the default timezone is UTC + timezone = &DateLUT::instance("UTC"); + timezone_name = ""; + is_utc_timezone = true; + } + } +}; + +} // namespace DB diff --git a/tests/mutable-test/expr/date_format.test b/tests/mutable-test/expr/date_format.test new file mode 100644 index 00000000000..7178ed4bada --- /dev/null +++ b/tests/mutable-test/expr/date_format.test @@ -0,0 +1,30 @@ +# Preparation. +=> DBGInvoke __enable_schema_sync_service('true') + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test + +=> DBGInvoke __set_flush_threshold(1000000, 1000000) + +# Data. +=> DBGInvoke __mock_tidb_table(default, test, 'a int, b MyDatetime(6), c MyDatetime') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __raft_insert_row(default, test, 4, 50, 1, '1988-04-17 01:59:59.123457', '1988-04-17 01:59:59') +=> DBGInvoke __raft_insert_row(default, test, 4, 51, 2, '1988-04-17 03:00:00.123456', '1988-04-17 03:00:00') + +# test date_format +=> DBGInvoke dag('select count(1) from default.test group by a, date_format(b,\'%b-%M-%m-%c-%D-%d-%e-%j-%H-%k-%h-%I-%l-%i-%p-%r-%T-%S-%s-%f-%U-%u-%V-%v-%a-%w-%W-%X-%x-%Y-%ydd\')', 4,'chunk') +┌─count(1)─┬─a─┬─from_UnixTime(b, \'%b-%M-%m-%c-%D-%d-%e-%j-%H-%k-%h-%I-%l-%i-%p-%r-%T-%S-%s-%f-%U-%u-%V-%v-%a-%w-%W-%X-%x-%Y-%ydd\')────────────┐ +│ 1 │ 1 │ Apr-April-04-4-17th-17-17-108-01-1-01-01-1-59-AM-01:59:59 AM-01:59:59-59-59-123457-16-15-16-15-Sun-0-Sunday-1988-1988-1988-88dd │ +│ 1 │ 2 │ Apr-April-04-4-17th-17-17-108-03-3-03-03-3-00-AM-03:00:00 AM-03:00:00-00-00-123456-16-15-16-15-Sun-0-Sunday-1988-1988-1988-88dd │ +└──────────┴───┴─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +=> DBGInvoke dag('select count(1) from default.test group by a, date_format(c,\'%b-%M-%m-%c-%D-%d-%e-%j-%H-%k-%h-%I-%l-%i-%p-%r-%T-%S-%s-%f-%U-%u-%V-%v-%a-%w-%W-%X-%x-%Y-%ydd\')', 4,'chunk') +┌─count(1)─┬─a─┬─from_UnixTime(c, \'%b-%M-%m-%c-%D-%d-%e-%j-%H-%k-%h-%I-%l-%i-%p-%r-%T-%S-%s-%f-%U-%u-%V-%v-%a-%w-%W-%X-%x-%Y-%ydd\')────────────┐ +│ 1 │ 1 │ Apr-April-04-4-17th-17-17-108-01-1-01-01-1-59-AM-01:59:59 AM-01:59:59-59-59-000000-16-15-16-15-Sun-0-Sunday-1988-1988-1988-88dd │ +│ 1 │ 2 │ Apr-April-04-4-17th-17-17-108-03-3-03-03-3-00-AM-03:00:00 AM-03:00:00-00-00-000000-16-15-16-15-Sun-0-Sunday-1988-1988-1988-88dd │ +└──────────┴───┴─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ + +# Clean up. +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test diff --git a/tests/mutable-test/expr/from_unixtime.test b/tests/mutable-test/expr/from_unixtime.test new file mode 100644 index 00000000000..dd4a0a8918a --- /dev/null +++ b/tests/mutable-test/expr/from_unixtime.test @@ -0,0 +1,42 @@ +# Preparation. +=> DBGInvoke __enable_schema_sync_service('true') + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test + +=> DBGInvoke __set_flush_threshold(1000000, 1000000) + +# Data. +=> DBGInvoke __mock_tidb_table(default, test, 'a int, b decimal(12,2), c decimal(65,30), d decimal(10,0)') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __raft_insert_row(default, test, 4, 50, 1, 295385399.12,295385399.123456789098765432123456789000, 295385399) +=> DBGInvoke __raft_insert_row(default, test, 4, 51, 2, 295385400.12,295385400.123456189098765432123456789000, 295385400) + +# test from_unixtime 1 arg +=> DBGInvoke dag('select count(1) from default.test group by a, from_UnixTime(b), from_UnixTime(c), from_UnixTime(d)', 4,'chunk',0, 'Asia/Hong_Kong') +┌─count(1)─┬─a─┬───────────from_UnixTime(b)─┬───────────from_UnixTime(c)─┬───────────from_UnixTime(d)─┐ +│ 1 │ 2 │ 1979-05-13 04:30:00.120000 │ 1979-05-13 04:30:00.123456 │ 1979-05-13 04:30:00.000000 │ +│ 1 │ 1 │ 1979-05-13 03:29:59.120000 │ 1979-05-13 03:29:59.123457 │ 1979-05-13 03:29:59.000000 │ +└──────────┴───┴────────────────────────────┴────────────────────────────┴────────────────────────────┘ +# test from_unixtime 2 arg +=> DBGInvoke dag('select count(1) from default.test group by a, from_UnixTime(b,\'%b-%M-%m-%c-%D-%d-%e-%j-%H-%k-%h-%I-%l-%i-%p-%r-%T-%S-%s-%f-%U-%u-%V-%v-%a-%w-%W-%X-%x-%Y-%ydd\')', 4,'chunk',28800, 'Asia/Hong_Kong') +┌─count(1)─┬─a─┬─from_UnixTime(b, \'%b-%M-%m-%c-%D-%d-%e-%j-%H-%k-%h-%I-%l-%i-%p-%r-%T-%S-%s-%f-%U-%u-%V-%v-%a-%w-%W-%X-%x-%Y-%ydd\')──────────┐ +│ 1 │ 2 │ May-May-05-5-13th-13-13-133-04-4-04-04-4-30-AM-04:30:00 AM-04:30:00-00-00-120000-19-19-19-19-Sun-0-Sunday-1979-1979-1979-79dd │ +│ 1 │ 1 │ May-May-05-5-13th-13-13-133-03-3-03-03-3-29-AM-03:29:59 AM-03:29:59-59-59-120000-19-19-19-19-Sun-0-Sunday-1979-1979-1979-79dd │ +└──────────┴───┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +=> DBGInvoke dag('select count(1) from default.test group by a, from_UnixTime(c,\'%b-%M-%m-%c-%D-%d-%e-%j-%H-%k-%h-%I-%l-%i-%p-%r-%T-%S-%s-%f-%U-%u-%V-%v-%a-%w-%W-%X-%x-%Y-%ydd\')', 4,'chunk',28800, 'Asia/Hong_Kong') +┌─count(1)─┬─a─┬─from_UnixTime(c, \'%b-%M-%m-%c-%D-%d-%e-%j-%H-%k-%h-%I-%l-%i-%p-%r-%T-%S-%s-%f-%U-%u-%V-%v-%a-%w-%W-%X-%x-%Y-%ydd\')──────────┐ +│ 1 │ 1 │ May-May-05-5-13th-13-13-133-03-3-03-03-3-29-AM-03:29:59 AM-03:29:59-59-59-123457-19-19-19-19-Sun-0-Sunday-1979-1979-1979-79dd │ +│ 1 │ 2 │ May-May-05-5-13th-13-13-133-04-4-04-04-4-30-AM-04:30:00 AM-04:30:00-00-00-123456-19-19-19-19-Sun-0-Sunday-1979-1979-1979-79dd │ +└──────────┴───┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ + +=> DBGInvoke dag('select count(1) from default.test group by a, from_UnixTime(d,\'%b-%M-%m-%c-%D-%d-%e-%j-%H-%k-%h-%I-%l-%i-%p-%r-%T-%S-%s-%f-%U-%u-%V-%v-%a-%w-%W-%X-%x-%Y-%ydd\')', 4,'chunk',28800, 'Asia/Hong_Kong') +┌─count(1)─┬─a─┬─from_UnixTime(d, \'%b-%M-%m-%c-%D-%d-%e-%j-%H-%k-%h-%I-%l-%i-%p-%r-%T-%S-%s-%f-%U-%u-%V-%v-%a-%w-%W-%X-%x-%Y-%ydd\')──────────┐ +│ 1 │ 2 │ May-May-05-5-13th-13-13-133-04-4-04-04-4-30-AM-04:30:00 AM-04:30:00-00-00-000000-19-19-19-19-Sun-0-Sunday-1979-1979-1979-79dd │ +│ 1 │ 1 │ May-May-05-5-13th-13-13-133-03-3-03-03-3-29-AM-03:29:59 AM-03:29:59-59-59-000000-19-19-19-19-Sun-0-Sunday-1979-1979-1979-79dd │ +└──────────┴───┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ + +# Clean up. +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test