From 4597300085def357fc5d689b007a006b0bd99baa Mon Sep 17 00:00:00 2001 From: Julian Hyde Date: Mon, 29 Jul 2019 13:16:56 -0700 Subject: [PATCH 01/19] Workaround --- .../java/org/apache/calcite/rel/rel2sql/SqlImplementor.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java index 9950674ea15..9eb88ca602f 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/SqlImplementor.java @@ -571,6 +571,8 @@ private boolean requiresAlias(SqlNode node) { case JOIN: case EXPLICIT_TABLE: return false; + case SELECT: + return node instanceof SqlSelect; // TODO workaround asIs hack default: return true; } From b338f1502f0c7d0988c088fe411bb9524acacd07 Mon Sep 17 00:00:00 2001 From: Julian Hyde Date: Tue, 10 Mar 2020 12:32:46 -0700 Subject: [PATCH 02/19] Looker instructions --- looker-release.sh | 98 +++++++++++++++++++++++++++ looker-snapshot.sh | 43 ++++++++++++ looker-upload-artifact-registry.sh | 43 ++++++++++++ looker.md | 105 +++++++++++++++++++++++++++++ 4 files changed, 289 insertions(+) create mode 100755 looker-release.sh create mode 100755 looker-snapshot.sh create mode 100755 looker-upload-artifact-registry.sh create mode 100644 looker.md diff --git a/looker-release.sh b/looker-release.sh new file mode 100755 index 00000000000..2d57ed9f8d7 --- /dev/null +++ b/looker-release.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# HOW TO RELEASE the Looker fork of Calcite: +# +# 1. Start working off the current fork branch: https://github.com/looker-open-source/calcite/tree/looker +# `git fetch git@github.com:looker-open-source/calcite.git looker && git checkout FETCH_HEAD` +# 2. Do whatever needs to be done (rebase on trunk, cherry-pick, remove fixups, etc.), +# but don't bother updating the `calcite.version` line in `gradle.properties`. +# There should only be "business logic" commits, +# and every commit that is not yet upstreamed (called "fix-up" commits) +# should reference a Calcite Jira ticket in its message, +# and have a PR open to upstream it to Apache's main repo, if possible. +# 3. Once the code looks good and your Git working directory is clean, run this script +# which will do all of this (mostly) automatically: +# a) Pull all Looker Calcite version tags from GitHub and show you what the latest one is. +# This is just for convenience. +# b) Ask you what the next version number should be. +# The major number should always be whatever's in Apache's `main` branch. +# The minor number should probably be 1 less than whatever's in Apache's `main` branch. +# Increment the patch number when you make backward compatible bug fixes. +# Generally, we just increment the patch number. +# c) Reset your local `looker` branch to the current HEAD. +# d) Update the version line in `gradle.properties` and create a version bump commit. +# e) Create a release tag pointing to the new version bump commit. +# f) Provide you with the commands to push your new branch and tag to GitHub, +# and publish release artifacts to Nexus. +# This script only automates local changes by design; double-check everything before pushing. + +echo "Fetching all tags from looker-open-source repo..." >&2 +git fetch git@github.com:looker-open-source/calcite.git --tags && ( + + echo -e "\nLatest Looker release tag was '$(git tag --list | grep -E '^calcite-[0-9]+(\.[0-9]+)*-looker$' | sort --version-sort --reverse | head --lines=1)'" >&2 + echo "What should the next version be called?" >&2 + read -p "Input just the numbers and dots (do not include 'calcite-' or '-looker'): " NEXT_NUMBER + export NEXT_VERSION="${NEXT_NUMBER}-looker" + export NEXT_TAG="calcite-${NEXT_VERSION}" + + echo -e "\nSetting version number in gradle.properties to '$NEXT_VERSION'." >&2 + # MacOS uses BSD sed, which works differently from GNU sed on Linux. + if [[ "$(uname -s)" == "Darwin" ]] + then + # https://stackoverflow.com/questions/5694228/sed-in-place-flag-that-works-both-on-mac-bsd-and-linux + # https://stackoverflow.com/questions/64373364/how-to-fix-sed-command-on-macos-with-error-extra-characters-after-at-the-end-o + sed -i.bak "/^calcite\\.version=.*/c\\ +calcite.version=$NEXT_VERSION +" gradle.properties && rm gradle.properties.bak + else + sed -i "/^calcite\\.version=.*/c\\calcite.version=$NEXT_VERSION" gradle.properties + fi + + # $(exit $?) has the same status code as the previous `sed` command. + $(exit $?) && ( + + echo -e "\nBuilding '$NEXT_VERSION'..." >&2 + ./gradlew build -x :redis:test && ( + + export LOOKER_COMMIT="$(git rev-parse --short HEAD)" + + export COMMIT_MSG="Prepare for $NEXT_TAG release" + echo -e "\nTests passed! Creating commit '$COMMIT_MSG'." >&2 + git add gradle.properties && git commit -m "$COMMIT_MSG" && ( + + echo -e "\nCreating new tag '$NEXT_TAG'." >&2 + git tag -f "$NEXT_TAG" + + echo -e "\nTake a look around." >&2 + echo -e "You should see (starting from the most recent tip of your history):" >&2 + echo -e "- A tagged release commit, which just updates the version and nothing else." >&2 + echo -e " There should be only 1 such commit in the whole history." >&2 + echo -e "- The latest fixup commit (business logic that is not upstreamed)." >&2 + echo -e " This becomes the new tip of the looker branch." >&2 + echo -e "- Prior fixups, if any..." >&2 + echo -e "- All commits from upstream..." >&2 + echo -e "\nIf everything looks good, you can publish to Nexus with this command:\n" >&2 + echo -e " ./gradlew -Prelease -PskipSign publishAllPublicationsToLookerNexusRepository\n" >&2 + echo -e "And you can push the release tag and force-push the looker branch to looker-open-source with these commands:\n" >&2 + echo -e " git push git@github.com:looker-open-source/calcite.git $NEXT_TAG" + echo -e " git push -f git@github.com:looker-open-source/calcite.git $LOOKER_COMMIT:looker" + ) + ) + ) +) diff --git a/looker-snapshot.sh b/looker-snapshot.sh new file mode 100755 index 00000000000..63b37ca66f2 --- /dev/null +++ b/looker-snapshot.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# $1 is the sub-project directory +# $2 is the artifact ID +# $3 is the version +function snapshot_upload { + mvn deploy:deploy-file \ + -DgroupId=org.apache.calcite \ + -DartifactId="$2" \ + -Dversion="$3" \ + -Dpackaging=jar \ + -Dfile="./$1/build/libs/$2-$3.jar" \ + -DgeneratePom=false \ + -DpomFile="./$1/build/publications/$1/pom-default.xml" \ + -DrepositoryId=nexus \ + -Durl=https://nexusrepo.looker.com/repository/maven-snapshots/ +} + +./gradlew build -x :redis:test && ./gradlew jar && ./gradlew generatePom && ( + VERSION="$(sed -n 's/^calcite\.version=\([^ ]*\).*/\1/p' gradle.properties)-SNAPSHOT" + snapshot_upload core calcite-core "$VERSION" + snapshot_upload babel calcite-babel "$VERSION" + snapshot_upload linq4j calcite-linq4j "$VERSION" + snapshot_upload testkit calcite-testkit "$VERSION" + echo + echo "Done uploading version ${VERSION} to Looker Nexus Snapshots!" +) diff --git a/looker-upload-artifact-registry.sh b/looker-upload-artifact-registry.sh new file mode 100755 index 00000000000..0881972192d --- /dev/null +++ b/looker-upload-artifact-registry.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# $1 is the sub-project directory +# $2 is the artifact ID +# $3 is the version +function artifact_registry_upload { + mvn deploy:deploy-file \ + -DgroupId=org.apache.calcite \ + -DartifactId="$2" \ + -Dversion="$3" \ + -Dpackaging=jar \ + -Dfile="./$1/build/libs/$2-$3.jar" \ + -DgeneratePom=false \ + -DpomFile="./$1/build/publications/$1/pom-default.xml" \ + -DrepositoryId=artifact-registry \ + -Durl=https://us-maven.pkg.dev/prow-build-looker/looker-maven-private +} + +./gradlew build -x :redis:test && ./gradlew jar && ./gradlew generatePom && ( + VERSION="$(sed -n 's/^calcite\.version=\([^ ]*\).*/\1/p' gradle.properties)" + artifact_registry_upload core calcite-core "$VERSION" + artifact_registry_upload babel calcite-babel "$VERSION" + artifact_registry_upload linq4j calcite-linq4j "$VERSION" + artifact_registry_upload testkit calcite-testkit "$VERSION" + echo + echo "Done uploading version ${VERSION} to Looker Artifact Registry!" +) diff --git a/looker.md b/looker.md new file mode 100644 index 00000000000..5cf2f199aae --- /dev/null +++ b/looker.md @@ -0,0 +1,105 @@ + +# Looker's branch of Calcite + +This document describes how to develop Looker's branch of Calcite. + +Do not merge to Calcite's master branch. + +## Development + +*Read the instructions* in the `looker-release.sh` script, +but do not run that script until you're ready to publish a production release. + +Looker has poor infrastructure for testing with local builds of Avatica. +The easiest way is to upload a snapshot version to Looker's Nexus repository and use it. +To upload a snapshot version, simply run `./looker-snapshot.sh`, +which runs `./gradlew build` and, if successful, +uploads the resulting snapshot artifacts to the repo +using the version number configured in `gradle.properties` (plus a "-SNAPSHOT" suffix). + +Then, use that snapshot version in Looker's `internal/repositories/maven_deps.bzl` and repin, +and you're ready to build. + +## Release + +Define Looker's Nexus repository in your `~/.gradle/init.gradle.kts` +file: + +```kotlin +allprojects { + plugins.withId("maven-publish") { + configure { + repositories { + maven { + name = "lookerNexus" + val baseUrl = "https://nexusrepo.looker.com" + val releasesUrl = "$baseUrl/repository/maven-releases" + val snapshotsUrl = "$baseUrl/repository/maven-snapshots" + val release = !project.version.toString().endsWith("-SNAPSHOT") + // val release = project.hasProperty("release") + url = uri(if (release) releasesUrl else snapshotsUrl) + credentials { + username = "xxx" + password = "xxx" + } + } + } + } + } +} +``` + +In the above fragment, replace the values of the `username` and +`password` properties with the secret credentials. + +*NOTE* This fragment *must* be in a file outside of your git sandbox. +If the file were in the git sandbox, it would be too easy to +accidentally commit the secret credentials and expose them on a +public site. + +*Read the instructions* in the `looker-release.sh` script, then run it. +The script will only make local changes. +You'll have a chance to review them before pushing anything to Nexus or GitHub. + +Each release will have a name like `1.21.1-looker` (if the most +recent official Calcite release is `1.21`) and have a git tag +`calcite-1.21.1-looker`. + +You should make it from a branch that differs from Calcite's +`master` branch in only minor ways: +* Cherry-pick commits from the previous `calcite-x.xx.x-looker` + release that set up Looker's repositories (or, if you prefer, + rebase the previous release branch onto the latest master) +* If necessary, include one or two commits for short-term fixes, but + log [JIRA cases](https://issues.apache.org/jira/browse/CALCITE) to + get them into `master`. +* Keep fixup commits coherent (squash incremental commits) + and to a minimum. +* The final commit for a release must only set the version + in `gradle.properties`, and it must be the only such commit + that differs from Calcite trunk. Note that Calcite increments + the version in trunk immediately after each release, so + that version is typically unreleased. The Looker version + should be named after the most recent Calcite release, + so the version in trunk is generally decremented + while adding the `-looker` suffix. + +Check the artifacts +[on Nexus](https://nexusproxy.looker.com/#browse/search=keyword%3Dorg.apache.calcite). From a941d290654e8dc99a985e41d7fc371adaa7d574 Mon Sep 17 00:00:00 2001 From: Julian Hyde Date: Wed, 16 Sep 2020 10:21:55 -0700 Subject: [PATCH 03/19] Temporary workaround: change SqlInternalOperator's syntax back to FUNCTION --- .../calcite/sql/SqlInternalOperator.java | 2 +- .../elasticsearch/PredicateAnalyzer.java | 23 +++++++++++-------- .../calcite/sql/parser/SqlParserTest.java | 10 ++++---- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlInternalOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlInternalOperator.java index 70c590650af..87d21667c8b 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlInternalOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlInternalOperator.java @@ -77,7 +77,7 @@ public SqlInternalOperator( //~ Methods ---------------------------------------------------------------- @Override public SqlSyntax getSyntax() { - return SqlSyntax.INTERNAL; + return SqlSyntax.FUNCTION; } @Override public RelDataType deriveType(SqlValidator validator, diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java index 9b9796857ca..5c42ac26113 100644 --- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java +++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java @@ -25,6 +25,7 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexVisitorImpl; +import org.apache.calcite.sql.SqlInternalOperator; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlSyntax; import org.apache.calcite.sql.type.SqlTypeFamily; @@ -172,6 +173,15 @@ private static boolean supportedRexCall(RexCall call) { return false; } case FUNCTION: + case INTERNAL: + if (call.getOperator() instanceof SqlInternalOperator) { + switch (call.getKind()) { + case SEARCH: + return canBeTranslatedToTermsQuery(call); + default: + return false; + } + } return true; case POSTFIX: switch (call.getKind()) { @@ -188,13 +198,6 @@ private static boolean supportedRexCall(RexCall call) { default: return false; } - case INTERNAL: - switch (call.getKind()) { - case SEARCH: - return canBeTranslatedToTermsQuery(call); - default: - return false; - } case FUNCTION_ID: case FUNCTION_STAR: default: @@ -248,8 +251,6 @@ static boolean isSearchWithComplementedPoints(RexCall search) { return postfix(call); case PREFIX: return prefix(call); - case INTERNAL: - return binary(call); case SPECIAL: switch (call.getKind()) { case CAST: @@ -267,6 +268,10 @@ static boolean isSearchWithComplementedPoints(RexCall search) { throw new PredicateAnalyzerException(message); } case FUNCTION: + case INTERNAL: + if (call.getOperator() instanceof SqlInternalOperator) { + return binary(call); + } if (call.getOperator().getName().equalsIgnoreCase("CONTAINS")) { List operands = visitList(call.getOperands()); String query = diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index ae0234fab59..eb88b808f3a 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -8867,15 +8867,15 @@ private static Consumer> checkWarnings( @Test void testJsonType() { expr("json_type('11.56')") - .ok("JSON_TYPE('11.56')"); + .ok("`JSON_TYPE`('11.56')"); expr("json_type('{}')") - .ok("JSON_TYPE('{}')"); + .ok("`JSON_TYPE`('{}')"); expr("json_type(null)") - .ok("JSON_TYPE(NULL)"); + .ok("`JSON_TYPE`(NULL)"); expr("json_type('[\"foo\",null]')") - .ok("JSON_TYPE('[\"foo\",null]')"); + .ok("`JSON_TYPE`('[\"foo\",null]')"); expr("json_type('{\"foo\": \"100\"}')") - .ok("JSON_TYPE('{\"foo\": \"100\"}')"); + .ok("`JSON_TYPE`('{\"foo\": \"100\"}')"); } @Test void testJsonDepth() { From d8933aa01c65e52ffc1bf8677054997e2b831c90 Mon Sep 17 00:00:00 2001 From: Marieke Gueye Date: Tue, 22 Mar 2022 16:22:12 +0000 Subject: [PATCH 04/19] [CALCITE-5052] Allow Source based on a URL with jar: protocol This allows dependent projects to run tests using Bazel. (Previously, DiffRepository would give errors because Bazel has packaged the .xml files it needs inside JAR files.) Close apache/calcite#2750 --- .../java/org/apache/calcite/util/Sources.java | 36 +++++++----- .../org/apache/calcite/util/SourceTest.java | 57 +++++++++++++++++++ .../apache/calcite/test/DiffRepository.java | 21 ++++++- 3 files changed, 97 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/util/Sources.java b/core/src/main/java/org/apache/calcite/util/Sources.java index d1846505564..53b2d4e8652 100644 --- a/core/src/main/java/org/apache/calcite/util/Sources.java +++ b/core/src/main/java/org/apache/calcite/util/Sources.java @@ -191,22 +191,30 @@ private File fileNonNull() { } private static @Nullable File urlToFile(URL url) { - if (!"file".equals(url.getProtocol())) { + switch (url.getProtocol()) { + case "jar": + try { + return urlToFile(new URL(url.getFile())); + } catch (MalformedURLException e) { + return null; + } + case "file": + URI uri; + try { + uri = url.toURI(); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Unable to convert URL " + url + " to URI", e); + } + if (uri.isOpaque()) { + // It is like file:test%20file.c++ + // getSchemeSpecificPart would return "test file.c++" + return new File(uri.getSchemeSpecificPart()); + } + // See https://stackoverflow.com/a/17870390/1261287 + return Paths.get(uri).toFile(); + default: return null; } - URI uri; - try { - uri = url.toURI(); - } catch (URISyntaxException e) { - throw new IllegalArgumentException("Unable to convert URL " + url + " to URI", e); - } - if (uri.isOpaque()) { - // It is like file:test%20file.c++ - // getSchemeSpecificPart would return "test file.c++" - return new File(uri.getSchemeSpecificPart()); - } - // See https://stackoverflow.com/a/17870390/1261287 - return Paths.get(uri).toFile(); } private static URL fileToUrl(File file) { diff --git a/core/src/test/java/org/apache/calcite/util/SourceTest.java b/core/src/test/java/org/apache/calcite/util/SourceTest.java index 4f4f775ebf2..90c8759e193 100644 --- a/core/src/test/java/org/apache/calcite/util/SourceTest.java +++ b/core/src/test/java/org/apache/calcite/util/SourceTest.java @@ -15,8 +15,13 @@ * limitations under the License. */ package org.apache.calcite.util; + +import org.apache.commons.lang3.StringUtils; + import com.google.common.io.CharSource; +import net.hydromatic.foodmart.queries.FoodmartQuerySet; + import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -25,13 +30,18 @@ import java.io.BufferedReader; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; import java.io.Reader; +import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.regex.Pattern; import java.util.stream.Stream; import static org.apache.calcite.util.Sources.file; @@ -39,6 +49,8 @@ import static org.apache.calcite.util.Sources.url; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasToString; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -118,6 +130,51 @@ void testAbsoluteFileToUrl(String path, String expectedUrl) throws URISyntaxExce () -> "Sources.of(Sources.of(file(" + path + ").absolutePath).url()).file().getPath()"); } + /** Test case for + * [CALCITE-5052] + * Allow Source based on a URL with jar: protocol. */ + @Test void testJarFileUrl() throws MalformedURLException { + // mock jar file + final String jarPath = "jar:file:sources!/abcdef.txt"; + final URL url = new URL(jarPath); + final Source source = of(url); + assertThat("No file retrieved for Sources.of(file " + jarPath + ")", + source.file(), notNullValue()); + assertThat("Sources.of(file " + jarPath + ").url()).file().getPath()", + slashify(source.file().getPath()), + is("sources!/abcdef.txt")); + + } + + /** Tests {@link Sources#of(URL)} with code similar to + * {@code DiffRepository.Key#toRepo()}. */ + @Test void testJarFileUrlWrite() { + final URL refFile = FoodmartQuerySet.class.getResource("/queries.json"); + assertThat(refFile, notNullValue()); + final Source source = of(refFile); + final String refFilePath = source.file().getAbsolutePath(); + final String logFilePath; + assertThat(StringUtils.containsIgnoreCase(refFilePath, ".jar!"), is(true)); + // If the file is located in a JAR, we cannot write the file in place + // so we add it to the /tmp directory + // the expected output is /tmp/[jarname]/[path-to-file-in-jar/filename]_actual.json + logFilePath = + Pattern.compile(".*\\/(.*)\\.jar\\!(.*)\\.json") + .matcher(refFilePath) + .replaceAll("/tmp/$1$2_actual.json"); + final File logFile = new File(logFilePath); + assertThat(refFile, not(is(logFile.getAbsolutePath()))); + boolean b = logFile.getParentFile().mkdirs(); + Util.discard(b); + try (FileOutputStream fos = new FileOutputStream(logFile); + OutputStreamWriter osw = new OutputStreamWriter(fos, StandardCharsets.UTF_8); + PrintWriter pw = new PrintWriter(osw)) { + pw.println("hello, world!"); + } catch (IOException e) { + throw Util.throwAsRuntime(e); + } + } + @Test void testAppendWithSpaces() { String fooRelative = "fo o+"; String fooAbsolute = ROOT_PREFIX + "fo o+"; diff --git a/testkit/src/main/java/org/apache/calcite/test/DiffRepository.java b/testkit/src/main/java/org/apache/calcite/test/DiffRepository.java index 160d42dd8c1..8a336686968 100644 --- a/testkit/src/main/java/org/apache/calcite/test/DiffRepository.java +++ b/testkit/src/main/java/org/apache/calcite/test/DiffRepository.java @@ -23,6 +23,8 @@ import org.apache.calcite.util.Util; import org.apache.calcite.util.XmlOutput; +import org.apache.commons.lang3.StringUtils; + import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; @@ -53,6 +55,7 @@ import java.util.Objects; import java.util.SortedMap; import java.util.TreeMap; +import java.util.regex.Pattern; import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -901,6 +904,9 @@ String filter( /** Cache key. */ private static class Key { + private static final Pattern JAR_EMBEDDED_XML_PATH = + Pattern.compile(".*\\/(.*)\\.jar\\!(.*)\\.xml"); + private final Class clazz; private final DiffRepository baseRepository; private final Filter filter; @@ -929,9 +935,18 @@ private static class Key { DiffRepository toRepo() { final URL refFile = findFile(clazz, ".xml"); final String refFilePath = Sources.of(refFile).file().getAbsolutePath(); - final String logFilePath = refFilePath - .replace("resources", "diffrepo") - .replace(".xml", "_actual.xml"); + final String logFilePath; + if (StringUtils.containsIgnoreCase(refFilePath, ".jar!")) { + // If the file is located in a JAR, we cannot write the file in place + // so we add it to the /tmp directory + // the expected output is /tmp/[jarname]/[path-to-file-in-jar/filename]_actual.xml + logFilePath = + JAR_EMBEDDED_XML_PATH.matcher(refFilePath).replaceAll("/tmp/$1$2_actual.xml"); + } else { + logFilePath = refFilePath + .replace("resources", "diffrepo") + .replace(".xml", "_actual.xml"); + } final File logFile = new File(logFilePath); assert !refFilePath.equals(logFile.getAbsolutePath()); return new DiffRepository(refFile, logFile, baseRepository, filter, From 327c3feb86c1d42202621f9441e0ccb41d35d644 Mon Sep 17 00:00:00 2001 From: TJ Banghart Date: Tue, 28 Mar 2023 15:02:24 -0700 Subject: [PATCH 05/19] [CALCITE-3312] WIP: TIMESTAMPDIFF cannot be converted to SQL --- babel/src/test/resources/sql/big-query.iq | 5 +- .../calcite/rel/externalize/RelJson.java | 1 + .../calcite/sql/fun/SqlStdOperatorTable.java | 4 + .../sql2rel/StandardConvertletTable.java | 25 +- .../java/org/apache/calcite/util/Bug.java | 5 + .../rel/rel2sql/RelToSqlConverterTest.java | 27 + .../apache/calcite/test/SqlOperatorTest.java | 737 +++++++++--------- 7 files changed, 434 insertions(+), 370 deletions(-) diff --git a/babel/src/test/resources/sql/big-query.iq b/babel/src/test/resources/sql/big-query.iq index 5ffa0bdf66d..6e0ed21ebbc 100755 --- a/babel/src/test/resources/sql/big-query.iq +++ b/babel/src/test/resources/sql/big-query.iq @@ -3428,7 +3428,8 @@ SELECT (1 row) !ok - +# Bug.CALCITE_2539_FIXED +!if (false) { ##################################################################### # DATE_DIFF # @@ -3739,6 +3740,8 @@ SELECT TIMESTAMP_DIFF(TIMESTAMP '2008-12-25', TIMESTAMP '2008-09-25', `quarter`) (1 row) !ok + +!} ##################################################################### # DATE_TRUNC # diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java index 3f7cbbca973..130ab18462e 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelJson.java @@ -630,6 +630,7 @@ public Object toJson(RexNode node) { } map.put("operands", list); switch (node.getKind()) { + case TIMESTAMP_DIFF: case MINUS: case CAST: case SAFE_CAST: diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java index 93f510518bc..3dd42bdfe78 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java @@ -1560,6 +1560,10 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { */ public static final SqlSpecialOperator REINTERPRET = new SqlSpecialOperator("Reinterpret", SqlKind.REINTERPRET) { + @Override public void unparse(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) { + call.operand(0).unparse(writer, leftPrec, rightPrec); + } + @Override public SqlOperandCountRange getOperandCountRange() { return SqlOperandCountRanges.between(1, 2); } diff --git a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java index 1490229ff97..2a8743bd1c0 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java @@ -79,6 +79,7 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.util.Bug; import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; @@ -202,26 +203,34 @@ private StandardConvertletTable() { registerOp(SqlLibraryOperators.DATE_ADD, new TimestampAddConvertlet()); - registerOp(SqlLibraryOperators.DATE_DIFF, - new TimestampDiffConvertlet()); + if (Bug.CALCITE_3312_FIXED) { + registerOp(SqlLibraryOperators.DATE_DIFF, + new TimestampDiffConvertlet()); + } registerOp(SqlLibraryOperators.DATE_SUB, new TimestampSubConvertlet()); registerOp(SqlLibraryOperators.DATETIME_ADD, new TimestampAddConvertlet()); - registerOp(SqlLibraryOperators.DATETIME_DIFF, - new TimestampDiffConvertlet()); + if (Bug.CALCITE_3312_FIXED) { + registerOp(SqlLibraryOperators.DATETIME_DIFF, + new TimestampDiffConvertlet()); + } registerOp(SqlLibraryOperators.DATETIME_SUB, new TimestampSubConvertlet()); registerOp(SqlLibraryOperators.TIME_ADD, new TimestampAddConvertlet()); - registerOp(SqlLibraryOperators.TIME_DIFF, - new TimestampDiffConvertlet()); + if (Bug.CALCITE_3312_FIXED) { + registerOp(SqlLibraryOperators.TIME_DIFF, + new TimestampDiffConvertlet()); + } registerOp(SqlLibraryOperators.TIME_SUB, new TimestampSubConvertlet()); registerOp(SqlLibraryOperators.TIMESTAMP_ADD2, new TimestampAddConvertlet()); - registerOp(SqlLibraryOperators.TIMESTAMP_DIFF3, - new TimestampDiffConvertlet()); + if (Bug.CALCITE_3312_FIXED) { + registerOp(SqlLibraryOperators.TIMESTAMP_DIFF3, + new TimestampDiffConvertlet()); + } registerOp(SqlLibraryOperators.TIMESTAMP_SUB, new TimestampSubConvertlet()); diff --git a/core/src/main/java/org/apache/calcite/util/Bug.java b/core/src/main/java/org/apache/calcite/util/Bug.java index e0eaa97a3ec..652c8146091 100644 --- a/core/src/main/java/org/apache/calcite/util/Bug.java +++ b/core/src/main/java/org/apache/calcite/util/Bug.java @@ -162,6 +162,11 @@ public abstract class Bug { * JSON data type support is fixed. */ public static final boolean CALCITE_2869_FIXED = false; + /** Whether + * [CALCITE-3312] + * TIMESTAMPDIFF cannot be converted to SQL is fixed. */ + public static final boolean CALCITE_3312_FIXED = false; + /** Whether * [CALCITE-3243] * Incomplete validation of operands in JSON functions is fixed. */ diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 6e55237bc9a..d062d2185fa 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -2351,6 +2351,33 @@ private SqlDialect nonOrdinalDialect() { sql(parseDatetime).withLibrary(SqlLibrary.BIG_QUERY).ok(expectedParseDatetime); } + @Test void testBigQueryDatetimeDiffFunctions() { + final String timeDiff = "select time_diff(time '15:30:00', time '14:35:00', minute)\n" + + "from \"foodmart\".\"product\"\n"; + final String expectedTimeDiff = "SELECT TIME_DIFF(TIME '15:30:00', TIME '14:35:00', MINUTE)\n" + + "FROM foodmart.product"; + + final String dateDiff = "select date_diff(date '2010-07-07', date '2008-12-25', day)\n" + + "from \"foodmart\".\"product\"\n"; + final String expectedDateDiff = "SELECT DATE_DIFF(DATE '2010-07-07', DATE '2008-12-25', DAY)\n" + + "FROM foodmart.product"; + + final String timestampDiff = + "select timestamp_diff(timestamp '2010-07-07 10:20:00', timestamp '2008-12-25 15:30:00', " + + "day)\n" + + "from \"foodmart\".\"product\"\n"; + final String expectedtimestampDiff = + "SELECT TIMESTAMP_DIFF(TIMESTAMP '2010-07-07 10:20:00', TIMESTAMP '2008-12-25 15:30:00', " + + "DAY)\n" + + "FROM foodmart.product"; + + + final Sql sql = fixture().withBigQuery().withLibrary(SqlLibrary.BIG_QUERY); + sql.withSql(timeDiff).ok(expectedTimeDiff); + sql.withSql(dateDiff).ok(expectedDateDiff); + sql.withSql(timestampDiff).ok(expectedtimestampDiff); + } + @Test void testBigQueryTimeTruncFunctions() { String timestampTrunc = "select timestamp_trunc(timestamp '2012-02-03 15:30:00', month)\n" + "from \"foodmart\".\"product\"\n"; diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index d0e529944d2..17b4debe68c 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -1285,8 +1285,10 @@ void testCastStringToDateTime(CastType castType, SqlOperatorFixture f) { f.checkScalar("cast('2004-02-29' as TIMESTAMP)", "2004-02-29 00:00:00", "TIMESTAMP(0) NOT NULL"); - f.checkScalar("cast('1945-02-24 12:42:25.34' as TIMESTAMP(2))", - "1945-02-24 12:42:25.34", "TIMESTAMP(2) NOT NULL"); + // TODO:[CALCITE-6282] enabled this test but + // it doesn't work with implementation of [CALCITE-6055] + // f.checkScalar("cast('1945-02-24 12:42:25.34' as TIMESTAMP(2))", + // "1945-02-24 12:42:25.34", "TIMESTAMP(2) NOT NULL"); if (castType == CastType.CAST) { f.checkFails("cast('1945-2-2 12:2:5' as TIMESTAMP)", "Invalid DATE value, '1945-2-2 12:2:5'", true); @@ -12038,11 +12040,13 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { f.checkScalar("timestampdiff(\"month4\", date '2016-02-24', " + "date '2016-02-23')", "0", "INTEGER NOT NULL"); - f.withLibrary(SqlLibrary.BIG_QUERY) - .setFor(SqlLibraryOperators.TIMESTAMP_DIFF3) - .checkScalar("timestamp_diff(timestamp '2008-12-25 15:30:00', " - + "timestamp '2008-12-25 16:30:00', \"minute15\")", - "-4", "INTEGER NOT NULL"); + if (Bug.CALCITE_2539_FIXED) { + f.withLibrary(SqlLibrary.BIG_QUERY) + .setFor(SqlLibraryOperators.TIMESTAMP_DIFF3) + .checkScalar("timestamp_diff(timestamp '2008-12-25 15:30:00', " + + "timestamp '2008-12-25 16:30:00', \"minute15\")", + "-4", "INTEGER NOT NULL"); + } } @Test void testFloorFuncInterval() { @@ -12354,102 +12358,104 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { final SqlOperatorFixture f = fixture() .withLibrary(SqlLibrary.BIG_QUERY) .setFor(SqlLibraryOperators.TIMESTAMP_DIFF3); - HOUR_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(timestamp '2016-02-24 12:42:25', " - + "timestamp '2016-02-24 15:42:25', " - + s + ")", - "-3", "INTEGER NOT NULL")); - MICROSECOND_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(timestamp '2016-02-24 12:42:25', " - + "timestamp '2016-02-24 12:42:20', " - + s + ")", - "5000000", "INTEGER NOT NULL")); - YEAR_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(timestamp '2014-02-24 12:42:25', " - + "timestamp '2016-02-24 12:42:25', " - + s + ")", - "-2", "INTEGER NOT NULL")); - WEEK_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(timestamp '2014-02-24 12:42:25', " - + "timestamp '2016-02-24 12:42:25', " - + s + ")", - "-104", "INTEGER NOT NULL")); - WEEK_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(timestamp '2014-02-19 12:42:25', " - + "timestamp '2016-02-24 12:42:25', " - + s + ")", - "-105", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(timestamp '2014-02-24 12:42:25', " - + "timestamp '2016-02-24 12:42:25', " - + s + ")", - "-24", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(timestamp '2019-09-01 12:42:25', " - + "timestamp '2020-03-01 12:42:25', " - + s + ")", - "-6", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(timestamp '2019-09-01 12:42:25', " - + "timestamp '2016-08-01 12:42:25', " - + s + ")", - "37", "INTEGER NOT NULL")); - QUARTER_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(timestamp '2014-02-24 12:42:25', " - + "timestamp '2016-02-24 12:42:25', " - + s + ")", - "-8", "INTEGER NOT NULL")); - f.checkScalar("timestamp_diff(timestamp '2014-02-24 12:42:25', " - + "timestamp '2614-02-24 12:42:25', " - + "CENTURY)", - "-6", "INTEGER NOT NULL"); - QUARTER_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(timestamp '2016-02-24 12:42:25', " - + "cast(null as timestamp), " - + s + ")", - isNullValue(), "INTEGER")); - - // timestamp_diff with date - MONTH_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(date '2016-03-15', " - + "date '2016-06-14', " - + s + ")", - "-3", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(date '2019-09-01', " - + "date '2020-03-01', " - + s + ")", - "-6", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(date '2019-09-01', " - + "date '2016-08-01', " - + s + ")", - "37", "INTEGER NOT NULL")); - DAY_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(date '2016-06-15', " - + "date '2016-06-14', " - + s + ")", - "1", "INTEGER NOT NULL")); - HOUR_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(date '2016-06-15', " - + "date '2016-06-14', " - + s + ")", - "24", "INTEGER NOT NULL")); - HOUR_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(date '2016-06-15', " - + "date '2016-06-15', " - + s + ")", - "0", "INTEGER NOT NULL")); - MINUTE_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(date '2016-06-15', " - + "date '2016-06-14', " - + s + ")", - "1440", "INTEGER NOT NULL")); - DAY_VARIANTS.forEach(s -> - f.checkScalar("timestamp_diff(date '2016-06-15', " - + "cast(null as date), " - + s + ")", - isNullValue(), "INTEGER")); + if (Bug.CALCITE_2539_FIXED) { + HOUR_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(timestamp '2016-02-24 12:42:25', " + + "timestamp '2016-02-24 15:42:25', " + + s + ")", + "-3", "INTEGER NOT NULL")); + MICROSECOND_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(timestamp '2016-02-24 12:42:25', " + + "timestamp '2016-02-24 12:42:20', " + + s + ")", + "5000000", "INTEGER NOT NULL")); + YEAR_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(timestamp '2014-02-24 12:42:25', " + + "timestamp '2016-02-24 12:42:25', " + + s + ")", + "-2", "INTEGER NOT NULL")); + WEEK_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(timestamp '2014-02-24 12:42:25', " + + "timestamp '2016-02-24 12:42:25', " + + s + ")", + "-104", "INTEGER NOT NULL")); + WEEK_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(timestamp '2014-02-19 12:42:25', " + + "timestamp '2016-02-24 12:42:25', " + + s + ")", + "-105", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(timestamp '2014-02-24 12:42:25', " + + "timestamp '2016-02-24 12:42:25', " + + s + ")", + "-24", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(timestamp '2019-09-01 12:42:25', " + + "timestamp '2020-03-01 12:42:25', " + + s + ")", + "-6", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(timestamp '2019-09-01 12:42:25', " + + "timestamp '2016-08-01 12:42:25', " + + s + ")", + "37", "INTEGER NOT NULL")); + QUARTER_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(timestamp '2014-02-24 12:42:25', " + + "timestamp '2016-02-24 12:42:25', " + + s + ")", + "-8", "INTEGER NOT NULL")); + f.checkScalar("timestamp_diff(timestamp '2014-02-24 12:42:25', " + + "timestamp '2614-02-24 12:42:25', " + + "CENTURY)", + "-6", "INTEGER NOT NULL"); + QUARTER_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(timestamp '2016-02-24 12:42:25', " + + "cast(null as timestamp), " + + s + ")", + isNullValue(), "INTEGER")); + + // timestamp_diff with date + MONTH_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(date '2016-03-15', " + + "date '2016-06-14', " + + s + ")", + "-3", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(date '2019-09-01', " + + "date '2020-03-01', " + + s + ")", + "-6", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(date '2019-09-01', " + + "date '2016-08-01', " + + s + ")", + "37", "INTEGER NOT NULL")); + DAY_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(date '2016-06-15', " + + "date '2016-06-14', " + + s + ")", + "1", "INTEGER NOT NULL")); + HOUR_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(date '2016-06-15', " + + "date '2016-06-14', " + + s + ")", + "24", "INTEGER NOT NULL")); + HOUR_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(date '2016-06-15', " + + "date '2016-06-15', " + + s + ")", + "0", "INTEGER NOT NULL")); + MINUTE_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(date '2016-06-15', " + + "date '2016-06-14', " + + s + ")", + "1440", "INTEGER NOT NULL")); + DAY_VARIANTS.forEach(s -> + f.checkScalar("timestamp_diff(date '2016-06-15', " + + "cast(null as date), " + + s + ")", + isNullValue(), "INTEGER")); + } } /** Tests BigQuery's {@code DATETIME_DIFF(timestamp, timestamp2, timeUnit)} @@ -12470,102 +12476,105 @@ private static void checkArrayConcatAggFuncFails(SqlOperatorFixture t) { final SqlOperatorFixture f = fixture() .withLibrary(SqlLibrary.BIG_QUERY) .setFor(SqlLibraryOperators.DATETIME_DIFF); - HOUR_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(timestamp '2016-02-24 12:42:25', " - + "timestamp '2016-02-24 15:42:25', " - + s + ")", - "-3", "INTEGER NOT NULL")); - MICROSECOND_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(timestamp '2016-02-24 12:42:25', " - + "timestamp '2016-02-24 12:42:20', " - + s + ")", - "5000000", "INTEGER NOT NULL")); - YEAR_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(timestamp '2014-02-24 12:42:25', " - + "timestamp '2016-02-24 12:42:25', " - + s + ")", - "-2", "INTEGER NOT NULL")); - WEEK_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(timestamp '2014-02-24 12:42:25', " - + "timestamp '2016-02-24 12:42:25', " - + s + ")", - "-104", "INTEGER NOT NULL")); - WEEK_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(timestamp '2014-02-19 12:42:25', " - + "timestamp '2016-02-24 12:42:25', " - + s + ")", - "-105", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(timestamp '2014-02-24 12:42:25', " - + "timestamp '2016-02-24 12:42:25', " - + s + ")", - "-24", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(timestamp '2019-09-01 12:42:25', " - + "timestamp '2020-03-01 12:42:25', " - + s + ")", - "-6", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(timestamp '2019-09-01 12:42:25', " - + "timestamp '2016-08-01 12:42:25', " - + s + ")", - "37", "INTEGER NOT NULL")); - QUARTER_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(timestamp '2014-02-24 12:42:25', " - + "timestamp '2016-02-24 12:42:25', " - + s + ")", - "-8", "INTEGER NOT NULL")); - f.checkScalar("datetime_diff(timestamp '2014-02-24 12:42:25', " - + "timestamp '2614-02-24 12:42:25', " - + "CENTURY)", - "-6", "INTEGER NOT NULL"); - QUARTER_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(timestamp '2016-02-24 12:42:25', " - + "cast(null as timestamp), " - + s + ")", - isNullValue(), "INTEGER")); - // datetime_diff with date - MONTH_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(date '2016-03-15', " - + "date '2016-06-14', " - + s + ")", - "-3", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(date '2019-09-01', " - + "date '2020-03-01', " - + s + ")", - "-6", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(date '2019-09-01', " - + "date '2016-08-01', " - + s + ")", - "37", "INTEGER NOT NULL")); - DAY_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(date '2016-06-15', " - + "date '2016-06-14', " - + s + ")", - "1", "INTEGER NOT NULL")); - HOUR_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(date '2016-06-15', " - + "date '2016-06-14', " - + s + ")", - "24", "INTEGER NOT NULL")); - HOUR_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(date '2016-06-15', " - + "date '2016-06-15', " - + s + ")", - "0", "INTEGER NOT NULL")); - MINUTE_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(date '2016-06-15', " - + "date '2016-06-14', " - + s + ")", - "1440", "INTEGER NOT NULL")); - DAY_VARIANTS.forEach(s -> - f.checkScalar("datetime_diff(date '2016-06-15', " - + "cast(null as date), " - + s + ")", - isNullValue(), "INTEGER")); + if (Bug.CALCITE_2539_FIXED) { + HOUR_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(timestamp '2016-02-24 12:42:25', " + + "timestamp '2016-02-24 15:42:25', " + + s + ")", + "-3", "INTEGER NOT NULL")); + MICROSECOND_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(timestamp '2016-02-24 12:42:25', " + + "timestamp '2016-02-24 12:42:20', " + + s + ")", + "5000000", "INTEGER NOT NULL")); + YEAR_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(timestamp '2014-02-24 12:42:25', " + + "timestamp '2016-02-24 12:42:25', " + + s + ")", + "-2", "INTEGER NOT NULL")); + WEEK_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(timestamp '2014-02-24 12:42:25', " + + "timestamp '2016-02-24 12:42:25', " + + s + ")", + "-104", "INTEGER NOT NULL")); + WEEK_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(timestamp '2014-02-19 12:42:25', " + + "timestamp '2016-02-24 12:42:25', " + + s + ")", + "-105", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(timestamp '2014-02-24 12:42:25', " + + "timestamp '2016-02-24 12:42:25', " + + s + ")", + "-24", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(timestamp '2019-09-01 12:42:25', " + + "timestamp '2020-03-01 12:42:25', " + + s + ")", + "-6", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(timestamp '2019-09-01 12:42:25', " + + "timestamp '2016-08-01 12:42:25', " + + s + ")", + "37", "INTEGER NOT NULL")); + QUARTER_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(timestamp '2014-02-24 12:42:25', " + + "timestamp '2016-02-24 12:42:25', " + + s + ")", + "-8", "INTEGER NOT NULL")); + f.checkScalar("datetime_diff(timestamp '2014-02-24 12:42:25', " + + "timestamp '2614-02-24 12:42:25', " + + "CENTURY)", + "-6", "INTEGER NOT NULL"); + QUARTER_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(timestamp '2016-02-24 12:42:25', " + + "cast(null as timestamp), " + + s + ")", + isNullValue(), "INTEGER")); + + // datetime_diff with date + MONTH_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(date '2016-03-15', " + + "date '2016-06-14', " + + s + ")", + "-3", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(date '2019-09-01', " + + "date '2020-03-01', " + + s + ")", + "-6", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(date '2019-09-01', " + + "date '2016-08-01', " + + s + ")", + "37", "INTEGER NOT NULL")); + DAY_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(date '2016-06-15', " + + "date '2016-06-14', " + + s + ")", + "1", "INTEGER NOT NULL")); + HOUR_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(date '2016-06-15', " + + "date '2016-06-14', " + + s + ")", + "24", "INTEGER NOT NULL")); + HOUR_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(date '2016-06-15', " + + "date '2016-06-15', " + + s + ")", + "0", "INTEGER NOT NULL")); + MINUTE_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(date '2016-06-15', " + + "date '2016-06-14', " + + s + ")", + "1440", "INTEGER NOT NULL")); + DAY_VARIANTS.forEach(s -> + f.checkScalar("datetime_diff(date '2016-06-15', " + + "cast(null as date), " + + s + ")", + isNullValue(), "INTEGER")); + } } @ValueSource(booleans = {true, false}) @@ -12574,121 +12583,124 @@ void testTimestampDiff(boolean coercionEnabled) { final SqlOperatorFixture f = fixture() .withValidatorConfig(c -> c.withTypeCoercionEnabled(coercionEnabled)); f.setFor(SqlStdOperatorTable.TIMESTAMP_DIFF, VmName.EXPAND); - HOUR_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "timestamp '2016-02-24 12:42:25', " - + "timestamp '2016-02-24 15:42:25')", - "3", "INTEGER NOT NULL")); - MICROSECOND_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "timestamp '2016-02-24 12:42:25', " - + "timestamp '2016-02-24 12:42:20')", - "-5000000", "INTEGER NOT NULL")); - NANOSECOND_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "timestamp '2016-02-24 12:42:25', " - + "timestamp '2016-02-24 12:42:20')", - "-5000000000", "BIGINT NOT NULL")); - YEAR_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "timestamp '2014-02-24 12:42:25', " - + "timestamp '2016-02-24 12:42:25')", - "2", "INTEGER NOT NULL")); - WEEK_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "timestamp '2014-02-24 12:42:25', " - + "timestamp '2016-02-24 12:42:25')", - "104", "INTEGER NOT NULL")); - WEEK_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "timestamp '2014-02-19 12:42:25', " - + "timestamp '2016-02-24 12:42:25')", - "105", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "timestamp '2014-02-24 12:42:25', " - + "timestamp '2016-02-24 12:42:25')", - "24", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "timestamp '2019-09-01 00:00:00', " - + "timestamp '2020-03-01 00:00:00')", - "6", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "timestamp '2019-09-01 00:00:00', " - + "timestamp '2016-08-01 00:00:00')", - "-37", "INTEGER NOT NULL")); - QUARTER_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "timestamp '2014-02-24 12:42:25', " - + "timestamp '2016-02-24 12:42:25')", - "8", "INTEGER NOT NULL")); - // Until 1.33, CENTURY was an invalid time frame for TIMESTAMPDIFF - f.checkScalar("timestampdiff(CENTURY, " - + "timestamp '2014-02-24 12:42:25', " - + "timestamp '2614-02-24 12:42:25')", - "6", "INTEGER NOT NULL"); - QUARTER_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "timestamp '2014-02-24 12:42:25', " - + "cast(null as timestamp))", - isNullValue(), "INTEGER")); - QUARTER_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "cast(null as timestamp), " - + "timestamp '2014-02-24 12:42:25')", - isNullValue(), "INTEGER")); - // timestampdiff with date - MONTH_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "date '2016-03-15', date '2016-06-14')", - "2", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "date '2019-09-01', date '2020-03-01')", - "6", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "date '2019-09-01', date '2016-08-01')", - "-37", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "time '12:42:25', time '12:42:25')", - "0", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "time '12:42:25', date '2016-06-14')", - "-1502389", "INTEGER NOT NULL")); - MONTH_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "date '2016-06-14', time '12:42:25')", - "1502389", "INTEGER NOT NULL")); - DAY_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "date '2016-06-15', date '2016-06-14')", - "-1", "INTEGER NOT NULL")); - HOUR_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "date '2016-06-15', date '2016-06-14')", - "-24", "INTEGER NOT NULL")); - HOUR_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "date '2016-06-15', date '2016-06-15')", - "0", "INTEGER NOT NULL")); - MINUTE_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "date '2016-06-15', date '2016-06-14')", - "-1440", "INTEGER NOT NULL")); - SECOND_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "cast(null as date), date '2016-06-15')", - isNullValue(), "INTEGER")); - DAY_VARIANTS.forEach(s -> - f.checkScalar("timestampdiff(" + s + ", " - + "date '2016-06-15', cast(null as date))", - isNullValue(), "INTEGER")); + if (Bug.CALCITE_2539_FIXED) { + HOUR_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "timestamp '2016-02-24 12:42:25', " + + "timestamp '2016-02-24 15:42:25')", + "3", "INTEGER NOT NULL")); + MICROSECOND_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "timestamp '2016-02-24 12:42:25', " + + "timestamp '2016-02-24 12:42:20')", + "-5000000", "INTEGER NOT NULL")); + NANOSECOND_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "timestamp '2016-02-24 12:42:25', " + + "timestamp '2016-02-24 12:42:20')", + "-5000000000", "BIGINT NOT NULL")); + YEAR_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "timestamp '2014-02-24 12:42:25', " + + "timestamp '2016-02-24 12:42:25')", + "2", "INTEGER NOT NULL")); + WEEK_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "timestamp '2014-02-24 12:42:25', " + + "timestamp '2016-02-24 12:42:25')", + "104", "INTEGER NOT NULL")); + WEEK_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "timestamp '2014-02-19 12:42:25', " + + "timestamp '2016-02-24 12:42:25')", + "105", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "timestamp '2014-02-24 12:42:25', " + + "timestamp '2016-02-24 12:42:25')", + "24", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "timestamp '2019-09-01 00:00:00', " + + "timestamp '2020-03-01 00:00:00')", + "6", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "timestamp '2019-09-01 00:00:00', " + + "timestamp '2016-08-01 00:00:00')", + "-37", "INTEGER NOT NULL")); + QUARTER_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "timestamp '2014-02-24 12:42:25', " + + "timestamp '2016-02-24 12:42:25')", + "8", "INTEGER NOT NULL")); + // Until 1.33, CENTURY was an invalid time frame for TIMESTAMPDIFF + f.checkScalar("timestampdiff(CENTURY, " + + "timestamp '2014-02-24 12:42:25', " + + "timestamp '2614-02-24 12:42:25')", + "6", "INTEGER NOT NULL"); + QUARTER_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "timestamp '2014-02-24 12:42:25', " + + "cast(null as timestamp))", + isNullValue(), "INTEGER")); + QUARTER_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "cast(null as timestamp), " + + "timestamp '2014-02-24 12:42:25')", + isNullValue(), "INTEGER")); + + // timestampdiff with date + MONTH_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "date '2016-03-15', date '2016-06-14')", + "2", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "date '2019-09-01', date '2020-03-01')", + "6", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "date '2019-09-01', date '2016-08-01')", + "-37", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "time '12:42:25', time '12:42:25')", + "0", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "time '12:42:25', date '2016-06-14')", + "-1502389", "INTEGER NOT NULL")); + MONTH_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "date '2016-06-14', time '12:42:25')", + "1502389", "INTEGER NOT NULL")); + DAY_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "date '2016-06-15', date '2016-06-14')", + "-1", "INTEGER NOT NULL")); + HOUR_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "date '2016-06-15', date '2016-06-14')", + "-24", "INTEGER NOT NULL")); + HOUR_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "date '2016-06-15', date '2016-06-15')", + "0", "INTEGER NOT NULL")); + MINUTE_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "date '2016-06-15', date '2016-06-14')", + "-1440", "INTEGER NOT NULL")); + SECOND_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "cast(null as date), date '2016-06-15')", + isNullValue(), "INTEGER")); + DAY_VARIANTS.forEach(s -> + f.checkScalar("timestampdiff(" + s + ", " + + "date '2016-06-15', cast(null as date))", + isNullValue(), "INTEGER")); + } } @Test void testTimestampSub() { @@ -12908,24 +12920,26 @@ void testTimestampDiff(boolean coercionEnabled) { "(?s)Column 'MONTH' not found in any table", false); final SqlOperatorFixture f0 = f.withLibrary(SqlLibrary.BIG_QUERY); - f0.checkScalar("date_diff(DATE '2010-07-07', DATE '2008-12-25', DAY)", - "559", - "INTEGER NOT NULL"); - f0.checkScalar("date_diff(DATE '2010-07-14', DATE '2010-07-07', WEEK)", - "1", - "INTEGER NOT NULL"); - f0.checkScalar("date_diff(DATE '2011-12-14', DATE '2011-07-14', MONTH)", - "5", - "INTEGER NOT NULL"); - f0.checkScalar("date_diff(DATE '2011-10-14', DATE '2011-07-14', QUARTER)", - "1", - "INTEGER NOT NULL"); - f0.checkScalar("date_diff(DATE '2021-07-14', DATE '2011-07-14', YEAR)", - "10", - "INTEGER NOT NULL"); - f0.checkNull("date_diff(CAST(NULL AS DATE), CAST(NULL AS DATE), DAY)"); - f0.checkNull("date_diff(DATE '2008-12-25', CAST(NULL AS DATE), DAY)"); - f0.checkNull("date_diff(CAST(NULL AS DATE), DATE '2008-12-25', DAY)"); + if (Bug.CALCITE_2539_FIXED) { + f0.checkScalar("date_diff(DATE '2010-07-07', DATE '2008-12-25', DAY)", + "559", + "INTEGER NOT NULL"); + f0.checkScalar("date_diff(DATE '2010-07-14', DATE '2010-07-07', WEEK)", + "1", + "INTEGER NOT NULL"); + f0.checkScalar("date_diff(DATE '2011-12-14', DATE '2011-07-14', MONTH)", + "5", + "INTEGER NOT NULL"); + f0.checkScalar("date_diff(DATE '2011-10-14', DATE '2011-07-14', QUARTER)", + "1", + "INTEGER NOT NULL"); + f0.checkScalar("date_diff(DATE '2021-07-14', DATE '2011-07-14', YEAR)", + "10", + "INTEGER NOT NULL"); + f0.checkNull("date_diff(CAST(NULL AS DATE), CAST(NULL AS DATE), DAY)"); + f0.checkNull("date_diff(DATE '2008-12-25', CAST(NULL AS DATE), DAY)"); + f0.checkNull("date_diff(CAST(NULL AS DATE), DATE '2008-12-25', DAY)"); + } } /** Tests BigQuery's {@code TIME_ADD}, which adds an interval to a time @@ -12966,37 +12980,38 @@ void testTimestampDiff(boolean coercionEnabled) { + "minute)^", "No match found for function signature " + "TIME_DIFF\\(