From d8ed972f4870ce477715d6d84d6dfe45f08c408c Mon Sep 17 00:00:00 2001 From: Noy <130386570+NoyException@users.noreply.github.com> Date: Wed, 18 Dec 2024 19:50:29 +0800 Subject: [PATCH] feat: support automatic redirection to __sys__ (#296) * feat: support automatic redirection to __sys__ Signed-off-by: Noy * ci: let clients compatibility tests recover Signed-off-by: Noy --------- Signed-off-by: Noy --- .github/workflows/clients-compatibility.yml | 10 +++++--- catalog/internal_tables.go | 21 ++++++++++++++++ pgserver/duck_handler.go | 2 +- pgserver/pg_catalog_handler.go | 8 ++---- pgserver/stmt.go | 27 +++++++++++++++++++++ 5 files changed, 57 insertions(+), 11 deletions(-) diff --git a/.github/workflows/clients-compatibility.yml b/.github/workflows/clients-compatibility.yml index 5b94317b..5185c139 100644 --- a/.github/workflows/clients-compatibility.yml +++ b/.github/workflows/clients-compatibility.yml @@ -33,8 +33,8 @@ jobs: - name: Install system packages uses: awalsh128/cache-apt-pkgs-action@latest with: - packages: bats cpanminus libmysqlclient-dev dotnet-sdk-8.0 dotnet-runtime-8.0 php-mysql r-base-core libblas-dev - version: 1.0 + packages: bats cpanminus libmysqlclient-dev dotnet-sdk-8.0 dotnet-runtime-8.0 php-mysql r-base-core + version: 1.1 - name: Install dependencies run: | @@ -52,6 +52,7 @@ jobs: npm install mysql sudo cpanm --notest DBD::mysql pip3 install mysql-connector-python + sudo apt-get install -y libblas-dev sudo R -e "install.packages('RMySQL', repos='http://cran.r-project.org')" sudo gem install mysql2 @@ -90,8 +91,8 @@ jobs: - name: Install system packages uses: awalsh128/cache-apt-pkgs-action@latest with: - packages: bats cpanminus libpq-dev postgresql-client dotnet-sdk-8.0 dotnet-runtime-8.0 r-base-core libblas-dev - version: 1.0 + packages: bats cpanminus libpq-dev postgresql-client dotnet-sdk-8.0 dotnet-runtime-8.0 r-base-core + version: 1.1 - name: Install dependencies run: | @@ -109,6 +110,7 @@ jobs: npm install pg sudo cpanm --notest DBD::Pg pip3 install "psycopg[binary]" pandas pyarrow polars + sudo apt-get install -y libblas-dev # TODO: Speed up the installation of RPostgres sudo R -e "install.packages('RPostgres', repos='http://cran.r-project.org')" sudo gem install pg diff --git a/catalog/internal_tables.go b/catalog/internal_tables.go index bd38c9be..ff424ba4 100644 --- a/catalog/internal_tables.go +++ b/catalog/internal_tables.go @@ -177,6 +177,7 @@ var InternalTables = struct { // Once we add 'pg_catalog' and support views for PG, replace this by a view. // https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-REPLICATION-VIEW PGStatReplication InternalTable + PGRange InternalTable }{ PersistentVariable: InternalTable{ Schema: "__sys__", @@ -287,6 +288,21 @@ var InternalTables = struct { }, DDL: "pid INTEGER PRIMARY KEY, usesysid TEXT, usename TEXT, application_name TEXT, client_addr TEXT, client_hostname TEXT, client_port INTEGER, backend_start TIMESTAMP, backend_xmin INTEGER, state TEXT, sent_lsn TEXT, write_lsn TEXT, flush_lsn TEXT, replay_lsn TEXT, write_lag INTERVAL, flush_lag INTERVAL, replay_lag INTERVAL, sync_priority INTEGER, sync_state TEXT, reply_time TIMESTAMP", }, + PGRange: InternalTable{ + Schema: "__sys__", + Name: "pg_range", + KeyColumns: []string{"rngtypid"}, + ValueColumns: []string{"rngsubtype", "rngmultitypid", "rngcollation", "rngsubopc", "rngcanonical", "rngsubdiff"}, + DDL: "rngtypid TEXT PRIMARY KEY, rngsubtype TEXT, rngmultitypid TEXT, rngcollation TEXT, rngsubopc TEXT, rngcanonical TEXT, rngsubdiff TEXT", + InitialData: [][]any{ + {"3904", "23", "4451", "0", "1978", "int4range_canonical", "int4range_subdiff"}, + {"3906", "1700", "4532", "0", "3125", "-", "numrange_subdiff"}, + {"3908", "1114", "4533", "0", "3128", "-", "tsrange_subdiff"}, + {"3910", "1184", "4534", "0", "3127", "-", "tstzrange_subdiff"}, + {"3912", "1082", "4535", "0", "3122", "daterange_canonical", "daterange_subdiff"}, + {"3926", "20", "4536", "0", "3124", "int8range_canonical", "int8range_subdiff"}, + }, + }, } var internalTables = []InternalTable{ @@ -295,4 +311,9 @@ var internalTables = []InternalTable{ InternalTables.PgSubscription, InternalTables.GlobalStatus, InternalTables.PGStatReplication, + InternalTables.PGRange, +} + +func GetInternalTables() []InternalTable { + return internalTables } diff --git a/pgserver/duck_handler.go b/pgserver/duck_handler.go index 7ce5cf85..2b3c7375 100644 --- a/pgserver/duck_handler.go +++ b/pgserver/duck_handler.go @@ -424,7 +424,7 @@ func (h *DuckHandler) executeQuery(ctx *sql.Context, query string, parsed tree.S })) default: - rows, err = adapter.QueryCatalog(ctx, query) + rows, err = adapter.QueryCatalog(ctx, ConvertToSys(query)) if err != nil { break } diff --git a/pgserver/pg_catalog_handler.go b/pgserver/pg_catalog_handler.go index beb789fd..84cb8781 100644 --- a/pgserver/pg_catalog_handler.go +++ b/pgserver/pg_catalog_handler.go @@ -25,9 +25,6 @@ var pgWALLSNRegex = regexp.MustCompile(`(?i)^\s*select\s+pg_catalog\.(pg_current // precompile a regex to match "select pg_catalog.current_setting('xxx');". var currentSettingRegex = regexp.MustCompile(`(?i)^\s*select\s+(pg_catalog.)?current_setting\(\s*'([^']+)'\s*\)\s*;?\s*$`) -// precompile a regex to match any "from pg_catalog.pg_stat_replication" in the query. -var pgCatalogRegex = regexp.MustCompile(`(?i)\s+from\s+pg_catalog\.(pg_stat_replication)`) - // isInRecovery will get the count of func (h *ConnectionHandler) isInRecovery() (string, error) { // Grab a sql.Context. @@ -167,9 +164,8 @@ func (h *ConnectionHandler) handleCurrentSetting(query ConvertedStatement) (bool // handler for pgCatalog func (h *ConnectionHandler) handlePgCatalog(query ConvertedStatement) (bool, error) { - sql := RemoveComments(query.String) return true, h.run(ConvertedStatement{ - String: pgCatalogRegex.ReplaceAllString(sql, " FROM __sys__.$1"), + String: ConvertToSys(query.String), Tag: "SELECT", }) } @@ -208,7 +204,7 @@ func isPgCurrentSetting(query ConvertedStatement) bool { func isSpecialPgCatalog(query ConvertedStatement) bool { sql := RemoveComments(query.String) - return pgCatalogRegex.MatchString(sql) + return getPgCatalogRegex().MatchString(sql) } // The key is the statement tag of the query. diff --git a/pgserver/stmt.go b/pgserver/stmt.go index 1fb0d0ac..27a4c8d1 100644 --- a/pgserver/stmt.go +++ b/pgserver/stmt.go @@ -2,7 +2,10 @@ package pgserver import ( "bytes" + "github.com/apecloud/myduckserver/catalog" + "regexp" "strings" + "sync" "unicode" "github.com/dolthub/go-mysql-server/sql" @@ -259,3 +262,27 @@ func RemoveComments(query string) string { return buf.String() } + +var ( + pgCatalogRegex *regexp.Regexp + initPgCatalogRegex sync.Once +) + +// get the regex to match any table in pg_catalog in the query. +func getPgCatalogRegex() *regexp.Regexp { + initPgCatalogRegex.Do(func() { + var tableNames []string + for _, table := range catalog.GetInternalTables() { + if table.Schema != "__sys__" { + continue + } + tableNames = append(tableNames, table.Name) + } + pgCatalogRegex = regexp.MustCompile(`(?i)\b(?:FROM|JOIN)\s+(?:pg_catalog\.)?(` + strings.Join(tableNames, "|") + `)`) + }) + return pgCatalogRegex +} + +func ConvertToSys(sql string) string { + return getPgCatalogRegex().ReplaceAllString(RemoveComments(sql), " __sys__.$1") +}