Merge remote-tracking branch 'refs/remotes/origin/main' into noy/feat…

…-multiple-databases

# Conflicts:
#	pgserver/server.go

.github/workflows/clients-compatibility.yml

@@ -6,6 +6,7 @@ on:
       - main
       - compatibility
       - test
+      - support_flightsql
   pull_request:
     branches: [ "main" ]
 
@@ -133,4 +134,39 @@ jobs:
 
       - name: Run the Compatibility Test for Python Data Tools
         run: |
-          bats ./compatibility/pg-pytools/test.bats
+          bats ./compatibility/pg-pytools/test.bats
+
+  test-flightsql:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Install dependencies
+        run: |
+          go get .
+
+          pip3 install "sqlglot[rs]"
+          pip3 install "psycopg[binary]" pandas pyarrow polars adbc_driver_flightsql
+
+      - name: Build
+        run: go build -v
+
+      - name: Start MyDuck Server
+        run: |
+          ./myduckserver --flightsql-port 47470 &
+          sleep 10
+
+      - name: Run the Compatibility Test for FlightSQL
+        run: |
+          go test -v ./compatibility/flightsql/go/flightsql_test.go
+          python3 -m unittest discover ./compatibility/flightsql/python -p "flightsql_test.py"

.github/workflows/password-auth.yml

@@ -0,0 +1,67 @@
+name: Password Auth Test
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: '1.23'
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+
+    - name: Install system packages
+      uses: awalsh128/cache-apt-pkgs-action@latest
+      with:
+        packages: postgresql-client mysql-client
+        version: 1.0
+
+    - name: Install dependencies
+      run: |
+        go get .
+
+        pip3 install "sqlglot[rs]"
+
+        curl -LJO https://github.com/duckdb/duckdb/releases/latest/download/duckdb_cli-linux-amd64.zip
+        unzip duckdb_cli-linux-amd64.zip
+        chmod +x duckdb
+        sudo mv duckdb /usr/local/bin
+  
+    - name: Build
+      run: go build -v
+
+    - name: Start MyDuck Server with password
+      run: |
+        ./myduckserver --superuser-password=testpass123 &
+        sleep 5
+
+    - name: Test PostgreSQL auth
+      run: |
+        export PGPASSWORD=testpass123
+        # Basic connection test
+        psql -h 127.0.0.1 -U postgres -d postgres -c "SELECT 1 as test;"
+        # Create and query a table
+        psql -h 127.0.0.1 -U postgres -d postgres -c "CREATE TABLE test (id int); INSERT INTO test VALUES (42); SELECT * FROM test;"
+        # Test wrong password
+        ! PGPASSWORD=wrongpass psql -h 127.0.0.1 -U postgres -d postgres -c "SELECT 1"
+
+    - name: Test MySQL auth
+      run: |
+        # Basic connection test
+        mysql -h127.0.0.1 -uroot -ptestpass123 -e "SELECT 1 as test;"
+        # Create and query a table
+        mysql -h127.0.0.1 -uroot -ptestpass123 -e "CREATE DATABASE IF NOT EXISTS test; USE test; CREATE TABLE t1 (id int); INSERT INTO t1 VALUES (42); SELECT * FROM t1;"
+        # Test wrong password
+        ! mysql -h127.0.0.1 -uroot -pwrongpass -e "SELECT 1"

.github/workflows/psql.yml

@@ -35,12 +35,10 @@ jobs:
 
         pip3 install "sqlglot[rs]" pyarrow pandas
 
-        curl -LJO https://github.com/duckdb/duckdb/releases/download/v1.1.3/duckdb_cli-linux-amd64.zip
+        curl -LJO https://github.com/duckdb/duckdb/releases/latest/download/duckdb_cli-linux-amd64.zip
         unzip duckdb_cli-linux-amd64.zip
         chmod +x duckdb
         sudo mv duckdb /usr/local/bin
-        duckdb -c 'INSTALL json from core'
-        duckdb -c 'SELECT extension_name, loaded, install_path FROM duckdb_extensions() where installed'
 
     - name: Build
       run: go build -v

README.md

@@ -163,18 +163,34 @@ With MyDuck's powerful analytics capabilities, you can create an hybrid transact
 To rename the default database, pass the `DEFAULT_DB` environment variable to the Docker container:
 
 ```bash
-docker run -p 13306:3306 -p 15432:5432 --env=DEFAULT_DB=mydbname apecloud/myduckserver:latest
+docker run -d -p 13306:3306 -p 15432:5432 \
+    --env=DEFAULT_DB=mydbname \
+    apecloud/myduckserver:latest
 ```
 
+
+To set the superuser password, pass the `SUPERUSER_PASSWORD` environment variable to the Docker container:
+
+```bash
+docker run -d -p 13306:3306 -p 15432:5432 \
+    --env=SUPERUSER_PASSWORD=mysecretpassword \
+    apecloud/myduckserver:latest
+```
+
+
 To initialize MyDuck Server with custom SQL statements, mount your `.sql` file to either `/docker-entrypoint-initdb.d/mysql/` or `/docker-entrypoint-initdb.d/postgres/` inside the Docker container, depending on the SQL dialect you're using.
 
 For example:
 ```bash
 # Execute `init.sql` via MySQL protocol
-docker run -d -p 13306:3306 --name=myduck -v ./init.sql:/docker-entrypoint-initdb.d/mysql/init.sql apecloud/myduckserver:latest
+docker run -d -p 13306:3306 --name=myduck \
+    -v ./init.sql:/docker-entrypoint-initdb.d/mysql/init.sql \
+    apecloud/myduckserver:latest
 
 # Execute `init.sql` via PostgreSQL protocol
-docker run -d -p 15432:5432 --name=myduck -v ./init.sql:/docker-entrypoint-initdb.d/postgres/init.sql apecloud/myduckserver:latest
+docker run -d -p 15432:5432 --name=myduck \
+    -v ./init.sql:/docker-entrypoint-initdb.d/postgres/init.sql \
+    apecloud/myduckserver:latest
 ```
 
 ### Query Parquet Files

compatibility/flightsql/go/flightsql_test.go

@@ -0,0 +1,152 @@
+package main
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	"github.com/apache/arrow-adbc/go/adbc"
+	"github.com/apache/arrow-adbc/go/adbc/driver/flightsql"
+	"github.com/apache/arrow-go/v18/arrow/array"
+	"github.com/apache/arrow-go/v18/arrow/memory"
+)
+
+// Set connection options
+var options = map[string]string{
+	adbc.OptionKeyURI:             "grpc://localhost:47470",
+	flightsql.OptionSSLSkipVerify: adbc.OptionValueEnabled,
+}
+
+// Create database connection
+func createDatabaseConnection(t *testing.T) adbc.Connection {
+	alloc := memory.NewGoAllocator()
+	drv := flightsql.NewDriver(alloc)
+	db, err := drv.NewDatabase(options)
+	if err != nil {
+		t.Fatalf("Error creating database: %v", err)
+	}
+
+	cnxn, err := db.Open(context.Background())
+	if err != nil {
+		t.Fatalf("Error opening connection: %v", err)
+	}
+
+	return cnxn
+}
+
+// Execute SQL statement
+func executeSQLStatement(cnxn adbc.Connection, query string, t *testing.T) {
+	stmt, err := cnxn.NewStatement()
+	if err != nil {
+		t.Fatalf("failed to create statement: %v", err)
+	}
+	defer stmt.Close()
+
+	err = stmt.SetSqlQuery(query)
+	if err != nil {
+		t.Fatalf("failed to set SQL query: %v", err)
+	}
+
+	_, err = stmt.ExecuteUpdate(context.Background())
+	if err != nil {
+		t.Fatalf("failed to execute SQL statement: %v", err)
+	}
+}
+
+// Execute query and verify results
+func executeQueryAndVerify(cnxn adbc.Connection, query string, expectedResults []struct {
+	id    int64
+	name  string
+	value int64
+}, t *testing.T) {
+	stmt, err := cnxn.NewStatement()
+	if err != nil {
+		t.Fatalf("failed to create statement: %v", err)
+	}
+	defer stmt.Close()
+
+	err = stmt.SetSqlQuery(query)
+	if err != nil {
+		t.Fatalf("failed to set SQL query: %v", err)
+	}
+
+	rows, _, err := stmt.ExecuteQuery(context.Background())
+	if err != nil {
+		t.Fatalf("failed to execute query: %v", err)
+	}
+	defer rows.Release()
+
+	var actualResults []struct {
+		id    int64
+		name  string
+		value int64
+	}
+
+	// Read query results and verify
+	for rows.Next() {
+		record := rows.Record()
+		numRows := record.NumRows()
+
+		id := record.Column(0).(*array.Int64)
+		name := record.Column(1).(*array.String)
+		value := record.Column(2).(*array.Int64)
+		for i := 0; i < int(numRows); i++ {
+			actualResults = append(actualResults, struct {
+				id    int64
+				name  string
+				value int64
+			}{
+				id:    id.Value(i),
+				name:  name.Value(i),
+				value: value.Value(i),
+			})
+		}
+	}
+
+	// Verify query results
+	if len(actualResults) != len(expectedResults) {
+		t.Errorf("Expected %d rows, but got %d", len(expectedResults), len(actualResults))
+	}
+
+	for i, result := range actualResults {
+		expected := expectedResults[i]
+		if !reflect.DeepEqual(result, expected) {
+			t.Errorf("Row %d: Expected %+v, but got %+v", i, expected, result)
+		}
+	}
+}
+
+// Go test function
+func TestSQLOperations(t *testing.T) {
+	cnxn := createDatabaseConnection(t)
+	defer cnxn.Close()
+
+	// 1. Execute DROP TABLE IF EXISTS intTable
+	executeSQLStatement(cnxn, "DROP TABLE IF EXISTS intTable", t)
+
+	// 2. Execute CREATE TABLE IF NOT EXISTS intTable
+	executeSQLStatement(cnxn, `CREATE TABLE IF NOT EXISTS intTable (
+		id INTEGER PRIMARY KEY,
+		name VARCHAR(50),
+		value INT
+	)`, t)
+
+	// 3. Execute INSERT INTO intTable
+	executeSQLStatement(cnxn, "INSERT INTO intTable (id, name, value) VALUES (1, 'TestName', 100)", t)
+	executeSQLStatement(cnxn, "INSERT INTO intTable (id, name, value) VALUES (2, 'AnotherName', 200)", t)
+
+	// 4. Query data and verify insertion was successful
+	expectedResults := []struct {
+		id    int64
+		name  string
+		value int64
+	}{
+		{id: 1, name: "TestName", value: 100},
+		{id: 2, name: "AnotherName", value: 200},
+	}
+	query := "SELECT id, name, value FROM intTable"
+	executeQueryAndVerify(cnxn, query, expectedResults, t)
+
+	// 5. Execute DROP TABLE IF EXISTS intTable
+	executeSQLStatement(cnxn, "DROP TABLE IF EXISTS intTable", t)
+}

compatibility/flightsql/python/flightsql_test.py

@@ -0,0 +1,61 @@
+import unittest
+from adbc_driver_flightsql import DatabaseOptions
+from adbc_driver_flightsql.dbapi import connect
+
+class TestFlightSQLDatabase(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        """Runs once before any tests are executed, used to set up the database connection."""
+        headers = {"foo": "bar"}
+        cls.conn = connect(
+            "grpc://localhost:47470",  # FlightSQL server address
+            db_kwargs={
+                DatabaseOptions.TLS_SKIP_VERIFY.value: "true",  # Skip TLS verification
+                **{f"{DatabaseOptions.RPC_CALL_HEADER_PREFIX.value}{k}": v for k, v in headers.items()}
+            }
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        """Runs once after all tests have been executed, used to close the database connection."""
+        cls.conn.close()
+
+    def setUp(self):
+        """Runs before each individual test to ensure a clean environment by resetting the database."""
+        with self.conn.cursor() as cursor:
+            cursor.execute("DROP TABLE IF EXISTS intTable")  # Drop the table if it exists
+            cursor.execute("""
+                CREATE TABLE IF NOT EXISTS intTable (
+                    id INTEGER PRIMARY KEY,
+                    name VARCHAR(50),
+                    value INT
+                )
+            """)  # Create the table
+
+    def test_insert_and_select(self):
+        """Test inserting data and selecting it back to verify correctness."""
+        with self.conn.cursor() as cursor:
+            # Insert sample data
+            cursor.execute("INSERT INTO intTable (id, name, value) VALUES (1, 'TestName', 100)")
+            cursor.execute("INSERT INTO intTable (id, name, value) VALUES (2, 'AnotherName', 200)")
+
+            # Select data from the table
+            cursor.execute("SELECT * FROM intTable")
+            rows = cursor.fetchall()
+
+            # Expected result after insertions
+            expected_rows = [(1, 'TestName', 100), (2, 'AnotherName', 200)]
+            self.assertEqual(rows, expected_rows, f"Expected rows: {expected_rows}, but got: {rows}")
+
+    def test_drop_table(self):
+        """Test dropping the table to ensure the table can be deleted successfully."""
+        with self.conn.cursor() as cursor:
+            cursor.execute("DROP TABLE IF EXISTS intTable")  # Drop the table
+            cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='intTable'")  # Check if the table exists
+            rows = cursor.fetchall()
+            self.assertEqual(len(rows), 0, "Table 'intTable' should be dropped and not exist in the database.")
+            cursor.execute("COMMIT;")
+
+if __name__ == "__main__":
+    unittest.main()

devtools/replica-setup-postgres/replica_setup.sh

@@ -7,7 +7,7 @@ usage() {
 
 MYDUCK_HOST=${MYDUCK_HOST:-127.0.0.1}
 MYDUCK_PORT=${MYDUCK_PORT:-5432}
-MYDUCK_USER=${MYDUCK_USER:-mysql}
+MYDUCK_USER=${MYDUCK_USER:-postgres}
 MYDUCK_PASSWORD=${MYDUCK_PASSWORD:-}
 MYDUCK_SERVER_ID=${MYDUCK_SERVER_ID:-2}
 MYDUCK_IN_DOCKER=${MYDUCK_IN_DOCKER:-false}