diff --git a/README.md b/README.md index c0ea48a1..03706e40 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,14 @@ MyDuck Server supports setting up replicas from common cloud-based MySQL offerin With MyDuck's powerful analytics capabilities, you can create an HTAP (Hybrid Transactional/Analytical Processing) system where high-frequency data writes are directed to a standard MySQL instance, while analytical queries are handled by a MyDuck Server instance. Follow our HTAP setup instructions based on [ProxySQL](docs/tutorial/htap-proxysql-setup.md) or [MariaDB MaxScale](docs/tutorial/htap-maxscale-setup.md) to easily set up an HTAP demonstration. +### Query & Load Parquet Files + +Looking to load Parquet files into MyDuck Server and start querying? Follow our [Parquet file loading guide](docs/tutorial/load-parquet-files.md) for easy setup. + +### Already Using DuckDB? + +Already have a DuckDB file? You can seamlessly bootstrap MyDuck Server with it. See our [DuckDB file bootstrapping guide](docs/tutorial/bootstrap.md) for more details. + ## 💡 Contributing Let’s make MySQL analytics fast and powerful—together! diff --git a/catalog/database.go b/catalog/database.go index e389a015..2c6f14e1 100644 --- a/catalog/database.go +++ b/catalog/database.go @@ -217,9 +217,9 @@ func (d *Database) RenameTable(ctx *sql.Context, oldName string, newName string) // extractViewDefinitions is a helper function to extract view definitions from DuckDB func (d *Database) extractViewDefinitions(ctx *sql.Context, schemaName string, viewName string) ([]sql.ViewDefinition, error) { query := ` - SELECT view_name, sql + SELECT DISTINCT view_name, sql FROM duckdb_views() - WHERE schema_name = ? + WHERE schema_name = ? AND NOT internal ` args := []interface{}{schemaName} @@ -240,6 +240,12 @@ func (d *Database) extractViewDefinitions(ctx *sql.Context, schemaName string, v if err := rows.Scan(&name, &createViewStmt); err != nil { return nil, ErrDuckDB.New(err) } + + // Skip system views directly + if IsSystemView(name) { + continue + } + views = append(views, sql.ViewDefinition{ Name: name, CreateViewStatement: createViewStmt, diff --git a/catalog/internal_tables.go b/catalog/internal_tables.go index 8845049d..ec349b5a 100644 --- a/catalog/internal_tables.go +++ b/catalog/internal_tables.go @@ -81,21 +81,21 @@ var InternalTables = struct { GlobalStatus InternalTable }{ PersistentVariable: InternalTable{ - Schema: "main", + Schema: "__sys__", Name: "persistent_variable", KeyColumns: []string{"name"}, ValueColumns: []string{"value", "vtype"}, DDL: "name TEXT PRIMARY KEY, value TEXT, vtype TEXT", }, BinlogPosition: InternalTable{ - Schema: "main", + Schema: "__sys__", Name: "binlog_position", KeyColumns: []string{"channel"}, ValueColumns: []string{"position"}, DDL: "channel TEXT PRIMARY KEY, position TEXT", }, PgReplicationLSN: InternalTable{ - Schema: "main", + Schema: "__sys__", Name: "pg_replication_lsn", KeyColumns: []string{"slot_name"}, ValueColumns: []string{"lsn"}, diff --git a/catalog/provider.go b/catalog/provider.go index 66357c93..47b73df1 100644 --- a/catalog/provider.go +++ b/catalog/provider.go @@ -155,7 +155,7 @@ func (prov *DatabaseProvider) AllDatabases(ctx *sql.Context) []sql.Database { } switch schemaName { - case "information_schema", "main", "pg_catalog": + case "information_schema", "pg_catalog", "__sys__": continue } diff --git a/catalog/system_views.go b/catalog/system_views.go new file mode 100644 index 00000000..720fc86d --- /dev/null +++ b/catalog/system_views.go @@ -0,0 +1,22 @@ +package catalog + +var SystemViews = map[string]struct{}{ + "duckdb_columns": {}, + "duckdb_constraints": {}, + "duckdb_databases": {}, + "duckdb_indexes": {}, + "duckdb_schemas": {}, + "duckdb_tables": {}, + "duckdb_types": {}, + "duckdb_views": {}, + "pragma_database_list": {}, + "sqlite_master": {}, + "sqlite_schema": {}, + "sqlite_temp_master": {}, + "sqlite_temp_schema": {}, +} + +func IsSystemView(viewName string) bool { + _, ok := SystemViews[viewName] + return ok +} diff --git a/docs/data/example.db b/docs/data/example.db new file mode 100644 index 00000000..6060df70 Binary files /dev/null and b/docs/data/example.db differ diff --git a/docs/data/example.parquet b/docs/data/example.parquet new file mode 100644 index 00000000..a2a87f31 Binary files /dev/null and b/docs/data/example.parquet differ diff --git a/docs/tutorial/bootstrap.md b/docs/tutorial/bootstrap.md new file mode 100644 index 00000000..514fdb6b --- /dev/null +++ b/docs/tutorial/bootstrap.md @@ -0,0 +1,34 @@ +# Bootstrapping from an existing DuckDB file + +Given an existing DuckDB file, it is possible to bootstrap MyDuck Server with it or serve it with MyDuck Server. +Here’s how to work with the `example.db` file located in `docs/data/`. + +### Steps + +1. **Prepare the data directory:** + ```bash + mkdir example_data + cp /path/to/example.db example_data/mysql.db # IMPORTANT: The attached filename must be `mysql.db` + ``` + +2. **Launch MyDuck Server and attach the data directory:** + ```bash + docker run \ + -p 13306:3306 \ + -p 15432:5432 \ + --volume=/path/to/example_data:/home/admin/data \ + apecloud/myduckserver:main + ``` + +3. **Connect to MyDuck Server and query:** + ```bash + # Using psql client & DuckDB syntax + psql -h 127.0.0.1 -p 15432 -U mysql <