build: make db drivers optional

Author: mmetc

.golangci.yml

@@ -1,5 +1,12 @@
 version: "2"
 
+# To lint the windows sources on a linux host, you need to cross-build and
+# disable the C-based sqlite driver:
+#
+# GOOS=windows golangci-lint run --build-tags=windows,sqlite_modernc
+#
+# In the CI, the linters are run on both linux and windows hosts.
+
 run:
   build-tags:
     - expr_debug

Makefile

@@ -18,7 +18,7 @@ BUILD_RE2_WASM ?= 0
 #expr_debug tag is required to enable the debug mode in expr
 GO_TAGS := netgo,osusergo,expr_debug
 
-# By default, build with CGO and sqlite3.
+# By default, build with sqlite3.
 BUILD_SQLITE ?= mattn
 SQLITE_MSG = Using mattn/go-sqlite3
 
@@ -159,15 +159,21 @@ COMPONENTS := \
 	datasource_s3 \
 	datasource_syslog \
 	datasource_wineventlog \
-	cscli_setup
+	cscli_setup \
+	db_mysql \
+	db_postgres \
+	db_sqlite
 
 comma := ,
 space := $(empty) $(empty)
 
 # Predefined profiles
 
-# keep only datasource-file
-EXCLUDE_MINIMAL := $(subst $(space),$(comma),$(filter-out datasource_file,,$(COMPONENTS)))
+# What we want to KEEP in the minimal build,
+# which should always include at least one data source and a sql driver.
+INCLUDE_MINIMAL := db_sqlite datasource_file
+
+EXCLUDE_MINIMAL := $(filter-out $(INCLUDE_MINIMAL),$(strip $(COMPONENTS)))
 
 # example
 # EXCLUDE_MEDIUM := datasource_kafka,datasource_kinesis,datasource_s3

pkg/csconfig/database.go

@@ -130,11 +130,9 @@ func (d *DatabaseCfg) ConnectionString() (string, error) {
 
 	switch d.Type {
 	case "sqlite":
-		var sqliteConnectionStringParameters string
+		sqliteConnectionStringParameters := "_busy_timeout=100000&_fk=1&_pragma=foreign_keys(1)"
 		if d.UseWal != nil && *d.UseWal {
-			sqliteConnectionStringParameters = "_busy_timeout=100000&_fk=1&_journal_mode=WAL"
-		} else {
-			sqliteConnectionStringParameters = "_busy_timeout=100000&_fk=1"
+			sqliteConnectionStringParameters += "&_journal_mode=WAL"
 		}
 
 		connString = fmt.Sprintf("file:%s?%s", d.DbPath, sqliteConnectionStringParameters)

pkg/cwversion/component/component.go

@@ -21,8 +21,11 @@ var Built = map[string]bool{
 	"datasource_syslog":       false,
 	"datasource_wineventlog":  false,
 	"datasource_victorialogs": false,
-  "datasource_http":        false,
+	"datasource_http":         false,
 	"cscli_setup":             false,
+	"db_mysql":                false,
+	"db_postgres":             false,
+	"db_sqlite":               false,
 }
 
 func Register(name string) {

pkg/database/database.go

@@ -8,9 +8,6 @@ import (
 	"os"
 
 	entsql "entgo.io/ent/dialect/sql"
-	// load database backends
-	_ "github.com/go-sql-driver/mysql"
-	_ "github.com/jackc/pgx/v4/stdlib"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
@@ -85,6 +82,7 @@ func NewClient(ctx context.Context, config *csconfig.DatabaseCfg) (*Client, erro
 	if err != nil {
 		return nil, fmt.Errorf("failed to generate DB connection string: %w", err)
 	}
+
 	drv, err := getEntDriver(typ, dia, dbConnectionString, config)
 	if err != nil {
 		return nil, fmt.Errorf("failed opening connection to %s: %w", config.Type, err)

pkg/database/drv_mysql.go

@@ -0,0 +1,15 @@
+//go:build !no_db_mysql
+
+package database
+
+import (
+	// register database driver as side effect
+	_ "github.com/go-sql-driver/mysql"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion/component"
+)
+
+//nolint:gochecknoinits
+func init() {
+	component.Register("db_mysql")
+}

pkg/database/drv_postgres.go

@@ -0,0 +1,15 @@
+//go:build !no_db_postgres
+
+package database
+
+import (
+	// register database driver as side effect
+	_ "github.com/jackc/pgx/v4/stdlib"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion/component"
+)
+
+//nolint:gochecknoinits
+func init() {
+	component.Register("db_postgres")
+}

pkg/database/sqlite_mattn.go renamed to pkg/database/drv_sqlite_mattn.go

@@ -1,20 +1,21 @@
-//go:build !sqlite_modernc
+//go:build !db_no_sqlite && !sqlite_modernc
 
 package database
 
 import (
 	"errors"
 
 	"github.com/mattn/go-sqlite3"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion/component"
 )
 
+//nolint:gochecknoinits
+func init() {
+	component.Register("db_sqlite")
+}
+
 func IsSqliteBusyError(err error) bool {
 	var sqliteErr sqlite3.Error
-	// sqlite3.Error{
-	//   Code:         5,
-	//   ExtendedCode: 5,
-	//   SystemErrno:  0,
-	//   err:          "database is locked",
-	// }
 	return errors.As(err, &sqliteErr) && sqliteErr.Code == sqlite3.ErrBusy
 }

pkg/database/drv_sqlite_modernc.go

@@ -0,0 +1,24 @@
+//go:build !db_no_sqlite && sqlite_modernc
+
+package database
+
+import (
+	"database/sql"
+	"errors"
+
+	"modernc.org/sqlite"
+	sqlite3 "modernc.org/sqlite/lib"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion/component"
+)
+
+//nolint:gochecknoinits
+func init() {
+	sql.Register("sqlite3", &sqlite.Driver{})
+	component.Register("db_sqlite")
+}
+
+func IsSqliteBusyError(err error) bool {
+	var se *sqlite.Error
+	return errors.As(err, &se) && se.Code() == sqlite3.SQLITE_BUSY
+}