Embed sqlite amalgamation v3.40.0 source code (#35)

* Embed sqlite amalgamation v3.40.0 source code
* Includes swift script for bumping to the latest patch version, inspecting
  the system's sqlite3 compile options, and updating to an arbitrary version
* Use sqlite serialized threading on linux
* Use async interfaces in vendor-sqlite3
* Prefix sqlite amalgamation symbols with "sqlite_nio_"
* Update CI to not depend on system libsqlite3
* Adjust CSQLite compile options to Ubuntu 22.04

Co-authored-by: Austin Payne <[email protected]>

Author: Austinpayne

.github/workflows/test.yml

@@ -13,13 +13,9 @@ jobs:
     runs-on: ubuntu-latest
     container: swift:5.7-jammy
     steps:
-      # N.B.: When we switch to embedded SQLite, these first two steps should be removed,
-      # and the version saved to the environment should come from the checked-out package.
-      - name: Install libsqlite3 dependency
-        run: apt-get -q update && apt-get -q install -y libsqlite3-dev
       - name: Save SQLite version to env
         run: |
-          echo SQLITE_VERSION="$(pkg-config --modversion sqlite3)" >> $GITHUB_ENV
+          echo SQLITE_VERSION="$(cat Sources/CSQLite/version.txt | grep -v '//')" >> $GITHUB_ENV
       - name: Check out package
         uses: actions/checkout@v3
       - name: Run local tests with coverage
@@ -39,8 +35,6 @@ jobs:
     runs-on: ubuntu-latest
     container: swift:5.7-jammy
     steps:
-      - name: Install libsqlite3 dependency
-        run: apt-get -q update && apt-get -q install -y libsqlite3-dev
       - name: Check out package
         uses: actions/checkout@v3
         with:
@@ -57,8 +51,6 @@ jobs:
     runs-on: ubuntu-latest
     container: swift:5.7-jammy
     steps:
-      - name: Install libsqlite3 dependency
-        run: apt-get -q update && apt-get -q install -y libsqlite3-dev
       - name: Check out package
         uses: actions/checkout@v3
         with:
@@ -96,8 +88,6 @@ jobs:
     container: ${{ matrix.runner }}
     runs-on: ubuntu-latest
     steps:
-      - name: Install libsqlite3 dependency
-        run: apt-get -q update && apt-get -q install -y libsqlite3-dev
       - name: Check out code
         uses: actions/checkout@v3
       - name: Run tests with Thread Sanitizer

Package.swift

@@ -9,19 +9,17 @@ let package = Package(
     ],
     products: [
         .library(name: "SQLiteNIO", targets: ["SQLiteNIO"]),
+        // This target is only used to add our vendor prefix and is added and removed automatically.
+        // See: scripts/vendor-sqlite3.swift
+        /* VENDOR_START
+        .library(name: "CSQLite", type: .static, targets: ["CSQLite"]),
+        VENDOR_END */
     ],
     dependencies: [
         .package(url: "https://github.com/apple/swift-nio.git", from: "2.42.0"),
         .package(url: "https://github.com/apple/swift-log.git", from: "1.0.0"),
     ],
     targets: [
-        .systemLibrary(
-            name: "CSQLite",
-            providers: [
-                .apt(["libsqlite3-dev"]),
-                .brew(["sqlite3"])
-            ]
-        ),
         .target(name: "SQLiteNIO", dependencies: [
             .target(name: "CSQLite"),
             .product(name: "Logging", package: "swift-log"),
@@ -30,3 +28,61 @@ let package = Package(
         .testTarget(name: "SQLiteNIOTests", dependencies: ["SQLiteNIO"]),
     ]
 )
+
+// Derived from sqlite3 version 3.37.2 2022-01-06 13:25:41 872ba256cbf61d9290b571c0e6d82a20c224ca3ad82971edc46b29818d5dalt1
+// compiled with gcc-11.3.0
+// on platform Ubuntu 22.04.1 LTS (Jammy Jellyfish)
+var cSQLiteSettings: [CSetting] = [
+    .define("SQLITE_ENABLE_COLUMN_METADATA"),
+    .define("SQLITE_ENABLE_DBSTAT_VTAB"),
+    .define("SQLITE_ENABLE_FTS3"),
+    .define("SQLITE_ENABLE_FTS3_PARENTHESIS"),
+    .define("SQLITE_ENABLE_FTS3_TOKENIZER"),
+    .define("SQLITE_ENABLE_FTS4"),
+    .define("SQLITE_ENABLE_FTS5"),
+    .define("SQLITE_ENABLE_JSON1"),
+    .define("SQLITE_ENABLE_PREUPDATE_HOOK"),
+    .define("SQLITE_ENABLE_RTREE"),
+    .define("SQLITE_ENABLE_SESSION"),
+    .define("SQLITE_ENABLE_STMTVTAB"),
+    .define("SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION"),
+    .define("SQLITE_ENABLE_UNLOCK_NOTIFY"),
+    .define("SQLITE_ENABLE_UPDATE_DELETE_LIMIT"),
+    .define("SQLITE_HAVE_ISNAN"),
+    .define("SQLITE_LIKE_DOESNT_MATCH_BLOBS"),
+    .define("SQLITE_MUTEX_PTHREADS"),
+    .define("SQLITE_OMIT_LOAD_EXTENSION"),
+    .define("SQLITE_OMIT_LOOKASIDE"),
+    .define("SQLITE_SECURE_DELETE"),
+    .define("SQLITE_SYSTEM_MALLOC"),
+    .define("SQLITE_USE_URI"),
+]
+
+#if os(macOS)
+// Derived from sqlite3 version 3.37.0 2021-12-09 01:34:53 9ff244ce0739f8ee52a3e9671adb4ee54c83c640b02e3f9d185fd2f9a179aapl
+// compiled with clang-13.1.6
+// on platform macOS 12.6 (21G115)
+cSQLiteSettings.append(contentsOf: [
+    .define("SQLITE_DEFAULT_CACHE_SIZE", to: "2000"),
+    .define("SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT", to: "32768"),
+    .define("SQLITE_DEFAULT_LOOKASIDE", to: "1200,102"),
+    .define("SQLITE_DEFAULT_MEMSTATUS", to: "0"),
+    .define("SQLITE_DEFAULT_MMAP_SIZE", to: "0"),
+    .define("SQLITE_DEFAULT_PAGE_SIZE", to: "4096"),
+    .define("SQLITE_DEFAULT_PCACHE_INITSZ", to: "20"),
+    .define("SQLITE_DEFAULT_SYNCHRONOUS", to: "2"),
+    .define("SQLITE_DEFAULT_WAL_AUTOCHECKPOINT", to: "1000"),
+    .define("SQLITE_DEFAULT_WAL_SYNCHRONOUS", to: "1"),
+    .define("SQLITE_DEFAULT_WORKER_THREADS", to: "0"),
+    .define("SQLITE_ENABLE_LOCKING_STYLE", to: "1"),
+    .define("SQLITE_MAX_MMAP_SIZE", to: "1073741824"),
+])
+#endif
+
+package.targets.append(
+    .target(
+        name: "CSQLite",
+        dependencies: [],
+        cSettings: cSQLiteSettings
+    )
+)