Merge #81

81: Switch to a fork of libsqlite3-sys in order to statically link libSQL r=MarinPostma a=psarna

Until (if?) we get official libSQL support from the great libsqlite3-sys crate, we can instead switch to a git dependency.

What it provides is as follows:
 - statically links libSQL, compiled from sqlite3.c and sqlite3.h amalgamation files
 - adds custom libSQL helper functions, like automatic initialization of the WebAssembly function table and opening a connection with virtual WAL

It also lets us drop the libsql git submodule, because we now only rely on the libsqlite3-sys crate to provide the library for us.

Ref: rusqlite/rusqlite#1283

Co-authored-by: Piotr Sarna <sarna@chiselstrike.com>

Author: bors[bot]

libsql-server/.gitmodules

@@ -1,6 +1,6 @@
 # install dependencies
 FROM rust:slim-bullseye AS compiler
-RUN apt update && apt install -y libclang-dev clang libsqlite3-dev \
+RUN apt update && apt install -y libclang-dev clang \
     build-essential tcl protobuf-compiler file
 RUN cargo install cargo-chef
 WORKDIR /sqld

libsql-server/Dockerfile

@@ -17,6 +17,7 @@ fallible-iterator = "0.2.0"
 futures = "0.3.25"
 hex = "0.4.3"
 hyper = { version = "0.14.23", features = ["http2"] }
+libsql-wasmtime-bindings = "0"
 # Regular mvfs prevents users from enabling WAL mode
 mvfs = { git = "https://github.com/psarna/mvsqlite", branch = "mwal", optional = true }
 mwal = { git = "https://github.com/psarna/mvsqlite", branch = "mwal", optional = true }
@@ -27,7 +28,10 @@ pin-project-lite = "0.2.9"
 postgres-protocol = "0.6.4"
 prost = "0.11.3"
 regex = "1.7.0"
-rusqlite = { version = "0.28.0", features = [ "buildtime_bindgen", "column_decltype" ] }
+rusqlite = { git = "https://github.com/psarna/rusqlite", branch = "libsql-dev", default-features = false, features = [
+    "bundled-libsql",
+    "column_decltype"
+] }
 serde = { version = "1.0.149", features = ["derive"] }
 serde_json = "1.0.91"
 smallvec = "1.10.0"

libsql-server/libsql

@@ -1,34 +1,6 @@
-use std::env;
-use std::fs;
-use std::process::Command;
-
 use prost_build::Config;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let mut pwd = env::current_dir().unwrap();
-    pwd.push("../libsql");
-    let libsql_dir = fs::canonicalize(pwd.as_path()).unwrap();
-    let mut bindings = Command::new("./configure");
-    let configure = bindings.current_dir(libsql_dir.as_path()).arg("--with-pic");
-    let profile = std::env::var("PROFILE").unwrap();
-    if profile.as_str() == "release" {
-        configure.arg("--enable-releasemode");
-    }
-    let output = configure.output().unwrap();
-    if !output.status.success() {
-        println!("{}", std::str::from_utf8(&output.stderr).unwrap());
-        panic!("failed to configure");
-    }
-
-    if !Command::new("make")
-        .current_dir(libsql_dir.as_path())
-        .status()
-        .unwrap()
-        .success()
-    {
-        panic!("failed to compile");
-    }
-
     let mut config = Config::new();
     config.bytes([".wal_log"]);
     tonic_build::configure()
@@ -41,9 +13,5 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     println!("cargo:rerun-if-changed=proto");
 
-    println!("cargo:rustc-link-search=native=libsql/.libs");
-    println!("cargo:rustc-link-lib=static=sqlite3");
-    println!("cargo:rerun-if-changed=../libsql/src");
-
     Ok(())
 }

libsql-server/sqld/Cargo.toml

@@ -10,7 +10,6 @@ use tokio::sync::oneshot;
 use tracing::warn;
 
 use crate::libsql::wal_hook::WalHook;
-use crate::libsql::WalConnection;
 use crate::query::{
     Column, ErrorCode, Params, Queries, Query, QueryError, QueryResponse, QueryResult, ResultSet,
     Row,
@@ -136,7 +135,7 @@ fn open_db(
         Arc<Mutex<mwal::ffi::libsql_wal_methods>>,
     >,
     wal_hook: impl WalHook + Send + Clone + 'static,
-) -> anyhow::Result<WalConnection> {
+) -> anyhow::Result<rusqlite::Connection> {
     let mut retries = 0;
     loop {
         #[cfg(feature = "mwal_backend")]

libsql-server/sqld/build.rs

@@ -31,15 +31,15 @@ use tokio::runtime::Handle;
 use tonic::transport::Channel;
 
 use crate::libsql::ffi::{types::XWalFrameFn, PgHdr, Wal};
+use crate::libsql::open_with_regular_wal;
 use crate::libsql::wal_hook::WalHook;
-use crate::libsql::{open_with_regular_wal, WalConnection};
 use crate::rpc::wal_log::wal_log_rpc::wal_log_entry::Payload;
 use crate::rpc::wal_log::wal_log_rpc::{wal_log_client::WalLogClient, LogOffset, WalLogEntry};
 use crate::rpc::wal_log::wal_log_rpc::{Commit, Frame};
 
 pub struct PeriodicDbUpdater {
     interval: Duration,
-    db: WalConnection,
+    db: rusqlite::Connection,
 }
 
 /// The `PeriodicUpdater` role is to periodically trigger a dummy write that will be intercepted by

libsql-server/sqld/src/database/libsql.rs

@@ -5,9 +5,12 @@ pub mod ffi;
 pub mod mwal;
 pub mod wal_hook;
 
+pub use wblibsql::{
+    libsql_compile_wasm_module, libsql_free_wasm_module, libsql_run_wasm, libsql_wasm_engine_new,
+};
+
 use anyhow::ensure;
 use rusqlite::Connection;
-use std::os::unix::ffi::OsStrExt;
 
 use crate::libsql::{ffi::libsql_wal_methods_register, wal_hook::WalMethodsHook};
 
@@ -16,37 +19,6 @@ use self::{
     wal_hook::WalHook,
 };
 
-pub struct WalConnection {
-    inner: rusqlite::Connection,
-}
-
-impl std::ops::Deref for WalConnection {
-    type Target = rusqlite::Connection;
-
-    fn deref(&self) -> &Self::Target {
-        &self.inner
-    }
-}
-
-impl std::ops::Drop for WalConnection {
-    fn drop(&mut self) {
-        unsafe {
-            rusqlite::ffi::sqlite3_close(self.inner.handle());
-        }
-        let _ = self.inner;
-    }
-}
-
-extern "C" {
-    fn libsql_open(
-        filename: *const u8,
-        ppdb: *mut *mut rusqlite::ffi::sqlite3,
-        flags: std::ffi::c_int,
-        vfs: *const u8,
-        wal: *const u8,
-    ) -> i32;
-}
-
 fn get_orig_wal_methods() -> anyhow::Result<*mut libsql_wal_methods> {
     let orig: *mut libsql_wal_methods = unsafe { libsql_wal_methods_find(0) };
     if orig.is_null() {
@@ -60,29 +32,17 @@ pub(crate) fn open_with_regular_wal(
     path: impl AsRef<std::path::Path>,
     flags: rusqlite::OpenFlags,
     wal_hook: impl WalHook + 'static,
-) -> anyhow::Result<WalConnection> {
+) -> anyhow::Result<Connection> {
     unsafe {
-        let mut pdb: *mut rusqlite::ffi::sqlite3 = std::ptr::null_mut();
-        let ppdb: *mut *mut rusqlite::ffi::sqlite3 = &mut pdb;
         let orig = get_orig_wal_methods()?;
         let wrapped = WalMethodsHook::wrap(orig, wal_hook);
         let res = libsql_wal_methods_register(wrapped);
         ensure!(res == 0, "failed to register WAL methods");
-
-        let open_err = libsql_open(
-            path.as_ref().as_os_str().as_bytes().as_ptr(),
-            ppdb,
-            flags.bits(),
-            std::ptr::null(),
-            WalMethodsHook::METHODS_NAME.as_ptr(),
-        );
-        assert_eq!(open_err, 0);
-        let conn = Connection::from_handle(pdb)?;
-        conn.pragma_update(None, "journal_mode", "wal")?;
-        tracing::trace!(
-            "Opening a connection with regular WAL at {}",
-            path.as_ref().display()
-        );
-        Ok(WalConnection { inner: conn })
     }
+    tracing::trace!(
+        "Opening a connection with regular WAL at {}",
+        path.as_ref().display()
+    );
+    let conn = Connection::open_with_flags_and_wal(path, flags, WalMethodsHook::METHODS_NAME_STR)?;
+    Ok(conn)
 }

libsql-server/sqld/src/database/write_proxy/replication.rs

@@ -6,30 +6,22 @@ pub(crate) fn open_with_virtual_wal(
     path: impl AsRef<std::path::Path>,
     flags: rusqlite::OpenFlags,
     vwal_methods: Arc<Mutex<mwal::ffi::libsql_wal_methods>>,
-) -> anyhow::Result<super::WalConnection> {
-    use std::os::unix::ffi::OsStrExt;
+) -> anyhow::Result<rusqlite::Connection> {
     let mut vwal_methods = vwal_methods.lock().map_err(|e| anyhow::anyhow!("{}", e))?;
     unsafe {
-        let mut pdb: *mut rusqlite::ffi::sqlite3 = std::ptr::null_mut();
-        let ppdb: *mut *mut rusqlite::ffi::sqlite3 = &mut pdb;
         let register_err = super::ffi::libsql_wal_methods_register(
             &mut *vwal_methods as *const mwal::ffi::libsql_wal_methods as _,
         );
         assert_eq!(register_err, 0);
-        let open_err = super::libsql_open(
-            path.as_ref().as_os_str().as_bytes().as_ptr(),
-            ppdb,
-            flags.bits(),
-            std::ptr::null(),
-            vwal_methods.name,
-        );
-        assert_eq!(open_err, 0);
-        let conn = super::Connection::from_handle(pdb)?;
-        conn.pragma_update(None, "journal_mode", "wal")?;
-        tracing::trace!(
-            "Opening a connection with virtual WAL at {}",
-            path.as_ref().display()
-        );
-        Ok(super::WalConnection { inner: conn })
     }
+    tracing::trace!(
+        "Opening a connection with virtual WAL at {}",
+        path.as_ref().display()
+    );
+    let conn = rusqlite::Connection::open_with_flags_and_wal(path, flags, unsafe {
+        std::ffi::CStr::from_ptr(vwal_methods.name as *const _)
+            .to_str()
+            .unwrap()
+    })?;
+    Ok(conn)
 }

libsql-server/sqld/src/libsql/mod.rs

@@ -53,6 +53,7 @@ pub unsafe trait WalHook {
 unsafe impl WalHook for () {}
 
 impl WalMethodsHook {
+    pub const METHODS_NAME_STR: &'static str = "wal_hook";
     pub const METHODS_NAME: &'static [u8] = b"wal_hook\0";
 
     pub fn wrap(