hatter/simple-rust-tests
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

feat: add sqlite extersion

Browse Source
This commit is contained in:
Hatter Jiang
2022-05-21 21:20:45 +08:00
parent 7034c0b86c
commit fe46aef388
6 changed files with 716 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

442
__database/sqlite_extersion/Cargo.lock generated Normal file
View File

@@ -0,0 +1,442 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "ahash"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
dependencies = [
"getrandom",
"once_cell",
"version_check",
]
[[package]]
name = "aho-corasick"
version = "0.7.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
dependencies = [
"memchr",
]
[[package]]
name = "anyhow"
version = "1.0.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc"
[[package]]
name = "atty"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
"hermit-abi",
"libc",
"winapi",
]
[[package]]
name = "bindgen"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8"
dependencies = [
"bitflags",
"cexpr",
"clang-sys",
"lazy_static",
"lazycell",
"peeking_take_while",
"proc-macro2",
"quote",
"regex",
"rustc-hash",
"shlex",
]
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "cc"
version = "1.0.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
[[package]]
name = "cexpr"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
dependencies = [
"nom",
]
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "clang-sys"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf6b561dcf059c85bbe388e0a7b0a1469acb3934cc0cfa148613a830629e3049"
dependencies = [
"glob",
"libc",
"libloading",
]
[[package]]
name = "either"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
[[package]]
name = "env_logger"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3"
dependencies = [
"atty",
"humantime",
"log",
"regex",
"termcolor",
]
[[package]]
name = "fallible-iterator"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "getrandom"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "glob"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
[[package]]
name = "hashbrown"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
dependencies = [
"ahash",
]
[[package]]
name = "hashlink"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf"
dependencies = [
"hashbrown",
]
[[package]]
name = "hermit-abi"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
dependencies = [
"libc",
]
[[package]]
name = "humantime"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "lazycell"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "libc"
version = "0.2.126"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"
[[package]]
name = "libloading"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd"
dependencies = [
"cfg-if",
"winapi",
]
[[package]]
name = "libsqlite3-sys"
version = "0.24.0"
source = "git+https://github.com/litements/rusqlite/?branch=loadable-extensions-release-2#cfca03a859ca1dcab01deb8210268674ba13a217"
dependencies = [
"bindgen",
"cc",
"pkg-config",
"proc-macro2",
"quote",
"regex",
"syn",
"vcpkg",
"version-compare",
"which",
]
[[package]]
name = "log"
version = "0.4.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
dependencies = [
"cfg-if",
]
[[package]]
name = "memchr"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "minimal-lexical"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "nom"
version = "7.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36"
dependencies = [
"memchr",
"minimal-lexical",
]
[[package]]
name = "once_cell"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b10983b38c53aebdf33f542c6275b0f58a238129d00c4ae0e6fb59738d783ca"
[[package]]
name = "peeking_take_while"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
[[package]]
name = "pkg-config"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
[[package]]
name = "proc-macro2"
version = "1.0.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
dependencies = [
"proc-macro2",
]
[[package]]
name = "regex"
version = "1.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.6.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64"
[[package]]
name = "rusqlite"
version = "0.27.0"
source = "git+https://github.com/litements/rusqlite/?branch=loadable-extensions-release-2#cfca03a859ca1dcab01deb8210268674ba13a217"
dependencies = [
"bitflags",
"fallible-iterator",
"fallible-streaming-iterator",
"hashlink",
"libsqlite3-sys",
"memchr",
"smallvec",
]
[[package]]
name = "rustc-hash"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "shlex"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
[[package]]
name = "smallvec"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
[[package]]
name = "sqlite-regex-ext"
version = "0.1.0"
dependencies = [
"anyhow",
"env_logger",
"log",
"regex",
"rusqlite",
]
[[package]]
name = "syn"
version = "1.0.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "termcolor"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
dependencies = [
"winapi-util",
]
[[package]]
name = "unicode-ident"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee"
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "version-compare"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe88247b92c1df6b6de80ddc290f3976dbdf2f5f5d3fd049a9fb598c6dd5ca73"
[[package]]
name = "version_check"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "wasi"
version = "0.10.2+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "which"
version = "4.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae"
dependencies = [
"either",
"lazy_static",
"libc",
]
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
dependencies = [
"winapi",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

40
__database/sqlite_extersion/Cargo.toml Normal file
View File

@@ -0,0 +1,40 @@
[package]
name = "sqlite-regex-ext"
version = "0.1.0"
edition = "2021"
# [features]
# default = []
# build_extension = [
# "rusqlite/bundled",
# "rusqlite/functions",
# "rusqlite/loadable_extension",
# ]
[lib]
crate-type = ["cdylib"]
[dependencies]
# once_cell = "1.9.0"
regex = "1.5.4"
log = "0.4.14"
env_logger = "0.9.0"
anyhow = "1.0.54"
[dependencies.rusqlite]
package = "rusqlite"
git = "https://github.com/litements/rusqlite/"
branch = "loadable-extensions-release-2"
# path = "../rusqlite/loadable-extensions-release-2"
default-features = false
features = [
"loadable_extension",
"vtab",
"functions",
"bundled",
"modern_sqlite",
"buildtime_bindgen",
]

6
__database/sqlite_extersion/README.md Normal file
View File

@@ -0,0 +1,6 @@
Link:
https://ricardoanderegg.com/posts/extending-sqlite-with-rust/
https://github.com/polyrand/rust-sqlite-ext-example

176
__database/sqlite_extersion/src/lib.rs Normal file
View File

@@ -0,0 +1,176 @@
// #![allow(
// dead_code,
// unused_imports,
// unused_variables,
// clippy::missing_safety_doc
// )]
#![allow(clippy::missing_safety_doc)]
use crate::ffi::loadable_extension_init;
use crate::ffi::sqlite3_auto_extension;
use anyhow::Context as ACtxt;
use log::LevelFilter;
use regex::bytes::Regex;
use rusqlite::ffi;
use rusqlite::functions::{Context, FunctionFlags};
use rusqlite::types::{ToSqlOutput, Value, ValueRef};
use rusqlite::Connection;
use std::os::raw::c_int;
fn ah(e: anyhow::Error) -> rusqlite::Error {
rusqlite::Error::UserFunctionError(format!("{:?}", e).into())
}
fn init_logging(default_level: LevelFilter) {
let lib_log_env = "SQLITE_REGEX_LOG";
if std::env::var(lib_log_env).is_err() {
std::env::set_var(lib_log_env, format!("{}", default_level))
}
let logger_env = env_logger::Env::new().filter(lib_log_env);
env_logger::try_init_from_env(logger_env).ok();
}
// Will use with ffi:sqlite3_auto_extension(arg1)
// https://www.sqlite.org/c3ref/auto_extension.html
// Example: https://sqlite.org/src/file/ext/misc/vfsstat.c
// https://www.sqlite.org/loadext.html
// #[no_mangle]
// pub unsafe extern "C" fn regex_register(
// db: *mut ffi::sqlite3,
// _pz_err_msg: &mut &mut std::os::raw::c_char,
// p_api: *mut ffi::sqlite3_api_routines,
// ) -> c_int {}
#[no_mangle]
pub unsafe extern "C" fn sqlite3_regex_init_internal(
db: *mut ffi::sqlite3,
_pz_err_msg: &mut &mut std::os::raw::c_char,
p_api: *mut ffi::sqlite3_api_routines,
) -> c_int {
// https://www.sqlite.org/loadext.html
// https://github.com/jgallagher/rusqlite/issues/524#issuecomment-507787350
// SQLITE_EXTENSION_INIT2 equivalent
loadable_extension_init(p_api);
/* Insert here calls to
** sqlite3_create_function_v2(),
** sqlite3_create_collation_v2(),
** sqlite3_create_module_v2(), and/or
** sqlite3_vfs_register()
** to register the new features that your extension adds.
*/
match init(db) {
Ok(()) => {
log::info!("[regex-extension] init ok");
// ffi::SQLITE_OK
ffi::SQLITE_OK_LOAD_PERMANENTLY
}
Err(e) => {
log::error!("[regex-extension] init error: {:?}", e);
ffi::SQLITE_ERROR
}
}
}
#[no_mangle]
pub unsafe extern "C" fn sqlite3_regex_init(
db: *mut ffi::sqlite3,
_pz_err_msg: &mut &mut std::os::raw::c_char,
p_api: *mut ffi::sqlite3_api_routines,
) -> c_int {
loadable_extension_init(p_api);
let ptr = sqlite3_regex_init_internal
as unsafe extern "C" fn(
*mut ffi::sqlite3,
&mut &mut std::os::raw::c_char,
*mut ffi::sqlite3_api_routines,
) -> c_int;
sqlite3_auto_extension(Some(std::mem::transmute(ptr)));
match init(db) {
Ok(()) => {
log::info!("[regex-extension] init ok");
ffi::SQLITE_OK_LOAD_PERMANENTLY
}
Err(e) => {
log::error!("[regex-extension] init error: {:?}", e);
ffi::SQLITE_ERROR
}
}
}
fn init(db_handle: *mut ffi::sqlite3) -> anyhow::Result<()> {
let db = unsafe { rusqlite::Connection::from_handle(db_handle)? };
load(&db)?;
Ok(())
}
fn load(c: &Connection) -> anyhow::Result<()> {
load_with_loglevel(c, LevelFilter::Info)
}
fn load_with_loglevel(c: &Connection, default_log_level: LevelFilter) -> anyhow::Result<()> {
init_logging(default_log_level);
add_functions(c)
}
fn add_functions(c: &Connection) -> anyhow::Result<()> {
let deterministic = FunctionFlags::SQLITE_DETERMINISTIC | FunctionFlags::SQLITE_UTF8;
// | FunctionFlags::SQLITE_INNOCUOUS;
c.create_scalar_function("regex_extract", 2, deterministic, |ctx: &Context| {
regex_extract(ctx).map_err(ah)
})?;
c.create_scalar_function("regex_extract", 3, deterministic, |ctx: &Context| {
regex_extract(ctx).map_err(ah)
})?;
Ok(())
}
fn regex_extract<'a>(ctx: &Context) -> anyhow::Result<ToSqlOutput<'a>> {
let arg_pat = 0;
let arg_input_data = 1;
let arg_cap_group = 2;
let empty_return = Ok(ToSqlOutput::Owned(Value::Null));
let pattern = match ctx.get_raw(arg_pat) {
ValueRef::Text(t) => t,
e => anyhow::bail!("regex pattern must be text, got {}", e.data_type()),
};
let re = Regex::new(std::str::from_utf8(pattern)?)?;
let input_value = match ctx.get_raw(arg_input_data) {
ValueRef::Text(t) => t,
ValueRef::Null => return empty_return,
e => anyhow::bail!("regex expects text as input, got {}", e.data_type()),
};
let cap_group: usize = if ctx.len() <= arg_cap_group {
// no capture group, use default
0
} else {
ctx.get(arg_cap_group).context("capture group")?
};
// let mut caploc = re.capture_locations();
// re.captures_read(&mut caploc, input_value);
if let Some(cap) = re.captures(input_value) {
match cap.get(cap_group) {
None => empty_return,
// String::from_utf8_lossy
Some(t) => {
let value = String::from_utf8_lossy(t.as_bytes());
return Ok(ToSqlOutput::Owned(Value::Text(value.to_string())));
}
}
} else {
empty_return
}
}

0
__database/sqlite_extersion/test.db Normal file
View File

52
__database/sqlite_extersion/test.py Normal file
View File

@@ -0,0 +1,52 @@
#!/usr/bin/env python3
import sqlite3
conn = sqlite3.connect("test.db", isolation_level=None)
print(f"Loading SQLite extension in connection: {conn}")
conn.enable_load_extension(True)
conn.execute(
"SELECT load_extension('target/release/libsqlite_regex_ext.dylib', 'sqlite3_regex_init');"
)
print("Running tests...")
print("Testing pattern 'x(ab)' WITHOUT capture group")
row = conn.execute("SELECT regex_extract('x(ab)', 'xxabaa')").fetchone()
assert row[0] == "xab", row[0]
print("Testing pattern 'x(ab)' WITH capture group = 1")
row = conn.execute("SELECT regex_extract('x(ab)', 'xxabaa', 1)").fetchone()
assert row[0] == "ab", row[0]
print("Testing pattern 'x(ab)' WITH capture group = 0")
row = conn.execute("SELECT regex_extract('x(ab)', 'xxabaa', 0)").fetchone()
assert row[0] == "xab", row[0]
print("Testing pattern 'g(oog)+le' WITHOUT capture group")
row = conn.execute("SELECT regex_extract('g(oog)+le', 'googoogoogle')").fetchone()
assert row[0] == "googoogoogle", row[0]
print("Testing pattern 'g(oog)+le' WITH capture group = 1")
row = conn.execute("SELECT regex_extract('g(oog)+le', 'googoogoogle', 1)").fetchone()
assert row[0] == "oog", row[0]
print("Testing pattern '[Cc]at' WITHOUT capture group")
row = conn.execute("SELECT regex_extract('[Cc]at', 'cat')").fetchone()
assert row[0] == "cat", row[0]
print("Testing pattern '[Cc]at' WITHOUT capture group, expecting empty return")
row = conn.execute("SELECT regex_extract('[Cc]at', 'hello')").fetchone()
assert row[0] is None, row[0]
conn.close()
conn2 = sqlite3.connect("test.db", isolation_level=None)
print(f"Testing connection 2: {conn2}")
row = conn2.execute("SELECT regex_extract('x(ab)', 'xxabaa')").fetchone()
assert row[0] == "xab", row[0]
print("All tests passed")