diff --git a/deps/Makefile b/deps/Makefile index d88e48642..560db98f1 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -15,8 +15,8 @@ $(error "cargo not found. Please install Rust toolchain") endif # SQLite environment variables for sqlite-rembed build -export SQLITE3_INCLUDE_DIR=$(shell pwd)/sqlite3/sqlite-amalgamation-3500400 -export SQLITE3_LIB_DIR=$(shell pwd)/sqlite3/sqlite-amalgamation-3500400 +export SQLITE3_INCLUDE_DIR=$(shell pwd)/sqlite3/sqlite3 +export SQLITE3_LIB_DIR=$(shell pwd)/sqlite3/sqlite3 export SQLITE3_STATIC=1 @@ -265,7 +265,10 @@ sqlite3/sqlite3/vec.o: sqlite3/sqlite3/sqlite3.o cd sqlite3/sqlite3 && cp ../sqlite-vec-source/sqlite-vec.c . && cp ../sqlite-vec-source/sqlite-vec.h . cd sqlite3/sqlite3 && ${CC} ${MYCFLAGS} -fPIC -c -o vec.o sqlite-vec.c -DSQLITE_CORE -DSQLITE_VEC_STATIC -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_JSON1 -DSQLITE_DLL=1 -sqlite3/libsqlite_rembed.a: sqlite3/sqlite-rembed-source/Cargo.toml $(shell find sqlite3/sqlite-rembed-source -type f -name '*.rs') +sqlite3/libsqlite_rembed.a: sqlite3/sqlite-rembed-0.0.1-alpha.9.tar.gz + cd sqlite3 && rm -rf sqlite-rembed-*/ sqlite-rembed-source/ || true + cd sqlite3 && tar -zxf sqlite-rembed-0.0.1-alpha.9.tar.gz + mv sqlite3/sqlite-rembed-0.0.1-alpha.9 sqlite3/sqlite-rembed-source cd sqlite3/sqlite-rembed-source && SQLITE3_INCLUDE_DIR=$(SQLITE3_INCLUDE_DIR) SQLITE3_LIB_DIR=$(SQLITE3_LIB_DIR) SQLITE3_STATIC=1 $(CARGO) build --release --features=sqlite-loadable/static --lib cp sqlite3/sqlite-rembed-source/target/release/libsqlite_rembed.a sqlite3/libsqlite_rembed.a @@ -361,6 +364,7 @@ cleanpart: cd mariadb-client-library && rm -rf mariadb-connector-c-*/ || true cd jemalloc && rm -rf jemalloc-*/ || true cd sqlite3 && rm -rf sqlite-amalgamation-*/ || true + cd sqlite3 && rm -rf libsqlite_rembed.a sqlite-rembed-source/ sqlite-rembed-*/ || true cd postgresql && rm -rf postgresql-*/ || true cd postgresql && rm -rf postgres-*/ || true .PHONY: cleanpart diff --git a/deps/sqlite3/sqlite-rembed-0.0.1-alpha.9.tar.gz b/deps/sqlite3/sqlite-rembed-0.0.1-alpha.9.tar.gz new file mode 100644 index 000000000..b3d9ebfe8 Binary files /dev/null and b/deps/sqlite3/sqlite-rembed-0.0.1-alpha.9.tar.gz differ diff --git a/deps/sqlite3/sqlite-rembed-source/.github/workflows/release.yaml b/deps/sqlite3/sqlite-rembed-source/.github/workflows/release.yaml deleted file mode 100644 index 97e26912c..000000000 --- a/deps/sqlite3/sqlite-rembed-source/.github/workflows/release.yaml +++ /dev/null @@ -1,122 +0,0 @@ -name: "Release" -on: - release: - types: [published] -permissions: - contents: read -jobs: - build-linux-x86_64-extension: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - run: make loadable-release - - uses: actions/upload-artifact@v4 - with: - name: sqlite-rembed-linux-x86_64-extension - path: dist/release/* - build-macos-x86_64-extension: - runs-on: macos-12 - steps: - - uses: actions/checkout@v4 - - run: make loadable-release - - uses: actions/upload-artifact@v4 - with: - name: sqlite-rembed-macos-x86_64-extension - path: dist/release/* - build-macos-aarch64-extension: - runs-on: macos-14 - steps: - - uses: actions/checkout@v4 - - run: make loadable-release - - uses: actions/upload-artifact@v4 - with: - name: sqlite-rembed-macos-aarch64-extension - path: dist/release/* - build-windows-x86_64-extension: - runs-on: windows-2019 - steps: - - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - run: make loadable-release - - uses: actions/upload-artifact@v4 - with: - name: sqlite-rembed-windows-x86_64-extension - path: dist/release/* - dist: - runs-on: ubuntu-latest - needs: - [ - build-linux-x86_64-extension, - build-macos-x86_64-extension, - build-macos-aarch64-extension, - build-windows-x86_64-extension, - ] - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - name: sqlite-rembed-linux-x86_64-extension - path: dist/linux-x86_64 - - uses: actions/download-artifact@v4 - with: - name: sqlite-rembed-macos-x86_64-extension - path: dist/macos-x86_64 - - uses: actions/download-artifact@v4 - with: - name: sqlite-rembed-macos-aarch64-extension - path: dist/macos-aarch64 - - uses: actions/download-artifact@v4 - with: - name: sqlite-rembed-windows-x86_64-extension - path: dist/windows-x86_64 - - run: | - curl -L https://github.com/asg017/sqlite-dist/releases/download/v0.0.1-alpha.7/sqlite-dist-x86_64-unknown-linux-gnu.tar.xz \ - | tar xfJ - --strip-components 1 - - run: make sqlite-rembed.h - - run: ./sqlite-dist ./sqlite-dist.toml --input dist/ --output distx/ --version $(cat VERSION) - - run: | - gh release upload ${{ github.ref_name }} \ - distx/github_releases/* \ - distx/spm/* \ - distx/sqlpkg/* \ - distx/checksums.txt \ - distx/sqlite-dist-manifest.json \ - distx/install.sh - env: - GH_TOKEN: ${{ github.token }} - - name: Install node - uses: actions/setup-node@v3 - with: - node-version: "16" - registry-url: "https://registry.npmjs.org" - - run: | - npm publish --access public distx/npm/sqlite-rembed-darwin-arm64.tar.gz - npm publish --access public distx/npm/sqlite-rembed-darwin-x64.tar.gz - npm publish --access public distx/npm/sqlite-rembed-linux-x64.tar.gz - npm publish --access public distx/npm/sqlite-rembed.tar.gz - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - uses: ruby/setup-ruby@v1 - with: - ruby-version: 3.2 - - run: | - for file in distx/gem/*; do - gem push "$file" - done - env: - GEM_HOST_API_KEY: ${{ secrets.GEM_HOST_API_KEY }} - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - run: pip install twine - - run: | - twine upload distx/pip/* - twine upload distx/datasette/* - twine upload distx/sqlite_utils/* - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} diff --git a/deps/sqlite3/sqlite-rembed-source/.github/workflows/test.yaml b/deps/sqlite3/sqlite-rembed-source/.github/workflows/test.yaml deleted file mode 100644 index 24f63c296..000000000 --- a/deps/sqlite3/sqlite-rembed-source/.github/workflows/test.yaml +++ /dev/null @@ -1,60 +0,0 @@ -name: "Test" -on: - push: - branches: - - main -permissions: - contents: read -jobs: - build-linux-x86_64-extension: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - run: make loadable static - #- run: pip install pytest numpy; make test-loadable - - uses: actions/upload-artifact@v4 - with: - name: sqlite-rembed-linux-x86_64-extension - path: dist/* - build-macos-x86_64-extension: - runs-on: macos-12 - steps: - - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - run: make loadable static - #- run: /usr/local/opt/python@3/libexec/bin/python -m pip install pytest numpy; make test-loadable python=/usr/local/opt/python@3/libexec/bin/python - - uses: actions/upload-artifact@v4 - with: - name: sqlite-rembed-macos-x86_64-extension - path: dist/* - build-macos-aarch64-extension: - runs-on: macos-14 - steps: - - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - run: make loadable static - #- run: /opt/homebrew/opt/python3/libexec/bin/python -m pip install pytest numpy --break-system-packages; make test-loadable python=/opt/homebrew/opt/python3/libexec/bin/python - - uses: actions/upload-artifact@v4 - with: - name: sqlite-rembed-macos-aarch64-extension - path: dist/* - build-windows-x86_64-extension: - runs-on: windows-2019 - steps: - - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - run: make loadable static - #- run: pip install pytest numpy; make test-loadable - - uses: actions/upload-artifact@v4 - with: - name: sqlite-rembed-windows-x86_64-extension - path: dist/* diff --git a/deps/sqlite3/sqlite-rembed-source/.gitignore b/deps/sqlite3/sqlite-rembed-source/.gitignore deleted file mode 100644 index bc97e80e2..000000000 --- a/deps/sqlite3/sqlite-rembed-source/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/target -.env -dist/ diff --git a/deps/sqlite3/sqlite-rembed-source/Cargo.lock b/deps/sqlite3/sqlite-rembed-source/Cargo.lock deleted file mode 100644 index ff31d5ae3..000000000 --- a/deps/sqlite3/sqlite-rembed-source/Cargo.lock +++ /dev/null @@ -1,847 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - -[[package]] -name = "autocfg" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "bindgen" -version = "0.60.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6" -dependencies = [ - "bitflags 1.3.2", - "cexpr", - "clang-sys", - "clap", - "env_logger", - "lazy_static", - "lazycell", - "log", - "peeking_take_while", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "which", -] - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "cc" -version = "1.0.98" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" - -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "clang-sys" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f803f94ecf597339c7a34eed2036ef83f86aaba937f001f7c5b5e251f043f1f9" -dependencies = [ - "glob", - "libc", - "libloading", -] - -[[package]] -name = "clap" -version = "3.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" -dependencies = [ - "atty", - "bitflags 1.3.2", - "clap_lex", - "indexmap", - "strsim", - "termcolor", - "textwrap", -] - -[[package]] -name = "clap_lex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" -dependencies = [ - "os_str_bytes", -] - -[[package]] -name = "crc32fast" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "either" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" - -[[package]] -name = "env_logger" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" -dependencies = [ - "atty", - "humantime", - "log", - "regex", - "termcolor", -] - -[[package]] -name = "errno" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" -dependencies = [ - "libc", - "windows-sys", -] - -[[package]] -name = "flate2" -version = "1.0.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - -[[package]] -name = "form_urlencoded" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "getrandom" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "home" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - -[[package]] -name = "idna" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown", -] - -[[package]] -name = "itoa" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - -[[package]] -name = "libc" -version = "0.2.155" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" - -[[package]] -name = "libloading" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" -dependencies = [ - "cfg-if", - "windows-targets", -] - -[[package]] -name = "linux-raw-sys" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" - -[[package]] -name = "log" -version = "0.4.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" - -[[package]] -name = "memchr" -version = "2.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "miniz_oxide" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" -dependencies = [ - "adler", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "once_cell" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" - -[[package]] -name = "os_str_bytes" -version = "6.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" - -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - -[[package]] -name = "percent-encoding" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" - -[[package]] -name = "proc-macro2" -version = "1.0.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "regex" -version = "1.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" - -[[package]] -name = "ring" -version = "0.17.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" -dependencies = [ - "cc", - "cfg-if", - "getrandom", - "libc", - "spin", - "untrusted", - "windows-sys", -] - -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - -[[package]] -name = "rustix" -version = "0.38.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" -dependencies = [ - "bitflags 2.5.0", - "errno", - "libc", - "linux-raw-sys", - "windows-sys", -] - -[[package]] -name = "rustls" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" -dependencies = [ - "log", - "ring", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-pki-types" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" - -[[package]] -name = "rustls-webpki" -version = "0.102.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted", -] - -[[package]] -name = "ryu" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" - -[[package]] -name = "serde" -version = "1.0.203" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.203" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "serde_json" -version = "1.0.117" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" -dependencies = [ - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" - -[[package]] -name = "sqlite-loadable" -version = "0.0.6-alpha.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daaaad0ad506b154a72bf01fde23235377c01256abd4bd25e17419dbfd4e28a0" -dependencies = [ - "bitflags 1.3.2", - "serde", - "serde_json", - "sqlite-loadable-macros", - "sqlite3ext-sys", -] - -[[package]] -name = "sqlite-loadable-macros" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96037a396115a2675db783f700faad878b44c8ff56c8a29c3404649a517a5e8f" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "sqlite-rembed" -version = "0.0.1-alpha.9" -dependencies = [ - "serde_json", - "sqlite-loadable", - "ureq", - "zerocopy", -] - -[[package]] -name = "sqlite3ext-sys" -version = "0.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3afdc2b3dc08f16d6eecf8aa07d19975a268603ab1cca67d3f9b4172c507cf16" -dependencies = [ - "bindgen", - "cc", -] - -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "subtle" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.66" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "textwrap" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" - -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "unicode-bidi" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" - -[[package]] -name = "unicode-ident" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" - -[[package]] -name = "unicode-normalization" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "ureq" -version = "2.9.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d11a831e3c0b56e438a28308e7c810799e3c118417f342d30ecec080105395cd" -dependencies = [ - "base64", - "flate2", - "log", - "once_cell", - "rustls", - "rustls-pki-types", - "rustls-webpki", - "serde", - "serde_json", - "url", - "webpki-roots", -] - -[[package]] -name = "url" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "webpki-roots" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" - -[[package]] -name = "zerocopy" -version = "0.7.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" -dependencies = [ - "byteorder", - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "zeroize" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" diff --git a/deps/sqlite3/sqlite-rembed-source/Cargo.toml b/deps/sqlite3/sqlite-rembed-source/Cargo.toml deleted file mode 100644 index 5d0bacb2f..000000000 --- a/deps/sqlite3/sqlite-rembed-source/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "sqlite-rembed" -version = "0.0.1-alpha.9" -edition = "2021" - -[dependencies] -serde_json = "1.0.117" -sqlite-loadable = "0.0.6-alpha.6" -ureq = {version="2.9.7", features=["json"]} -zerocopy = "0.7.34" - -[lib] -crate-type=["cdylib", "staticlib", "lib"] - diff --git a/deps/sqlite3/sqlite-rembed-source/LICENSE-APACHE b/deps/sqlite3/sqlite-rembed-source/LICENSE-APACHE deleted file mode 100644 index f49a4e16e..000000000 --- a/deps/sqlite3/sqlite-rembed-source/LICENSE-APACHE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/deps/sqlite3/sqlite-rembed-source/LICENSE-MIT b/deps/sqlite3/sqlite-rembed-source/LICENSE-MIT deleted file mode 100644 index 9736ab442..000000000 --- a/deps/sqlite3/sqlite-rembed-source/LICENSE-MIT +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2022 Alex Garcia - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/deps/sqlite3/sqlite-rembed-source/Makefile b/deps/sqlite3/sqlite-rembed-source/Makefile deleted file mode 100644 index 9bd7661aa..000000000 --- a/deps/sqlite3/sqlite-rembed-source/Makefile +++ /dev/null @@ -1,141 +0,0 @@ -SHELL := /bin/bash - -VERSION=$(shell cat VERSION) - -ifeq ($(shell uname -s),Darwin) -CONFIG_DARWIN=y -else ifeq ($(OS),Windows_NT) -CONFIG_WINDOWS=y -else -CONFIG_LINUX=y -endif - -LIBRARY_PREFIX=lib -ifdef CONFIG_DARWIN -LOADABLE_EXTENSION=dylib -STATIC_EXTENSION=a -endif - -ifdef CONFIG_LINUX -LOADABLE_EXTENSION=so -STATIC_EXTENSION=a -endif - - -ifdef CONFIG_WINDOWS -LOADABLE_EXTENSION=dll -LIBRARY_PREFIX= -STATIC_EXTENSION=lib -endif - -prefix=dist -TARGET_LOADABLE=$(prefix)/debug/rembed0.$(LOADABLE_EXTENSION) -TARGET_LOADABLE_RELEASE=$(prefix)/release/rembed0.$(LOADABLE_EXTENSION) - -TARGET_STATIC=$(prefix)/debug/$(LIBRARY_PREFIX)sqlite_rembed0.$(STATIC_EXTENSION) -TARGET_STATIC_RELEASE=$(prefix)/release/$(LIBRARY_PREFIX)sqlite_rembed0.$(STATIC_EXTENSION) - -TARGET_H=$(prefix)/debug/sqlite-rembed.h -TARGET_H_RELEASE=$(prefix)/release/sqlite-rembed.h - -TARGET_WHEELS=$(prefix)/debug/wheels -TARGET_WHEELS_RELEASE=$(prefix)/release/wheels - -INTERMEDIATE_PYPACKAGE_EXTENSION=python/sqlite_rembed/sqlite_rembed/rembed0.$(LOADABLE_EXTENSION) - -ifdef target -CARGO_TARGET=--target=$(target) -BUILT_LOCATION=target/$(target)/debug/$(LIBRARY_PREFIX)sqlite_rembed.$(LOADABLE_EXTENSION) -BUILT_LOCATION_RELEASE=target/$(target)/release/$(LIBRARY_PREFIX)sqlite_rembed.$(LOADABLE_EXTENSION) -BUILT_LOCATION_STATIC=target/$(target)/debug/$(LIBRARY_PREFIX)sqlite_rembed.$(STATIC_EXTENSION) -BUILT_LOCATION_STATIC_RELEASE=target/$(target)/release/$(LIBRARY_PREFIX)sqlite_rembed.$(STATIC_EXTENSION) -else -CARGO_TARGET= -BUILT_LOCATION=target/debug/$(LIBRARY_PREFIX)sqlite_rembed.$(LOADABLE_EXTENSION) -BUILT_LOCATION_RELEASE=target/release/$(LIBRARY_PREFIX)sqlite_rembed.$(LOADABLE_EXTENSION) -BUILT_LOCATION_STATIC=target/debug/$(LIBRARY_PREFIX)sqlite_rembed.$(STATIC_EXTENSION) -BUILT_LOCATION_STATIC_RELEASE=target/release/$(LIBRARY_PREFIX)sqlite_rembed.$(STATIC_EXTENSION) -endif - -ifdef python -PYTHON=$(python) -else -PYTHON=python3 -endif - -ifdef IS_MACOS_ARM -RENAME_WHEELS_ARGS=--is-macos-arm -else -RENAME_WHEELS_ARGS= -endif - -$(prefix): - mkdir -p $(prefix)/debug - mkdir -p $(prefix)/release - -$(TARGET_WHEELS): $(prefix) - mkdir -p $(TARGET_WHEELS) - -$(TARGET_WHEELS_RELEASE): $(prefix) - mkdir -p $(TARGET_WHEELS_RELEASE) - -$(TARGET_LOADABLE): $(prefix) $(shell find . -type f -name '*.rs') - cargo build --verbose $(CARGO_TARGET) - cp $(BUILT_LOCATION) $@ - -$(TARGET_LOADABLE_RELEASE): $(prefix) $(shell find . -type f -name '*.rs') - cargo build --verbose --release $(CARGO_TARGET) - cp $(BUILT_LOCATION_RELEASE) $@ - -$(TARGET_STATIC): $(prefix) $(shell find . -type f -name '*.rs') - cargo build --verbose $(CARGO_TARGET) --features=sqlite-loadable/static - ls target - ls target/$(target)/debug - cp $(BUILT_LOCATION_STATIC) $@ - -$(TARGET_STATIC_RELEASE): $(prefix) $(shell find . -type f -name '*.rs') - cargo build --verbose --release $(CARGO_TARGET) --features=sqlite-loadable/static - cp $(BUILT_LOCATION_STATIC_RELEASE) $@ - -$(TARGET_H): sqlite-rembed.h - cp $< $@ - -$(TARGET_H_RELEASE): sqlite-rembed.h - cp $< $@ - -Cargo.toml: VERSION - cargo set-version `cat VERSION` - -version: - make Cargo.toml - -format: - cargo fmt - -release: $(TARGET_LOADABLE_RELEASE) $(TARGET_STATIC_RELEASE) - -loadable: $(TARGET_LOADABLE) -loadable-release: $(TARGET_LOADABLE_RELEASE) - -static: $(TARGET_STATIC) $(TARGET_H) -static-release: $(TARGET_STATIC_RELEASE) $(TARGET_H_RELEASE) - -debug: loadable static python datasette -release: loadable-release static-release python-release datasette-release - -clean: - rm dist/* - cargo clean - -test-loadable: - $(PYTHON) tests/test-loadable.py - -publish-release: - ./scripts/publish_release.sh - -.PHONY: clean \ - test test-loadable test-python test-npm test-deno \ - loadable loadable-release \ - static static-release \ - debug release \ - format version publish-release diff --git a/deps/sqlite3/sqlite-rembed-source/README.md b/deps/sqlite3/sqlite-rembed-source/README.md deleted file mode 100644 index d59a4fc0c..000000000 --- a/deps/sqlite3/sqlite-rembed-source/README.md +++ /dev/null @@ -1,134 +0,0 @@ -# `sqlite-rembed` - -A SQLite extension for generating text embeddings from remote APIs (OpenAI, Nomic, Cohere, llamafile, Ollama, etc.). A sister project to [`sqlite-vec`](https://github.com/asg017/sqlite-vec) and [`sqlite-lembed`](https://github.com/asg017/sqlite-lembed). A work-in-progress! - -## Usage - -```sql -.load ./rembed0 - -INSERT INTO temp.rembed_clients(name, options) - VALUES ('text-embedding-3-small', 'openai'); - -select rembed( - 'text-embedding-3-small', - 'The United States Postal Service is an independent agency...' -); -``` - -The `temp.rembed_clients` virtual table lets you "register" clients with pure `INSERT INTO` statements. The `name` field is a unique identifier for a given client, and `options` allows you to specify which 3rd party embedding service you want to use. - -In this case, `openai` is a pre-defined client that will default to OpenAI's `https://api.openai.com/v1/embeddings` endpoint and will source your API key from the `OPENAI_API_KEY` environment variable. The name of the client, `text-embedding-3-small`, will be used as the embeddings model. - -Other pre-defined clients include: - -| Client name | Provider | Endpoint | API Key | -| ------------ | ------------------------------------------------------------------------------------ | ---------------------------------------------- | -------------------- | -| `openai` | [OpenAI](https://platform.openai.com/docs/guides/embeddings) | `https://api.openai.com/v1/embeddings` | `OPENAI_API_KEY` | -| `nomic` | [Nomic](https://docs.nomic.ai/reference/endpoints/nomic-embed-text) | `https://api-atlas.nomic.ai/v1/embedding/text` | `NOMIC_API_KEY` | -| `cohere` | [Cohere](https://docs.cohere.com/reference/embed) | `https://api.cohere.com/v1/embed` | `CO_API_KEY` | -| `jina` | [Jina](https://api.jina.ai/redoc#tag/embeddings) | `https://api.jina.ai/v1/embeddings` | `JINA_API_KEY` | -| `mixedbread` | [MixedBread](https://www.mixedbread.ai/api-reference#quick-start-guide) | `https://api.mixedbread.ai/v1/embeddings/` | `MIXEDBREAD_API_KEY` | -| `llamafile` | [llamafile](https://github.com/Mozilla-Ocho/llamafile) | `http://localhost:8080/embedding` | None | -| `ollama` | [Ollama](https://github.com/ollama/ollama/blob/main/docs/api.md#generate-embeddings) | `http://localhost:11434/api/embeddings` | None | - -Different client options can be specified with `remebed_client_options()`. For example, if you have a different OpenAI-compatible service you want to use, then you can use: - -```sql -INSERT INTO temp.rembed_clients(name, options) VALUES - ( - 'xyz-small-1', - rembed_client_options( - 'format', 'openai', - 'url', 'https://api.xyz.com/v1/embeddings', - 'key', 'xyz-ca865ece65-hunter2' - ) - ); -``` - -Or to use a llamafile server that's on a different port: - -```sql -INSERT INTO temp.rembed_clients(name, options) VALUES - ( - 'xyz-small-1', - rembed_client_options( - 'format', 'lamafile', - 'url', 'http://localhost:9999/embedding' - ) - ); -``` - -### Using with `sqlite-vec` - -`sqlite-rembed` works well with [`sqlite-vec`](https://github.com/asg017/sqlite-vec), a SQLite extension for vector search. Embeddings generated with `rembed()` use the same BLOB format for vectors that `sqlite-vec` uses. - -Here's a sample "semantic search" application, made from a sample dataset of news article headlines. - -```sql -create table articles( - headline text -); - --- Random NPR headlines from 2024-06-04 -insert into articles VALUES - ('Shohei Ohtani''s ex-interpreter pleads guilty to charges related to gambling and theft'), - ('The jury has been selected in Hunter Biden''s gun trial'), - ('Larry Allen, a Super Bowl champion and famed Dallas Cowboy, has died at age 52'), - ('After saying Charlotte, a lone stingray, was pregnant, aquarium now says she''s sick'), - ('An Epoch Times executive is facing money laundering charge'); - - --- Build a vector table with embeddings of article headlines, using OpenAI's API -create virtual table vec_articles using vec0( - headline_embeddings float[1536] -); - -insert into vec_articles(rowid, headline_embeddings) - select rowid, rembed('text-embedding-3-small', headline) - from articles; - -``` - -Now we have a regular `articles` table that stores text headlines, and a `vec_articles` virtual table that stores embeddings of the article headlines, using OpenAI's `text-embedding-3-small` model. - -To perform a "semantic search" on the embeddings, we can query the `vec_articles` table with an embedding of our query, and join the results back to our `articles` table to retrieve the original headlines. - -```sql -param set :query 'firearm courtroom' - -with matches as ( - select - rowid, - distance - from vec_articles - where headline_embeddings match rembed('text-embedding-3-small', :query) - order by distance - limit 3 -) -select - headline, - distance -from matches -left join articles on articles.rowid = matches.rowid; - -/* -+--------------------------------------------------------------+------------------+ -| headline | distance | -+--------------------------------------------------------------+------------------+ -| The jury has been selected in Hunter Biden's gun trial | 1.05906391143799 | -+--------------------------------------------------------------+------------------+ -| Shohei Ohtani's ex-interpreter pleads guilty to charges rela | 1.2574303150177 | -| ted to gambling and theft | | -+--------------------------------------------------------------+------------------+ -| An Epoch Times executive is facing money laundering charge | 1.27144026756287 | -+--------------------------------------------------------------+------------------+ -*/ -``` - -Notice how "firearm courtroom" doesn't appear in any of these headlines, but it can still figure out that "Hunter Biden's gun trial" is related, and the other two justice-related articles appear on top. - -## Drawbacks - -1. **No batch support yet.** If you use `rembed()` in a batch UPDATE or INSERT in 1,000 rows, then 1,000 HTTP requests will be made. Add a :+1: to [Issue #1](https://github.com/asg017/sqlite-rembed/issues/1) if you want to see this fixed. -2. **No builtin rate limiting.** Requests are sent sequentially so this may not come up in small demos, but `sqlite-rembed` could add features that handles rate limiting/retries implicitly. Add a :+1: to [Issue #2](https://github.com/asg017/sqlite-rembed/issues/2) if you want to see this implemented. diff --git a/deps/sqlite3/sqlite-rembed-source/VERSION b/deps/sqlite3/sqlite-rembed-source/VERSION deleted file mode 100644 index 1429ae318..000000000 --- a/deps/sqlite3/sqlite-rembed-source/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.0.1-alpha.9 \ No newline at end of file diff --git a/deps/sqlite3/sqlite-rembed-source/build.rs b/deps/sqlite3/sqlite-rembed-source/build.rs deleted file mode 100644 index c5c0c3b4a..000000000 --- a/deps/sqlite3/sqlite-rembed-source/build.rs +++ /dev/null @@ -1,9 +0,0 @@ -use std::process::Command; -fn main() { - let output = Command::new("git") - .args(["rev-parse", "HEAD"]) - .output() - .unwrap(); - let git_hash = String::from_utf8(output.stdout).unwrap(); - println!("cargo:rustc-env=GIT_HASH={}", git_hash); -} diff --git a/deps/sqlite3/sqlite-rembed-source/examples/simple-search/demo.sql b/deps/sqlite3/sqlite-rembed-source/examples/simple-search/demo.sql deleted file mode 100644 index 20ee88b0e..000000000 --- a/deps/sqlite3/sqlite-rembed-source/examples/simple-search/demo.sql +++ /dev/null @@ -1,48 +0,0 @@ -.bail on -.mode table -.header on - -.timer on - -.load ../../dist/debug/rembed0 -.load ../../../sqlite-vec/dist/vec0 - -INSERT INTO temp.rembed_clients(name, options) - VALUES ('text-embedding-3-small', 'openai'); - -create table articles(headline text); - - --- Random NPR headlines from 2024-06-04 -insert into articles VALUES - ('Shohei Ohtani''s ex-interpreter pleads guilty to charges related to gambling and theft'), - ('The jury has been selected in Hunter Biden''s gun trial'), - ('Larry Allen, a Super Bowl champion and famed Dallas Cowboy, has died at age 52'), - ('After saying Charlotte, a lone stingray, was pregnant, aquarium now says she''s sick'), - ('An Epoch Times executive is facing money laundering charge'); - - --- Seed a vector table with embeddings of article headlines, using OpenAI's API -create virtual table vec_articles using vec0(headline_embeddings float[1536]); - -insert into vec_articles(rowid, headline_embeddings) - select rowid, rembed('text-embedding-3-small', headline) - from articles; - - -.param set :query 'firearm courtroom' - -with matches as ( - select - rowid, - distance - from vec_articles - where headline_embeddings match rembed('text-embedding-3-small', :query) - order by distance - limit 3 -) -select - headline, - distance -from matches -left join articles on articles.rowid = matches.rowid; diff --git a/deps/sqlite3/sqlite-rembed-source/scripts/publish-release.sh b/deps/sqlite3/sqlite-rembed-source/scripts/publish-release.sh deleted file mode 100755 index 0bfecc192..000000000 --- a/deps/sqlite3/sqlite-rembed-source/scripts/publish-release.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -set -euo pipefail xtrace - -if [[ -n $(git status --porcelain | grep -v VERSION | grep -v sqlite-dist.toml) ]]; then - echo "❌ There are other un-staged changes to the repository besides VERSION and sqlite-dist.toml" - exit 1 -fi - -VERSION="$(cat VERSION)" - -echo "Publishing version v$VERSION..." - -make version -git add --all -git commit -m "v$VERSION" -git tag v$VERSION -git push origin main v$VERSION - -if grep -qE "alpha|beta" VERSION; then - gh release create v$VERSION --title=v$VERSION --prerelease --notes="" -else - gh release create v$VERSION --title=v$VERSION -fi - - -echo "✅ Published! version v$VERSION" diff --git a/deps/sqlite3/sqlite-rembed-source/sqlite-dist.toml b/deps/sqlite3/sqlite-rembed-source/sqlite-dist.toml deleted file mode 100644 index d3671aaca..000000000 --- a/deps/sqlite3/sqlite-rembed-source/sqlite-dist.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "sqlite-rembed" -license = "MIT OR Apache" -homepage = "https://alexgarcia.xyz/sqlite-rembed" -repo = "https://github.com/asg017/sqlite-rembed" -description = "A SQLite extension for generating text embeddings from remote sources (OpenAI, Cohere, localhost, etc.)" -authors = ["Alex Garcia"] -git_tag_format = "v$VERSION" - -[targets] -github_releases = {} -sqlpkg = {} -spm = {} - -pip = {} -datasette = {} -sqlite_utils = {} - -npm = {} - -gem = { module_name = "SqliteRembed" } diff --git a/deps/sqlite3/sqlite-rembed-source/sqlite-rembed.h b/deps/sqlite3/sqlite-rembed-source/sqlite-rembed.h deleted file mode 100644 index b47a3f24c..000000000 --- a/deps/sqlite3/sqlite-rembed-source/sqlite-rembed.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef _SQLITE_REMBED_H -#define _SQLITE_REMBED_H - -#ifdef __cplusplus -extern "C" { -#endif - -int sqlite3_rembed_init(sqlite3*, char**, const sqlite3_api_routines*); - -#ifdef __cplusplus -} /* end of the 'extern "C"' block */ -#endif - -#endif /* ifndef _SQLITE_REMBED_H */ diff --git a/deps/sqlite3/sqlite-rembed-source/src/clients.rs b/deps/sqlite3/sqlite-rembed-source/src/clients.rs deleted file mode 100644 index 5f83b9a38..000000000 --- a/deps/sqlite3/sqlite-rembed-source/src/clients.rs +++ /dev/null @@ -1,516 +0,0 @@ -use sqlite_loadable::{Error, Result}; - -pub(crate) fn try_env_var(key: &str) -> Result { - std::env::var(key) - .map_err(|_| Error::new_message(format!("{} environment variable not define. Alternatively, pass in an API key with rembed_client_options", DEFAULT_OPENAI_API_KEY_ENV))) -} - -#[derive(Clone)] -pub struct OpenAiClient { - model: String, - url: String, - key: String, -} -const DEFAULT_OPENAI_URL: &str = "https://api.openai.com/v1/embeddings"; -const DEFAULT_OPENAI_API_KEY_ENV: &str = "OPENAI_API_KEY"; - -impl OpenAiClient { - pub fn new>( - model: S, - url: Option, - key: Option, - ) -> Result { - Ok(Self { - model: model.into(), - url: url.unwrap_or(DEFAULT_OPENAI_URL.to_owned()), - key: match key { - Some(key) => key, - None => try_env_var(DEFAULT_OPENAI_API_KEY_ENV)?, - }, - }) - } - pub fn infer_single(&self, input: &str) -> Result> { - let body = serde_json::json!({ - "input": input, - "model": self.model - }); - - let data: serde_json::Value = ureq::post(&self.url) - .set("Content-Type", "application/json") - .set("Authorization", format!("Bearer {}", self.key).as_str()) - .send_bytes( - serde_json::to_vec(&body) - .map_err(|error| { - Error::new_message(format!("Error serializing body to JSON: {error}")) - })? - .as_ref(), - ) - .map_err(|error| Error::new_message(format!("Error sending HTTP request: {error}")))? - .into_json() - .map_err(|error| { - Error::new_message(format!("Error parsing HTTP response as JSON: {error}")) - })?; - OpenAiClient::parse_single_response(data) - } - - pub fn parse_single_response(value: serde_json::Value) -> Result> { - value - .get("data") - .ok_or_else(|| Error::new_message("expected 'data' key in response body")) - .and_then(|v| { - v.get(0) - .ok_or_else(|| Error::new_message("expected 'data.0' path in response body")) - }) - .and_then(|v| { - v.get("embedding").ok_or_else(|| { - Error::new_message("expected 'data.0.embedding' path in response body") - }) - }) - .and_then(|v| { - v.as_array().ok_or_else(|| { - Error::new_message("expected 'data.0.embedding' path to be an array") - }) - }) - .and_then(|arr| { - arr.iter() - .map(|v| { - v.as_f64() - .ok_or_else(|| { - Error::new_message( - "expected 'data.0.embedding' array to contain floats", - ) - }) - .map(|f| f as f32) - }) - .collect() - }) - } -} - -#[derive(Clone)] -pub struct NomicClient { - model: String, - url: String, - key: String, -} -const DEFAULT_NOMIC_URL: &str = "https://api-atlas.nomic.ai/v1/embedding/text"; -const DEFAULT_NOMIC_API_KEY_ENV: &str = "NOMIC_API_KEY"; - -impl NomicClient { - pub fn new>( - model: S, - url: Option, - key: Option, - ) -> Result { - Ok(Self { - model: model.into(), - url: url.unwrap_or(DEFAULT_NOMIC_URL.to_owned()), - key: match key { - Some(key) => key, - None => try_env_var(DEFAULT_NOMIC_API_KEY_ENV)?, - }, - }) - } - - pub fn infer_single(&self, input: &str, input_type: Option<&str>) -> Result> { - let mut body = serde_json::Map::new(); - body.insert("texts".to_owned(), vec![input.to_owned()].into()); - body.insert("model".to_owned(), self.model.to_owned().into()); - - if let Some(input_type) = input_type { - body.insert("input_type".to_owned(), input_type.to_owned().into()); - } - - let data: serde_json::Value = ureq::post(&self.url) - .set("Content-Type", "application/json") - .set("Authorization", format!("Bearer {}", self.key).as_str()) - .send_bytes( - serde_json::to_vec(&body) - .map_err(|error| { - Error::new_message(format!("Error serializing body to JSON: {error}")) - })? - .as_ref(), - ) - .map_err(|error| Error::new_message(format!("Error sending HTTP request: {error}")))? - .into_json() - .map_err(|error| { - Error::new_message(format!("Error parsing HTTP response as JSON: {error}")) - })?; - NomicClient::parse_single_response(data) - } - pub fn parse_single_response(value: serde_json::Value) -> Result> { - value - .get("embeddings") - .ok_or_else(|| Error::new_message("expected 'embeddings' key in response body")) - .and_then(|v| { - v.get(0).ok_or_else(|| { - Error::new_message("expected 'embeddings.0' path in response body") - }) - }) - .and_then(|v| { - v.as_array().ok_or_else(|| { - Error::new_message("expected 'embeddings.0' path to be an array") - }) - }) - .and_then(|arr| { - arr.iter() - .map(|v| { - v.as_f64() - .ok_or_else(|| { - Error::new_message( - "expected 'embeddings.0' array to contain floats", - ) - }) - .map(|f| f as f32) - }) - .collect() - }) - } -} - -#[derive(Clone)] -pub struct CohereClient { - url: String, - model: String, - key: String, -} -const DEFAULT_COHERE_URL: &str = "https://api.cohere.com/v1/embed"; -const DEFAULT_COHERE_API_KEY_ENV: &str = "CO_API_KEY"; - -impl CohereClient { - pub fn new>( - model: S, - url: Option, - key: Option, - ) -> Result { - Ok(Self { - model: model.into(), - url: url.unwrap_or(DEFAULT_COHERE_URL.to_owned()), - key: match key { - Some(key) => key, - None => try_env_var(DEFAULT_COHERE_API_KEY_ENV)?, - }, - }) - } - - pub fn infer_single(&self, input: &str, input_type: Option<&str>) -> Result> { - let mut body = serde_json::Map::new(); - body.insert("texts".to_owned(), vec![input.to_owned()].into()); - body.insert("model".to_owned(), self.model.to_owned().into()); - - if let Some(input_type) = input_type { - body.insert("input_type".to_owned(), input_type.to_owned().into()); - } - - let data: serde_json::Value = ureq::post(&self.url) - .set("Content-Type", "application/json") - .set("Accept", "application/json") - .set("Authorization", format!("Bearer {}", self.key).as_str()) - .send_bytes( - serde_json::to_vec(&body) - .map_err(|error| { - Error::new_message(format!("Error serializing body to JSON: {error}")) - })? - .as_ref(), - ) - .map_err(|error| Error::new_message(format!("Error sending HTTP request: {error}")))? - .into_json() - .map_err(|error| { - Error::new_message(format!("Error parsing HTTP response as JSON: {error}")) - })?; - CohereClient::parse_single_response(data) - } - pub fn parse_single_response(value: serde_json::Value) -> Result> { - value - .get("embeddings") - .ok_or_else(|| Error::new_message("expected 'embeddings' key in response body")) - .and_then(|v| { - v.get(0).ok_or_else(|| { - Error::new_message("expected 'embeddings.0' path in response body") - }) - }) - .and_then(|v| { - v.as_array().ok_or_else(|| { - Error::new_message("expected 'embeddings.0' path to be an array") - }) - }) - .and_then(|arr| { - arr.iter() - .map(|v| { - v.as_f64() - .ok_or_else(|| { - Error::new_message( - "expected 'embeddings.0' array to contain floats", - ) - }) - .map(|f| f as f32) - }) - .collect() - }) - } -} -#[derive(Clone)] -pub struct JinaClient { - url: String, - model: String, - key: String, -} -const DEFAULT_JINA_URL: &str = "https://api.jina.ai/v1/embeddings"; -const DEFAULT_JINA_API_KEY_ENV: &str = "JINA_API_KEY"; - -impl JinaClient { - pub fn new>( - model: S, - url: Option, - key: Option, - ) -> Result { - Ok(Self { - model: model.into(), - url: url.unwrap_or(DEFAULT_JINA_URL.to_owned()), - key: match key { - Some(key) => key, - None => try_env_var(DEFAULT_JINA_API_KEY_ENV)?, - }, - }) - } - - pub fn infer_single(&self, input: &str) -> Result> { - let mut body = serde_json::Map::new(); - body.insert("input".to_owned(), vec![input.to_owned()].into()); - body.insert("model".to_owned(), self.model.to_owned().into()); - - let data: serde_json::Value = ureq::post(&self.url) - .set("Content-Type", "application/json") - .set("Accept", "application/json") - .set("Authorization", format!("Bearer {}", self.key).as_str()) - .send_bytes( - serde_json::to_vec(&body) - .map_err(|error| { - Error::new_message(format!("Error serializing body to JSON: {error}")) - })? - .as_ref(), - ) - .map_err(|error| Error::new_message(format!("Error sending HTTP request: {error}")))? - .into_json() - .map_err(|error| { - Error::new_message(format!("Error parsing HTTP response as JSON: {error}")) - })?; - JinaClient::parse_single_response(data) - } - pub fn parse_single_response(value: serde_json::Value) -> Result> { - value - .get("data") - .ok_or_else(|| Error::new_message("expected 'data' key in response body")) - .and_then(|v| { - v.get(0) - .ok_or_else(|| Error::new_message("expected 'data.0' path in response body")) - }) - .and_then(|v| { - v.get("embedding").ok_or_else(|| { - Error::new_message("expected 'data.0.embedding' path in response body") - }) - }) - .and_then(|v| { - v.as_array().ok_or_else(|| { - Error::new_message("expected 'data.0.embedding' path to be an array") - }) - }) - .and_then(|arr| { - arr.iter() - .map(|v| { - v.as_f64() - .ok_or_else(|| { - Error::new_message( - "expected 'data.0.embedding' array to contain floats", - ) - }) - .map(|f| f as f32) - }) - .collect() - }) - } -} -#[derive(Clone)] -pub struct MixedbreadClient { - url: String, - model: String, - key: String, -} -const DEFAULT_MIXEDBREAD_URL: &str = "https://api.mixedbread.ai/v1/embeddings/"; -const DEFAULT_MIXEDBREAD_API_KEY_ENV: &str = "MIXEDBREAD_API_KEY"; - -impl MixedbreadClient { - pub fn new>( - model: S, - url: Option, - key: Option, - ) -> Result { - Ok(Self { - model: model.into(), - url: url.unwrap_or(DEFAULT_MIXEDBREAD_URL.to_owned()), - key: match key { - Some(key) => key, - None => try_env_var(DEFAULT_MIXEDBREAD_API_KEY_ENV)?, - }, - }) - } - - pub fn infer_single(&self, input: &str) -> Result> { - let mut body = serde_json::Map::new(); - body.insert("input".to_owned(), vec![input.to_owned()].into()); - body.insert("model".to_owned(), self.model.to_owned().into()); - - let data: serde_json::Value = ureq::post(&self.url) - .set("Content-Type", "application/json") - .set("Accept", "application/json") - .set("Authorization", format!("Bearer {}", self.key).as_str()) - .send_bytes( - serde_json::to_vec(&body) - .map_err(|error| { - Error::new_message(format!("Error serializing body to JSON: {error}")) - })? - .as_ref(), - ) - .map_err(|error| Error::new_message(format!("Error sending HTTP request: {error}")))? - .into_json() - .map_err(|error| { - Error::new_message(format!("Error parsing HTTP response as JSON: {error}")) - })?; - JinaClient::parse_single_response(data) - } - pub fn parse_single_response(value: serde_json::Value) -> Result> { - value - .get("data") - .ok_or_else(|| Error::new_message("expected 'data' key in response body")) - .and_then(|v| { - v.get(0) - .ok_or_else(|| Error::new_message("expected 'data.0' path in response body")) - }) - .and_then(|v| { - v.get("embedding").ok_or_else(|| { - Error::new_message("expected 'data.0.embedding' path in response body") - }) - }) - .and_then(|v| { - v.as_array().ok_or_else(|| { - Error::new_message("expected 'data.0.embedding' path to be an array") - }) - }) - .and_then(|arr| { - arr.iter() - .map(|v| { - v.as_f64() - .ok_or_else(|| { - Error::new_message( - "expected 'data.0.embedding' array to contain floats", - ) - }) - .map(|f| f as f32) - }) - .collect() - }) - } -} - -#[derive(Clone)] -pub struct OllamaClient { - url: String, - model: String, -} -const DEFAULT_OLLAMA_URL: &str = "http://localhost:11434/api/embeddings"; -impl OllamaClient { - pub fn new>(model: S, url: Option) -> Self { - Self { - model: model.into(), - url: url.unwrap_or(DEFAULT_OLLAMA_URL.to_owned()), - } - } - - pub fn infer_single(&self, input: &str) -> Result> { - let mut body = serde_json::Map::new(); - body.insert("prompt".to_owned(), input.to_owned().into()); - body.insert("model".to_owned(), self.model.to_owned().into()); - - let data: serde_json::Value = ureq::post(&self.url) - .set("Content-Type", "application/json") - .send_bytes( - serde_json::to_vec(&body) - .map_err(|error| { - Error::new_message(format!("Error serializing body to JSON: {error}")) - })? - .as_ref(), - ) - .map_err(|error| Error::new_message(format!("Error sending HTTP request: {error}")))? - .into_json() - .map_err(|error| { - Error::new_message(format!("Error parsing HTTP response as JSON: {error}")) - })?; - OllamaClient::parse_single_response(data) - } - pub fn parse_single_response(value: serde_json::Value) -> Result> { - value - .get("embedding") - .ok_or_else(|| Error::new_message("expected 'embedding' key in response body")) - .and_then(|v| { - v.as_array() - .ok_or_else(|| Error::new_message("expected 'embedding' path to be an array")) - }) - .and_then(|arr| { - arr.iter() - .map(|v| { - v.as_f64() - .ok_or_else(|| { - Error::new_message("expected 'embedding' array to contain floats") - }) - .map(|f| f as f32) - }) - .collect() - }) - } -} - -#[derive(Clone)] -pub struct LlamafileClient { - url: String, -} -const DEFAULT_LLAMAFILE_URL: &str = "http://localhost:8080/embedding"; - -impl LlamafileClient { - pub fn new(url: Option) -> Self { - Self { - url: url.unwrap_or(DEFAULT_LLAMAFILE_URL.to_owned()), - } - } - - pub fn infer_single(&self, input: &str) -> Result> { - let mut body = serde_json::Map::new(); - body.insert("content".to_owned(), input.to_owned().into()); - - let data: serde_json::Value = ureq::post(&self.url) - .set("Content-Type", "application/json") - .send_bytes( - serde_json::to_vec(&body) - .map_err(|error| { - Error::new_message(format!("Error serializing body to JSON: {error}")) - })? - .as_ref(), - ) - .map_err(|error| Error::new_message(format!("Error sending HTTP request: {error}")))? - .into_json() - .map_err(|error| { - Error::new_message(format!("Error parsing HTTP response as JSON: {error}")) - })?; - OllamaClient::parse_single_response(data) - } -} - -#[derive(Clone)] -pub enum Client { - OpenAI(OpenAiClient), - Nomic(NomicClient), - Cohere(CohereClient), - Ollama(OllamaClient), - Llamafile(LlamafileClient), - Jina(JinaClient), - Mixedbread(MixedbreadClient), -} diff --git a/deps/sqlite3/sqlite-rembed-source/src/clients_vtab.rs b/deps/sqlite3/sqlite-rembed-source/src/clients_vtab.rs deleted file mode 100644 index 101c95c6f..000000000 --- a/deps/sqlite3/sqlite-rembed-source/src/clients_vtab.rs +++ /dev/null @@ -1,184 +0,0 @@ -use sqlite_loadable::table::UpdateOperation; -use sqlite_loadable::{api, prelude::*, Error}; -use sqlite_loadable::{ - api::ValueType, - table::{IndexInfo, VTab, VTabArguments, VTabCursor, VTabWriteable}, - BestIndexError, Result, -}; -use std::{cell::RefCell, collections::HashMap, marker::PhantomData, mem, os::raw::c_int, rc::Rc}; - -use crate::clients::MixedbreadClient; -use crate::{ - clients::{ - Client, CohereClient, JinaClient, LlamafileClient, NomicClient, OllamaClient, OpenAiClient, - }, - CLIENT_OPTIONS_POINTER_NAME, -}; - -enum Columns { - Name, - Options, -} -fn column(index: i32) -> Option { - match index { - 0 => Some(Columns::Name), - 1 => Some(Columns::Options), - _ => None, - } -} -#[repr(C)] -pub struct ClientsTable { - /// must be first - base: sqlite3_vtab, - clients: Rc>>, -} - -impl<'vtab> VTab<'vtab> for ClientsTable { - type Aux = Rc>>; - type Cursor = ClientsCursor<'vtab>; - - fn create( - db: *mut sqlite3, - aux: Option<&Self::Aux>, - args: VTabArguments, - ) -> Result<(String, Self)> { - Self::connect(db, aux, args) - } - fn connect( - _db: *mut sqlite3, - aux: Option<&Self::Aux>, - _args: VTabArguments, - ) -> Result<(String, ClientsTable)> { - let base: sqlite3_vtab = unsafe { mem::zeroed() }; - let clients = aux.expect("Required aux").to_owned(); - - let vtab = ClientsTable { base, clients }; - let sql = "create table x(name text primary key, options)".to_owned(); - - Ok((sql, vtab)) - } - fn destroy(&self) -> Result<()> { - Ok(()) - } - - fn best_index(&self, mut info: IndexInfo) -> core::result::Result<(), BestIndexError> { - info.set_estimated_cost(10000.0); - info.set_estimated_rows(10000); - info.set_idxnum(1); - Ok(()) - } - - fn open(&'vtab mut self) -> Result> { - ClientsCursor::new(self) - } -} - -impl<'vtab> VTabWriteable<'vtab> for ClientsTable { - fn update(&'vtab mut self, operation: UpdateOperation<'_>, _p_rowid: *mut i64) -> Result<()> { - match operation { - UpdateOperation::Delete(_) => { - return Err(Error::new_message( - "DELETE operations on rembed_clients is not supported yet", - )) - } - UpdateOperation::Update { _values } => { - return Err(Error::new_message( - "DELETE operations on rembed_clients is not supported yet", - )) - } - UpdateOperation::Insert { values, rowid: _ } => { - let name = api::value_text(&values[0])?; - let client = match api::value_type(&values[1]) { - ValueType::Text => match api::value_text(&values[1])? { - "openai" => Client::OpenAI(OpenAiClient::new(name, None, None)?), - "mixedbread" => { - Client::Mixedbread(MixedbreadClient::new(name, None, None)?) - } - "jina" => Client::Jina(JinaClient::new(name, None, None)?), - "nomic" => Client::Nomic(NomicClient::new(name, None, None)?), - "cohere" => Client::Cohere(CohereClient::new(name, None, None)?), - "ollama" => Client::Ollama(OllamaClient::new(name, None)), - "llamafile" => Client::Llamafile(LlamafileClient::new(None)), - text => { - return Err(Error::new_message(format!( - "'{text}' is not a valid rembed client." - ))) - } - }, - ValueType::Null => unsafe { - if let Some(client) = - api::value_pointer::(&values[1], CLIENT_OPTIONS_POINTER_NAME) - { - (*client).clone() - } else { - return Err(Error::new_message("client options required")); - } - }, - _ => return Err(Error::new_message("client options required")), - }; - self.clients.borrow_mut().insert(name.to_owned(), client); - } - } - Ok(()) - } -} - -#[repr(C)] -pub struct ClientsCursor<'vtab> { - /// Base class. Must be first - base: sqlite3_vtab_cursor, - keys: Vec, - rowid: i64, - phantom: PhantomData<&'vtab ClientsTable>, -} -impl ClientsCursor<'_> { - fn new(table: &mut ClientsTable) -> Result { - let base: sqlite3_vtab_cursor = unsafe { mem::zeroed() }; - let c = table.clients.borrow(); - let keys = c.keys().map(|k| k.to_string()).collect(); - let cursor = ClientsCursor { - base, - keys, - rowid: 0, - phantom: PhantomData, - }; - Ok(cursor) - } -} - -impl VTabCursor for ClientsCursor<'_> { - fn filter( - &mut self, - _idx_num: c_int, - _idx_str: Option<&str>, - _values: &[*mut sqlite3_value], - ) -> Result<()> { - Ok(()) - } - - fn next(&mut self) -> Result<()> { - self.rowid += 1; - Ok(()) - } - - fn eof(&self) -> bool { - (self.rowid as usize) >= self.keys.len() - } - - fn column(&self, context: *mut sqlite3_context, i: c_int) -> Result<()> { - let key = self - .keys - .get(self.rowid as usize) - .expect("Internal rembed_clients logic error"); - match column(i) { - Some(Columns::Name) => api::result_text(context, key)?, - Some(Columns::Options) => (), - None => (), - }; - Ok(()) - } - - fn rowid(&self) -> Result { - Ok(self.rowid) - } -} diff --git a/deps/sqlite3/sqlite-rembed-source/src/lib.rs b/deps/sqlite3/sqlite-rembed-source/src/lib.rs deleted file mode 100644 index 192452526..000000000 --- a/deps/sqlite3/sqlite-rembed-source/src/lib.rs +++ /dev/null @@ -1,169 +0,0 @@ -mod clients; -mod clients_vtab; - -use std::cell::RefCell; -use std::collections::HashMap; -use std::rc::Rc; - -use clients::{Client, CohereClient, LlamafileClient, NomicClient, OllamaClient, OpenAiClient}; -use clients_vtab::ClientsTable; -use sqlite_loadable::{ - api, define_scalar_function, define_scalar_function_with_aux, define_virtual_table_writeablex, - prelude::*, Error, Result, -}; -use zerocopy::AsBytes; - -const FLOAT32_VECTOR_SUBTYPE: u8 = 223; -const CLIENT_OPTIONS_POINTER_NAME: &[u8] = b"sqlite-rembed-client-options\0"; - -pub fn rembed_version(context: *mut sqlite3_context, _values: &[*mut sqlite3_value]) -> Result<()> { - api::result_text(context, format!("v{}", env!("CARGO_PKG_VERSION")))?; - Ok(()) -} - -pub fn rembed_debug(context: *mut sqlite3_context, _values: &[*mut sqlite3_value]) -> Result<()> { - api::result_text( - context, - format!( - "Version: v{} -Source: {} -", - env!("CARGO_PKG_VERSION"), - env!("GIT_HASH") - ), - )?; - Ok(()) -} - -pub fn rembed_client_options( - context: *mut sqlite3_context, - values: &[*mut sqlite3_value], -) -> Result<()> { - if (values.len() % 2) != 0 { - return Err(Error::new_message( - "Must have an even number of arguments to rembed_client_options, as key/value pairs.", - )); - } - let mut options: HashMap = HashMap::new(); - let mut format: Option = None; - for pair in values.chunks(2) { - let key = api::value_text(&pair[0])?; - let value = api::value_text(&pair[1])?; - if key == "format" { - format = Some(value.to_owned()); - } else { - options.insert(key.to_owned(), value.to_owned()); - } - } - - let format = match format { - Some(format) => format, - None => { - return Err(Error::new_message("'format' key is required.")); - } - }; - let client: Client = match format.as_str() { - "openai" => Client::OpenAI(OpenAiClient::new( - options - .get("model") - .ok_or_else(|| Error::new_message("'model' option is required"))?, - options.get("url").cloned(), - options.get("key").cloned(), - )?), - "nomic" => Client::Nomic(NomicClient::new( - options - .get("model") - .ok_or_else(|| Error::new_message("'model' option is required"))?, - options.get("url").cloned(), - options.get("key").cloned(), - )?), - "cohere" => Client::Cohere(CohereClient::new( - options - .get("model") - .ok_or_else(|| Error::new_message("'model' option is required"))?, - options.get("url").cloned(), - options.get("key").cloned(), - )?), - "ollama" => Client::Ollama(OllamaClient::new( - options - .get("model") - .ok_or_else(|| Error::new_message("'model' option is required"))?, - options.get("url").cloned(), - )), - "llamafile" => Client::Llamafile(LlamafileClient::new(options.get("url").cloned())), - format => return Err(Error::new_message(format!("Unknown format '{format}'"))), - }; - - api::result_pointer(context, CLIENT_OPTIONS_POINTER_NAME, client); - - Ok(()) -} -pub fn rembed( - context: *mut sqlite3_context, - values: &[*mut sqlite3_value], - clients: &Rc>>, -) -> Result<()> { - let client_name = api::value_text(&values[0])?; - let input = api::value_text(&values[1])?; - let x = clients.borrow(); - let client = x.get(client_name).ok_or_else(|| { - Error::new_message(format!( - "Client with name {client_name} was not registered with rembed_clients." - )) - })?; - - let embedding = match client { - Client::OpenAI(client) => client.infer_single(input)?, - Client::Jina(client) => client.infer_single(input)?, - Client::Mixedbread(client) => client.infer_single(input)?, - Client::Ollama(client) => client.infer_single(input)?, - Client::Llamafile(client) => client.infer_single(input)?, - Client::Nomic(client) => { - let input_type = values.get(2).and_then(|v| api::value_text(v).ok()); - client.infer_single(input, input_type)? - } - Client::Cohere(client) => { - let input_type = values.get(2).and_then(|v| api::value_text(v).ok()); - client.infer_single(input, input_type)? - } - }; - - api::result_blob(context, embedding.as_bytes()); - api::result_subtype(context, FLOAT32_VECTOR_SUBTYPE); - Ok(()) -} - -#[sqlite_entrypoint] -pub fn sqlite3_rembed_init(db: *mut sqlite3) -> Result<()> { - let flags = FunctionFlags::UTF8 - | FunctionFlags::DETERMINISTIC - | unsafe { FunctionFlags::from_bits_unchecked(0x001000000) }; - - let c = Rc::new(RefCell::new(HashMap::new())); - - define_scalar_function( - db, - "rembed_version", - 0, - rembed_version, - FunctionFlags::UTF8 | FunctionFlags::DETERMINISTIC, - )?; - define_scalar_function( - db, - "rembed_debug", - 0, - rembed_debug, - FunctionFlags::UTF8 | FunctionFlags::DETERMINISTIC, - )?; - define_scalar_function_with_aux(db, "rembed", 2, rembed, flags, Rc::clone(&c))?; - define_scalar_function_with_aux(db, "rembed", 3, rembed, flags, Rc::clone(&c))?; - define_scalar_function( - db, - "rembed_client_options", - -1, - rembed_client_options, - flags, - )?; - define_virtual_table_writeablex::(db, "rembed_clients", Some(Rc::clone(&c)))?; - Ok(()) -} diff --git a/deps/sqlite3/sqlite-rembed-source/test.sql b/deps/sqlite3/sqlite-rembed-source/test.sql deleted file mode 100644 index d1e8e8515..000000000 --- a/deps/sqlite3/sqlite-rembed-source/test.sql +++ /dev/null @@ -1,37 +0,0 @@ -.load dist/debug/rembed0 -.bail on -.mode box -.header on -.timer on -.echo on - -INSERT INTO temp.rembed_clients(name, options) VALUES - ('text-embedding-3-small','openai'), - ('jina-embeddings-v2-base-en','jina'), - ('mixedbread-ai/mxbai-embed-large-v1','mixedbread'), - ('nomic-embed-text-v1.5', 'nomic'), - ('embed-english-v3.0', 'cohere'), - ('snowflake-arctic-embed:s', 'ollama'), - ('llamafile', 'llamafile'), - ( - 'mxbai-embed-large-v1-f16', - rembed_client_options( - 'format', 'llamafile', - --'url', 'http://mm1:8080/v1/embeddings' - 'url', 'http://mm1:8080/embedding' - ) - ); - -select length(rembed('mixedbread-ai/mxbai-embed-large-v1', 'obama the person')); -.exit -select length(rembed('jina-embeddings-v2-base-en', 'obama the person')); - -.exit - -select length(rembed('text-embedding-3-small', 'obama the person')); -select length(rembed('llamafile', 'obama the person')); -select length(rembed('snowflake-arctic-embed:s', 'obama the person')); -select length(rembed('embed-english-v3.0', 'obama the person', 'search_document')); -select length(rembed('mxbai-embed-large-v1-f16', 'obama the person')); - - diff --git a/doc/sqlite-rembed-integration.md b/doc/sqlite-rembed-integration.md index d05a51e53..2dba500bd 100644 --- a/doc/sqlite-rembed-integration.md +++ b/doc/sqlite-rembed-integration.md @@ -169,9 +169,6 @@ cd deps && make sqlite3 # Verify symbol exists nm deps/sqlite3/libsqlite_rembed.a | grep sqlite3_rembed_init - -# Test compilation (without ClickHouse) -make PROXYSQLCLICKHOUSE=0 ``` ### Functional Testing @@ -208,7 +205,6 @@ VALUES ('test', 'ollama', 'nomic-embed-text'); 1. **Missing clang**: Install `clang` and `libclang-dev` 2. **Rust not found**: Install Rust toolchain via `rustup` 3. **SQLite headers**: Ensure `sqlite-amalgamation` is extracted -4. **ClickHouse errors**: Build with `PROXYSQLCLICKHOUSE=0` ### Runtime Issues 1. **Client not found**: Verify `temp.rembed_clients` entry exists @@ -232,4 +228,4 @@ VALUES ('test', 'ollama', 'nomic-embed-text'); - sqlite-rembed: Apache 2.0 / MIT (see `deps/sqlite3/sqlite-rembed-source/LICENSE-*`) - ProxySQL: GPL v3 -- Integration code: Same as ProxySQL \ No newline at end of file +- Integration code: Same as ProxySQL