mirror of https://github.com/sysown/proxysql
- Download official sqlite-rembed-0.0.1-alpha.9.tar.gz from GitHub - Update deps/Makefile to extract from tar.gz instead of using local source - Add tar.gz to git repository, remove sqlite-rembed-source from git cache - Update documentation to remove ClickHouse troubleshooting reference - Clean targets now remove sqlite-rembed-*/ and sqlite-rembed-source/ Following the pattern of other ProxySQL dependencies, the Rust extension is now packaged as a compressed tarball that gets extracted during build.pull/5310/head
parent
01d654692d
commit
9f30d85e10
Binary file not shown.
@ -1,122 +0,0 @@
|
||||
name: "Release"
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
build-linux-x86_64-extension:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: make loadable-release
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: sqlite-rembed-linux-x86_64-extension
|
||||
path: dist/release/*
|
||||
build-macos-x86_64-extension:
|
||||
runs-on: macos-12
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: make loadable-release
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: sqlite-rembed-macos-x86_64-extension
|
||||
path: dist/release/*
|
||||
build-macos-aarch64-extension:
|
||||
runs-on: macos-14
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: make loadable-release
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: sqlite-rembed-macos-aarch64-extension
|
||||
path: dist/release/*
|
||||
build-windows-x86_64-extension:
|
||||
runs-on: windows-2019
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
- run: make loadable-release
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: sqlite-rembed-windows-x86_64-extension
|
||||
path: dist/release/*
|
||||
dist:
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
[
|
||||
build-linux-x86_64-extension,
|
||||
build-macos-x86_64-extension,
|
||||
build-macos-aarch64-extension,
|
||||
build-windows-x86_64-extension,
|
||||
]
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: sqlite-rembed-linux-x86_64-extension
|
||||
path: dist/linux-x86_64
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: sqlite-rembed-macos-x86_64-extension
|
||||
path: dist/macos-x86_64
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: sqlite-rembed-macos-aarch64-extension
|
||||
path: dist/macos-aarch64
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: sqlite-rembed-windows-x86_64-extension
|
||||
path: dist/windows-x86_64
|
||||
- run: |
|
||||
curl -L https://github.com/asg017/sqlite-dist/releases/download/v0.0.1-alpha.7/sqlite-dist-x86_64-unknown-linux-gnu.tar.xz \
|
||||
| tar xfJ - --strip-components 1
|
||||
- run: make sqlite-rembed.h
|
||||
- run: ./sqlite-dist ./sqlite-dist.toml --input dist/ --output distx/ --version $(cat VERSION)
|
||||
- run: |
|
||||
gh release upload ${{ github.ref_name }} \
|
||||
distx/github_releases/* \
|
||||
distx/spm/* \
|
||||
distx/sqlpkg/* \
|
||||
distx/checksums.txt \
|
||||
distx/sqlite-dist-manifest.json \
|
||||
distx/install.sh
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
- name: Install node
|
||||
uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: "16"
|
||||
registry-url: "https://registry.npmjs.org"
|
||||
- run: |
|
||||
npm publish --access public distx/npm/sqlite-rembed-darwin-arm64.tar.gz
|
||||
npm publish --access public distx/npm/sqlite-rembed-darwin-x64.tar.gz
|
||||
npm publish --access public distx/npm/sqlite-rembed-linux-x64.tar.gz
|
||||
npm publish --access public distx/npm/sqlite-rembed.tar.gz
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
- uses: ruby/setup-ruby@v1
|
||||
with:
|
||||
ruby-version: 3.2
|
||||
- run: |
|
||||
for file in distx/gem/*; do
|
||||
gem push "$file"
|
||||
done
|
||||
env:
|
||||
GEM_HOST_API_KEY: ${{ secrets.GEM_HOST_API_KEY }}
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: pip install twine
|
||||
- run: |
|
||||
twine upload distx/pip/*
|
||||
twine upload distx/datasette/*
|
||||
twine upload distx/sqlite_utils/*
|
||||
env:
|
||||
TWINE_USERNAME: __token__
|
||||
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
|
||||
@ -1,60 +0,0 @@
|
||||
name: "Test"
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
build-linux-x86_64-extension:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
- run: make loadable static
|
||||
#- run: pip install pytest numpy; make test-loadable
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: sqlite-rembed-linux-x86_64-extension
|
||||
path: dist/*
|
||||
build-macos-x86_64-extension:
|
||||
runs-on: macos-12
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
- run: make loadable static
|
||||
#- run: /usr/local/opt/python@3/libexec/bin/python -m pip install pytest numpy; make test-loadable python=/usr/local/opt/python@3/libexec/bin/python
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: sqlite-rembed-macos-x86_64-extension
|
||||
path: dist/*
|
||||
build-macos-aarch64-extension:
|
||||
runs-on: macos-14
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
- run: make loadable static
|
||||
#- run: /opt/homebrew/opt/python3/libexec/bin/python -m pip install pytest numpy --break-system-packages; make test-loadable python=/opt/homebrew/opt/python3/libexec/bin/python
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: sqlite-rembed-macos-aarch64-extension
|
||||
path: dist/*
|
||||
build-windows-x86_64-extension:
|
||||
runs-on: windows-2019
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
- run: make loadable static
|
||||
#- run: pip install pytest numpy; make test-loadable
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: sqlite-rembed-windows-x86_64-extension
|
||||
path: dist/*
|
||||
@ -1,3 +0,0 @@
|
||||
/target
|
||||
.env
|
||||
dist/
|
||||
@ -1,847 +0,0 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
|
||||
[[package]]
|
||||
name = "adler"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "atty"
|
||||
version = "0.2.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "bindgen"
|
||||
version = "0.60.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
"cexpr",
|
||||
"clang-sys",
|
||||
"clap",
|
||||
"env_logger",
|
||||
"lazy_static",
|
||||
"lazycell",
|
||||
"log",
|
||||
"peeking_take_while",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
"shlex",
|
||||
"which",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.98"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f"
|
||||
|
||||
[[package]]
|
||||
name = "cexpr"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
|
||||
dependencies = [
|
||||
"nom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "clang-sys"
|
||||
version = "1.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f803f94ecf597339c7a34eed2036ef83f86aaba937f001f7c5b5e251f043f1f9"
|
||||
dependencies = [
|
||||
"glob",
|
||||
"libc",
|
||||
"libloading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "3.2.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"bitflags 1.3.2",
|
||||
"clap_lex",
|
||||
"indexmap",
|
||||
"strsim",
|
||||
"termcolor",
|
||||
"textwrap",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
|
||||
dependencies = [
|
||||
"os_str_bytes",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b"
|
||||
|
||||
[[package]]
|
||||
name = "env_logger"
|
||||
version = "0.9.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"humantime",
|
||||
"log",
|
||||
"regex",
|
||||
"termcolor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.0.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "form_urlencoded"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
|
||||
dependencies = [
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.1.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "home"
|
||||
version = "0.5.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "humantime"
|
||||
version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
|
||||
dependencies = [
|
||||
"unicode-bidi",
|
||||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "1.9.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||
|
||||
[[package]]
|
||||
name = "lazycell"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.155"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
|
||||
|
||||
[[package]]
|
||||
name = "libloading"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.4.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d"
|
||||
|
||||
[[package]]
|
||||
name = "minimal-lexical"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae"
|
||||
dependencies = [
|
||||
"adler",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "7.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"minimal-lexical",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
|
||||
|
||||
[[package]]
|
||||
name = "os_str_bytes"
|
||||
version = "6.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1"
|
||||
|
||||
[[package]]
|
||||
name = "peeking_take_while"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.84"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.36"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56"
|
||||
|
||||
[[package]]
|
||||
name = "ring"
|
||||
version = "0.17.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"getrandom",
|
||||
"libc",
|
||||
"spin",
|
||||
"untrusted",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.38.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
|
||||
dependencies = [
|
||||
"bitflags 2.5.0",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.22.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432"
|
||||
dependencies = [
|
||||
"log",
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"rustls-webpki",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-pki-types"
|
||||
version = "1.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d"
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.102.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e"
|
||||
dependencies = [
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"untrusted",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.203"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.203"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.66",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.117"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||
|
||||
[[package]]
|
||||
name = "spin"
|
||||
version = "0.9.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
|
||||
|
||||
[[package]]
|
||||
name = "sqlite-loadable"
|
||||
version = "0.0.6-alpha.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "daaaad0ad506b154a72bf01fde23235377c01256abd4bd25e17419dbfd4e28a0"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sqlite-loadable-macros",
|
||||
"sqlite3ext-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sqlite-loadable-macros"
|
||||
version = "0.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96037a396115a2675db783f700faad878b44c8ff56c8a29c3404649a517a5e8f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sqlite-rembed"
|
||||
version = "0.0.1-alpha.9"
|
||||
dependencies = [
|
||||
"serde_json",
|
||||
"sqlite-loadable",
|
||||
"ureq",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sqlite3ext-sys"
|
||||
version = "0.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3afdc2b3dc08f16d6eecf8aa07d19975a268603ab1cca67d3f9b4172c507cf16"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
|
||||
|
||||
[[package]]
|
||||
name = "subtle"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.109"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.66"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "termcolor"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
|
||||
dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "textwrap"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9"
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
|
||||
dependencies = [
|
||||
"tinyvec_macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec_macros"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-bidi"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization"
|
||||
version = "0.1.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"
|
||||
dependencies = [
|
||||
"tinyvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
|
||||
|
||||
[[package]]
|
||||
name = "ureq"
|
||||
version = "2.9.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d11a831e3c0b56e438a28308e7c810799e3c118417f342d30ecec080105395cd"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"flate2",
|
||||
"log",
|
||||
"once_cell",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"rustls-webpki",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"url",
|
||||
"webpki-roots",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633"
|
||||
dependencies = [
|
||||
"form_urlencoded",
|
||||
"idna",
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "0.26.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "which"
|
||||
version = "4.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
|
||||
dependencies = [
|
||||
"either",
|
||||
"home",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-util"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.52.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
|
||||
dependencies = [
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.52.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm",
|
||||
"windows_aarch64_msvc",
|
||||
"windows_i686_gnu",
|
||||
"windows_i686_gnullvm",
|
||||
"windows_i686_msvc",
|
||||
"windows_x86_64_gnu",
|
||||
"windows_x86_64_gnullvm",
|
||||
"windows_x86_64_msvc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.52.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.52.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.52.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnullvm"
|
||||
version = "0.52.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.52.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.52.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.52.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.52.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.7.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.7.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.66",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zeroize"
|
||||
version = "1.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
|
||||
@ -1,14 +0,0 @@
|
||||
[package]
|
||||
name = "sqlite-rembed"
|
||||
version = "0.0.1-alpha.9"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
serde_json = "1.0.117"
|
||||
sqlite-loadable = "0.0.6-alpha.6"
|
||||
ureq = {version="2.9.7", features=["json"]}
|
||||
zerocopy = "0.7.34"
|
||||
|
||||
[lib]
|
||||
crate-type=["cdylib", "staticlib", "lib"]
|
||||
|
||||
@ -1,201 +0,0 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@ -1,21 +0,0 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2022 Alex Garcia
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@ -1,141 +0,0 @@
|
||||
SHELL := /bin/bash
|
||||
|
||||
VERSION=$(shell cat VERSION)
|
||||
|
||||
ifeq ($(shell uname -s),Darwin)
|
||||
CONFIG_DARWIN=y
|
||||
else ifeq ($(OS),Windows_NT)
|
||||
CONFIG_WINDOWS=y
|
||||
else
|
||||
CONFIG_LINUX=y
|
||||
endif
|
||||
|
||||
LIBRARY_PREFIX=lib
|
||||
ifdef CONFIG_DARWIN
|
||||
LOADABLE_EXTENSION=dylib
|
||||
STATIC_EXTENSION=a
|
||||
endif
|
||||
|
||||
ifdef CONFIG_LINUX
|
||||
LOADABLE_EXTENSION=so
|
||||
STATIC_EXTENSION=a
|
||||
endif
|
||||
|
||||
|
||||
ifdef CONFIG_WINDOWS
|
||||
LOADABLE_EXTENSION=dll
|
||||
LIBRARY_PREFIX=
|
||||
STATIC_EXTENSION=lib
|
||||
endif
|
||||
|
||||
prefix=dist
|
||||
TARGET_LOADABLE=$(prefix)/debug/rembed0.$(LOADABLE_EXTENSION)
|
||||
TARGET_LOADABLE_RELEASE=$(prefix)/release/rembed0.$(LOADABLE_EXTENSION)
|
||||
|
||||
TARGET_STATIC=$(prefix)/debug/$(LIBRARY_PREFIX)sqlite_rembed0.$(STATIC_EXTENSION)
|
||||
TARGET_STATIC_RELEASE=$(prefix)/release/$(LIBRARY_PREFIX)sqlite_rembed0.$(STATIC_EXTENSION)
|
||||
|
||||
TARGET_H=$(prefix)/debug/sqlite-rembed.h
|
||||
TARGET_H_RELEASE=$(prefix)/release/sqlite-rembed.h
|
||||
|
||||
TARGET_WHEELS=$(prefix)/debug/wheels
|
||||
TARGET_WHEELS_RELEASE=$(prefix)/release/wheels
|
||||
|
||||
INTERMEDIATE_PYPACKAGE_EXTENSION=python/sqlite_rembed/sqlite_rembed/rembed0.$(LOADABLE_EXTENSION)
|
||||
|
||||
ifdef target
|
||||
CARGO_TARGET=--target=$(target)
|
||||
BUILT_LOCATION=target/$(target)/debug/$(LIBRARY_PREFIX)sqlite_rembed.$(LOADABLE_EXTENSION)
|
||||
BUILT_LOCATION_RELEASE=target/$(target)/release/$(LIBRARY_PREFIX)sqlite_rembed.$(LOADABLE_EXTENSION)
|
||||
BUILT_LOCATION_STATIC=target/$(target)/debug/$(LIBRARY_PREFIX)sqlite_rembed.$(STATIC_EXTENSION)
|
||||
BUILT_LOCATION_STATIC_RELEASE=target/$(target)/release/$(LIBRARY_PREFIX)sqlite_rembed.$(STATIC_EXTENSION)
|
||||
else
|
||||
CARGO_TARGET=
|
||||
BUILT_LOCATION=target/debug/$(LIBRARY_PREFIX)sqlite_rembed.$(LOADABLE_EXTENSION)
|
||||
BUILT_LOCATION_RELEASE=target/release/$(LIBRARY_PREFIX)sqlite_rembed.$(LOADABLE_EXTENSION)
|
||||
BUILT_LOCATION_STATIC=target/debug/$(LIBRARY_PREFIX)sqlite_rembed.$(STATIC_EXTENSION)
|
||||
BUILT_LOCATION_STATIC_RELEASE=target/release/$(LIBRARY_PREFIX)sqlite_rembed.$(STATIC_EXTENSION)
|
||||
endif
|
||||
|
||||
ifdef python
|
||||
PYTHON=$(python)
|
||||
else
|
||||
PYTHON=python3
|
||||
endif
|
||||
|
||||
ifdef IS_MACOS_ARM
|
||||
RENAME_WHEELS_ARGS=--is-macos-arm
|
||||
else
|
||||
RENAME_WHEELS_ARGS=
|
||||
endif
|
||||
|
||||
$(prefix):
|
||||
mkdir -p $(prefix)/debug
|
||||
mkdir -p $(prefix)/release
|
||||
|
||||
$(TARGET_WHEELS): $(prefix)
|
||||
mkdir -p $(TARGET_WHEELS)
|
||||
|
||||
$(TARGET_WHEELS_RELEASE): $(prefix)
|
||||
mkdir -p $(TARGET_WHEELS_RELEASE)
|
||||
|
||||
$(TARGET_LOADABLE): $(prefix) $(shell find . -type f -name '*.rs')
|
||||
cargo build --verbose $(CARGO_TARGET)
|
||||
cp $(BUILT_LOCATION) $@
|
||||
|
||||
$(TARGET_LOADABLE_RELEASE): $(prefix) $(shell find . -type f -name '*.rs')
|
||||
cargo build --verbose --release $(CARGO_TARGET)
|
||||
cp $(BUILT_LOCATION_RELEASE) $@
|
||||
|
||||
$(TARGET_STATIC): $(prefix) $(shell find . -type f -name '*.rs')
|
||||
cargo build --verbose $(CARGO_TARGET) --features=sqlite-loadable/static
|
||||
ls target
|
||||
ls target/$(target)/debug
|
||||
cp $(BUILT_LOCATION_STATIC) $@
|
||||
|
||||
$(TARGET_STATIC_RELEASE): $(prefix) $(shell find . -type f -name '*.rs')
|
||||
cargo build --verbose --release $(CARGO_TARGET) --features=sqlite-loadable/static
|
||||
cp $(BUILT_LOCATION_STATIC_RELEASE) $@
|
||||
|
||||
$(TARGET_H): sqlite-rembed.h
|
||||
cp $< $@
|
||||
|
||||
$(TARGET_H_RELEASE): sqlite-rembed.h
|
||||
cp $< $@
|
||||
|
||||
Cargo.toml: VERSION
|
||||
cargo set-version `cat VERSION`
|
||||
|
||||
version:
|
||||
make Cargo.toml
|
||||
|
||||
format:
|
||||
cargo fmt
|
||||
|
||||
release: $(TARGET_LOADABLE_RELEASE) $(TARGET_STATIC_RELEASE)
|
||||
|
||||
loadable: $(TARGET_LOADABLE)
|
||||
loadable-release: $(TARGET_LOADABLE_RELEASE)
|
||||
|
||||
static: $(TARGET_STATIC) $(TARGET_H)
|
||||
static-release: $(TARGET_STATIC_RELEASE) $(TARGET_H_RELEASE)
|
||||
|
||||
debug: loadable static python datasette
|
||||
release: loadable-release static-release python-release datasette-release
|
||||
|
||||
clean:
|
||||
rm dist/*
|
||||
cargo clean
|
||||
|
||||
test-loadable:
|
||||
$(PYTHON) tests/test-loadable.py
|
||||
|
||||
publish-release:
|
||||
./scripts/publish_release.sh
|
||||
|
||||
.PHONY: clean \
|
||||
test test-loadable test-python test-npm test-deno \
|
||||
loadable loadable-release \
|
||||
static static-release \
|
||||
debug release \
|
||||
format version publish-release
|
||||
@ -1,134 +0,0 @@
|
||||
# `sqlite-rembed`
|
||||
|
||||
A SQLite extension for generating text embeddings from remote APIs (OpenAI, Nomic, Cohere, llamafile, Ollama, etc.). A sister project to [`sqlite-vec`](https://github.com/asg017/sqlite-vec) and [`sqlite-lembed`](https://github.com/asg017/sqlite-lembed). A work-in-progress!
|
||||
|
||||
## Usage
|
||||
|
||||
```sql
|
||||
.load ./rembed0
|
||||
|
||||
INSERT INTO temp.rembed_clients(name, options)
|
||||
VALUES ('text-embedding-3-small', 'openai');
|
||||
|
||||
select rembed(
|
||||
'text-embedding-3-small',
|
||||
'The United States Postal Service is an independent agency...'
|
||||
);
|
||||
```
|
||||
|
||||
The `temp.rembed_clients` virtual table lets you "register" clients with pure `INSERT INTO` statements. The `name` field is a unique identifier for a given client, and `options` allows you to specify which 3rd party embedding service you want to use.
|
||||
|
||||
In this case, `openai` is a pre-defined client that will default to OpenAI's `https://api.openai.com/v1/embeddings` endpoint and will source your API key from the `OPENAI_API_KEY` environment variable. The name of the client, `text-embedding-3-small`, will be used as the embeddings model.
|
||||
|
||||
Other pre-defined clients include:
|
||||
|
||||
| Client name | Provider | Endpoint | API Key |
|
||||
| ------------ | ------------------------------------------------------------------------------------ | ---------------------------------------------- | -------------------- |
|
||||
| `openai` | [OpenAI](https://platform.openai.com/docs/guides/embeddings) | `https://api.openai.com/v1/embeddings` | `OPENAI_API_KEY` |
|
||||
| `nomic` | [Nomic](https://docs.nomic.ai/reference/endpoints/nomic-embed-text) | `https://api-atlas.nomic.ai/v1/embedding/text` | `NOMIC_API_KEY` |
|
||||
| `cohere` | [Cohere](https://docs.cohere.com/reference/embed) | `https://api.cohere.com/v1/embed` | `CO_API_KEY` |
|
||||
| `jina` | [Jina](https://api.jina.ai/redoc#tag/embeddings) | `https://api.jina.ai/v1/embeddings` | `JINA_API_KEY` |
|
||||
| `mixedbread` | [MixedBread](https://www.mixedbread.ai/api-reference#quick-start-guide) | `https://api.mixedbread.ai/v1/embeddings/` | `MIXEDBREAD_API_KEY` |
|
||||
| `llamafile` | [llamafile](https://github.com/Mozilla-Ocho/llamafile) | `http://localhost:8080/embedding` | None |
|
||||
| `ollama` | [Ollama](https://github.com/ollama/ollama/blob/main/docs/api.md#generate-embeddings) | `http://localhost:11434/api/embeddings` | None |
|
||||
|
||||
Different client options can be specified with `remebed_client_options()`. For example, if you have a different OpenAI-compatible service you want to use, then you can use:
|
||||
|
||||
```sql
|
||||
INSERT INTO temp.rembed_clients(name, options) VALUES
|
||||
(
|
||||
'xyz-small-1',
|
||||
rembed_client_options(
|
||||
'format', 'openai',
|
||||
'url', 'https://api.xyz.com/v1/embeddings',
|
||||
'key', 'xyz-ca865ece65-hunter2'
|
||||
)
|
||||
);
|
||||
```
|
||||
|
||||
Or to use a llamafile server that's on a different port:
|
||||
|
||||
```sql
|
||||
INSERT INTO temp.rembed_clients(name, options) VALUES
|
||||
(
|
||||
'xyz-small-1',
|
||||
rembed_client_options(
|
||||
'format', 'lamafile',
|
||||
'url', 'http://localhost:9999/embedding'
|
||||
)
|
||||
);
|
||||
```
|
||||
|
||||
### Using with `sqlite-vec`
|
||||
|
||||
`sqlite-rembed` works well with [`sqlite-vec`](https://github.com/asg017/sqlite-vec), a SQLite extension for vector search. Embeddings generated with `rembed()` use the same BLOB format for vectors that `sqlite-vec` uses.
|
||||
|
||||
Here's a sample "semantic search" application, made from a sample dataset of news article headlines.
|
||||
|
||||
```sql
|
||||
create table articles(
|
||||
headline text
|
||||
);
|
||||
|
||||
-- Random NPR headlines from 2024-06-04
|
||||
insert into articles VALUES
|
||||
('Shohei Ohtani''s ex-interpreter pleads guilty to charges related to gambling and theft'),
|
||||
('The jury has been selected in Hunter Biden''s gun trial'),
|
||||
('Larry Allen, a Super Bowl champion and famed Dallas Cowboy, has died at age 52'),
|
||||
('After saying Charlotte, a lone stingray, was pregnant, aquarium now says she''s sick'),
|
||||
('An Epoch Times executive is facing money laundering charge');
|
||||
|
||||
|
||||
-- Build a vector table with embeddings of article headlines, using OpenAI's API
|
||||
create virtual table vec_articles using vec0(
|
||||
headline_embeddings float[1536]
|
||||
);
|
||||
|
||||
insert into vec_articles(rowid, headline_embeddings)
|
||||
select rowid, rembed('text-embedding-3-small', headline)
|
||||
from articles;
|
||||
|
||||
```
|
||||
|
||||
Now we have a regular `articles` table that stores text headlines, and a `vec_articles` virtual table that stores embeddings of the article headlines, using OpenAI's `text-embedding-3-small` model.
|
||||
|
||||
To perform a "semantic search" on the embeddings, we can query the `vec_articles` table with an embedding of our query, and join the results back to our `articles` table to retrieve the original headlines.
|
||||
|
||||
```sql
|
||||
param set :query 'firearm courtroom'
|
||||
|
||||
with matches as (
|
||||
select
|
||||
rowid,
|
||||
distance
|
||||
from vec_articles
|
||||
where headline_embeddings match rembed('text-embedding-3-small', :query)
|
||||
order by distance
|
||||
limit 3
|
||||
)
|
||||
select
|
||||
headline,
|
||||
distance
|
||||
from matches
|
||||
left join articles on articles.rowid = matches.rowid;
|
||||
|
||||
/*
|
||||
+--------------------------------------------------------------+------------------+
|
||||
| headline | distance |
|
||||
+--------------------------------------------------------------+------------------+
|
||||
| The jury has been selected in Hunter Biden's gun trial | 1.05906391143799 |
|
||||
+--------------------------------------------------------------+------------------+
|
||||
| Shohei Ohtani's ex-interpreter pleads guilty to charges rela | 1.2574303150177 |
|
||||
| ted to gambling and theft | |
|
||||
+--------------------------------------------------------------+------------------+
|
||||
| An Epoch Times executive is facing money laundering charge | 1.27144026756287 |
|
||||
+--------------------------------------------------------------+------------------+
|
||||
*/
|
||||
```
|
||||
|
||||
Notice how "firearm courtroom" doesn't appear in any of these headlines, but it can still figure out that "Hunter Biden's gun trial" is related, and the other two justice-related articles appear on top.
|
||||
|
||||
## Drawbacks
|
||||
|
||||
1. **No batch support yet.** If you use `rembed()` in a batch UPDATE or INSERT in 1,000 rows, then 1,000 HTTP requests will be made. Add a :+1: to [Issue #1](https://github.com/asg017/sqlite-rembed/issues/1) if you want to see this fixed.
|
||||
2. **No builtin rate limiting.** Requests are sent sequentially so this may not come up in small demos, but `sqlite-rembed` could add features that handles rate limiting/retries implicitly. Add a :+1: to [Issue #2](https://github.com/asg017/sqlite-rembed/issues/2) if you want to see this implemented.
|
||||
@ -1 +0,0 @@
|
||||
0.0.1-alpha.9
|
||||
@ -1,9 +0,0 @@
|
||||
use std::process::Command;
|
||||
fn main() {
|
||||
let output = Command::new("git")
|
||||
.args(["rev-parse", "HEAD"])
|
||||
.output()
|
||||
.unwrap();
|
||||
let git_hash = String::from_utf8(output.stdout).unwrap();
|
||||
println!("cargo:rustc-env=GIT_HASH={}", git_hash);
|
||||
}
|
||||
@ -1,48 +0,0 @@
|
||||
.bail on
|
||||
.mode table
|
||||
.header on
|
||||
|
||||
.timer on
|
||||
|
||||
.load ../../dist/debug/rembed0
|
||||
.load ../../../sqlite-vec/dist/vec0
|
||||
|
||||
INSERT INTO temp.rembed_clients(name, options)
|
||||
VALUES ('text-embedding-3-small', 'openai');
|
||||
|
||||
create table articles(headline text);
|
||||
|
||||
|
||||
-- Random NPR headlines from 2024-06-04
|
||||
insert into articles VALUES
|
||||
('Shohei Ohtani''s ex-interpreter pleads guilty to charges related to gambling and theft'),
|
||||
('The jury has been selected in Hunter Biden''s gun trial'),
|
||||
('Larry Allen, a Super Bowl champion and famed Dallas Cowboy, has died at age 52'),
|
||||
('After saying Charlotte, a lone stingray, was pregnant, aquarium now says she''s sick'),
|
||||
('An Epoch Times executive is facing money laundering charge');
|
||||
|
||||
|
||||
-- Seed a vector table with embeddings of article headlines, using OpenAI's API
|
||||
create virtual table vec_articles using vec0(headline_embeddings float[1536]);
|
||||
|
||||
insert into vec_articles(rowid, headline_embeddings)
|
||||
select rowid, rembed('text-embedding-3-small', headline)
|
||||
from articles;
|
||||
|
||||
|
||||
.param set :query 'firearm courtroom'
|
||||
|
||||
with matches as (
|
||||
select
|
||||
rowid,
|
||||
distance
|
||||
from vec_articles
|
||||
where headline_embeddings match rembed('text-embedding-3-small', :query)
|
||||
order by distance
|
||||
limit 3
|
||||
)
|
||||
select
|
||||
headline,
|
||||
distance
|
||||
from matches
|
||||
left join articles on articles.rowid = matches.rowid;
|
||||
@ -1,27 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail xtrace
|
||||
|
||||
if [[ -n $(git status --porcelain | grep -v VERSION | grep -v sqlite-dist.toml) ]]; then
|
||||
echo "❌ There are other un-staged changes to the repository besides VERSION and sqlite-dist.toml"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
VERSION="$(cat VERSION)"
|
||||
|
||||
echo "Publishing version v$VERSION..."
|
||||
|
||||
make version
|
||||
git add --all
|
||||
git commit -m "v$VERSION"
|
||||
git tag v$VERSION
|
||||
git push origin main v$VERSION
|
||||
|
||||
if grep -qE "alpha|beta" VERSION; then
|
||||
gh release create v$VERSION --title=v$VERSION --prerelease --notes=""
|
||||
else
|
||||
gh release create v$VERSION --title=v$VERSION
|
||||
fi
|
||||
|
||||
|
||||
echo "✅ Published! version v$VERSION"
|
||||
@ -1,21 +0,0 @@
|
||||
[package]
|
||||
name = "sqlite-rembed"
|
||||
license = "MIT OR Apache"
|
||||
homepage = "https://alexgarcia.xyz/sqlite-rembed"
|
||||
repo = "https://github.com/asg017/sqlite-rembed"
|
||||
description = "A SQLite extension for generating text embeddings from remote sources (OpenAI, Cohere, localhost, etc.)"
|
||||
authors = ["Alex Garcia"]
|
||||
git_tag_format = "v$VERSION"
|
||||
|
||||
[targets]
|
||||
github_releases = {}
|
||||
sqlpkg = {}
|
||||
spm = {}
|
||||
|
||||
pip = {}
|
||||
datasette = {}
|
||||
sqlite_utils = {}
|
||||
|
||||
npm = {}
|
||||
|
||||
gem = { module_name = "SqliteRembed" }
|
||||
@ -1,14 +0,0 @@
|
||||
#ifndef _SQLITE_REMBED_H
|
||||
#define _SQLITE_REMBED_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
int sqlite3_rembed_init(sqlite3*, char**, const sqlite3_api_routines*);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} /* end of the 'extern "C"' block */
|
||||
#endif
|
||||
|
||||
#endif /* ifndef _SQLITE_REMBED_H */
|
||||
@ -1,516 +0,0 @@
|
||||
use sqlite_loadable::{Error, Result};
|
||||
|
||||
pub(crate) fn try_env_var(key: &str) -> Result<String> {
|
||||
std::env::var(key)
|
||||
.map_err(|_| Error::new_message(format!("{} environment variable not define. Alternatively, pass in an API key with rembed_client_options", DEFAULT_OPENAI_API_KEY_ENV)))
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct OpenAiClient {
|
||||
model: String,
|
||||
url: String,
|
||||
key: String,
|
||||
}
|
||||
const DEFAULT_OPENAI_URL: &str = "https://api.openai.com/v1/embeddings";
|
||||
const DEFAULT_OPENAI_API_KEY_ENV: &str = "OPENAI_API_KEY";
|
||||
|
||||
impl OpenAiClient {
|
||||
pub fn new<S: Into<String>>(
|
||||
model: S,
|
||||
url: Option<String>,
|
||||
key: Option<String>,
|
||||
) -> Result<Self> {
|
||||
Ok(Self {
|
||||
model: model.into(),
|
||||
url: url.unwrap_or(DEFAULT_OPENAI_URL.to_owned()),
|
||||
key: match key {
|
||||
Some(key) => key,
|
||||
None => try_env_var(DEFAULT_OPENAI_API_KEY_ENV)?,
|
||||
},
|
||||
})
|
||||
}
|
||||
pub fn infer_single(&self, input: &str) -> Result<Vec<f32>> {
|
||||
let body = serde_json::json!({
|
||||
"input": input,
|
||||
"model": self.model
|
||||
});
|
||||
|
||||
let data: serde_json::Value = ureq::post(&self.url)
|
||||
.set("Content-Type", "application/json")
|
||||
.set("Authorization", format!("Bearer {}", self.key).as_str())
|
||||
.send_bytes(
|
||||
serde_json::to_vec(&body)
|
||||
.map_err(|error| {
|
||||
Error::new_message(format!("Error serializing body to JSON: {error}"))
|
||||
})?
|
||||
.as_ref(),
|
||||
)
|
||||
.map_err(|error| Error::new_message(format!("Error sending HTTP request: {error}")))?
|
||||
.into_json()
|
||||
.map_err(|error| {
|
||||
Error::new_message(format!("Error parsing HTTP response as JSON: {error}"))
|
||||
})?;
|
||||
OpenAiClient::parse_single_response(data)
|
||||
}
|
||||
|
||||
pub fn parse_single_response(value: serde_json::Value) -> Result<Vec<f32>> {
|
||||
value
|
||||
.get("data")
|
||||
.ok_or_else(|| Error::new_message("expected 'data' key in response body"))
|
||||
.and_then(|v| {
|
||||
v.get(0)
|
||||
.ok_or_else(|| Error::new_message("expected 'data.0' path in response body"))
|
||||
})
|
||||
.and_then(|v| {
|
||||
v.get("embedding").ok_or_else(|| {
|
||||
Error::new_message("expected 'data.0.embedding' path in response body")
|
||||
})
|
||||
})
|
||||
.and_then(|v| {
|
||||
v.as_array().ok_or_else(|| {
|
||||
Error::new_message("expected 'data.0.embedding' path to be an array")
|
||||
})
|
||||
})
|
||||
.and_then(|arr| {
|
||||
arr.iter()
|
||||
.map(|v| {
|
||||
v.as_f64()
|
||||
.ok_or_else(|| {
|
||||
Error::new_message(
|
||||
"expected 'data.0.embedding' array to contain floats",
|
||||
)
|
||||
})
|
||||
.map(|f| f as f32)
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NomicClient {
|
||||
model: String,
|
||||
url: String,
|
||||
key: String,
|
||||
}
|
||||
const DEFAULT_NOMIC_URL: &str = "https://api-atlas.nomic.ai/v1/embedding/text";
|
||||
const DEFAULT_NOMIC_API_KEY_ENV: &str = "NOMIC_API_KEY";
|
||||
|
||||
impl NomicClient {
|
||||
pub fn new<S: Into<String>>(
|
||||
model: S,
|
||||
url: Option<String>,
|
||||
key: Option<String>,
|
||||
) -> Result<Self> {
|
||||
Ok(Self {
|
||||
model: model.into(),
|
||||
url: url.unwrap_or(DEFAULT_NOMIC_URL.to_owned()),
|
||||
key: match key {
|
||||
Some(key) => key,
|
||||
None => try_env_var(DEFAULT_NOMIC_API_KEY_ENV)?,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
pub fn infer_single(&self, input: &str, input_type: Option<&str>) -> Result<Vec<f32>> {
|
||||
let mut body = serde_json::Map::new();
|
||||
body.insert("texts".to_owned(), vec![input.to_owned()].into());
|
||||
body.insert("model".to_owned(), self.model.to_owned().into());
|
||||
|
||||
if let Some(input_type) = input_type {
|
||||
body.insert("input_type".to_owned(), input_type.to_owned().into());
|
||||
}
|
||||
|
||||
let data: serde_json::Value = ureq::post(&self.url)
|
||||
.set("Content-Type", "application/json")
|
||||
.set("Authorization", format!("Bearer {}", self.key).as_str())
|
||||
.send_bytes(
|
||||
serde_json::to_vec(&body)
|
||||
.map_err(|error| {
|
||||
Error::new_message(format!("Error serializing body to JSON: {error}"))
|
||||
})?
|
||||
.as_ref(),
|
||||
)
|
||||
.map_err(|error| Error::new_message(format!("Error sending HTTP request: {error}")))?
|
||||
.into_json()
|
||||
.map_err(|error| {
|
||||
Error::new_message(format!("Error parsing HTTP response as JSON: {error}"))
|
||||
})?;
|
||||
NomicClient::parse_single_response(data)
|
||||
}
|
||||
pub fn parse_single_response(value: serde_json::Value) -> Result<Vec<f32>> {
|
||||
value
|
||||
.get("embeddings")
|
||||
.ok_or_else(|| Error::new_message("expected 'embeddings' key in response body"))
|
||||
.and_then(|v| {
|
||||
v.get(0).ok_or_else(|| {
|
||||
Error::new_message("expected 'embeddings.0' path in response body")
|
||||
})
|
||||
})
|
||||
.and_then(|v| {
|
||||
v.as_array().ok_or_else(|| {
|
||||
Error::new_message("expected 'embeddings.0' path to be an array")
|
||||
})
|
||||
})
|
||||
.and_then(|arr| {
|
||||
arr.iter()
|
||||
.map(|v| {
|
||||
v.as_f64()
|
||||
.ok_or_else(|| {
|
||||
Error::new_message(
|
||||
"expected 'embeddings.0' array to contain floats",
|
||||
)
|
||||
})
|
||||
.map(|f| f as f32)
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct CohereClient {
|
||||
url: String,
|
||||
model: String,
|
||||
key: String,
|
||||
}
|
||||
const DEFAULT_COHERE_URL: &str = "https://api.cohere.com/v1/embed";
|
||||
const DEFAULT_COHERE_API_KEY_ENV: &str = "CO_API_KEY";
|
||||
|
||||
impl CohereClient {
|
||||
pub fn new<S: Into<String>>(
|
||||
model: S,
|
||||
url: Option<String>,
|
||||
key: Option<String>,
|
||||
) -> Result<Self> {
|
||||
Ok(Self {
|
||||
model: model.into(),
|
||||
url: url.unwrap_or(DEFAULT_COHERE_URL.to_owned()),
|
||||
key: match key {
|
||||
Some(key) => key,
|
||||
None => try_env_var(DEFAULT_COHERE_API_KEY_ENV)?,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
pub fn infer_single(&self, input: &str, input_type: Option<&str>) -> Result<Vec<f32>> {
|
||||
let mut body = serde_json::Map::new();
|
||||
body.insert("texts".to_owned(), vec![input.to_owned()].into());
|
||||
body.insert("model".to_owned(), self.model.to_owned().into());
|
||||
|
||||
if let Some(input_type) = input_type {
|
||||
body.insert("input_type".to_owned(), input_type.to_owned().into());
|
||||
}
|
||||
|
||||
let data: serde_json::Value = ureq::post(&self.url)
|
||||
.set("Content-Type", "application/json")
|
||||
.set("Accept", "application/json")
|
||||
.set("Authorization", format!("Bearer {}", self.key).as_str())
|
||||
.send_bytes(
|
||||
serde_json::to_vec(&body)
|
||||
.map_err(|error| {
|
||||
Error::new_message(format!("Error serializing body to JSON: {error}"))
|
||||
})?
|
||||
.as_ref(),
|
||||
)
|
||||
.map_err(|error| Error::new_message(format!("Error sending HTTP request: {error}")))?
|
||||
.into_json()
|
||||
.map_err(|error| {
|
||||
Error::new_message(format!("Error parsing HTTP response as JSON: {error}"))
|
||||
})?;
|
||||
CohereClient::parse_single_response(data)
|
||||
}
|
||||
pub fn parse_single_response(value: serde_json::Value) -> Result<Vec<f32>> {
|
||||
value
|
||||
.get("embeddings")
|
||||
.ok_or_else(|| Error::new_message("expected 'embeddings' key in response body"))
|
||||
.and_then(|v| {
|
||||
v.get(0).ok_or_else(|| {
|
||||
Error::new_message("expected 'embeddings.0' path in response body")
|
||||
})
|
||||
})
|
||||
.and_then(|v| {
|
||||
v.as_array().ok_or_else(|| {
|
||||
Error::new_message("expected 'embeddings.0' path to be an array")
|
||||
})
|
||||
})
|
||||
.and_then(|arr| {
|
||||
arr.iter()
|
||||
.map(|v| {
|
||||
v.as_f64()
|
||||
.ok_or_else(|| {
|
||||
Error::new_message(
|
||||
"expected 'embeddings.0' array to contain floats",
|
||||
)
|
||||
})
|
||||
.map(|f| f as f32)
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
}
|
||||
#[derive(Clone)]
|
||||
pub struct JinaClient {
|
||||
url: String,
|
||||
model: String,
|
||||
key: String,
|
||||
}
|
||||
const DEFAULT_JINA_URL: &str = "https://api.jina.ai/v1/embeddings";
|
||||
const DEFAULT_JINA_API_KEY_ENV: &str = "JINA_API_KEY";
|
||||
|
||||
impl JinaClient {
|
||||
pub fn new<S: Into<String>>(
|
||||
model: S,
|
||||
url: Option<String>,
|
||||
key: Option<String>,
|
||||
) -> Result<Self> {
|
||||
Ok(Self {
|
||||
model: model.into(),
|
||||
url: url.unwrap_or(DEFAULT_JINA_URL.to_owned()),
|
||||
key: match key {
|
||||
Some(key) => key,
|
||||
None => try_env_var(DEFAULT_JINA_API_KEY_ENV)?,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
pub fn infer_single(&self, input: &str) -> Result<Vec<f32>> {
|
||||
let mut body = serde_json::Map::new();
|
||||
body.insert("input".to_owned(), vec![input.to_owned()].into());
|
||||
body.insert("model".to_owned(), self.model.to_owned().into());
|
||||
|
||||
let data: serde_json::Value = ureq::post(&self.url)
|
||||
.set("Content-Type", "application/json")
|
||||
.set("Accept", "application/json")
|
||||
.set("Authorization", format!("Bearer {}", self.key).as_str())
|
||||
.send_bytes(
|
||||
serde_json::to_vec(&body)
|
||||
.map_err(|error| {
|
||||
Error::new_message(format!("Error serializing body to JSON: {error}"))
|
||||
})?
|
||||
.as_ref(),
|
||||
)
|
||||
.map_err(|error| Error::new_message(format!("Error sending HTTP request: {error}")))?
|
||||
.into_json()
|
||||
.map_err(|error| {
|
||||
Error::new_message(format!("Error parsing HTTP response as JSON: {error}"))
|
||||
})?;
|
||||
JinaClient::parse_single_response(data)
|
||||
}
|
||||
pub fn parse_single_response(value: serde_json::Value) -> Result<Vec<f32>> {
|
||||
value
|
||||
.get("data")
|
||||
.ok_or_else(|| Error::new_message("expected 'data' key in response body"))
|
||||
.and_then(|v| {
|
||||
v.get(0)
|
||||
.ok_or_else(|| Error::new_message("expected 'data.0' path in response body"))
|
||||
})
|
||||
.and_then(|v| {
|
||||
v.get("embedding").ok_or_else(|| {
|
||||
Error::new_message("expected 'data.0.embedding' path in response body")
|
||||
})
|
||||
})
|
||||
.and_then(|v| {
|
||||
v.as_array().ok_or_else(|| {
|
||||
Error::new_message("expected 'data.0.embedding' path to be an array")
|
||||
})
|
||||
})
|
||||
.and_then(|arr| {
|
||||
arr.iter()
|
||||
.map(|v| {
|
||||
v.as_f64()
|
||||
.ok_or_else(|| {
|
||||
Error::new_message(
|
||||
"expected 'data.0.embedding' array to contain floats",
|
||||
)
|
||||
})
|
||||
.map(|f| f as f32)
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
}
|
||||
#[derive(Clone)]
|
||||
pub struct MixedbreadClient {
|
||||
url: String,
|
||||
model: String,
|
||||
key: String,
|
||||
}
|
||||
const DEFAULT_MIXEDBREAD_URL: &str = "https://api.mixedbread.ai/v1/embeddings/";
|
||||
const DEFAULT_MIXEDBREAD_API_KEY_ENV: &str = "MIXEDBREAD_API_KEY";
|
||||
|
||||
impl MixedbreadClient {
|
||||
pub fn new<S: Into<String>>(
|
||||
model: S,
|
||||
url: Option<String>,
|
||||
key: Option<String>,
|
||||
) -> Result<Self> {
|
||||
Ok(Self {
|
||||
model: model.into(),
|
||||
url: url.unwrap_or(DEFAULT_MIXEDBREAD_URL.to_owned()),
|
||||
key: match key {
|
||||
Some(key) => key,
|
||||
None => try_env_var(DEFAULT_MIXEDBREAD_API_KEY_ENV)?,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
pub fn infer_single(&self, input: &str) -> Result<Vec<f32>> {
|
||||
let mut body = serde_json::Map::new();
|
||||
body.insert("input".to_owned(), vec![input.to_owned()].into());
|
||||
body.insert("model".to_owned(), self.model.to_owned().into());
|
||||
|
||||
let data: serde_json::Value = ureq::post(&self.url)
|
||||
.set("Content-Type", "application/json")
|
||||
.set("Accept", "application/json")
|
||||
.set("Authorization", format!("Bearer {}", self.key).as_str())
|
||||
.send_bytes(
|
||||
serde_json::to_vec(&body)
|
||||
.map_err(|error| {
|
||||
Error::new_message(format!("Error serializing body to JSON: {error}"))
|
||||
})?
|
||||
.as_ref(),
|
||||
)
|
||||
.map_err(|error| Error::new_message(format!("Error sending HTTP request: {error}")))?
|
||||
.into_json()
|
||||
.map_err(|error| {
|
||||
Error::new_message(format!("Error parsing HTTP response as JSON: {error}"))
|
||||
})?;
|
||||
JinaClient::parse_single_response(data)
|
||||
}
|
||||
pub fn parse_single_response(value: serde_json::Value) -> Result<Vec<f32>> {
|
||||
value
|
||||
.get("data")
|
||||
.ok_or_else(|| Error::new_message("expected 'data' key in response body"))
|
||||
.and_then(|v| {
|
||||
v.get(0)
|
||||
.ok_or_else(|| Error::new_message("expected 'data.0' path in response body"))
|
||||
})
|
||||
.and_then(|v| {
|
||||
v.get("embedding").ok_or_else(|| {
|
||||
Error::new_message("expected 'data.0.embedding' path in response body")
|
||||
})
|
||||
})
|
||||
.and_then(|v| {
|
||||
v.as_array().ok_or_else(|| {
|
||||
Error::new_message("expected 'data.0.embedding' path to be an array")
|
||||
})
|
||||
})
|
||||
.and_then(|arr| {
|
||||
arr.iter()
|
||||
.map(|v| {
|
||||
v.as_f64()
|
||||
.ok_or_else(|| {
|
||||
Error::new_message(
|
||||
"expected 'data.0.embedding' array to contain floats",
|
||||
)
|
||||
})
|
||||
.map(|f| f as f32)
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct OllamaClient {
|
||||
url: String,
|
||||
model: String,
|
||||
}
|
||||
const DEFAULT_OLLAMA_URL: &str = "http://localhost:11434/api/embeddings";
|
||||
impl OllamaClient {
|
||||
pub fn new<S: Into<String>>(model: S, url: Option<String>) -> Self {
|
||||
Self {
|
||||
model: model.into(),
|
||||
url: url.unwrap_or(DEFAULT_OLLAMA_URL.to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn infer_single(&self, input: &str) -> Result<Vec<f32>> {
|
||||
let mut body = serde_json::Map::new();
|
||||
body.insert("prompt".to_owned(), input.to_owned().into());
|
||||
body.insert("model".to_owned(), self.model.to_owned().into());
|
||||
|
||||
let data: serde_json::Value = ureq::post(&self.url)
|
||||
.set("Content-Type", "application/json")
|
||||
.send_bytes(
|
||||
serde_json::to_vec(&body)
|
||||
.map_err(|error| {
|
||||
Error::new_message(format!("Error serializing body to JSON: {error}"))
|
||||
})?
|
||||
.as_ref(),
|
||||
)
|
||||
.map_err(|error| Error::new_message(format!("Error sending HTTP request: {error}")))?
|
||||
.into_json()
|
||||
.map_err(|error| {
|
||||
Error::new_message(format!("Error parsing HTTP response as JSON: {error}"))
|
||||
})?;
|
||||
OllamaClient::parse_single_response(data)
|
||||
}
|
||||
pub fn parse_single_response(value: serde_json::Value) -> Result<Vec<f32>> {
|
||||
value
|
||||
.get("embedding")
|
||||
.ok_or_else(|| Error::new_message("expected 'embedding' key in response body"))
|
||||
.and_then(|v| {
|
||||
v.as_array()
|
||||
.ok_or_else(|| Error::new_message("expected 'embedding' path to be an array"))
|
||||
})
|
||||
.and_then(|arr| {
|
||||
arr.iter()
|
||||
.map(|v| {
|
||||
v.as_f64()
|
||||
.ok_or_else(|| {
|
||||
Error::new_message("expected 'embedding' array to contain floats")
|
||||
})
|
||||
.map(|f| f as f32)
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct LlamafileClient {
|
||||
url: String,
|
||||
}
|
||||
const DEFAULT_LLAMAFILE_URL: &str = "http://localhost:8080/embedding";
|
||||
|
||||
impl LlamafileClient {
|
||||
pub fn new(url: Option<String>) -> Self {
|
||||
Self {
|
||||
url: url.unwrap_or(DEFAULT_LLAMAFILE_URL.to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn infer_single(&self, input: &str) -> Result<Vec<f32>> {
|
||||
let mut body = serde_json::Map::new();
|
||||
body.insert("content".to_owned(), input.to_owned().into());
|
||||
|
||||
let data: serde_json::Value = ureq::post(&self.url)
|
||||
.set("Content-Type", "application/json")
|
||||
.send_bytes(
|
||||
serde_json::to_vec(&body)
|
||||
.map_err(|error| {
|
||||
Error::new_message(format!("Error serializing body to JSON: {error}"))
|
||||
})?
|
||||
.as_ref(),
|
||||
)
|
||||
.map_err(|error| Error::new_message(format!("Error sending HTTP request: {error}")))?
|
||||
.into_json()
|
||||
.map_err(|error| {
|
||||
Error::new_message(format!("Error parsing HTTP response as JSON: {error}"))
|
||||
})?;
|
||||
OllamaClient::parse_single_response(data)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub enum Client {
|
||||
OpenAI(OpenAiClient),
|
||||
Nomic(NomicClient),
|
||||
Cohere(CohereClient),
|
||||
Ollama(OllamaClient),
|
||||
Llamafile(LlamafileClient),
|
||||
Jina(JinaClient),
|
||||
Mixedbread(MixedbreadClient),
|
||||
}
|
||||
@ -1,184 +0,0 @@
|
||||
use sqlite_loadable::table::UpdateOperation;
|
||||
use sqlite_loadable::{api, prelude::*, Error};
|
||||
use sqlite_loadable::{
|
||||
api::ValueType,
|
||||
table::{IndexInfo, VTab, VTabArguments, VTabCursor, VTabWriteable},
|
||||
BestIndexError, Result,
|
||||
};
|
||||
use std::{cell::RefCell, collections::HashMap, marker::PhantomData, mem, os::raw::c_int, rc::Rc};
|
||||
|
||||
use crate::clients::MixedbreadClient;
|
||||
use crate::{
|
||||
clients::{
|
||||
Client, CohereClient, JinaClient, LlamafileClient, NomicClient, OllamaClient, OpenAiClient,
|
||||
},
|
||||
CLIENT_OPTIONS_POINTER_NAME,
|
||||
};
|
||||
|
||||
enum Columns {
|
||||
Name,
|
||||
Options,
|
||||
}
|
||||
fn column(index: i32) -> Option<Columns> {
|
||||
match index {
|
||||
0 => Some(Columns::Name),
|
||||
1 => Some(Columns::Options),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
#[repr(C)]
|
||||
pub struct ClientsTable {
|
||||
/// must be first
|
||||
base: sqlite3_vtab,
|
||||
clients: Rc<RefCell<HashMap<String, Client>>>,
|
||||
}
|
||||
|
||||
impl<'vtab> VTab<'vtab> for ClientsTable {
|
||||
type Aux = Rc<RefCell<HashMap<String, Client>>>;
|
||||
type Cursor = ClientsCursor<'vtab>;
|
||||
|
||||
fn create(
|
||||
db: *mut sqlite3,
|
||||
aux: Option<&Self::Aux>,
|
||||
args: VTabArguments,
|
||||
) -> Result<(String, Self)> {
|
||||
Self::connect(db, aux, args)
|
||||
}
|
||||
fn connect(
|
||||
_db: *mut sqlite3,
|
||||
aux: Option<&Self::Aux>,
|
||||
_args: VTabArguments,
|
||||
) -> Result<(String, ClientsTable)> {
|
||||
let base: sqlite3_vtab = unsafe { mem::zeroed() };
|
||||
let clients = aux.expect("Required aux").to_owned();
|
||||
|
||||
let vtab = ClientsTable { base, clients };
|
||||
let sql = "create table x(name text primary key, options)".to_owned();
|
||||
|
||||
Ok((sql, vtab))
|
||||
}
|
||||
fn destroy(&self) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn best_index(&self, mut info: IndexInfo) -> core::result::Result<(), BestIndexError> {
|
||||
info.set_estimated_cost(10000.0);
|
||||
info.set_estimated_rows(10000);
|
||||
info.set_idxnum(1);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn open(&'vtab mut self) -> Result<ClientsCursor<'vtab>> {
|
||||
ClientsCursor::new(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'vtab> VTabWriteable<'vtab> for ClientsTable {
|
||||
fn update(&'vtab mut self, operation: UpdateOperation<'_>, _p_rowid: *mut i64) -> Result<()> {
|
||||
match operation {
|
||||
UpdateOperation::Delete(_) => {
|
||||
return Err(Error::new_message(
|
||||
"DELETE operations on rembed_clients is not supported yet",
|
||||
))
|
||||
}
|
||||
UpdateOperation::Update { _values } => {
|
||||
return Err(Error::new_message(
|
||||
"DELETE operations on rembed_clients is not supported yet",
|
||||
))
|
||||
}
|
||||
UpdateOperation::Insert { values, rowid: _ } => {
|
||||
let name = api::value_text(&values[0])?;
|
||||
let client = match api::value_type(&values[1]) {
|
||||
ValueType::Text => match api::value_text(&values[1])? {
|
||||
"openai" => Client::OpenAI(OpenAiClient::new(name, None, None)?),
|
||||
"mixedbread" => {
|
||||
Client::Mixedbread(MixedbreadClient::new(name, None, None)?)
|
||||
}
|
||||
"jina" => Client::Jina(JinaClient::new(name, None, None)?),
|
||||
"nomic" => Client::Nomic(NomicClient::new(name, None, None)?),
|
||||
"cohere" => Client::Cohere(CohereClient::new(name, None, None)?),
|
||||
"ollama" => Client::Ollama(OllamaClient::new(name, None)),
|
||||
"llamafile" => Client::Llamafile(LlamafileClient::new(None)),
|
||||
text => {
|
||||
return Err(Error::new_message(format!(
|
||||
"'{text}' is not a valid rembed client."
|
||||
)))
|
||||
}
|
||||
},
|
||||
ValueType::Null => unsafe {
|
||||
if let Some(client) =
|
||||
api::value_pointer::<Client>(&values[1], CLIENT_OPTIONS_POINTER_NAME)
|
||||
{
|
||||
(*client).clone()
|
||||
} else {
|
||||
return Err(Error::new_message("client options required"));
|
||||
}
|
||||
},
|
||||
_ => return Err(Error::new_message("client options required")),
|
||||
};
|
||||
self.clients.borrow_mut().insert(name.to_owned(), client);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
pub struct ClientsCursor<'vtab> {
|
||||
/// Base class. Must be first
|
||||
base: sqlite3_vtab_cursor,
|
||||
keys: Vec<String>,
|
||||
rowid: i64,
|
||||
phantom: PhantomData<&'vtab ClientsTable>,
|
||||
}
|
||||
impl ClientsCursor<'_> {
|
||||
fn new(table: &mut ClientsTable) -> Result<ClientsCursor> {
|
||||
let base: sqlite3_vtab_cursor = unsafe { mem::zeroed() };
|
||||
let c = table.clients.borrow();
|
||||
let keys = c.keys().map(|k| k.to_string()).collect();
|
||||
let cursor = ClientsCursor {
|
||||
base,
|
||||
keys,
|
||||
rowid: 0,
|
||||
phantom: PhantomData,
|
||||
};
|
||||
Ok(cursor)
|
||||
}
|
||||
}
|
||||
|
||||
impl VTabCursor for ClientsCursor<'_> {
|
||||
fn filter(
|
||||
&mut self,
|
||||
_idx_num: c_int,
|
||||
_idx_str: Option<&str>,
|
||||
_values: &[*mut sqlite3_value],
|
||||
) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn next(&mut self) -> Result<()> {
|
||||
self.rowid += 1;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn eof(&self) -> bool {
|
||||
(self.rowid as usize) >= self.keys.len()
|
||||
}
|
||||
|
||||
fn column(&self, context: *mut sqlite3_context, i: c_int) -> Result<()> {
|
||||
let key = self
|
||||
.keys
|
||||
.get(self.rowid as usize)
|
||||
.expect("Internal rembed_clients logic error");
|
||||
match column(i) {
|
||||
Some(Columns::Name) => api::result_text(context, key)?,
|
||||
Some(Columns::Options) => (),
|
||||
None => (),
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn rowid(&self) -> Result<i64> {
|
||||
Ok(self.rowid)
|
||||
}
|
||||
}
|
||||
@ -1,169 +0,0 @@
|
||||
mod clients;
|
||||
mod clients_vtab;
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::collections::HashMap;
|
||||
use std::rc::Rc;
|
||||
|
||||
use clients::{Client, CohereClient, LlamafileClient, NomicClient, OllamaClient, OpenAiClient};
|
||||
use clients_vtab::ClientsTable;
|
||||
use sqlite_loadable::{
|
||||
api, define_scalar_function, define_scalar_function_with_aux, define_virtual_table_writeablex,
|
||||
prelude::*, Error, Result,
|
||||
};
|
||||
use zerocopy::AsBytes;
|
||||
|
||||
const FLOAT32_VECTOR_SUBTYPE: u8 = 223;
|
||||
const CLIENT_OPTIONS_POINTER_NAME: &[u8] = b"sqlite-rembed-client-options\0";
|
||||
|
||||
pub fn rembed_version(context: *mut sqlite3_context, _values: &[*mut sqlite3_value]) -> Result<()> {
|
||||
api::result_text(context, format!("v{}", env!("CARGO_PKG_VERSION")))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn rembed_debug(context: *mut sqlite3_context, _values: &[*mut sqlite3_value]) -> Result<()> {
|
||||
api::result_text(
|
||||
context,
|
||||
format!(
|
||||
"Version: v{}
|
||||
Source: {}
|
||||
",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
env!("GIT_HASH")
|
||||
),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn rembed_client_options(
|
||||
context: *mut sqlite3_context,
|
||||
values: &[*mut sqlite3_value],
|
||||
) -> Result<()> {
|
||||
if (values.len() % 2) != 0 {
|
||||
return Err(Error::new_message(
|
||||
"Must have an even number of arguments to rembed_client_options, as key/value pairs.",
|
||||
));
|
||||
}
|
||||
let mut options: HashMap<String, String> = HashMap::new();
|
||||
let mut format: Option<String> = None;
|
||||
for pair in values.chunks(2) {
|
||||
let key = api::value_text(&pair[0])?;
|
||||
let value = api::value_text(&pair[1])?;
|
||||
if key == "format" {
|
||||
format = Some(value.to_owned());
|
||||
} else {
|
||||
options.insert(key.to_owned(), value.to_owned());
|
||||
}
|
||||
}
|
||||
|
||||
let format = match format {
|
||||
Some(format) => format,
|
||||
None => {
|
||||
return Err(Error::new_message("'format' key is required."));
|
||||
}
|
||||
};
|
||||
let client: Client = match format.as_str() {
|
||||
"openai" => Client::OpenAI(OpenAiClient::new(
|
||||
options
|
||||
.get("model")
|
||||
.ok_or_else(|| Error::new_message("'model' option is required"))?,
|
||||
options.get("url").cloned(),
|
||||
options.get("key").cloned(),
|
||||
)?),
|
||||
"nomic" => Client::Nomic(NomicClient::new(
|
||||
options
|
||||
.get("model")
|
||||
.ok_or_else(|| Error::new_message("'model' option is required"))?,
|
||||
options.get("url").cloned(),
|
||||
options.get("key").cloned(),
|
||||
)?),
|
||||
"cohere" => Client::Cohere(CohereClient::new(
|
||||
options
|
||||
.get("model")
|
||||
.ok_or_else(|| Error::new_message("'model' option is required"))?,
|
||||
options.get("url").cloned(),
|
||||
options.get("key").cloned(),
|
||||
)?),
|
||||
"ollama" => Client::Ollama(OllamaClient::new(
|
||||
options
|
||||
.get("model")
|
||||
.ok_or_else(|| Error::new_message("'model' option is required"))?,
|
||||
options.get("url").cloned(),
|
||||
)),
|
||||
"llamafile" => Client::Llamafile(LlamafileClient::new(options.get("url").cloned())),
|
||||
format => return Err(Error::new_message(format!("Unknown format '{format}'"))),
|
||||
};
|
||||
|
||||
api::result_pointer(context, CLIENT_OPTIONS_POINTER_NAME, client);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
pub fn rembed(
|
||||
context: *mut sqlite3_context,
|
||||
values: &[*mut sqlite3_value],
|
||||
clients: &Rc<RefCell<HashMap<String, Client>>>,
|
||||
) -> Result<()> {
|
||||
let client_name = api::value_text(&values[0])?;
|
||||
let input = api::value_text(&values[1])?;
|
||||
let x = clients.borrow();
|
||||
let client = x.get(client_name).ok_or_else(|| {
|
||||
Error::new_message(format!(
|
||||
"Client with name {client_name} was not registered with rembed_clients."
|
||||
))
|
||||
})?;
|
||||
|
||||
let embedding = match client {
|
||||
Client::OpenAI(client) => client.infer_single(input)?,
|
||||
Client::Jina(client) => client.infer_single(input)?,
|
||||
Client::Mixedbread(client) => client.infer_single(input)?,
|
||||
Client::Ollama(client) => client.infer_single(input)?,
|
||||
Client::Llamafile(client) => client.infer_single(input)?,
|
||||
Client::Nomic(client) => {
|
||||
let input_type = values.get(2).and_then(|v| api::value_text(v).ok());
|
||||
client.infer_single(input, input_type)?
|
||||
}
|
||||
Client::Cohere(client) => {
|
||||
let input_type = values.get(2).and_then(|v| api::value_text(v).ok());
|
||||
client.infer_single(input, input_type)?
|
||||
}
|
||||
};
|
||||
|
||||
api::result_blob(context, embedding.as_bytes());
|
||||
api::result_subtype(context, FLOAT32_VECTOR_SUBTYPE);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[sqlite_entrypoint]
|
||||
pub fn sqlite3_rembed_init(db: *mut sqlite3) -> Result<()> {
|
||||
let flags = FunctionFlags::UTF8
|
||||
| FunctionFlags::DETERMINISTIC
|
||||
| unsafe { FunctionFlags::from_bits_unchecked(0x001000000) };
|
||||
|
||||
let c = Rc::new(RefCell::new(HashMap::new()));
|
||||
|
||||
define_scalar_function(
|
||||
db,
|
||||
"rembed_version",
|
||||
0,
|
||||
rembed_version,
|
||||
FunctionFlags::UTF8 | FunctionFlags::DETERMINISTIC,
|
||||
)?;
|
||||
define_scalar_function(
|
||||
db,
|
||||
"rembed_debug",
|
||||
0,
|
||||
rembed_debug,
|
||||
FunctionFlags::UTF8 | FunctionFlags::DETERMINISTIC,
|
||||
)?;
|
||||
define_scalar_function_with_aux(db, "rembed", 2, rembed, flags, Rc::clone(&c))?;
|
||||
define_scalar_function_with_aux(db, "rembed", 3, rembed, flags, Rc::clone(&c))?;
|
||||
define_scalar_function(
|
||||
db,
|
||||
"rembed_client_options",
|
||||
-1,
|
||||
rembed_client_options,
|
||||
flags,
|
||||
)?;
|
||||
define_virtual_table_writeablex::<ClientsTable>(db, "rembed_clients", Some(Rc::clone(&c)))?;
|
||||
Ok(())
|
||||
}
|
||||
@ -1,37 +0,0 @@
|
||||
.load dist/debug/rembed0
|
||||
.bail on
|
||||
.mode box
|
||||
.header on
|
||||
.timer on
|
||||
.echo on
|
||||
|
||||
INSERT INTO temp.rembed_clients(name, options) VALUES
|
||||
('text-embedding-3-small','openai'),
|
||||
('jina-embeddings-v2-base-en','jina'),
|
||||
('mixedbread-ai/mxbai-embed-large-v1','mixedbread'),
|
||||
('nomic-embed-text-v1.5', 'nomic'),
|
||||
('embed-english-v3.0', 'cohere'),
|
||||
('snowflake-arctic-embed:s', 'ollama'),
|
||||
('llamafile', 'llamafile'),
|
||||
(
|
||||
'mxbai-embed-large-v1-f16',
|
||||
rembed_client_options(
|
||||
'format', 'llamafile',
|
||||
--'url', 'http://mm1:8080/v1/embeddings'
|
||||
'url', 'http://mm1:8080/embedding'
|
||||
)
|
||||
);
|
||||
|
||||
select length(rembed('mixedbread-ai/mxbai-embed-large-v1', 'obama the person'));
|
||||
.exit
|
||||
select length(rembed('jina-embeddings-v2-base-en', 'obama the person'));
|
||||
|
||||
.exit
|
||||
|
||||
select length(rembed('text-embedding-3-small', 'obama the person'));
|
||||
select length(rembed('llamafile', 'obama the person'));
|
||||
select length(rembed('snowflake-arctic-embed:s', 'obama the person'));
|
||||
select length(rembed('embed-english-v3.0', 'obama the person', 'search_document'));
|
||||
select length(rembed('mxbai-embed-large-v1-f16', 'obama the person'));
|
||||
|
||||
|
||||
Loading…
Reference in new issue