rag_ingest: Rewrite to use MySQL protocol (Sqlite_Server) instead of SQLite3 API

Convert rag_ingest from direct SQLite database access to MySQL protocol communication with SQLite Server.

Major changes:
- Replace SQLite3 direct API with MySQL/mariadb client library
- Update connection flow: rag_ingest -> MySQL protocol -> SQLite Server -> RAG DB
- Remove sqlite3 amalgamation dependency from rag_ingest target
- Keep all ingestion functionality intact (chunking, FTS5, vec0 embeddings)

Schema initialization:
- Add init command to create RAG schema via MySQL protocol
- Add ingest command to run data ingestion pipeline
- Add query command for vector similarity search with embeddings
pull/5325/head
Rahim Kanji 4 weeks ago
parent c5c726ea98
commit 211179bdcb

@ -6,14 +6,11 @@ ROOT_DIR := ..
INCLUDES := \
-I$(ROOT_DIR)/deps/json \
-I$(ROOT_DIR)/deps/mariadb-client-library/mariadb_client/include \
-I$(ROOT_DIR)/deps/sqlite3/sqlite-amalgamation-3500400 \
-I$(ROOT_DIR)/deps/curl/curl/include
LIBDIRS := \
-L$(ROOT_DIR)/deps/mariadb-client-library/mariadb_client/libmariadb
SQLITE3_OBJ := $(ROOT_DIR)/deps/sqlite3/sqlite-amalgamation-3500400/sqlite3.o
# Use static libcurl
CURL_STATIC_LIB := $(ROOT_DIR)/deps/curl/curl/lib/.libs/libcurl.a
@ -27,7 +24,7 @@ SOURCES := rag_ingest.cpp
all: $(TARGET)
$(TARGET): $(SOURCES)
$(CXX) $(CXXFLAGS) $(INCLUDES) $(LIBDIRS) $(SQLITE3_OBJ) $^ -o $@ $(LIBS)
$(CXX) $(CXXFLAGS) $(INCLUDES) $(LIBDIRS) $^ -o $@ $(LIBS)
clean:
rm -f $(TARGET)

File diff suppressed because it is too large Load Diff

@ -50,9 +50,17 @@ int (*proxy_sqlite3_prepare_v2)(sqlite3*, const char*, int, sqlite3_stmt**, cons
int (*proxy_sqlite3_open_v2)(const char*, sqlite3**, int, const char*) = sqlite3_open_v2;
int (*proxy_sqlite3_exec)(sqlite3*, const char*, int (*)(void*,int,char**,char**), void*, char**) = sqlite3_exec;
// Optional hooks used by sqlite-vec (function pointers will be set by LoadPlugin or remain NULL)
void (*proxy_sqlite3_vec_init)(sqlite3*, char**, const sqlite3_api_routines*) = NULL;
void (*proxy_sqlite3_rembed_init)(sqlite3*, char**, const sqlite3_api_routines*) = NULL;
// Optional hooks used by sqlite-vec and sqlite-rembed
// These are statically linked extensions that need to be initialized
// Forward declarations for the extension init functions
extern "C" int sqlite3_vec_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi);
extern "C" int sqlite3_rembed_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi);
// Initialize the extension pointers to the statically linked functions
// These allow Admin_Bootstrap.cpp to register them as auto-extensions
int (*proxy_sqlite3_vec_init)(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) = sqlite3_vec_init;
int (*proxy_sqlite3_rembed_init)(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) = sqlite3_rembed_init;
// Internal helpers used by admin stats batching; keep defaults as NULL

Loading…
Cancel
Save