π¦ ClawHub
The Librarian
by @rochyroch
Build and search lightweight quantized document indexes with TurboVec. Use when you need to create searchable indexes from documents for RAG applications wit...
β‘ When to Use
π‘ Examples
Prerequisites
# Install BLAS library (required for TurboVec)
sudo apt install libblas3Create venv and install dependencies
cd /path/to/the-librarian
python3 -m venv venv
source venv/bin/activate
pip install turbovec numpy requests rank-bm25 flashrank
Build an Index
# Using the wrapper (recommended)
./scripts/librarian build /path/to/documents/ index/my_libraryWith options
./scripts/librarian build /path/to/docs/ index/my_library --bits 3 --chunk-size 800Direct Python
LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libblas.so.3 \
python scripts/build_index.py --input /path/to/docs/ --output index/my_library
Search
# Pure vector search
./scripts/librarian search "habit formation" index/my_libraryHybrid (vector + BM25)
./scripts/librarian search "habit formation" index/my_library --hybridHybrid + rerank (best accuracy)
./scripts/librarian search "habit formation" index/my_library --hybrid --rerankWith context expansion
./scripts/librarian search "habit formation" index/my_library --hybrid --rerank --expand 1JSON output
./scripts/librarian search "habit formation" index/my_library --json
βοΈ Configuration
# Install BLAS library (required for TurboVec)
sudo apt install libblas3Create venv and install dependencies
cd /path/to/the-librarian
python3 -m venv venv
source venv/bin/activate
pip install turbovec numpy requests rank-bm25 flashrank
Build an Index
# Using the wrapper (recommended)
./scripts/librarian build /path/to/documents/ index/my_libraryWith options
./scripts/librarian build /path/to/docs/ index/my_library --bits 3 --chunk-size 800Direct Python
LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libblas.so.3 \
python scripts/build_index.py --input /path/to/docs/ --output index/my_library
Search
# Pure vector search
./scripts/librarian search "habit formation" index/my_libraryHybrid (vector + BM25)
./scripts/librarian search "habit formation" index/my_library --hybridHybrid + rerank (best accuracy)
./scripts/librarian search "habit formation" index/my_library --hybrid --rerankWith context expansion
./scripts/librarian search "habit formation" index/my_library --hybrid --rerank --expand 1JSON output
./scripts/librarian search "habit formation" index/my_library --json
TERMINAL
clawhub install thelibrarian