Using Polymathy as a Library

Polymathy can be used as a Rust library in your own projects.

Setup

Add Polymathy to your Cargo.toml:

[dependencies]
polymathy = { git = "https://github.com/skelfresearch/polymathy" }
tokio = { version = "1", features = ["full"] }

Basic Usage

Running the Server

The simplest use case is running the built-in server:

use polymathy::run;

#[tokio::main]
async fn main() -> std::io::Result<()> {
    // Set required environment variables
    std::env::set_var("SEARXNG_URL", "http://searx.example.com/search");
    std::env::set_var("PROCESSOR_URL", "http://processor:8081/v1/process");

    // Start the server
    run().await
}

Using Individual Modules

Vector Index

Create and use the vector index directly:

use polymathy::index::create_index;

fn main() {
    // Create a new index
    let index = create_index();

    // The index is configured for:
    // - 384 dimensions (AllMiniLML6V2 embeddings)
    // - Inner Product metric
    // - F32 quantization

    // Example: Add a vector (you'd typically get these from embeddings)
    let embedding: Vec<f32> = vec![0.0; 384];
    if let Err(e) = index.add(0, &embedding) {
        eprintln!("Failed to add vector: {}", e);
    }

    // Example: Search for similar vectors
    let query: Vec<f32> = vec![0.0; 384];
    match index.search(&query, 10) {
        Ok(results) => {
            for (key, distance) in results.keys.iter().zip(results.distances.iter()) {
                println!("Key: {}, Distance: {}", key, distance);
            }
        }
        Err(e) => eprintln!("Search failed: {}", e),
    }
}

Search Structures

Use the data structures for your own processing:

use polymathy::search::{SearchQuery, ProcessedContent, Config};
use std::collections::HashMap;

fn process_content() {
    // Create a mock ProcessedContent structure
    let content = ProcessedContent {
        url: "https://example.com".to_string(),
        config: Config {
            chunking_type: "words".to_string(),
            chunking_size: 100,
            embedding_model: "AllMiniLML6V2".to_string(),
        },
        chunks: HashMap::from([
            ("chunk_0".to_string(), "First chunk content".to_string()),
            ("chunk_1".to_string(), "Second chunk content".to_string()),
        ]),
        embeddings: HashMap::from([
            ("chunk_0".to_string(), vec![0.1; 384]),
            ("chunk_1".to_string(), vec![0.2; 384]),
        ]),
        error: None,
    };

    // Process the content
    for (chunk_id, text) in &content.chunks {
        if let Some(embedding) = content.embeddings.get(chunk_id) {
            println!("Chunk {}: {} ({} dimensions)",
                chunk_id, text, embedding.len());
        }
    }
}

Custom Server Setup

Build your own server using Polymathy components:

use actix_web::{web, App, HttpServer, HttpResponse};
use polymathy::index::create_index;
use polymathy::search::SearchQuery;
use std::sync::{Arc, Mutex};
use usearch::Index;

async fn custom_search(
    query: web::Query<SearchQuery>,
    index: web::Data<Arc<Mutex<Index>>>,
) -> HttpResponse {
    // Your custom search logic here
    let q = &query.q;

    // Use the index for similarity search
    // Process results...

    HttpResponse::Ok().json(serde_json::json!({
        "query": q,
        "status": "processed"
    }))
}

#[actix_web::main]
async fn main() -> std::io::Result<()> {
    let index = Arc::new(Mutex::new(create_index()));

    HttpServer::new(move || {
        App::new()
            .app_data(web::Data::new(index.clone()))
            .route("/search", web::get().to(custom_search))
    })
    .bind("127.0.0.1:8080")?
    .run()
    .await
}

Integration Patterns

With Your Own Embedding Service

use polymathy::index::create_index;
use reqwest::Client;
use serde::{Deserialize, Serialize};

#[derive(Serialize)]
struct EmbeddingRequest {
    text: String,
}

#[derive(Deserialize)]
struct EmbeddingResponse {
    embedding: Vec<f32>,
}

async fn get_embedding(client: &Client, text: &str) -> Result<Vec<f32>, reqwest::Error> {
    let response: EmbeddingResponse = client
        .post("http://your-embedding-service/embed")
        .json(&EmbeddingRequest { text: text.to_string() })
        .send()
        .await?
        .json()
        .await?;

    Ok(response.embedding)
}

async fn index_content(texts: Vec<String>) {
    let client = Client::new();
    let index = create_index();

    for (i, text) in texts.iter().enumerate() {
        if let Ok(embedding) = get_embedding(&client, text).await {
            if let Err(e) = index.add(i as u64, &embedding) {
                eprintln!("Failed to index text {}: {}", i, e);
            }
        }
    }
}

Error Handling

Use anyhow for comprehensive error handling:

use anyhow::{Context, Result};
use polymathy::index::create_index;

fn process_with_errors() -> Result<()> {
    let index = create_index();

    let embedding: Vec<f32> = vec![0.0; 384];
    index.add(0, &embedding)
        .context("Failed to add vector to index")?;

    let results = index.search(&embedding, 10)
        .context("Failed to search index")?;

    println!("Found {} results", results.keys.len());
    Ok(())
}