Multi-Provider Examples

Switch between OpenAI, Ollama, Anthropic, and Gemini seamlessly.

Ollama (Local) 

1
2
3
4
5
6
7
8
9
10
11

# Ensure ollama is running
ollama serve

# Pull model
ollama pull llama3

# Use CLI
fortified-llm-client \
  --api-url http://localhost:11434/v1/chat/completions \
  --model llama3 \
  --user-text "Hello"

OpenAI (Cloud) 

1
2
3
4
5
6
7

export OPENAI_API_KEY=sk-...

fortified-llm-client \
  --api-url https://api.openai.com/v1/chat/completions \
  --model gpt-4 \
  --api-key-name OPENAI_API_KEY \
  --user-text "Hello"

Anthropic (Direct API) 

1
2
3
4
5
6
7
8
9

export ANTHROPIC_API_KEY=sk-ant-...

fortified-llm-client \
  --api-url https://api.anthropic.com/v1/messages \
  --model claude-sonnet-4-6 \
  --api-key-name ANTHROPIC_API_KEY \
  --system-text "You are a helpful assistant." \
  --user-text "Hello" \
  --max-tokens 1000

Anthropic (Vertex AI) 

1
2
3
4
5
6
7

fortified-llm-client \
  --api-url "https://global-aiplatform.googleapis.com/v1/projects/MY_PROJECT/locations/global/publishers/anthropic/models/claude-sonnet-4-6:streamRawPredict" \
  --model claude-sonnet-4-6 \
  --api-key "$(gcloud auth print-access-token)" \
  --system-text "You are a helpful assistant." \
  --user-text "Hello" \
  --max-tokens 1000

Gemini (Vertex AI) 

1
2
3
4
5
6

fortified-llm-client \
  --api-url "https://us-central1-aiplatform.googleapis.com/v1/projects/MY_PROJECT/locations/us-central1/publishers/google/models/gemini-2.0-flash:generateContent" \
  --model gemini-2.0-flash \
  --api-key "$(gcloud auth print-access-token)" \
  --system-text "You are a helpful assistant." \
  --user-text "Hello"

Provider-Specific Config Files

ollama.toml:

1
2
3

api_url = "http://localhost:11434/v1/chat/completions"
model = "llama3"
temperature = 0.7

openai.toml:

1
2
3
4

api_url = "https://api.openai.com/v1/chat/completions"
model = "gpt-4"
api_key_name = "OPENAI_API_KEY"
temperature = 0.7

anthropic.toml:

1
2
3
4
5

api_url = "https://api.anthropic.com/v1/messages"
model = "claude-sonnet-4-6"
api_key_name = "ANTHROPIC_API_KEY"
temperature = 0.7
max_tokens = 1000

vertex.toml:

1
2
3
4

api_url = "https://global-aiplatform.googleapis.com/v1/projects/MY_PROJECT/locations/global/publishers/anthropic/models/claude-sonnet-4-6:streamRawPredict"
provider = "anthropic"
temperature = 0.7
max_tokens = 1000

gemini-vertex.toml:

1
2
3
4

api_url = "https://us-central1-aiplatform.googleapis.com/v1/projects/MY_PROJECT/locations/us-central1/publishers/google/models/gemini-2.0-flash:generateContent"
model = "gemini-2.0-flash"
temperature = 0.7
max_tokens = 1000

vertex-proxy.toml (when Vertex AI is behind a proxy/gateway):

1
2
3
4

api_url = "https://my-proxy.example.com/claude"
provider = "anthropic-vertex"
temperature = 0.7
max_tokens = 1000

Usage:

1
2
3
4
5
6
7
8
9
10
11
12
13
14

# Use Ollama
fortified-llm-client -c ollama.toml --user-text "prompt"

# Use OpenAI
fortified-llm-client -c openai.toml --user-text "prompt"

# Use Anthropic
fortified-llm-client -c anthropic.toml --user-text "prompt"

# Use Vertex AI Anthropic (pass OAuth token via CLI)
fortified-llm-client -c vertex.toml --api-key "$(gcloud auth print-access-token)" --user-text "prompt"

# Use Vertex AI Gemini (pass OAuth token via CLI)
fortified-llm-client -c gemini-vertex.toml --api-key "$(gcloud auth print-access-token)" --user-text "prompt"

Library Multi-Provider 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

async fn call_llm(provider: &str, prompt: &str) -> Result<String, Box<dyn std::error::Error>> {
    let (api_url, model, api_key_name) = match provider {
        "ollama" => ("http://localhost:11434/v1/chat/completions", "llama3", None),
        "openai" => ("https://api.openai.com/v1/chat/completions", "gpt-4", Some("OPENAI_API_KEY")),
        "anthropic" => ("https://api.anthropic.com/v1/messages", "claude-sonnet-4-6", Some("ANTHROPIC_API_KEY")),
        _ => return Err("Unknown provider".into()),
    };

    let config = EvaluationConfig {
        api_url: api_url.to_string(),
        model: model.to_string(),
        user_prompt: prompt.to_string(),
        api_key_name: api_key_name.map(String::from),
        ..Default::default()
    };

    let result = evaluate(config).await?;
    Ok(result.content)
}