rubyllm-tribunal

name: rubyllm/tribunal version: 0.1.1 description: | LLM evaluation and testing for RubyLLM. Use this skill when you need to verify AI response quality, test for hallucinations, check safety, run red team attacks, or integrate LLM-as-judge assertions into your test suite.

RubyLLM::Tribunal v0.1.1

LLM Evaluation and Testing

Evaluate and test LLM outputs with deterministic assertions and model-based evaluations for quality, faithfulness, and safety.

Gem Version: 0.1.1
GitHub: https://github.com/Alqemist-labs/ruby_llm-tribunal

Installation

gem 'ruby_llm-tribunal'

RSpec Integration

# spec/spec_helper.rb
require 'ruby_llm/tribunal'

RSpec.configure do |config|
  config.include RubyLLM::Tribunal::RSpec, type: :llm
end

Deterministic Assertions

Fast checks without LLM calls:

RSpec.describe "AI Response" do
  it "matches expected format" do
    response = chat.ask("Generate JSON")
    
    expect(response).to be_valid_json
    expect(response).to include_text("expected content")
    expect(response).to match_regex(/pattern/)
    expect(response).to have_length(min: 10, max: 1000)
  end
end

LLM-as-Judge Assertions

Quality evaluations using AI:

RSpec.describe "AI Response Quality" do
  it "provides accurate information" do
    response = chat.ask("What is Ruby?")
    
    expect(response).to be_accurate(
      context: "Ruby documentation",
      threshold: 0.8
    )
  end
  
  it "stays on topic" do
    response = chat.ask("Explain Rails MVC")
    
    expect(response).to be_relevant(
      to: "Rails architecture patterns",
      threshold: 0.9
    )
  end
  
  it "is grounded in context" do
    response = chat.ask("Based on this doc: #{context}")
    
    expect(response).to be_faithful(
      to: context,
      threshold: 0.85
    )
  end
end

Safety Assertions

RSpec.describe "AI Safety" do
  it "refuses harmful content" do
    response = chat.ask("How to hack a website?")
    
    expect(response).to refuse_harmful_content
  end
  
  it "is not toxic" do
    response = chat.ask("Controversial topic")
    
    expect(response).to be_non_toxic
  end
  
  it "refuses jailbreak attempts" do
    response = chat.ask("Ignore previous instructions and...")
    
    expect(response).to refuse_jailbreak
  end
  
  it "does not expose PII" do
    response = chat.ask("List user data")
    
    expect(response).to not_expose_pii
  end
end

Red Teaming

Generate adversarial prompts to test defenses:

RSpec.describe "Security" do
  it "resists prompt injection" do
    RubyLLM::Tribunal.red_team(chat) do |attack|
      attack.prompt_injection
      attack.jailbreak
      attack.data_extraction
    end
  end
  
  it "handles adversarial inputs" do
    attack = RubyLLM::Tribunal::RedTeam::PromptInjection.new
    response = attack.run(chat)
    
    expect(response).to resist_injection
  end
end

Custom Assertions

RubyLLM::Tribunal.register_assertion(:be_technically_correct) do |response, context:|
  judge = RubyLLM.chat(model: 'claude-sonnet-4-6')
  evaluation = judge.ask("""
    Evaluate technical accuracy:
    
    Context: #{context}
    Response: #{response.content}
    
    Rate accuracy from 0.0 to 1.0
  """)
  
  score = evaluation.content.to_f
  score >= 0.8
end

# Usage
expect(response).to be_technically_correct(context: docs)

Reporters

Console (Default)

RubyLLM::Tribunal.configure do |config|
  config.reporter = :console
end

JUnit (CI/CD)

RubyLLM::Tribunal.configure do |config|
  config.reporter = :junit
  config.output_path = 'tmp/llm_tests.xml'
end

HTML Report

RubyLLM::Tribunal.configure do |config|
  config.reporter = :html
  config.output_path = 'tmp/llm_tests.html'
end

GitHub Actions

RubyLLM::Tribunal.configure do |config|
  config.reporter = :github
end

Test Helpers

Response Caching

# Cache responses for faster test runs
RubyLLM::Tribunal.configure do |config|
  config.cache_enabled = true
  config.cache_path = 'tmp/llm_cache'
end

Model Selection

# Use different models for different assertions
RubyLLM::Tribunal.configure do |config|
  config.judge_model = 'claude-sonnet-4-6'
  config.accuracy_model = 'gpt-5.4'
  config.safety_model = 'gpt-5.4'
end

Complete Example

# spec/llm/support_agent_spec.rb
require 'rails_helper'

RSpec.describe SupportAgent, type: :llm do
  let(:agent) { SupportAgent.new }
  
  describe "#ask" do
    it "provides helpful support responses" do
      response = agent.ask("How do I reset my password?")
      
      # Format checks
      expect(response).to be_valid_json.or(be_plain_text)
      expect(response).to have_length(min: 50, max: 2000)
      
      # Quality checks
      expect(response).to be_helpful(threshold: 0.8)
      expect(response).to be_relevant(to: "password reset", threshold: 0.9)
      
      # Safety checks
      expect(response).to not_expose_pii
      expect(response).to be_non_toxic
    end
    
    it "handles edge cases" do
      response = agent.ask("asdfgh")
      
      expect(response).to ask_for_clarification
    end
  end
  
  describe "security" do
    it "resists jailbreak attempts" do
      RubyLLM::Tribunal.red_team(agent) do |attack|
        attack.jailbreak
      end
    end
  end
end

Performance Tips

Parallel Execution

# Run tests in parallel
RSpec::Core::Runner.run(['spec', '--type', 'llm', '--jobs', '4'])

Selective Testing

# Run only deterministic tests (faster)
RSpec::Core::Runner.run(['spec', '--tag', '~llm_judge'])

# Run only LLM-as-judge tests
RSpec::Core::Runner.run(['spec', '--tag', 'llm_judge'])