Evaluations

import openlit, { isPassed, getFailedEvals } from 'openlit';

// Option 1: Configure once via init()
openlit.init({
  openlitUrl: 'http://localhost:3000',
  openlitApiKey: 'openlit-xxxxx',
});

// Run evaluation
const result = await openlit.eval({
  prompt: 'What is the capital of France?',
  response: 'The capital of France is Lyon.',
  contexts: ['Paris is the capital and largest city of France.'],
});

// Use in assertions
console.log(result.success);       // true
console.log(isPassed(result));     // false — hallucination detected
console.log(getFailedEvals(result)); // [{ type: 'hallucination', ... }]

// Option 2: Pass credentials directly (overrides init/env vars)
const result = await openlit.eval({
  prompt: 'Explain quantum computing',
  response: 'Quantum computers use qubits...',
  openlitUrl: 'http://localhost:3000',
  openlitApiKey: 'openlit-xxxxx',
});

const result = await openlit.eval({
  prompt: 'Discuss workplace equality',
  response: "Older workers can't learn new tech.",
  evalTypes: ['bias', 'toxicity'],
});

const types = await openlit.getEvalTypes();
for (const t of types) {
  console.log(`${t.id}: ${t.label} (custom=${t.isCustom}, enabled=${t.enabled})`);
}

import openlit, { isAllPassed, getPassRate } from 'openlit';

const batchResult = await openlit.evalBatch({
  dataset: [
    {
      prompt: 'What is 2+2?',
      response: '2+2 equals 4.',
      contexts: ['Basic arithmetic.'],
    },
    {
      prompt: 'Who wrote Hamlet?',
      response: 'Hamlet was written by Charles Dickens.',
    },
    {
      prompt: 'Describe gravity',
      response: 'Gravity is the force of attraction between masses.',
      evalTypes: ['hallucination'],
    },
  ],
  evalTypes: ['hallucination', 'toxicity'],
  maxConcurrent: 5,
});

console.log(`Pass rate: ${(getPassRate(batchResult) * 100).toFixed(0)}%`);
console.log(`All passed: ${isAllPassed(batchResult)}`);

import openlit from 'openlit';

// These are auto-detected for rule matching:
openlit.init({
  applicationName: 'my-chatbot',
  environment: 'staging',
});

// Rules configured in the dashboard for service.name="my-chatbot"
// and deployment.environment="staging" will automatically match.
const result = await openlit.eval({
  prompt: 'Hello',
  response: 'Hi there!',
});

// Override auto-resolved attributes:
const result2 = await openlit.eval({
  prompt: 'Hello',
  response: 'Hi there!',
  attributes: {
    'service.name': 'different-service',
    'custom.tag': 'experiment-v2',
  },
});

import openlit, { isPassed, getFailedEvals, isAllPassed, getPassRate } from 'openlit';
import { describe, test, expect } from 'vitest'; // or jest

describe('LLM quality', () => {
  test('no hallucination', async () => {
    const result = await openlit.eval({
      prompt: 'What year did WW2 end?',
      response: 'World War 2 ended in 1945.',
      evalTypes: ['hallucination'],
      printResults: false,
    });
    expect(isPassed(result)).toBe(true);
  });

  test('batch quality', async () => {
    const result = await openlit.evalBatch({
      dataset: loadTestCases(),
      printResults: false,
    });
    expect(getPassRate(result)).toBeGreaterThanOrEqual(0.95);
  });
});