Métriques d'Évaluation

import {
  contextPrecision,
  contextRecall,
  answerRelevance,
  semanticSimilarity,
  cosineSimilarity,
  ragasMetrics,
} from '@orka-js/evaluation';
import { OpenAIAdapter } from '@orka-js/openai';
 
const llm = new OpenAIAdapter({ apiKey: process.env.OPENAI_API_KEY! });
 
// Use individually
const precision = await contextPrecision({
  input: 'What is the capital of France?',
  output: 'Paris is the capital of France.',
  context: ['France is a country in Europe. Its capital is Paris.', 'Unrelated text.'],
  llm,
});
console.log(precision.score); // e.g. 0.87
 
// Use the ragasMetrics bundle with orka.evaluate()
const summary = await orka.evaluate({
  dataset,
  metrics: Object.values(ragasMetrics),
});
 
// cosineSimilarity helper
const sim = cosineSimilarity([0.1, 0.9, 0.3], [0.2, 0.8, 0.4]);
console.log(sim); // 0.998

import type { MetricFn } from 'orkajs';
 
// Custom metric: Check professionalism of tone
const toneCheck: MetricFn = async ({ input, output, context, llm }) => {
  const result = await llm.generate(
    `Rate the professionalism of this response on a scale of 0.0 to 1.0.
     Question: ${input}
     Response: ${output}
     Reply with ONLY a number.`,
    { temperature: 0, maxTokens: 10 }
  );
 
  const score = parseFloat(result.content.trim());
  return {
    name: 'professionalism',
    score: isNaN(score) ? 0 : Math.min(1, Math.max(0, score)),
  };
};
 
// Custom metric: Check response length
const lengthCheck: MetricFn = async ({ output }) => {
  const wordCount = output.split(/\s+/).length;
  // Penalize very short or very long responses
  const idealLength = 50;
  const score = Math.max(0, 1 - Math.abs(wordCount - idealLength) / idealLength);
  return { name: 'length_appropriateness', score };
};
 
// Use alongside built-in metrics
const summary = await orka.evaluate({
  dataset: [...],
  metrics: ['relevance', 'faithfulness', toneCheck, lengthCheck],
});