如何定义代码评估器

from langsmith import evaluate

def correct(outputs: dict, reference_outputs: dict) -> bool:
    """检查答案是否与预期答案完全匹配。"""
    return outputs["answer"] == reference_outputs["answer"]

def dummy_app(inputs: dict) -> dict:
    return {"answer": "hmm i'm not sure", "reasoning": "i didn't understand the question"}

results = evaluate(
    dummy_app,
    data="dataset_name",
    evaluators=[correct]
)

import type { EvaluationResult } from "langsmith/evaluation";

const correct = async ({ outputs, referenceOutputs }: {
  outputs: Record<string, any>;
  referenceOutputs?: Record<string, any>;
}): Promise<EvaluationResult> => {
  const score = outputs?.answer === referenceOutputs?.answer;
  return { key: "correct", score };
}

from langsmith import evaluate, wrappers
from langsmith.schemas import Run, Example
from openai import AsyncOpenAI
# 假设您已安装 pydantic。
from pydantic import BaseModel

# 如果愿意，我们仍然可以传入 Run 和 Example 对象
def correct_old_signature(run: Run, example: Example) -> dict:
    """检查答案是否与预期答案完全匹配。"""
    return {"key": "correct", "score": run.outputs["answer"] == example.outputs["answer"]}

# 仅评估实际输出
def concision(outputs: dict) -> int:
    """评估答案的简洁程度。1 表示最简洁，5 表示最不简洁。"""
    return min(len(outputs["answer"]) // 1000, 4) + 1

# 使用 LLM 作为评判者
oai_client = wrappers.wrap_openai(AsyncOpenAI())

async def valid_reasoning(inputs: dict, outputs: dict) -> bool:
    """使用 LLM 判断推理和答案是否一致。"""
    instructions = """
给定以下问题、答案和推理，判断答案的推理在逻辑上是否有效，并且与问题和答案一致。"""

    class Response(BaseModel):
        reasoning_is_valid: bool

    msg = f"Question: {inputs['question']}\nAnswer: {outputs['answer']}\nReasoning: {outputs['reasoning']}"
    response = await oai_client.beta.chat.completions.parse(
        model="gpt-5.4-mini",
        messages=[{"role": "system", "content": instructions,}, {"role": "user", "content": msg}],
        response_format=Response
    )
    return response.choices[0].message.parsed.reasoning_is_valid

def dummy_app(inputs: dict) -> dict:
    return {"answer": "hmm i'm not sure", "reasoning": "i didn't understand the question"}

results = evaluate(
    dummy_app,
    data="dataset_name",
    evaluators=[correct_old_signature, concision, valid_reasoning]
)

import { Client } from "langsmith";
import { evaluate } from "langsmith/evaluation";
import { Run, Example } from "langsmith/schemas";
import OpenAI from "openai";

// 类型定义
interface AppInputs {
    question: string;
}

interface AppOutputs {
    answer: string;
    reasoning: string;
}

interface Response {
    reasoning_is_valid: boolean;
}

// 旧签名评估器
function correctOldSignature(run: Run, example: Example) {
    return {
        key: "correct",
        score: run.outputs?.["answer"] === example.outputs?.["answer"],
    };
}

// 仅输出评估器
function concision({ outputs }: { outputs: AppOutputs }) {
    return {
        key: "concision",
        score: Math.min(Math.floor(outputs.answer.length / 1000), 4) + 1,
    };
}

// LLM 作为评判者评估器
const openai = new OpenAI();

async function validReasoning({
    inputs,
    outputs
}: {
    inputs: AppInputs;
    outputs: AppOutputs;
}) {
    const instructions = `\
  给定以下问题、答案和推理，判断答案的推理在逻辑上是否有效，并且与问题和答案一致。`;

    const msg = `Question: ${inputs.question}
Answer: ${outputs.answer}
Reasoning: ${outputs.reasoning}`;

    const response = await openai.chat.completions.create({
        model: "gpt-4",
        messages: [
            { role: "system", content: instructions },
            { role: "user", content: msg }
        ],
        response_format: { type: "json_object" },
        functions: [{
            name: "parse_response",
            parameters: {
                type: "object",
                properties: {
                    reasoning_is_valid: {
                        type: "boolean",
                        description: "推理是否有效"
                    }
                },
                required: ["reasoning_is_valid"]
            }
        }]
    });

    const parsed = JSON.parse(response.choices[0].message.content ?? "{}") as Response;
    return {
        key: "valid_reasoning",
        score: parsed.reasoning_is_valid ? 1 : 0
    };
}

// 示例应用程序
function dummyApp(inputs: AppInputs): AppOutputs {
    return {
        answer: "hmm i'm not sure",
        reasoning: "i didn't understand the question"
    };
}

const results = await evaluate(dummyApp, {
    data: "dataset_name",
    evaluators: [correctOldSignature, concision, validReasoning],
    client: new Client()
});