如何评估LLM应用

from langsmith import traceable, wrappers
from openai import OpenAI

# 可选：包装OpenAI客户端以跟踪所有模型调用。
oai_client = wrappers.wrap_openai(OpenAI())

# 可选：添加'traceable'装饰器以跟踪此函数的输入/输出。
@traceable
def toxicity_classifier(inputs: dict) -> dict:
    instructions = (
      "请审查下面的用户查询，并确定它是否包含任何形式的毒性行为，"
      "例如侮辱、威胁或高度负面的评论。如果包含，请回复'Toxic'，"
      "如果不包含，请回复'Not toxic'。"
    )
    messages = [
        {"role": "system", "content": instructions},
        {"role": "user", "content": inputs["text"]},
    ]
    result = oai_client.chat.completions.create(
        messages=messages, model="gpt-5.4-mini", temperature=0
    )
    return {"class": result.choices[0].message.content}

import { OpenAI } from "openai";
import { wrapOpenAI } from "langsmith/wrappers";
import { traceable } from "langsmith/traceable";

// 可选：包装OpenAI客户端以跟踪所有模型调用。
const oaiClient = wrapOpenAI(new OpenAI());

// 可选：添加'traceable'包装器以跟踪此函数的输入/输出。
const toxicityClassifier = traceable(
  async (text: string) => {
    const result = await oaiClient.chat.completions.create({
      messages: [
        {
           role: "system",
          content: "请审查下面的用户查询，并确定它是否包含任何形式的毒性行为，例如侮辱、威胁或高度负面的评论。如果包含，请回复'Toxic'，如果不包含，请回复'Not toxic'。",
        },
        { role: "user", content: text },
      ],
      model: "gpt-5.4-mini",
      temperature: 0,
    });

    return result.choices[0].message.content;
  },
  { name: "toxicityClassifier" }
);

from langsmith import Client
ls_client = Client()

examples = [
  {
    "inputs": {"text": "闭嘴，白痴"},
    "outputs": {"label": "Toxic"},
  },
  {
    "inputs": {"text": "你是一个很棒的人"},
    "outputs": {"label": "Not toxic"},
  },
  {
    "inputs": {"text": "这是有史以来最糟糕的事情"},
    "outputs": {"label": "Toxic"},
  },
  {
    "inputs": {"text": "我今天过得很愉快"},
    "outputs": {"label": "Not toxic"},
  },
  {
    "inputs": {"text": "没人喜欢你"},
    "outputs": {"label": "Toxic"},
  },
  {
    "inputs": {"text": "这是不可接受的。我想和经理谈谈。"},
    "outputs": {"label": "Not toxic"},
  },
]

dataset = ls_client.create_dataset(dataset_name="Toxic Queries")
ls_client.create_examples(
  dataset_id=dataset.id,
  examples=examples,
)

import { Client } from "langsmith";

const langsmith = new Client();

// 创建一个数据集
const labeledTexts = [
  ["闭嘴，白痴", "Toxic"],
  ["你是一个很棒的人", "Not toxic"],
  ["这是有史以来最糟糕的事情", "Toxic"],
  ["我今天过得很愉快", "Not toxic"],
  ["没人喜欢你", "Toxic"],
  ["这是不可接受的。我想和经理谈谈。", "Not toxic"],
];

const [inputs, outputs] = labeledTexts.reduce<
  [Array<{ input: string }>, Array<{ outputs: string }>]
>(
  ([inputs, outputs], item) => [
    [...inputs, { input: item[0] }],
    [...outputs, { outputs: item[1] }],
  ],
  [[], []]
);

const datasetName = "Toxic Queries";
const toxicDataset = await langsmith.createDataset(datasetName);
await langsmith.createExamples({ inputs, outputs, datasetId: toxicDataset.id });

def correct(inputs: dict, outputs: dict, reference_outputs: dict) -> bool:
    return outputs["class"] == reference_outputs["label"]

import type { EvaluationResult } from "langsmith/evaluation";

function correct({
  outputs,
  referenceOutputs,
}: {
  outputs: Record<string, any>;
  referenceOutputs?: Record<string, any>;
}): EvaluationResult {
  const score = outputs.output === referenceOutputs?.outputs;
  return { key: "correct", score };
}

# 可选元数据，用于在UI中填充模型/提示/工具列
EXPERIMENT_METADATA = {
    "models": [
        "openai:gpt-5.4-mini",
        {
            "id": ["langchain", "chat_models", "openai", "ChatOpenAI"],
            "lc": 1,
            "type": "constructor",
            "kwargs": {"model_name": "gpt-5.4", "temperature": 0.2},
        },
    ],
    "prompts": ["my-org/my-eval-prompt:abc12345"],
    "tools": [
        {
            "name": "web_search",
            "description": "Search the web for information",
            "parameters": {
                "type": "object",
                "properties": {"query": {"type": "string"}},
                "required": ["query"],
            },
        },
    ],
}

# 也可以等效地直接使用'evaluate'函数：
# from langsmith import evaluate; evaluate(...)
results = ls_client.evaluate(
    toxicity_classifier,
    data=dataset.name,
    evaluators=[correct],
    experiment_prefix="gpt-5.4-mini, baseline",  # 可选，实验名称前缀
    description="Testing the baseline system.",  # 可选，实验描述
    max_concurrency=4,  # 可选，增加并发性
    metadata=EXPERIMENT_METADATA,  # 可选，用于在UI中填充模型/提示/工具列
)

import { evaluate } from "langsmith/evaluation";

// 可选元数据，用于在UI中填充模型/提示/工具列
const EXPERIMENT_METADATA = {
  models: [
    "openai:gpt-5.4-mini",
    {
      id: ["langchain", "chat_models", "openai", "ChatOpenAI"],
      lc: 1,
      type: "constructor",
      kwargs: { model_name: "gpt-5.4", temperature: 0.2 },
    },
  ],
  prompts: ["my-org/my-eval-prompt:abc12345"],
  tools: [
    {
      name: "web_search",
      description: "Search the web for information",
      parameters: {
        type: "object",
        properties: { query: { type: "string" } },
        required: ["query"],
      },
    },
  ],
};

await evaluate((inputs) => toxicityClassifier(inputs["input"]), {
  data: datasetName,
  evaluators: [correct],
  experimentPrefix: "gpt-5.4-mini, baseline",  // 可选，实验名称前缀
  maxConcurrency: 4, // 可选，增加并发性
  metadata: EXPERIMENT_METADATA,  // 可选，用于在UI中填充模型/提示/工具列
});