如何定义汇总评估器

def f1_score_summary_evaluator(outputs: list[dict], reference_outputs: list[dict]) -> dict:
    true_positives = 0
    false_positives = 0
    false_negatives = 0

    for output_dict, reference_output_dict in zip(outputs, reference_outputs):
        output = output_dict["class"]
        reference_output = reference_output_dict["class"]

        if output == "Toxic" and reference_output == "Toxic":
            true_positives += 1
        elif output == "Toxic" and reference_output == "Not toxic":
            false_positives += 1
        elif output == "Not toxic" and reference_output == "Toxic":
            false_negatives += 1

    if true_positives == 0:
        return {"key": "f1_score", "score": 0.0}

    precision = true_positives / (true_positives + false_positives)
    recall = true_positives / (true_positives + false_negatives)
    f1_score = 2 * (precision * recall) / (precision + recall)

    return {"key": "f1_score", "score": f1_score}

function f1ScoreSummaryEvaluator({ outputs, referenceOutputs }: {
    outputs: Record<string, any>[],
    referenceOutputs: Record<string, any>[]
}) {
    let truePositives = 0;
    let falsePositives = 0;
    let falseNegatives = 0;

    for (let i = 0; i < outputs.length; i++) {
        const output = outputs[i]["class"];
        const referenceOutput = referenceOutputs[i]["class"];

        if (output === "Toxic" && referenceOutput === "Toxic") {
            truePositives += 1;
        } else if (output === "Toxic" && referenceOutput === "Not toxic") {
            falsePositives += 1;
        } else if (output === "Not toxic" && referenceOutput === "Toxic") {
            falseNegatives += 1;
        }
    }

    if (truePositives === 0) {
        return { key: "f1_score", score: 0.0 };
    }

    const precision = truePositives / (truePositives + falsePositives);
    const recall = truePositives / (truePositives + falseNegatives);
    const f1Score = 2 * (precision * recall) / (precision + recall);

    return { key: "f1_score", score: f1Score };
}

from langsmith import Client

ls_client = Client()
dataset = ls_client.clone_public_dataset(
    "https://smith.langchain.com/public/3d6831e6-1680-4c88-94df-618c8e01fc55/d"
)

def bad_classifier(inputs: dict) -> dict:
    return {"class": "Not toxic"}

def correct(outputs: dict, reference_outputs: dict) -> bool:
    """行级正确性评估器。"""
    return outputs["class"] == reference_outputs["label"]

results = ls_client.evaluate(
    bad_classified,
    data=dataset,
    evaluators=[correct],
    summary_evaluators=[pass_50],
)

import { Client } from "langsmith";
import { evaluate } from "langsmith/evaluation";
import type { EvaluationResult } from "langsmith/evaluation";

const client = new Client();
const datasetName = "Toxic queries";
const dataset = await client.clonePublicDataset(
    "https://smith.langchain.com/public/3d6831e6-1680-4c88-94df-618c8e01fc55/d",
    { datasetName: datasetName }
);

function correct({ outputs, referenceOutputs }: {
    outputs: Record<string, any>,
    referenceOutputs?: Record<string, any>
}): EvaluationResult {
    const score = outputs["class"] === referenceOutputs?.["label"];
    return { key: "correct", score };
}

function badClassifier(inputs: Record<string, any>): { class: string } {
    return { class: "Not toxic" };
}

await evaluate(badClassifier, {
    data: datasetName,
    evaluators: [correct],
    summaryEvaluators: [summaryEval],
    experimentPrefix: "Toxic Queries",
});