执行轨迹收集与评测运行器

/*
 * 题目：执行轨迹收集与评测运行器
 * 要求：收集 run/step 的耗时、Token、成本和错误；评测器并发运行数据集，
 * 单用例失败不影响其他用例，并汇总成功率等指标。
 * 示例：三个评测用例中一个失败，应保留三条结果而不是中断整个批次。
 */
class TraceCollector {
  constructor(now = Date.now) {}
  start(runId, metadata = {}) {}
  step(runId, step) {}
  end(runId, status, error = null) {}
}
async function runEval(dataset, { agent, scorer, concurrency = 2 }) {}

(async () => {
  let time = 0;
  const trace = new TraceCollector(() => time);
  trace.start("r");
  time = 5;
  trace.step("r", { name: "tool", tokens: 10, cost: 0.1 });
  time = 9;
  const finished = trace.end("r", "completed");
  console.log(finished.duration); // 预期：9
  console.log(finished.usage.tokens); // 预期：10
  const report = await runEval(
    [
      { id: "1", input: 2, expected: 4 },
      { id: "2", input: 3, expected: 6 },
    ],
    {
      agent: async (n) => ({ output: n * 2, cost: 0.01 }),
      scorer: async (actual, expected) => Number(actual === expected),
    },
  );
  console.log(report.summary.averageScore); // 预期：1
})().catch(console.error);

// 关键机制：每个 case 按输入索引写回；agent、scorer、clock 可注入，保证评测可重复。
class TraceCollector {
  constructor(now = Date.now) {
    this.now = now;
    this.runs = new Map();
  }
  start(runId, metadata = {}) {
    if (this.runs.has(runId)) throw new Error("duplicate run");
    this.runs.set(runId, {
      runId,
      metadata,
      startedAt: this.now(),
      endedAt: null,
      steps: [],
      usage: { tokens: 0, cost: 0 },
    });
  }
  step(runId, step) {
    const run = this.runs.get(runId);
    if (!run || run.endedAt !== null) throw new Error("run not active");
    run.steps.push(structuredClone(step));
    run.usage.tokens += step.tokens || 0;
    run.usage.cost += step.cost || 0;
  }
  end(runId, status, error = null) {
    const run = this.runs.get(runId);
    if (!run || run.endedAt !== null) throw new Error("run not active");
    run.endedAt = this.now();
    run.status = status;
    run.error = error ? String(error.message || error) : null;
    return structuredClone({ ...run, duration: run.endedAt - run.startedAt });
  }
}
async function runEval(dataset, { agent, scorer, concurrency = 2 }) {
  const results = new Array(dataset.length);
  let next = 0;
  async function worker() {
    while (true) {
      const index = next++;
      if (index >= dataset.length) return;
      const item = dataset[index];
      const started = Date.now();
      try {
        const execution = await agent(item.input, item);
        const score = await scorer(execution.output, item.expected, execution);
        results[index] = {
          id: item.id,
          ok: true,
          score,
          latency: Date.now() - started,
          cost: execution.cost || 0,
        };
      } catch (error) {
        results[index] = {
          id: item.id,
          ok: false,
          score: 0,
          latency: Date.now() - started,
          cost: 0,
          error: String(error.message || error),
        };
      }
    }
  }
  await Promise.all(
    Array.from({ length: Math.min(concurrency, dataset.length) }, worker),
  );
  const totals = results.reduce(
    (a, r) => ({
      score: a.score + r.score,
      latency: a.latency + r.latency,
      cost: a.cost + r.cost,
      success: a.success + Number(r.ok),
    }),
    { score: 0, latency: 0, cost: 0, success: 0 },
  );
  const count = results.length || 1;
  return {
    results,
    summary: {
      cases: results.length,
      successRate: totals.success / count,
      averageScore: totals.score / count,
      averageLatency: totals.latency / count,
      totalCost: totals.cost,
    },
  };
}

// 三组以上可直接运行的验证；异步题使用独立执行块。
(async () => {
  let time = 0;
  const trace = new TraceCollector(() => time);
  trace.start("r");
  time = 5;
  trace.step("r", { name: "tool", tokens: 10, cost: 0.1 });
  time = 9;
  const finished = trace.end("r", "completed");
  console.log(finished.duration); // 预期：9
  console.log(finished.usage.tokens); // 预期：10
  const report = await runEval(
    [
      { id: "1", input: 2, expected: 4 },
      { id: "2", input: 3, expected: 6 },
    ],
    {
      agent: async (n) => ({ output: n * 2, cost: 0.01 }),
      scorer: async (actual, expected) => Number(actual === expected),
    },
  );
  console.log(report.summary.averageScore); // 预期：1
  console.log(report.summary.totalCost); // 预期：0.02


})().catch(console.error);