Compare two agent versions

import os
from atlanai import AtlanClient, Eval, push_dataset

client = AtlanClient(
    os.environ.get("ATLAN_BASE_URL", "https://api.atlan.com"),
    bearer_token=os.environ["ATLAN_API_KEY"],
    workspace=os.environ["ATLAN_WORKSPACE_ID"],
)

suite = push_dataset(client, "release-regression-cases", [
    {"name": "arithmetic", "input": {"question": "What is 2 + 2?"}, "expected": {"value": "4"}},
    {"name": "geography", "input": {"question": "What is the capital of France?"}, "expected": {"value": "Paris"}},
])


def accuracy(*, output: str, expected: str, **_) -> float:
    return float(output == expected)


def baseline_task(question: str) -> str:
    return "4"  # Replace with the deployed version.


def candidate_task(question: str) -> str:
    return "4" if "2 + 2" in question else "Paris"  # Replace with the candidate.


baseline = Eval(
    "baseline", dataset=suite.dataset.id, task=baseline_task, scores=[accuracy],
    config={"agent_revision": "example-v1"},
)
candidate = Eval(
    "candidate", dataset=suite.dataset.id, task=candidate_task, scores=[accuracy],
    config={"agent_revision": "example-v2"},
    baseline_experiment_id=baseline.experiment_id,
)

base_score = baseline.summary["scores"]["accuracy"]["mean"]
new_score = candidate.summary["scores"]["accuracy"]["mean"]
print(baseline.experiment_id, base_score)
print(candidate.experiment_id, new_score)
if new_score < base_score:
    raise SystemExit("candidate regressed")