Recipe: grade LLM outputs

curl https://api.thinqit.ai/v1/decide \
  -H "authorization: Bearer $DEX_API_KEY" \
  -H "content-type: application/json" \
  --data-binary @- <<'DEX_REQUEST'
{
  "model": "dex-1.0.1",
  "state": {
    "task": {
      "instruction": "Answer in exactly three bullet points: how does a customer reset their password?",
      "answer": "Sure! Resetting your password is easy. First, open the app and tap Settings. Then choose Account and tap Reset password. We will email you a link that works for 30 minutes. Let me know if you need anything else!"
    }
  },
  "questions": {
    "uses_bullets": {
      "type": "check",
      "instructions": "Is {{task.answer}} written as bullet points, as {{task.instruction}} asks?",
      "min_confidence": 0.6
    },
    "answers_question": {
      "type": "check",
      "instructions": "Does {{task.answer}} tell the customer how to reset their password?",
      "min_confidence": 0.6
    },
    "refusal": {
      "type": "check",
      "instructions": "Does {{task.answer}} refuse or avoid the task in {{task.instruction}}?",
      "min_confidence": 0.6
    },
    "helpfulness": {
      "type": "rate",
      "instructions": "How helpful is {{task.answer}} for a customer who wants to reset their password? Judge the content, not the form.",
      "levels": ["Unhelpful", "Somewhat helpful", "Helpful", "Very helpful"],
      "min_confidence": 0.3
    }
  }
}
DEX_REQUEST

# Save as dex_request.py, then run: python dex_request.py
import json

from thinqit_dex import Client

client = Client()  # reads DEX_API_KEY from the environment

request = json.loads(r'''
{
  "model": "dex-1.0.1",
  "state": {
    "task": {
      "instruction": "Answer in exactly three bullet points: how does a customer reset their password?",
      "answer": "Sure! Resetting your password is easy. First, open the app and tap Settings. Then choose Account and tap Reset password. We will email you a link that works for 30 minutes. Let me know if you need anything else!"
    }
  },
  "questions": {
    "uses_bullets": {
      "type": "check",
      "instructions": "Is {{task.answer}} written as bullet points, as {{task.instruction}} asks?",
      "min_confidence": 0.6
    },
    "answers_question": {
      "type": "check",
      "instructions": "Does {{task.answer}} tell the customer how to reset their password?",
      "min_confidence": 0.6
    },
    "refusal": {
      "type": "check",
      "instructions": "Does {{task.answer}} refuse or avoid the task in {{task.instruction}}?",
      "min_confidence": 0.6
    },
    "helpfulness": {
      "type": "rate",
      "instructions": "How helpful is {{task.answer}} for a customer who wants to reset their password? Judge the content, not the form.",
      "levels": ["Unhelpful", "Somewhat helpful", "Helpful", "Very helpful"],
      "min_confidence": 0.3
    }
  }
}
''')

decision = client.decide(
    request["state"],
    request["questions"],
    model=request["model"],
)
for question_id, answer in decision.answers.items():
    print(question_id, answer)

// Save as dex-request.mts, then run: npx tsx dex-request.mts (Node.js 18 or newer)
import { Client, parseRequest } from "@thinqit/dex";

const client = new Client(); // reads DEX_API_KEY from the environment

// parseRequest keeps the key order of the text (JSON.parse would move labels such as "1" to the front).
const request = parseRequest(`{
  "model": "dex-1.0.1",
  "state": {
    "task": {
      "instruction": "Answer in exactly three bullet points: how does a customer reset their password?",
      "answer": "Sure! Resetting your password is easy. First, open the app and tap Settings. Then choose Account and tap Reset password. We will email you a link that works for 30 minutes. Let me know if you need anything else!"
    }
  },
  "questions": {
    "uses_bullets": {
      "type": "check",
      "instructions": "Is {{task.answer}} written as bullet points, as {{task.instruction}} asks?",
      "min_confidence": 0.6
    },
    "answers_question": {
      "type": "check",
      "instructions": "Does {{task.answer}} tell the customer how to reset their password?",
      "min_confidence": 0.6
    },
    "refusal": {
      "type": "check",
      "instructions": "Does {{task.answer}} refuse or avoid the task in {{task.instruction}}?",
      "min_confidence": 0.6
    },
    "helpfulness": {
      "type": "rate",
      "instructions": "How helpful is {{task.answer}} for a customer who wants to reset their password? Judge the content, not the form.",
      "levels": ["Unhelpful", "Somewhat helpful", "Helpful", "Very helpful"],
      "min_confidence": 0.3
    }
  }
}`);

const decision = await client.decide(request);
console.log(decision.answers);

grade = {}
for question_id in ("uses_bullets", "answers_question", "refusal"):
    answer = decision.check(question_id)
    grade[question_id] = None if answer.abstained else answer.probability >= 0.5
helpfulness = decision.rate("helpfulness")
grade["helpfulness"] = None if helpfulness.abstained else helpfulness.rating

passed = grade["uses_bullets"] is True and grade["answers_question"] is True and grade["refusal"] is False
record_grade(case_id, decision.model, grade, passed)  # None: a person grades that property

const grade: Record<string, boolean | number | null> = {};
for (const id of ["uses_bullets", "answers_question", "refusal"]) {
  const answer = decision.answers[id];
  grade[id] = answer?.type === "check" && !answer.abstained ? answer.probability >= 0.5 : null;
}
const { helpfulness } = decision.answers;
grade.helpfulness = helpfulness?.type === "rate" && !helpfulness.abstained ? helpfulness.rating : null;

const passed = grade.uses_bullets === true && grade.answers_question === true && grade.refusal === false;
recordGrade(caseId, decision.model, grade, passed); // null: a person grades that property