Eval runs

Author eval cases from a document

The sibling of cases/generate: generation invents cases from the suite’s tools, import reads them out of something a person already wrote — a test plan, a spreadsheet of scenarios, a JSON export. MCPJam’s own model turns the document into full cases (ordered steps, tool calls, assertions), so the caller does not have to guess the case shape. No format is declared: the model reads the document’s shape itself, so Markdown, JSON, CSV and plain notes all work.

Spends the organization’s credits, unlike generation, which is on MCPJam. Pass an idempotency key so a retry replays the same job instead of authoring — and billing — the document twice.

Like cases/generate, this discovers the suite’s tools over a live MCP connection first, so the authored cases are grounded in tools the suite can actually call.

Answers 200 when the job finishes inside the wait window and 202 with a jobId when it does not — poll the job, then commit it. A case the commit could not write comes back under skipped, with a reviewUrl that opens the app on exactly those drafts.

POST

projects

{projectId}

eval-suites

{suiteId}

cases

import

Author eval cases from a document

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/import \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "content": "<string>",
  "fileName": "<string>",
  "servers": [
    "<string>"
  ],
  "environmentId": "<string>",
  "caseModels": [
    {
      "model": "<string>",
      "provider": "<string>"
    }
  ],
  "overrideReason": "<string>",
  "idempotencyKey": "<string>"
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/import"

payload = {
    "content": "<string>",
    "fileName": "<string>",
    "servers": ["<string>"],
    "environmentId": "<string>",
    "caseModels": [
        {
            "model": "<string>",
            "provider": "<string>"
        }
    ],
    "overrideReason": "<string>",
    "idempotencyKey": "<string>"
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    content: '<string>',
    fileName: '<string>',
    servers: ['<string>'],
    environmentId: '<string>',
    caseModels: [{model: '<string>', provider: '<string>'}],
    overrideReason: '<string>',
    idempotencyKey: '<string>'
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/import', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/import",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'content' => '<string>',
    'fileName' => '<string>',
    'servers' => [
        '<string>'
    ],
    'environmentId' => '<string>',
    'caseModels' => [
        [
                'model' => '<string>',
                'provider' => '<string>'
        ]
    ],
    'overrideReason' => '<string>',
    'idempotencyKey' => '<string>'
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/import"

	payload := strings.NewReader("{\n  \"content\": \"<string>\",\n  \"fileName\": \"<string>\",\n  \"servers\": [\n    \"<string>\"\n  ],\n  \"environmentId\": \"<string>\",\n  \"caseModels\": [\n    {\n      \"model\": \"<string>\",\n      \"provider\": \"<string>\"\n    }\n  ],\n  \"overrideReason\": \"<string>\",\n  \"idempotencyKey\": \"<string>\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/import")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"content\": \"<string>\",\n  \"fileName\": \"<string>\",\n  \"servers\": [\n    \"<string>\"\n  ],\n  \"environmentId\": \"<string>\",\n  \"caseModels\": [\n    {\n      \"model\": \"<string>\",\n      \"provider\": \"<string>\"\n    }\n  ],\n  \"overrideReason\": \"<string>\",\n  \"idempotencyKey\": \"<string>\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/import")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"content\": \"<string>\",\n  \"fileName\": \"<string>\",\n  \"servers\": [\n    \"<string>\"\n  ],\n  \"environmentId\": \"<string>\",\n  \"caseModels\": [\n    {\n      \"model\": \"<string>\",\n      \"provider\": \"<string>\"\n    }\n  ],\n  \"overrideReason\": \"<string>\",\n  \"idempotencyKey\": \"<string>\"\n}"

response = http.request(request)
puts response.read_body

{
  "generationModel": "<string>",
  "created": [
    {
      "id": "<string>",
      "title": "<string>",
      "steps": [
        {
          "id": "<string>",
          "kind": "prompt",
          "prompt": "<string>",
          "serverName": "<string>",
          "toolName": "<string>",
          "arguments": {},
          "action": {},
          "assertion": {}
        }
      ],
      "iterations": 5,
      "isNegative": true,
      "models": [
        {
          "model": "<string>",
          "provider": "<string>"
        }
      ],
      "judge": {
        "enabled": true
      },
      "declaredId": "<string>",
      "expectedOutput": "<string>",
      "repetitions": 2,
      "passThreshold": 0.5,
      "scenario": "<string>",
      "intent": "<string>",
      "kind": "capability",
      "matchOptions": {},
      "checks": {
        "mode": "inherit",
        "list": [
          {
            "type": "toolDescriptionsPresent",
            "minLength": 123,
            "role": "gating",
            "severity": "warn"
          }
        ]
      },
      "import": {
        "status": "exact",
        "sourceCaseKey": "<string>",
        "note": "<string>"
      },
      "source": {
        "format": "markdown",
        "method": "ai",
        "fileName": "<string>",
        "fileHash": "<string>",
        "excerpt": "<string>",
        "startLine": 2,
        "endLine": 2,
        "extractorVersion": "<string>"
      },
      "createdAt": 123,
      "updatedAt": 123,
      "suppressedSuiteStandardCheckIds": [
        "<string>"
      ]
    }
  ],
  "counts": {
    "normal": 123,
    "negative": 123
  },
  "skipped": [
    {}
  ],
  "reviewUrl": "<string>"
}

Authorizations

Authorization

string

header

required

MCPJam API key (sk_…). Create one at Settings → API keys. Guest sessions cannot use the API, and API keys cannot manage other API keys.

Headers

x-mcpjam-idempotency-key

string

Makes a retry replay the recorded job instead of re-authoring — and re-billing — the document.

Path Parameters

projectId

string

required

ID of the hosted project that contains the server.

suiteId

string

required

Eval suite ID, as returned by POST /eval-runs.

Body

application/json

content

string

required

The document, as UTF-8 text. Markdown, JSON, CSV or plain notes — no format is declared, because the authoring model reads the document's shape itself. At most 100 KiB.

Maximum string length: 102400

fileName

string

Recorded on each case's provenance so a reviewer can find the source. Optional: a pasted document has no file behind it.

Required string length: 1 - 255

servers

string[]

Server ids or names to discover tools from. Mutually exclusive with environmentId.

environmentId

string

Author against this environment's closed server set. Mutually exclusive with servers.

caseModels

object[]

Show child attributes

duplicatePolicy

enum<string>

Applied when the cases are written, not when the job starts. Anything but block requires overrideReason.

Available options:

block,

warn,

create_anyway

overrideReason

string

Minimum string length: 1

idempotencyKey

string

Makes a retry replay the recorded job instead of authoring — and billing — the document a second time.

Required string length: 1 - 256

Response

The cases the job authored and committed, or the job's own status when it finished without committing (failed, cancelled, or still pending at the end of the wait).

Option 1
Option 2

generationModel

string

required

created

object[]

required

Show child attributes

counts

object

required

Show child attributes

skipped

object[]

Cases the commit could not write, each with its reason. They stay on the job rather than being lost.

reviewUrl

string<uri>

Opens the app on exactly the skipped drafts. Present only when something was skipped — finishing them there costs nothing, whereas re-sending the document re-authors and re-bills every case in it.

⌘I

Author eval cases from a document

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/import \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "content": "<string>",
  "fileName": "<string>",
  "servers": [
    "<string>"
  ],
  "environmentId": "<string>",
  "caseModels": [
    {
      "model": "<string>",
      "provider": "<string>"
    }
  ],
  "overrideReason": "<string>",
  "idempotencyKey": "<string>"
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/import"

payload = {
    "content": "<string>",
    "fileName": "<string>",
    "servers": ["<string>"],
    "environmentId": "<string>",
    "caseModels": [
        {
            "model": "<string>",
            "provider": "<string>"
        }
    ],
    "overrideReason": "<string>",
    "idempotencyKey": "<string>"
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    content: '<string>',
    fileName: '<string>',
    servers: ['<string>'],
    environmentId: '<string>',
    caseModels: [{model: '<string>', provider: '<string>'}],
    overrideReason: '<string>',
    idempotencyKey: '<string>'
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/import', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/import",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'content' => '<string>',
    'fileName' => '<string>',
    'servers' => [
        '<string>'
    ],
    'environmentId' => '<string>',
    'caseModels' => [
        [
                'model' => '<string>',
                'provider' => '<string>'
        ]
    ],
    'overrideReason' => '<string>',
    'idempotencyKey' => '<string>'
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/import"

	payload := strings.NewReader("{\n  \"content\": \"<string>\",\n  \"fileName\": \"<string>\",\n  \"servers\": [\n    \"<string>\"\n  ],\n  \"environmentId\": \"<string>\",\n  \"caseModels\": [\n    {\n      \"model\": \"<string>\",\n      \"provider\": \"<string>\"\n    }\n  ],\n  \"overrideReason\": \"<string>\",\n  \"idempotencyKey\": \"<string>\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/import")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"content\": \"<string>\",\n  \"fileName\": \"<string>\",\n  \"servers\": [\n    \"<string>\"\n  ],\n  \"environmentId\": \"<string>\",\n  \"caseModels\": [\n    {\n      \"model\": \"<string>\",\n      \"provider\": \"<string>\"\n    }\n  ],\n  \"overrideReason\": \"<string>\",\n  \"idempotencyKey\": \"<string>\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/{suiteId}/cases/import")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"content\": \"<string>\",\n  \"fileName\": \"<string>\",\n  \"servers\": [\n    \"<string>\"\n  ],\n  \"environmentId\": \"<string>\",\n  \"caseModels\": [\n    {\n      \"model\": \"<string>\",\n      \"provider\": \"<string>\"\n    }\n  ],\n  \"overrideReason\": \"<string>\",\n  \"idempotencyKey\": \"<string>\"\n}"

response = http.request(request)
puts response.read_body

{
  "generationModel": "<string>",
  "created": [
    {
      "id": "<string>",
      "title": "<string>",
      "steps": [
        {
          "id": "<string>",
          "kind": "prompt",
          "prompt": "<string>",
          "serverName": "<string>",
          "toolName": "<string>",
          "arguments": {},
          "action": {},
          "assertion": {}
        }
      ],
      "iterations": 5,
      "isNegative": true,
      "models": [
        {
          "model": "<string>",
          "provider": "<string>"
        }
      ],
      "judge": {
        "enabled": true
      },
      "declaredId": "<string>",
      "expectedOutput": "<string>",
      "repetitions": 2,
      "passThreshold": 0.5,
      "scenario": "<string>",
      "intent": "<string>",
      "kind": "capability",
      "matchOptions": {},
      "checks": {
        "mode": "inherit",
        "list": [
          {
            "type": "toolDescriptionsPresent",
            "minLength": 123,
            "role": "gating",
            "severity": "warn"
          }
        ]
      },
      "import": {
        "status": "exact",
        "sourceCaseKey": "<string>",
        "note": "<string>"
      },
      "source": {
        "format": "markdown",
        "method": "ai",
        "fileName": "<string>",
        "fileHash": "<string>",
        "excerpt": "<string>",
        "startLine": 2,
        "endLine": 2,
        "extractorVersion": "<string>"
      },
      "createdAt": 123,
      "updatedAt": 123,
      "suppressedSuiteStandardCheckIds": [
        "<string>"
      ]
    }
  ],
  "counts": {
    "normal": 123,
    "negative": 123
  },
  "skipped": [
    {}
  ],
  "reviewUrl": "<string>"
}

API Guides

Endpoints

Author eval cases from a document

Authorizations

Headers

Path Parameters

Body

Response