{"openapi":"3.1.0","info":{"title":"Reactor Bench","version":"1","description":"Submit a scored Reactor Bench run (open, rate limited per IP, an email required), and review submissions (reviewer API keys). See /docs/api."},"servers":[{"url":"https://reactorbench.com"}],"components":{"securitySchemes":{"reviewerKey":{"type":"http","scheme":"bearer","description":"A reviewer API key, rbk_..."}},"schemas":{"File":{"type":"object","required":["name","size","sha256"],"properties":{"name":{"enum":["submission.zip","model/workspace.tar.gz","model/weights.tar"]},"size":{"type":"integer"},"sha256":{"type":"string"}}},"Origin":{"oneOf":[{"enum":["ref","upload"]},{"type":"object","required":["kind"],"properties":{"kind":{"enum":["ref","upload"]},"repo":{"type":"string","description":"The pinned repository's URL; it must match entry.json"},"token":{"type":"string","description":"A read-only token, for a private repository"}}}]}}},"paths":{"/api/benchmarks":{"get":{"summary":"The benchmarks, tracks and scorers that take submissions","responses":{"200":{"description":"reactor-bench's manifest"}}}},"/api/submissions":{"post":{"summary":"Submit a scored run","requestBody":{"content":{"application/json":{"schema":{"type":"object","required":["type","contact","source"],"properties":{"type":{"enum":["results-only","full"]},"contact":{"type":"object","description":"The submitter: an email, and optionally a GitHub handle and a name","required":["email"],"properties":{"email":{"type":"string"},"github":{"type":"string"},"first_name":{"type":"string","maxLength":100},"last_name":{"type":"string","maxLength":100}}},"source":{"oneOf":[{"type":"object","required":["kind","link"],"properties":{"kind":{"const":"hf"},"link":{"type":"string","description":"hf://datasets/<owner>/<name>@<commit>, as publish() returns"},"token":{"type":"string","description":"A read-only token, for a private dataset"},"files":{"type":"array","description":"A full submission's uploaded code or weights, when they are not in repositories","items":{"$ref":"#/components/schemas/File"}}}},{"type":"object","required":["kind","entry_json","result_json","files"],"properties":{"kind":{"const":"upload"},"entry_json":{"type":"string","description":"entry.json, as text"},"result_json":{"type":"string","description":"result.json, as text"},"files":{"type":"array","items":{"$ref":"#/components/schemas/File"}}}}]},"model":{"type":"object","description":"A full submission only: where its code and weights come from. Each is the repository entry.json pins (\"ref\", the default), optionally with that repository and a read-only token when it is private, or an uploaded file (\"upload\").","properties":{"workspace":{"$ref":"#/components/schemas/Origin"},"weights":{"$ref":"#/components/schemas/Origin"}}}}}}}},"responses":{"201":{"description":"The submission's ID; for an upload, the files to send","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string","format":"uuid"},"status":{"enum":["uploading","received"]},"uploads":{"type":"array","items":{"type":"object"}}}}}}},"400":{"description":"What to fix","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"}}}}}},"429":{"description":"What to fix","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"}}}}}}}}},"/api/submissions/{id}/parts":{"post":{"summary":"Presigned URLs to PUT parts of a file to R2","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string","format":"uuid"}}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","required":["name"],"properties":{"name":{"type":"string"},"from":{"type":"integer"},"to":{"type":"integer"}}}}}},"responses":{"200":{"description":"One URL per part; PUT each part's bytes and keep the ETag header"},"400":{"description":"What to fix","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"}}}}}},"409":{"description":"What to fix","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"}}}}}}}}},"/api/submissions/{id}/files":{"post":{"summary":"Close a file's upload","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string","format":"uuid"}}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","required":["name","parts"],"properties":{"name":{"type":"string"},"parts":{"type":"array","items":{"type":"object","properties":{"part":{"type":"integer"},"etag":{"type":"string"}}}}}}}}},"responses":{"200":{"description":"The file arrived"},"400":{"description":"What to fix","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"}}}}}},"409":{"description":"What to fix","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"}}}}}}}}},"/api/review/me":{"get":{"summary":"Who the reviewer key belongs to","security":[{"reviewerKey":[]}],"responses":{"200":{"description":"The reviewer"},"401":{"description":"What to fix","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"}}}}}}}}},"/api/review/submissions":{"get":{"summary":"The review queue","security":[{"reviewerKey":[]}],"parameters":[{"name":"status","in":"query","schema":{"enum":["uploading","received","claimed","verified","published","rejected"]}}],"responses":{"200":{"description":"Submissions, newest first"},"401":{"description":"What to fix","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"}}}}}}}}},"/api/review/submissions/{id}":{"get":{"summary":"Everything about a submission, and its stages in order","security":[{"reviewerKey":[]}],"parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string","format":"uuid"}}],"responses":{"200":{"description":"The submission"},"404":{"description":"What to fix","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"}}}}}}}}},"/api/review/submissions/{id}/claim":{"post":{"summary":"Claim a submission for review","security":[{"reviewerKey":[]}],"parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string","format":"uuid"}}],"responses":{"200":{"description":"Claimed"},"409":{"description":"What to fix","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"}}}}}}}}},"/api/review/submissions/{id}/release":{"post":{"summary":"Give a claimed submission back to the queue","security":[{"reviewerKey":[]}],"parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string","format":"uuid"}}],"responses":{"200":{"description":"Released"},"409":{"description":"What to fix","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"}}}}}}}}},"/api/review/submissions/{id}/download":{"post":{"summary":"The link and token, or presigned download URLs; recorded","security":[{"reviewerKey":[]}],"parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string","format":"uuid"}}],"responses":{"200":{"description":"How to fetch the submission"}}}},"/api/review/submissions/{id}/stages/{stage}":{"put":{"summary":"Report one review stage","security":[{"reviewerKey":[]}],"parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string","format":"uuid"}},{"name":"stage","in":"path","required":true,"schema":{"type":"string"}}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","required":["status"],"properties":{"status":{"enum":["pending","running","completed","failed","skipped"]},"detail":{"type":"string"},"done":{"type":"integer"},"total":{"type":"integer"}}}}}},"responses":{"200":{"description":"The stage"},"400":{"description":"What to fix","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"}}}}}},"409":{"description":"What to fix","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"}}}}}}}}},"/api/review/submissions/{id}/verification":{"post":{"summary":"Upload verification.json","security":[{"reviewerKey":[]}],"parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string","format":"uuid"}}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","description":"verification.json, format reactor-bench/verification"}}}},"responses":{"200":{"description":"Whether it was accepted, and the problems"},"400":{"description":"What to fix","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"}}}}}},"409":{"description":"What to fix","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"}}}}}}}}},"/api/submissions/{id}/complete":{"post":{"summary":"Queue an uploaded submission for review","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string","format":"uuid"}}],"responses":{"200":{"description":"Received"},"409":{"description":"What to fix","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"}}}}}}}}}}}