noulscorechoiceHarness
evaluate — an MCP server that hands agents typed judgments instead of prose
An MCP server, written in Go, that lets Claude Code, Claude Desktop, Codex and pi call Jev directly and get back probabilities they can branch on.
The problem it targets is specific. When an agent needs a yes/no call, a routing decision or a severity rating, it usually asks an LLM, parses the prose and hopes the format holds — and the answer carries no probability, so the agent cannot tell a confident yes from a coin flip.
evaluate exposes one tool. State plus typed questions go to Jev; structured answers with probabilities come back. Nothing to parse, and no prompt formatting to keep in sync.
Claude Code / Desktop / Codex / pi --evaluate (stdio)--> evaluate (MCP)
--POST /v1/systemone--> TypeSafe API
retries 429 / 529 or OpenRouter
It carries all three primitives through to the caller rather than fixing a question shape, so the agent decides whether it wants a Noul, a Score or a Choice. Retries on 429 and 529 are handled in the server, and evaluate setup mcp registers it with whichever of the three clients it detects.
Source (7 files)
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
)
// maxBody caps response size so an oversized reply cannot exhaust memory.
const maxBody = 16 << 20
// Client calls a Jev evaluation endpoint: the TypeSafe API directly, or
// OpenRouter's Decisions router. Both take the same request body.
type Client struct {
// URL is the full endpoint, not a base. Model is the route's default.
URL, APIKey, Model string
HTTP *http.Client
// Backoff is the first retry delay for 429/529; it doubles each attempt.
Backoff time.Duration
}
// Evaluate posts a System One request and returns the raw response JSON.
// 429 and 529 are retried with exponential backoff; other non-2xx statuses
// become errors carrying the API's error body.
func (c *Client) Evaluate(ctx context.Context, req any) ([]byte, error) {
body, err := json.Marshal(req)
if err != nil {
return nil, err
}
delay := c.Backoff
for attempt := 0; ; attempt++ {
r, err := http.NewRequestWithContext(ctx, http.MethodPost, c.URL, bytes.NewReader(body))
if err != nil {
return nil, err
}
r.Header.Set("Authorization", "Bearer "+c.APIKey)
r.Header.Set("Content-Type", "application/json")
resp, err := c.HTTP.Do(r)
if err != nil {
return nil, err
}
b, err := io.ReadAll(io.LimitReader(resp.Body, maxBody))
resp.Body.Close()
if err != nil {
return nil, err
}
if resp.StatusCode/100 == 2 {
return b, nil
}
retryable := resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == 529
if !retryable || attempt == 3 {
return nil, fmt.Errorf("evaluate: %s: %s", resp.Status, b)
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(delay):
}
delay *= 2
}
}
package main
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"github.com/modelcontextprotocol/go-sdk/mcp"
"os"
"path/filepath"
"slices"
"strings"
"testing"
)
func TestEvaluate(t *testing.T) {
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
if r.URL.Path != "/v1/systemone" || r.Header.Get("Authorization") != "Bearer k" {
t.Errorf("bad request: %s %q", r.URL.Path, r.Header.Get("Authorization"))
}
var in evaluateIn
if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
t.Fatal(err)
}
switch in.State {
case "busy":
if calls == 1 {
w.WriteHeader(529)
return
}
case "bad":
w.WriteHeader(http.StatusUnprocessableEntity)
w.Write([]byte(`{"detail":"criteria required"}`))
return
}
w.Write([]byte(`{"model":"` + in.Model + `","answers":{"q":{"type":"noul","noul":0.9}}}`))
}))
defer srv.Close()
c := &Client{URL: srv.URL + "/v1/systemone", APIKey: "k", HTTP: srv.Client()}
req := evaluateIn{Model: "jev-latest", Questions: map[string]question{"q": {Type: "noul", Instructions: "urgent?"}}}
req.State = "busy"
b, err := c.Evaluate(context.Background(), req)
if err != nil || !strings.Contains(string(b), `"noul":0.9`) || calls != 2 {
t.Fatalf("retry: calls=%d b=%s err=%v", calls, b, err)
}
req.State = "bad"
if _, err := c.Evaluate(context.Background(), req); err == nil || !strings.Contains(err.Error(), "criteria required") {
t.Fatalf("422: err=%v", err)
}
}
// The endpoint is taken verbatim, so a route's full URL reaches the server.
func TestEvaluatePostsToURL(t *testing.T) {
var got string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
got = r.URL.Path
w.Write([]byte(`{}`))
}))
defer srv.Close()
c := &Client{URL: srv.URL + "/api/alpha/decisions", APIKey: "k", HTTP: srv.Client()}
if _, err := c.Evaluate(context.Background(), evaluateIn{State: "x"}); err != nil {
t.Fatal(err)
}
if got != "/api/alpha/decisions" {
t.Fatalf("path = %q", got)
}
}
func TestRoute(t *testing.T) {
for _, tc := range []struct{ typesafe, openrouter, url, model, key string }{
{"t", "", "https://api.typesafe.ai/v1/systemone", "jev-latest", "t"},
{"", "o", "https://openrouter.ai/api/alpha/decisions", "~typesafe/jev-latest", "o"},
// TypeSafe wins so a stray OpenRouter key cannot reroute an existing setup.
{"t", "o", "https://api.typesafe.ai/v1/systemone", "jev-latest", "t"},
} {
t.Setenv("TYPESAFE_API_KEY", tc.typesafe)
t.Setenv("OPENROUTER_API_KEY", tc.openrouter)
c, err := route()
if err != nil {
t.Fatalf("%+v: %v", tc, err)
}
// APIKey too: each route must send the key that selected it.
if c.URL != tc.url || c.Model != tc.model || c.APIKey != tc.key {
t.Errorf("%+v: got %s %s %s", tc, c.URL, c.Model, c.APIKey)
}
}
t.Setenv("TYPESAFE_API_KEY", "")
t.Setenv("OPENROUTER_API_KEY", "")
if _, err := route(); err == nil {
t.Fatal("no keys: want error")
}
}
func TestSetupEnv(t *testing.T) {
got := setupEnv([]string{
"PATH=/bin", "TYPESAFE_API_KEY=k", "OPENROUTER_API_KEY_OTHER=no",
"TYPESAFE_OTHER=s", "OPENROUTER_BASE_URL=no", "OPENROUTER_API_KEY=o=o",
})
want := []string{"TYPESAFE_API_KEY=k", "TYPESAFE_OTHER=s", "OPENROUTER_API_KEY=o=o"}
if !slices.Equal(got, want) {
t.Fatalf("got %q\nwant %q", got, want)
}
}
func TestSetupCommands(t *testing.T) {
cmds := setupCommands("/bin/evaluate", []string{"TYPESAFE_API_KEY=k", "OPENROUTER_API_KEY=o"})
want := [][]string{
{"mcp", "remove", "evaluate", "-s", "user"},
{"mcp", "add", "evaluate", "-s", "user", "-e", "TYPESAFE_API_KEY=k", "-e", "OPENROUTER_API_KEY=o", "--", "/bin/evaluate", "mcp"},
{"mcp", "remove", "jev", "-s", "user"},
nil,
{"mcp", "add", "evaluate", "--env", "TYPESAFE_API_KEY=k", "--env", "OPENROUTER_API_KEY=o", "--", "/bin/evaluate", "mcp"},
{"mcp", "remove", "jev"},
}
got := [][]string{cmds[0].reset, cmds[0].add, cmds[0].legacy, cmds[1].reset, cmds[1].add, cmds[1].legacy}
if !slices.EqualFunc(got, want, slices.Equal) {
t.Fatalf("got %q\nwant %q", got, want)
}
}
func TestSetupClaudeDesktop(t *testing.T) {
path := filepath.Join(t.TempDir(), "claude_desktop_config.json")
seed := `{"mcpServers":{"lumi":{"command":"/bin/lumi"},"evaluate":{"command":"/old"},"jev":{"command":"/gone"}},"preferences":{"sidebarMode":"chat"}}`
if err := os.WriteFile(path, []byte(seed), 0o600); err != nil {
t.Fatal(err)
}
if err := setupClaudeDesktop(path, "/bin/evaluate", []string{"TYPESAFE_API_KEY=k"}); err != nil {
t.Fatal(err)
}
b, _ := os.ReadFile(path)
var got struct {
MCPServers map[string]struct {
Command string `json:"command"`
Args []string `json:"args"`
Env map[string]string `json:"env"`
} `json:"mcpServers"`
Preferences map[string]string `json:"preferences"`
}
if err := json.Unmarshal(b, &got); err != nil {
t.Fatal(err)
}
s := got.MCPServers["evaluate"]
if s.Command != "/bin/evaluate" || !slices.Equal(s.Args, []string{"mcp"}) || s.Env["TYPESAFE_API_KEY"] != "k" {
t.Fatalf("evaluate entry = %+v", s)
}
if got.MCPServers["lumi"].Command != "/bin/lumi" || got.Preferences["sidebarMode"] != "chat" {
t.Fatalf("other keys lost: %s", b)
}
// The pre-rename entry has to go, or the client keeps launching /gone.
if _, ok := got.MCPServers["jev"]; ok {
t.Fatalf("legacy jev entry kept: %s", b)
}
for _, seed := range []string{`null`, `{"mcpServers":null}`} {
if err := os.WriteFile(path, []byte(seed), 0o600); err != nil {
t.Fatal(err)
}
if err := setupClaudeDesktop(path, "/bin/evaluate", nil); err != nil {
t.Fatalf("seed %s: %v", seed, err)
}
}
}
// A tar member named "evaluate" that is a symlink (or any other non-regular entry)
// must not be extracted and installed over the running binary.
func TestExtractBinaryRejectsNonRegularMember(t *testing.T) {
var buf bytes.Buffer
gw := gzip.NewWriter(&buf)
tw := tar.NewWriter(gw)
if err := tw.WriteHeader(&tar.Header{
Name: "evaluate",
Typeflag: tar.TypeSymlink,
Linkname: "/etc/passwd",
Mode: 0o777,
}); err != nil {
t.Fatal(err)
}
for _, closer := range []func() error{tw.Close, gw.Close} {
if err := closer(); err != nil {
t.Fatal(err)
}
}
if err := extractBinaryFromTar(buf.Bytes(), "evaluate", io.Discard); err == nil || !strings.Contains(err.Error(), "not a regular file") {
t.Fatalf("expected non-regular member to be rejected, got %v", err)
}
}
func TestWritePiExtension(t *testing.T) {
dir := t.TempDir()
// A pre-rename install: both files would register the tool `evaluate`.
legacy := filepath.Join(dir, "extensions", "jev.ts")
if err := os.MkdirAll(filepath.Dir(legacy), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(legacy, []byte("// old"), 0o600); err != nil {
t.Fatal(err)
}
path, err := writePiExtension(dir, `/bin/je"v`)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(legacy); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("legacy jev.ts kept: %v", err)
}
if want := filepath.Join(dir, "extensions", "evaluate.ts"); path != want {
t.Fatalf("path = %q, want %q", path, want)
}
b, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
src := string(b)
// The quote in the path must come back escaped, not as a broken literal.
if !strings.Contains(src, `const BINARY = "/bin/je\"v"`) {
t.Fatalf("binary path not rendered: %s", src)
}
if !strings.Contains(src, "A noul near 0.5 means uncertain") {
t.Fatal("instructions not rendered")
}
if strings.Contains(src, "__EVALUATE_") {
t.Fatal("placeholder left behind")
}
}
func TestPiDir(t *testing.T) {
home, err := os.UserHomeDir()
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct{ env, want string }{
{"", filepath.Join(home, ".pi", "agent")},
// pi expands "~" itself, so taking it literally would miss a real install.
{"~", home},
{"~/.pi/agent", filepath.Join(home, ".pi", "agent")},
{"/tmp/pi", "/tmp/pi"},
} {
t.Setenv("PI_CODING_AGENT_DIR", tc.env)
got, err := piDir()
if err != nil {
t.Fatal(err)
}
if got != tc.want {
t.Errorf("piDir() with %q = %q, want %q", tc.env, got, tc.want)
}
}
// An absolute override must not need a home directory: sanitized
// environments (env -i PI_CODING_AGENT_DIR=...) have none.
t.Setenv("HOME", "")
t.Setenv("PI_CODING_AGENT_DIR", "/tmp/pi")
if got, err := piDir(); err != nil || got != "/tmp/pi" {
t.Errorf("piDir() without HOME = %q, %v, want %q, nil", got, err, "/tmp/pi")
}
}
// Malformed criteria must be caught here, with the path the caller wrote. The
// API reports this as "questions.<id>.score.criteria" — a union branch, not a
// property the request ever had.
func TestValidate(t *testing.T) {
obj := map[string]any{"0": "low", "1": "high"}
arr := []any{"low", "high"}
for _, tc := range []struct {
name string
q question
want string
}{
{"score object", question{Type: "score", Criteria: obj}, `questions["q"].criteria: score criteria must be an array of level descriptions, ordered low to high, got an object`},
{"score missing", question{Type: "score"}, "got nothing"},
{"score string", question{Type: "score", Criteria: "high"}, "got a string"},
{"choice array", question{Type: "choice", Criteria: arr}, `questions["q"].criteria: choice criteria must be an object`},
{"noul array", question{Type: "noul", Criteria: arr}, `questions["q"].criteria: noul criteria must be an object`},
{"score array", question{Type: "score", Criteria: arr}, ""},
// One level is accepted by the API, so it must not be rejected here.
{"score one level", question{Type: "score", Criteria: []any{"only"}}, ""},
{"choice object", question{Type: "choice", Criteria: obj}, ""},
{"noul object", question{Type: "noul", Criteria: obj}, ""},
{"noul omitted", question{Type: "noul"}, ""},
// The server enumerates types this tool does not document; rejecting an
// unknown type here would break every one of them.
{"unknown type", question{Type: "bounding_box", Criteria: obj}, ""},
} {
tc.q.Instructions = "x"
err := validate(evaluateIn{State: "s", Questions: map[string]question{"q": tc.q}})
switch {
case tc.want == "" && err != nil:
t.Errorf("%s: want nil, got %v", tc.name, err)
case tc.want != "" && err == nil:
t.Errorf("%s: want %q, got nil", tc.name, tc.want)
case tc.want != "" && err != nil && !strings.Contains(err.Error(), tc.want):
t.Errorf("%s: got %q, want it to contain %q", tc.name, err, tc.want)
}
}
}
// TestValidateBlocksRequest drives the registered tool with real JSON arguments,
// so it covers what TestValidate cannot: that the SDK decodes criteria into the
// types validate type-switches on, that validate runs *before* the HTTP call,
// and that a type this tool does not know still reaches the API.
func TestValidateBlocksRequest(t *testing.T) {
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
calls++
w.Write([]byte(`{"answers":{}}`))
}))
defer srv.Close()
s := mcp.NewServer(&mcp.Implementation{Name: "evaluate", Version: "test"}, nil)
registerTools(s, &Client{URL: srv.URL, APIKey: "k", HTTP: srv.Client(), Model: "m"})
ct, st := mcp.NewInMemoryTransports()
ctx := context.Background()
ss, err := s.Connect(ctx, st, nil)
if err != nil {
t.Fatal(err)
}
defer ss.Close()
cs, err := mcp.NewClient(&mcp.Implementation{Name: "t", Version: "1"}, nil).Connect(ctx, ct, nil)
if err != nil {
t.Fatal(err)
}
defer cs.Close()
for _, tc := range []struct {
name, criteria, qtype, wantErr string
wantCalls int
}{
// The reported bug: an index-keyed object, the shape the response legend
// comes back as. It must never reach the API.
{"score object", `{"0":"low","1":"high"}`, "score", `questions["q"].criteria`, 0},
{"score array", `["low","high"]`, "score", "", 1},
// The API accepts one level, so this must not be rejected locally.
{"score one level", `["only"]`, "score", "", 1},
// The API enumerates types this tool does not document; they pass through.
{"unknown type", `{"0":"low"}`, "bounding_box", "", 1},
} {
calls = 0
args := `{"state":"s","questions":{"q":{"type":"` + tc.qtype +
`","instructions":"i","criteria":` + tc.criteria + `}}}`
res, err := cs.CallTool(ctx, &mcp.CallToolParams{Name: "evaluate", Arguments: json.RawMessage(args)})
if err != nil {
t.Fatalf("%s: %v", tc.name, err)
}
got := res.Content[0].(*mcp.TextContent).Text
if (tc.wantErr != "") != res.IsError {
t.Errorf("%s: IsError=%v, got %q", tc.name, res.IsError, got)
}
if tc.wantErr != "" && !strings.Contains(got, tc.wantErr) {
t.Errorf("%s: got %q, want it to contain %q", tc.name, got, tc.wantErr)
}
if calls != tc.wantCalls {
t.Errorf("%s: %d HTTP calls, want %d", tc.name, calls, tc.wantCalls)
}
}
}
// Command evaluate is an MCP stdio server for running TypeSafe Jev prompts.
package main
import (
"context"
"errors"
"fmt"
"net/http"
"os"
"os/signal"
"runtime/debug"
"syscall"
"time"
"github.com/modelcontextprotocol/go-sdk/mcp"
"github.com/spf13/cobra"
)
// version is set by release builds via -ldflags "-X main.version=...".
var version = "dev"
func init() {
if info, ok := debug.ReadBuildInfo(); ok && version == "dev" && info.Main.Version != "" && info.Main.Version != "(devel)" {
version = info.Main.Version
}
}
const instructions = `The evaluate tool runs Jev, a TypeSafe System One model that returns typed judgments and probabilities, not generated text.
- Question types: noul (probability a yes/no condition holds), choice (one option from a criteria map), score (probability-weighted position on ordered criteria levels).
- Ask one narrow judgment per question. Question ids are NOT sent to the model, so instructions must carry the full meaning.
- Put everything the judgment needs in state; prefer a JSON object with named fields, and reference nested fields with backticked paths like ` + "`ticket.messages[0].text`" + `.
- State is what you observed — the raw ticket, diff, log, or field values, or a faithful condensation of them. Keep the uncertainty and the counterevidence; leave out your own verdict, which Jev reads as evidence.
- Write instructions that name the condition to test, not the conclusion you expect. "Does the message report a failed payout?", not "Confirm this urgent payout failure."
- A conclusion asserted in state biases the answer toward it, and the confidence that comes back is then agreement with yourself, not independent corroboration.
- Batch independent questions over the same state into one call; they run in parallel and cannot see each other's answers.
- Include a no-match option in a choice when nothing may fit. Score levels must describe concrete situations.
- A noul near 0.5 means uncertain, not medium intensity. Confidence measures how concentrated the distribution is, not correctness.
- Score criteria are an ordered array; the answer is 0-indexed, so N levels score 0 to N-1. A 3.87 over 5 levels sits between levels 3 and 4, not 3.87/5. Report it with the labels from the response ` + "`legend`" + `, and read ` + "`probabilities`" + ` alongside it.
Docs: https://docs.typesafe.ai/llms.txt`
func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
err := newRootCmd().ExecuteContext(ctx)
stop()
if err != nil {
fmt.Fprintln(os.Stderr, "evaluate:", err)
os.Exit(1)
}
}
func newRootCmd() *cobra.Command {
root := &cobra.Command{
Use: "evaluate",
Short: "MCP server for TypeSafe Jev prompts",
Version: version,
// Stdout carries the MCP protocol; main reports errors on stderr.
SilenceUsage: true,
SilenceErrors: true,
}
root.SetVersionTemplate("{{.Version}}\n")
mcpCmd := &cobra.Command{Use: "mcp", Short: "Run the MCP server over stdio", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error {
return serve(cmd.Context())
}}
// Args+RunE, not a bare parent: cobra checks Runnable before validating args,
// so without both `evaluate setup typo` prints help and exits 0.
setupCmd := &cobra.Command{
Use: "setup",
Short: "Register this binary with your agents",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error { return cmd.Help() },
}
setupCmd.AddCommand(
&cobra.Command{
Use: "mcp",
Short: "Register with Claude Code, Claude Desktop, and Codex",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
return runMCPSetup(cmd.Context())
},
},
&cobra.Command{
Use: "pi",
Short: "Install the evaluate extension for pi",
Args: cobra.NoArgs,
RunE: func(*cobra.Command, []string) error {
return runPiSetup()
},
},
)
root.AddCommand(
mcpCmd,
setupCmd,
&cobra.Command{Use: "update", Short: "Update evaluate to the latest release", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error {
return runUpdate(cmd.Context())
}},
&cobra.Command{Use: "version", Short: "Print the version", Args: cobra.NoArgs, Run: func(*cobra.Command, []string) {
fmt.Println(version)
}},
)
return root
}
// route picks the evaluation endpoint from the environment: the TypeSafe API
// when TYPESAFE_API_KEY is set, otherwise OpenRouter's Decisions router.
// TypeSafe wins when both are set, so an OPENROUTER_API_KEY left in the shell
// by another tool cannot silently reroute and re-bill an existing setup.
func route() (*Client, error) {
switch {
case os.Getenv("TYPESAFE_API_KEY") != "":
return &Client{
URL: "https://api.typesafe.ai/v1/systemone",
APIKey: os.Getenv("TYPESAFE_API_KEY"),
Model: "jev-latest",
}, nil
case os.Getenv("OPENROUTER_API_KEY") != "":
return &Client{
URL: "https://openrouter.ai/api/alpha/decisions",
APIKey: os.Getenv("OPENROUTER_API_KEY"),
Model: "~typesafe/jev-latest",
}, nil
}
return nil, errors.New("set TYPESAFE_API_KEY (https://console.typesafe.ai/) or OPENROUTER_API_KEY (https://openrouter.ai/keys)")
}
func serve(ctx context.Context) error {
c, err := route()
if err != nil {
return err
}
c.HTTP = &http.Client{Timeout: 60 * time.Second}
c.Backoff = time.Second
s := mcp.NewServer(&mcp.Implementation{Name: "evaluate", Version: version}, &mcp.ServerOptions{Instructions: instructions})
registerTools(s, c)
return s.Run(ctx, &mcp.StdioTransport{})
}
// evaluate extension for pi. Generated by `evaluate setup pi` — do not edit by hand, as
// re-running that command overwrites this file.
//
// pi has no built-in MCP client, so this extension registers `evaluate` as a
// native pi tool and talks to the evaluate binary over MCP stdio. The tool schema and
// the TypeSafe request logic stay in one place: the Go server.
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"
import { Type } from "typebox"
import { spawn } from "node:child_process"
const BINARY = __EVALUATE_BINARY__
const INSTRUCTIONS = __EVALUATE_INSTRUCTIONS__
const PROTOCOL_VERSION = "2025-06-18"
// callEvaluate runs one `evaluate` tool call against a fresh `evaluate mcp` process and
// resolves with the raw TypeSafe JSON.
function callEvaluate(args: unknown, signal?: AbortSignal): Promise<string> {
return new Promise<string>((resolve, reject) => {
// addEventListener does not replay an abort that already happened, so an
// aborted call would otherwise spawn evaluate and bill a request nobody wants.
if (signal?.aborted) {
reject(new Error("evaluate: aborted"))
return
}
const child = spawn(BINARY, ["mcp"], { stdio: ["pipe", "pipe", "pipe"], env: process.env })
let settled = false
const finish = (err: Error | null, text?: string) => {
// kill() below makes the child close with code null, which lands back here.
if (settled) return
settled = true
signal?.removeEventListener("abort", onAbort)
child.kill()
if (err) reject(err)
else resolve(text ?? "")
}
const onAbort = () => finish(new Error("evaluate: aborted"))
signal?.addEventListener("abort", onAbort, { once: true })
// A write error lands on the stream, not on the child, and an unhandled one
// takes down the whole pi process. EPIPE only means evaluate exited first, and the
// close handler below reports why, so it needs no message of its own.
child.stdin.on("error", (err: NodeJS.ErrnoException) => {
if (err.code !== "EPIPE") finish(err)
})
let stderr = ""
child.stderr.setEncoding("utf8")
child.stderr.on("data", (chunk: string) => {
stderr += chunk
})
let buf = ""
child.stdout.setEncoding("utf8")
child.stdout.on("data", (chunk: string) => {
buf += chunk
// Newline-delimited JSON-RPC: keep any trailing partial line buffered.
const lines = buf.split("\n")
buf = lines.pop() ?? ""
for (const line of lines) {
if (!line.trim()) continue
let msg: any
try {
msg = JSON.parse(line)
} catch {
continue
}
if (msg.id !== 2) continue
if (msg.error) {
return finish(new Error(`evaluate: ${msg.error.message ?? JSON.stringify(msg.error)}`))
}
const text = (msg.result?.content ?? []).map((c: any) => c.text ?? "").join("")
// Tool errors arrive as a result with isError set, not as a JSON-RPC error.
if (msg.result?.isError) return finish(new Error(`evaluate: ${text || "evaluate failed"}`))
return finish(null, text)
}
})
child.on("error", (err: Error) => finish(new Error(`evaluate: cannot run ${BINARY}: ${err.message}`)))
child.on("close", (code: number | null) => {
// evaluate already prefixes its stderr with "evaluate:", so pass it through as-is.
const detail = stderr.trim()
finish(new Error(detail || `evaluate: exited with code ${code}`))
})
try {
const send = (msg: unknown) => child.stdin.write(`${JSON.stringify(msg)}\n`)
send({
jsonrpc: "2.0",
id: 1,
method: "initialize",
params: {
protocolVersion: PROTOCOL_VERSION,
capabilities: {},
clientInfo: { name: "pi", version: "1" },
},
})
send({ jsonrpc: "2.0", method: "notifications/initialized" })
send({ jsonrpc: "2.0", id: 2, method: "tools/call", params: { name: "evaluate", arguments: args } })
// stdin deliberately stays open: evaluate shuts down on EOF, and would do so
// before flushing the reply.
} catch (err) {
finish(err instanceof Error ? err : new Error(String(err)))
}
})
}
export default function (pi: ExtensionAPI) {
pi.registerTool({
name: "evaluate",
label: "Jev",
description:
"Jev is a fast structured-decision model: unstructured state in, typed answers " +
"(noul, choice, score) with calibrated confidence out; 70-500ms, schema-enforced. " +
"Use for classification, routing, scoring, extraction, branching, guardrails/judging, " +
"and map-reduce over large data — wherever hand-written logic is too brittle or latency matters. " +
"Not for prose, code, or free-form text: the answer space must be enumerable up front (max 255 options). " +
"Pass raw evidence as state, not your read of it — a conclusion asserted in state biases the answer toward it, and the confidence is then not independent corroboration.",
promptSnippet:
"Classify, route, score, extract, or guard with Jev: typed answers and calibrated confidence to branch on, in 70-500ms, instead of parsing prose.",
// filter: an empty INSTRUCTIONS must not inject a blank guideline.
promptGuidelines: INSTRUCTIONS.split("\n").filter(Boolean),
parameters: Type.Object({
state: Type.Any({
description:
"content to judge: plain text, or a JSON object/array with named fields; observed evidence or a faithful condensation of it, not your verdict about it",
}),
questions: Type.Object(
{},
{
description:
"map of question id to question; answers come back under the same ids, which are not sent to the model",
// Not Type.Record: that emits patternProperties, which some providers reject.
additionalProperties: Type.Object({
type: Type.String({
description:
"noul (probability a yes/no condition holds), choice (one option from the criteria map), or score (probability-weighted position on ordered criteria levels)",
}),
instructions: Type.Any({
description:
"the judgment to make, with its full meaning; a string, or an object/array for definitions, contrasts, and examples; name the condition to test, not the conclusion you expect",
}),
criteria: Type.Optional(
Type.Any({
description:
'noul: optional {"true": ..., "false": ...} descriptions; choice (required): map of option to description or null; score (required): ordered array of at least 2 level descriptions, e.g. ["poor", "fair", "good"] — an array, not the index-keyed object the response legend comes back as',
}),
),
}),
},
),
model: Type.Optional(Type.String({ description: "model to use; default jev-latest" })),
}),
async execute(_toolCallId, params, signal) {
return { content: [{ type: "text", text: await callEvaluate(params, signal) }], details: {} }
},
})
}
package main
import (
"bytes"
"context"
_ "embed"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
)
// goRunDir matches the temp directory `go run` builds into, deleted on exit.
var goRunDir = regexp.MustCompile(`/go-build\d+/`)
// evaluateBinary returns the absolute path clients should be pointed at.
func evaluateBinary() (string, error) {
exe, err := os.Executable()
if err != nil {
return "", fmt.Errorf("finding current executable: %w", err)
}
if goRunDir.MatchString(exe) {
return "", fmt.Errorf("refusing to configure %s: `go run` binaries are deleted on exit; build or install evaluate first", exe)
}
return exe, nil
}
// runMCPSetup registers this binary as the "evaluate" MCP server with Claude Code
// and Codex, via their own CLIs, baking in the TYPESAFE_* variables and
// OPENROUTER_API_KEY from the current environment: clients launch the server
// without the user's shell env.
func runMCPSetup(ctx context.Context) error {
if _, err := route(); err != nil {
return err
}
exe, err := evaluateBinary()
if err != nil {
return err
}
env := setupEnv(os.Environ())
var errs []error
fail := func(name string, err error) {
fmt.Printf("❌ %s setup failed\n", name)
errs = append(errs, fmt.Errorf("%s: %w", name, err))
}
for _, c := range setupCommands(exe, env) {
if _, err := exec.LookPath(c.cli); err != nil {
fmt.Printf("➖ %s not found, skipped (`%s` not on PATH; see README to configure by hand)\n", c.name, c.cli)
continue
}
fmt.Printf("🔎 %s detected\n", c.name)
var prev []byte
if c.reset != nil {
prev = claudeUserEntry()
// Fails when there is no entry yet; nothing to report either way.
exec.CommandContext(ctx, c.cli, c.reset...).Run()
}
// Captured so the CLIs' own chatter stays out of the list; shown on failure.
if out, err := exec.CommandContext(ctx, c.cli, c.add...).CombinedOutput(); err != nil {
fail(c.name, fmt.Errorf("%w\n%s", err, bytes.TrimSpace(out)))
if prev != nil {
// Not ctx: an interrupted add must still put the old entry back.
if err := exec.Command(c.cli, "mcp", "add-json", "evaluate", string(prev), "-s", "user").Run(); err != nil {
errs = append(errs, fmt.Errorf("%s: restoring previous entry: %w", c.name, err))
}
}
} else {
// Only now the replacement is registered: rollback restores the
// "evaluate" entry, so dropping "jev" before a failed add would
// leave a pre-rename install with no server at all. Best effort,
// since a missing entry is the normal case.
exec.CommandContext(ctx, c.cli, c.legacy...).Run()
}
}
// Claude Desktop has no CLI; edit its config file if the app is installed.
desktop := false
if dir, err := os.UserConfigDir(); err == nil {
if _, err := os.Stat(filepath.Join(dir, "Claude")); err != nil {
fmt.Println("➖ Claude Desktop not found, skipped (see README to configure by hand)")
} else {
fmt.Println("🔎 Claude Desktop detected")
if err := setupClaudeDesktop(filepath.Join(dir, "Claude", "claude_desktop_config.json"), exe, env); err != nil {
fail("Claude Desktop", err)
} else {
desktop = true
}
}
}
if len(errs) > 0 {
return errors.Join(errs...)
}
fmt.Println("\n✅ Setup complete!")
if desktop {
fmt.Println("Restart Claude Desktop to load the evaluate server.")
}
return nil
}
// setupClaudeDesktop sets the evaluate entry in Claude Desktop's config at path,
// keeping every other key and server intact.
func setupClaudeDesktop(path, exe string, env []string) error {
cfg := map[string]json.RawMessage{}
b, err := os.ReadFile(path)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
if len(b) > 0 {
if err := json.Unmarshal(b, &cfg); err != nil {
return fmt.Errorf("parsing %s: %w", path, err)
}
}
servers := map[string]json.RawMessage{}
if raw, ok := cfg["mcpServers"]; ok {
if err := json.Unmarshal(raw, &servers); err != nil {
return fmt.Errorf("parsing mcpServers in %s: %w", path, err)
}
}
// A JSON null unmarshals to a nil map, which panics on assignment.
if cfg == nil {
cfg = map[string]json.RawMessage{}
}
if servers == nil {
servers = map[string]json.RawMessage{}
}
entry := struct {
Command string `json:"command"`
Args []string `json:"args"`
Env map[string]string `json:"env,omitempty"`
}{Command: exe, Args: []string{"mcp"}}
for _, kv := range env {
k, v, _ := strings.Cut(kv, "=")
if entry.Env == nil {
entry.Env = map[string]string{}
}
entry.Env[k] = v
}
// Same rename cleanup as setupCommand.legacy, for the client with no CLI.
// Safe to do before the write: the whole config lands atomically below.
delete(servers, "jev")
if servers["evaluate"], err = json.Marshal(entry); err != nil {
return err
}
if cfg["mcpServers"], err = json.Marshal(servers); err != nil {
return err
}
out, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
return err
}
// Write to a temp file and rename, so a failed write can't truncate the
// app's config (it also holds the user's preferences).
tmp, err := os.CreateTemp(filepath.Dir(path), ".claude_desktop_config-*.json")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
if _, err := tmp.Write(append(out, '\n')); err != nil {
tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
return os.Rename(tmp.Name(), path)
}
// claudeUserEntry returns Claude Code's current user-scope evaluate entry, or nil
// if there is none or the config cannot be read.
func claudeUserEntry() []byte {
dir := os.Getenv("CLAUDE_CONFIG_DIR")
if dir == "" {
dir, _ = os.UserHomeDir()
}
b, err := os.ReadFile(filepath.Join(dir, ".claude.json"))
if err != nil {
return nil
}
var cfg struct {
MCPServers map[string]json.RawMessage `json:"mcpServers"`
}
if json.Unmarshal(b, &cfg) != nil {
return nil
}
return cfg.MCPServers["evaluate"]
}
// setupEnv picks the variables to bake into the client configs: every
// TYPESAFE_* knob, plus the OpenRouter key for that route. The "=" anchors the
// name, so OPENROUTER_API_KEY_OTHER is left behind.
func setupEnv(environ []string) []string {
var env []string
for _, kv := range environ {
if strings.HasPrefix(kv, "TYPESAFE_") || strings.HasPrefix(kv, "OPENROUTER_API_KEY=") {
env = append(env, kv)
}
}
return env
}
type setupCommand struct {
name, cli string
// legacy removes the "jev" entry this binary registered before it was
// renamed; left behind, it fails to launch a path that no longer exists
// every time the client starts. The name is the only signal available —
// codex has no config read path — so a server someone else named "jev"
// would go too. Acceptable: this tool owned that name.
reset, add, legacy []string
}
func setupCommands(exe string, env []string) []setupCommand {
claude := []string{"mcp", "add", "evaluate", "-s", "user"}
codex := []string{"mcp", "add", "evaluate"}
for _, kv := range env {
// One flag per pair: claude's -e is variadic and would swallow the name.
claude = append(claude, "-e", kv)
codex = append(codex, "--env", kv)
}
return []setupCommand{
// `claude mcp add` refuses an existing name; `codex mcp add` overwrites.
{"Claude Code", "claude", []string{"mcp", "remove", "evaluate", "-s", "user"}, append(claude, "--", exe, "mcp"), []string{"mcp", "remove", "jev", "-s", "user"}},
{"Codex", "codex", nil, append(codex, "--", exe, "mcp"), []string{"mcp", "remove", "jev"}},
}
}
//go:embed pi.ts
var piExtension string
// piDir returns pi's config directory. PI_CODING_AGENT_DIR wins, and a leading
// "~" is expanded because pi expands it too: taking it literally would make the
// override silently miss an existing install.
func piDir() (string, error) {
dir := os.Getenv("PI_CODING_AGENT_DIR")
// Only the tilde and default cases need a home directory; an explicit path
// has to keep working where HOME is unset.
if dir != "" && !strings.HasPrefix(dir, "~") {
return dir, nil
}
home, err := os.UserHomeDir()
if err != nil {
return "", err
}
switch {
case dir == "":
return filepath.Join(home, ".pi", "agent"), nil
case dir == "~":
return home, nil
case strings.HasPrefix(dir, "~/"):
return filepath.Join(home, dir[2:]), nil
}
return dir, nil
}
// runPiSetup installs the evaluate extension into pi. pi has no MCP client, so the
// extension registers `evaluate` as a native pi tool and speaks MCP to this
// binary itself.
func runPiSetup() error {
exe, err := evaluateBinary()
if err != nil {
return err
}
dir, err := piDir()
if err != nil {
return err
}
if _, err := os.Stat(dir); err != nil {
fmt.Println("➖ pi not found, skipped (see README to install by hand)")
return nil
}
fmt.Println("🔎 pi detected")
path, err := writePiExtension(dir, exe)
if err != nil {
return err
}
fmt.Printf(" wrote %s\n", path)
fmt.Println("\n✅ Setup complete!")
fmt.Println("Run /reload in pi, or restart it, to load the evaluate extension.")
// Unlike the MCP clients, nothing is baked in: the extension reads the key
// from the shell pi runs in, so a missing one is a hint, not a failure.
if _, err := route(); err != nil {
fmt.Printf("\nNote: %v\n", err)
}
return nil
}
// writePiExtension renders the embedded extension into dir and returns its path.
func writePiExtension(dir, exe string) (string, error) {
// json.Marshal, not a bare quoted placeholder: it escapes a path, or the
// instructions' newlines and backticks, into a valid JS string literal.
binary, err := json.Marshal(exe)
if err != nil {
return "", err
}
guide, err := json.Marshal(instructions)
if err != nil {
return "", err
}
src := strings.NewReplacer(
"__EVALUATE_BINARY__", string(binary),
"__EVALUATE_INSTRUCTIONS__", string(guide),
).Replace(piExtension)
out := filepath.Join(dir, "extensions")
if err := os.MkdirAll(out, 0o755); err != nil {
return "", err
}
path := filepath.Join(out, "evaluate.ts")
if err := os.WriteFile(path, []byte(src), 0o600); err != nil {
return "", err
}
// Both files register a tool named `evaluate`, so a leftover jev.ts from
// before the rename would collide with the one just written. Removed after
// the write, so a failed write leaves pi with the old extension, not none.
if err := os.Remove(filepath.Join(out, "jev.ts")); err != nil && !errors.Is(err, os.ErrNotExist) {
return path, err
}
return path, nil
}
package main
import (
"context"
"errors"
"fmt"
"github.com/modelcontextprotocol/go-sdk/mcp"
)
// add registers a tool whose handler returns raw text (usually TypeSafe JSON).
// A returned error becomes a tool result with IsError set.
func add[In any](s *mcp.Server, t *mcp.Tool, fn func(ctx context.Context, in In) ([]byte, error)) {
mcp.AddTool(s, t, func(ctx context.Context, _ *mcp.CallToolRequest, in In) (*mcp.CallToolResult, any, error) {
b, err := fn(ctx, in)
if err != nil {
return nil, nil, err
}
return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: string(b)}}}, nil, nil
})
}
type question struct {
Type string `json:"type" jsonschema:"noul (probability a yes/no condition holds), choice (one option from the criteria map), or score (probability-weighted position on ordered criteria levels)"`
Instructions any `json:"instructions" jsonschema:"the judgment to make, with its full meaning; a string, or an object/array for definitions, contrasts, and examples; name the condition to test, not the conclusion you expect"`
Criteria any `json:"criteria,omitempty" jsonschema:"noul: optional {\"true\": ..., \"false\": ...} descriptions; choice (required): map of option to description or null; score (required): ordered array of at least 2 level descriptions, e.g. [\"poor\", \"fair\", \"good\"] — an array, not the index-keyed object the response legend comes back as"`
}
type evaluateIn struct {
State any `json:"state" jsonschema:"content to judge: plain text, or a JSON object/array with named fields; observed evidence or a faithful condensation of it, not your verdict about it"`
Questions map[string]question `json:"questions" jsonschema:"map of question id to question; answers come back under the same ids, which are not sent to the model"`
Model string `json:"model,omitempty" jsonschema:"model to use; defaults to the latest Jev on whichever endpoint is configured"`
}
func registerTools(s *mcp.Server, c *Client) {
add(s, &mcp.Tool{
Name: "evaluate",
Description: "Jev is a fast structured-decision model: unstructured state in, typed answers " +
"(noul, choice, score) with calibrated confidence out; 70-500ms, schema-enforced. " +
"Use for classification, routing, scoring, extraction, branching, guardrails/judging, " +
"and map-reduce over large data — wherever hand-written logic is too brittle or latency matters. " +
"Not for prose, code, or free-form text: the answer space must be enumerable up front (max 255 options). " +
"Pass raw evidence as state, not your read of it — a conclusion asserted in state biases the answer toward it, and the confidence is then not independent corroboration.",
Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true},
}, func(ctx context.Context, in evaluateIn) ([]byte, error) {
if in.State == nil {
return nil, errors.New("state is required")
}
if len(in.Questions) == 0 {
return nil, errors.New("questions must not be empty")
}
if err := validate(in); err != nil {
return nil, err
}
if in.Model == "" {
in.Model = c.Model
}
return c.Evaluate(ctx, in)
})
}
// validate rejects the criteria shapes the API is known to refuse, so the caller
// sees its own JSON path instead of the union-branch path the API reports
// ("questions.<id>.score.criteria" for input that has no score property at all).
//
// Only shapes the API definitely rejects are checked. An unknown type passes
// through untouched: the server enumerates more types than this tool documents,
// and guessing at the list here would break the ones it does not name. The
// two-level minimum stays guidance, not a rule, because the API accepts one level.
func validate(in evaluateIn) error {
for id, q := range in.Questions {
var want string
switch q.Type {
case "score":
if v, ok := q.Criteria.([]any); ok && len(v) > 0 {
continue
}
want = "an array of level descriptions, ordered low to high"
case "choice":
if v, ok := q.Criteria.(map[string]any); ok && len(v) > 0 {
continue
}
want = "an object mapping each option to a description or null"
case "noul":
if q.Criteria == nil {
continue
}
if _, ok := q.Criteria.(map[string]any); ok {
continue
}
want = `an object with "true" and "false" descriptions, or omitted`
default:
continue
}
// Bracket-quoted, not questions.%s.criteria: an id containing a dot
// would otherwise read as nesting that the request never had, which is
// the exact ambiguity this check exists to remove.
return fmt.Errorf("questions[%q].criteria: %s criteria must be %s, got %s",
id, q.Type, want, jsonKind(q.Criteria))
}
return nil
}
// jsonKind names a decoded JSON value the way the caller wrote it, so the error
// says "got object" rather than a Go type the caller never typed.
func jsonKind(v any) string {
switch v := v.(type) {
case nil:
return "nothing"
case []any:
if len(v) == 0 {
return "an empty array"
}
return "an array"
case map[string]any:
if len(v) == 0 {
return "an empty object"
}
return "an object"
case string:
return "a string"
case float64:
return "a number"
case bool:
return "a boolean"
}
return "an unsupported value"
}
package main
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"runtime"
"strings"
"time"
)
const githubRepo = "itsmostafa/typesafe-mcp"
const (
// One deadline covers connect, headers and body: a stalled mirror must not
// hang `evaluate update` forever.
updateTimeout = 5 * time.Minute
// Release JSON and SHA256SUMS.txt are a few KB; the archive is one
// compressed binary. Both caps are generous by orders of magnitude.
maxMetadataBytes = 1 << 20
maxArchiveBytes = 100 << 20
)
var updateClient = &http.Client{Timeout: updateTimeout}
// httpGet fetches url with the command's context and a bounded client, and
// rejects non-200 responses so no caller parses an error page as data.
func httpGet(ctx context.Context, url string) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := updateClient.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, fmt.Errorf("%s returned %s", url, resp.Status)
}
return resp, nil
}
// download reads url into memory, failing if the body exceeds limit bytes.
func download(ctx context.Context, url string, limit int64) ([]byte, error) {
resp, err := httpGet(ctx, url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
b, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
if err == nil && int64(len(b)) > limit {
err = fmt.Errorf("%s exceeds %d bytes", url, limit)
}
return b, err
}
// runUpdate replaces the running binary with the latest GitHub release.
func runUpdate(ctx context.Context) error {
exe, err := os.Executable()
if err == nil {
// Depending on the OS, exe can be the symlink the binary was started
// through; renaming over it would replace the link, not the binary.
exe, err = filepath.EvalSymlinks(exe)
}
if err != nil {
return fmt.Errorf("finding current executable: %w", err)
}
// Stage beside exe so the final rename never crosses filesystems, and an
// unwritable install dir fails before anything is downloaded. CreateTemp's
// exclusive random name can't be redirected by a planted symlink.
staged, err := os.CreateTemp(filepath.Dir(exe), ".evaluate-update-*")
if err != nil {
if errors.Is(err, os.ErrPermission) {
return fmt.Errorf("%s is not writable, re-run with: sudo evaluate update", filepath.Dir(exe))
}
return fmt.Errorf("staging update: %w", err)
}
defer os.Remove(staged.Name()) // no-op after a successful rename
defer staged.Close()
release, err := fetchLatestRelease(ctx)
if err != nil {
return fmt.Errorf("fetching latest release: %w", err)
}
if version == release.TagName {
fmt.Printf("Already up to date (%s).\n", version)
return nil
}
fmt.Printf("Updating %s → %s\n", version, release.TagName)
archiveName := fmt.Sprintf("evaluate-%s-%s.tar.gz", runtime.GOOS, runtime.GOARCH)
archiveURL, err := findAssetURL(release.Assets, archiveName)
if err != nil {
return err
}
sumsURL, err := findAssetURL(release.Assets, "SHA256SUMS.txt")
if err != nil {
return err
}
expectedHash, err := fetchExpectedChecksum(ctx, sumsURL, archiveName)
if err != nil {
return fmt.Errorf("fetching checksums: %w", err)
}
archive, err := download(ctx, archiveURL, maxArchiveBytes)
if err != nil {
return fmt.Errorf("downloading archive: %w", err)
}
sum := sha256.Sum256(archive)
if got := hex.EncodeToString(sum[:]); got != expectedHash {
return fmt.Errorf("checksum mismatch: got %s, want %s", got, expectedHash)
}
if err := extractBinaryFromTar(archive, "evaluate", staged); err != nil {
return fmt.Errorf("extracting binary: %w", err)
}
if err := staged.Close(); err != nil {
return err
}
if err := os.Chmod(staged.Name(), 0755); err != nil {
return err
}
if err := os.Rename(staged.Name(), exe); err != nil {
return fmt.Errorf("replacing executable: %w", err)
}
fmt.Printf("Updated to %s. Run `evaluate version` to confirm.\n", release.TagName)
return nil
}
type githubRelease struct {
TagName string `json:"tag_name"`
Assets []githubAsset `json:"assets"`
}
type githubAsset struct {
Name string `json:"name"`
BrowserDownloadURL string `json:"browser_download_url"`
}
func fetchLatestRelease(ctx context.Context) (*githubRelease, error) {
body, err := download(ctx, fmt.Sprintf("https://api.github.com/repos/%s/releases/latest", githubRepo), maxMetadataBytes)
if err != nil {
return nil, err
}
var rel githubRelease
return &rel, json.Unmarshal(body, &rel)
}
func findAssetURL(assets []githubAsset, name string) (string, error) {
for _, a := range assets {
if a.Name == name {
return a.BrowserDownloadURL, nil
}
}
return "", fmt.Errorf("asset %q not found in release", name)
}
func fetchExpectedChecksum(ctx context.Context, sumsURL, assetName string) (string, error) {
body, err := download(ctx, sumsURL, maxMetadataBytes)
if err != nil {
return "", err
}
for line := range strings.Lines(string(body)) {
fields := strings.Fields(line)
if len(fields) == 2 && fields[1] == assetName {
return fields[0], nil
}
}
return "", fmt.Errorf("no checksum found for %q", assetName)
}
// extractBinaryFromTar copies the regular file binaryName from a .tar.gz
// archive into dst.
func extractBinaryFromTar(archive []byte, binaryName string, dst io.Writer) error {
gr, err := gzip.NewReader(bytes.NewReader(archive))
if err != nil {
return err
}
defer gr.Close()
tr := tar.NewReader(gr)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
if hdr.Name != binaryName {
continue
}
if hdr.Typeflag != tar.TypeReg {
return fmt.Errorf("archive member %q is not a regular file", hdr.Name)
}
// tar.Reader stops at hdr.Size, so this bounds the copy too.
if hdr.Size > maxArchiveBytes {
return fmt.Errorf("archive member %q declares %d bytes, over the %d byte limit", hdr.Name, hdr.Size, int64(maxArchiveBytes))
}
_, err = io.Copy(dst, tr)
return err
}
return fmt.Errorf("binary %q not found in archive", binaryName)
}