noulscorechoiceTriageVerification

Jev Review — a staged code review that narrows before it scores

jev.directorylast month1 likemain@31f8960

A code-review workflow that reviews a Git diff or scans a whole codebase, built as a pipeline of bounded Jev judgments rather than one big ask.

The stages run in order, and the order is the design:

Noul risk matrix
  -> Choice + Score file profiles
  -> Choice evidence selection
  -> Choice mechanism classification
  -> Score severity
  -> conditional Choice reviewer routing

It narrows before it judges. A Noul risk matrix screens correctness, security, reliability, compatibility and test coverage; file profiles rank what deserves attention; then evidence selection picks the concrete diff hunks or source regions before anything scores impact. Severity is only asked once there is a specific piece of code to be severe about, which is what stops a reviewer from rating a file it never really looked at.

The last stage is conditional: reviewer routing is a Choice that only runs when the earlier stages justify it.

Thresholds and workflow policy stay in code. Changed and related tests are fed in as context when judging test gaps, so a missing test is judged against what the change actually touched. Results land in a local dashboard bound to 127.0.0.1 that never serves environment files.

Source (3 files)

src/dashboard/server.ts

// Local-only dashboard server: a fixed list of static assets from ./public
// plus the saved review report as JSON. Binds to loopback only.
import { readFile } from "node:fs/promises";
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { join, relative } from "node:path";
import { readReport, reportPath } from "../adapters/report-store.ts";

const HOST = "127.0.0.1";
const PORT = Number(process.env.PORT ?? 4317);
const REPORT = reportPath();
const PUBLIC_DIR = join(import.meta.dirname, "public");

// Only these files are served; nothing else on disk is reachable.
const assets: Record<string, [file: string, type: string]> = {
  "/": ["index.html", "text/html; charset=utf-8"],
  "/style.css": ["style.css", "text/css; charset=utf-8"],
  "/app.js": ["app.js", "text/javascript; charset=utf-8"],
};

function send(res: ServerResponse, status: number, type: string, body: string | Buffer) {
  res.writeHead(status, {
    "Content-Type": type,
    "Cache-Control": "no-store",
    "X-Content-Type-Options": "nosniff",
    "Content-Security-Policy": "default-src 'self'; img-src 'self' data:",
  });
  res.end(body);
}

function json(res: ServerResponse, status: number, body: unknown) {
  send(res, status, "application/json; charset=utf-8", JSON.stringify(body));
}

export async function handle(req: IncomingMessage, res: ServerResponse) {
  if (req.method !== "GET" && req.method !== "HEAD") {
    return send(res, 405, "text/plain; charset=utf-8", "Method not allowed");
  }

  const { pathname } = new URL(req.url ?? "/", "http://localhost");
  if (pathname === "/api/review") {
    const source = relative(process.cwd(), REPORT) || REPORT;
    return json(res, 200, { source, ...(await readReport(REPORT)) });
  }

  const asset = assets[pathname];
  if (!asset) return send(res, 404, "text/plain; charset=utf-8", "Not found");

  const [file, type] = asset;
  send(res, 200, type, await readFile(join(PUBLIC_DIR, file)));
}

if (import.meta.main) {
  createServer(handle).listen(PORT, HOST, () => {
    console.log(`Jev review dashboard: http://${HOST}:${PORT}`);
    console.log(`report: ${relative(process.cwd(), REPORT) || REPORT}`);
  });
}

src/review/codebase-judgments.ts

// Codebase-scan judgments. These ask whether an issue exists in complete
// source, rather than whether a patch introduced one.
import { choice, noul, score, TypeSafeClient } from "@typesafe-ai/sdk";
import { basename, dirname } from "node:path";
import {
  BLOCKING_SEVERITY,
  type Dimension,
  dimensions,
  mechanisms,
  MIN_LOCATION_CONFIDENCE,
  owners,
  reviewPriorityRubric,
  ROUTE_SEVERITY,
  severityRubric,
} from "../domain/config.ts";
import type {
  FileProfile,
  Finding,
  Screening,
  Signal,
  SourceFile,
} from "../domain/types.ts";

const client = new TypeSafeClient();
const REGION_LINES = 80;
const SCREEN_REGION_LINES = 160;
const MAX_RELATED_TESTS = 4;
const MAX_TEST_SNIPPET_CHARS = 1_800;

const fileRoles = {
  entrypoint: "Application, command, route, or public package entry point",
  boundary: "Authentication, validation, serialization, or external-system boundary",
  domain: "Core business rules, state transitions, or domain behavior",
  persistence: "Database, cache, filesystem, migration, or durable state",
  infrastructure: "Runtime, scheduling, networking, build, or operational plumbing",
  utility: "Shared helper, adapter, formatting, or low-level utility",
};

export async function screenSourceFile(
  file: SourceFile,
  testFiles: SourceFile[],
): Promise<Screening<SourceFile>> {
  const relatedTests = selectRelatedTests(file, testFiles);
  const results: Array<Record<Dimension, number>> = [];

  for (const region of sourceRegions(file.content, SCREEN_REGION_LINES)) {
    const response = await client.systemOne({
      state: {
        file: { path: file.path, startLine: region.startLine, content: region.content },
        relatedTests,
      },
      questions: {
        correctness: noul(
          {
            question: "Does file.content directly support that this code contains incorrect runtime behavior?",
            inspect: "file.content",
            focus: "Concrete behavior, state, data-flow, or async errors reachable in realistic use",
            ignore: ["Style preferences", "Naming concerns", "Missing context with no concrete failure path"],
          },
          {
            true: {
              what: "The source contains a realistic path to a wrong runtime result",
              examples: ["A condition handles the opposite case", "State is updated under the wrong key"],
            },
            false: {
              what: "The implementation is coherent or no concrete incorrect path is supported",
              not_for: "Unusual code that is still internally consistent",
            },
          },
        ),
        security: noul(
          {
            question: "Does file.content directly support that this code weakens a security boundary?",
            inspect: "file.content",
            focus: "Authorization, injection, secret exposure, trust boundaries, and unsafe defaults",
          },
          {
            true: {
              what: "The source contains a concrete path around a control or into an unsafe sink",
              examples: ["A privileged action lacks authorization", "Untrusted input reaches command execution"],
            },
            false: {
              what: "No concrete security weakness is supported by this file",
              not_for: "Code that merely handles credentials or permissions safely",
            },
          },
        ),
        reliability: noul(
          {
            question: "Does file.content directly support that this code can crash, race, leak, deadlock, or recover poorly?",
            inspect: "file.content",
            focus: "Realistic resource, concurrency, cancellation, and failure paths",
          },
          {
            true: {
              what: "A reachable path can lose work, leak resources, hang, crash, or leave inconsistent state",
              examples: ["Cleanup is skipped after failure", "Concurrent work mutates shared state unsafely"],
            },
            false: { what: "Lifecycle and failure handling appear safe, or no concrete failure path is supported" },
          },
        ),
        compatibility: noul(
          {
            question: "Does file.content directly support an internal inconsistency that can break a caller, format, protocol, or documented behavior?",
            inspect: "file.content",
            focus: "Contradictions visible in this source, not guesses about unknown historical versions",
          },
          {
            true: {
              what: "The source contains conflicting contracts or a concrete caller-facing mismatch",
              examples: ["A parser and serializer disagree on a required field", "An exported type contradicts runtime behavior"],
            },
            false: {
              what: "The visible contracts are internally consistent",
              not_for: "Speculation that an API may once have behaved differently",
            },
          },
        ),
        testGap: noul(
          {
            question: "Does file.content contain important behavior without adequate targeted evidence in relatedTests?",
            compare: ["file.content", "relatedTests"],
            focus: "Critical branches, boundaries, failure paths, and component interactions",
            caution: "A filename mismatch alone is not enough; identify behavior that specifically needs a test",
          },
          {
            true: {
              what: "Important behavior is present and the related tests do not exercise it",
              examples: ["An error-recovery branch has no assertion", "Authorization behavior lacks a denial test"],
            },
            false: {
              what: "Related tests cover the important behavior, or this file has no behavior needing direct tests",
              examples: ["A focused test covers the boundary", "A declarative constants module"],
            },
          },
        ),
      },
  });

    results.push({
      correctness: response.answers.correctness.noul,
      security: response.answers.security.noul,
      reliability: response.answers.reliability.noul,
      compatibility: response.answers.compatibility.noul,
      testGap: response.answers.testGap.noul,
    });
  }

  return {
    file,
    probabilities: {
      correctness: Math.max(...results.map((result) => result.correctness)),
      security: Math.max(...results.map((result) => result.security)),
      reliability: Math.max(...results.map((result) => result.reliability)),
      compatibility: Math.max(...results.map((result) => result.compatibility)),
      testGap: Math.max(...results.map((result) => result.testGap)),
    },
  };
}

export async function profileSourceFile(
  file: SourceFile,
  screeningProbabilities: Record<Dimension, number>,
): Promise<FileProfile> {
  const response = await client.systemOne({
    state: { file, screeningProbabilities },
    questions: {
      category: choice(
        { question: "Which role best describes this source file?", focus: "Primary runtime responsibility" },
        fileRoles,
      ),
      reviewPriority: score(
        "Rate how closely a human should review this complete file, considering its role and screeningProbabilities.",
        [...reviewPriorityRubric],
      ),
    },
  });

  return {
    file: file.path,
    category: response.answers.category.choice,
    categoryConfidence: response.answers.category.confidence,
    reviewPriority: response.answers.reviewPriority.score,
    reviewPriorityConfidence: response.answers.reviewPriority.confidence,
  };
}

export async function locateSourceSignal(
  signal: Signal<SourceFile>,
): Promise<Finding<SourceFile> | null> {
  const regions = sourceRegions(signal.file.content);
  if (regions.length === 0) return null;

  const suspectedConcern = {
    dimension: signal.dimension,
    definition: dimensions[signal.dimension],
  };
  const location = await client.systemOne({
    state: {
      file: signal.file.path,
      suspectedConcern: { ...suspectedConcern, screeningProbability: signal.probability },
      candidateRegions: regions,
    },
    questions: {
      evidence: choice(
        {
          question: "Which candidate region provides the strongest direct evidence for suspectedConcern?",
          fallback: "Select noMatch when no region provides sufficient evidence",
        },
        {
          ...Object.fromEntries(
            regions.map((region) => [region.id, "Source beginning at line " + region.startLine]),
          ),
          noMatch: "No source region directly supports the suspected concern",
        },
      ),
    },
  });

  const selected = location.answers.evidence;
  if (selected.choice === "noMatch" || selected.confidence < MIN_LOCATION_CONFIDENCE) return null;
  const region = regions.find((candidate) => candidate.id === selected.choice);
  if (!region) return null;

  const classification = await client.systemOne({
    state: { file: signal.file.path, suspectedConcern, selectedEvidence: region },
    questions: {
      mechanism: choice(
        "Which mechanism best describes the suspected concern supported by selectedEvidence?",
        mechanisms[signal.dimension],
      ),
    },
  });
  const mechanism = classification.answers.mechanism;
  if (mechanism.choice === "noIssue") return null;

  const impact = await client.systemOne({
    state: { file: signal.file.path, suspectedConcern, selectedEvidence: region },
    questions: {
      severity: score(
        "Assuming selectedEvidence exhibits suspectedConcern, rate the likely production impact.",
        [...severityRubric],
      ),
    },
  });

  const severity = impact.answers.severity;
  let owner: string | null = null;
  let ownerConfidence: number | null = null;
  if (severity.score >= ROUTE_SEVERITY) {
    const routing = await client.systemOne({
      state: {
        file: signal.file.path,
        concern: {
          dimension: signal.dimension,
          mechanism: mechanism.choice,
          severity: severity.score,
        },
        selectedEvidence: region,
      },
      questions: {
        owner: choice("Which reviewer is best suited to investigate this concern?", owners),
      },
    });
    owner = routing.answers.owner.choice;
    ownerConfidence = routing.answers.owner.confidence;
  }

  return {
    ...signal,
    line: region.startLine,
    locationConfidence: selected.confidence,
    mechanism: mechanism.choice,
    mechanismConfidence: mechanism.confidence,
    severity: severity.score,
    severityConfidence: severity.confidence,
    owner,
    ownerConfidence,
    action: severity.score >= BLOCKING_SEVERITY ? "request_changes" : "comment",
  };
}

function sourceRegions(content: string, linesPerRegion = REGION_LINES) {
  const lines = content.split("\n");
  const count = Math.ceil(lines.length / linesPerRegion);
  return Array.from({ length: count }, (_, index) => ({
    id: "R" + (index + 1),
    startLine: index * linesPerRegion + 1,
    content: lines.slice(index * linesPerRegion, (index + 1) * linesPerRegion).join("\n"),
  }));
}

function selectRelatedTests(file: SourceFile, testFiles: SourceFile[]): SourceFile[] {
  const stem = basename(file.path).replace(/\.[^.]+$/, "");
  const directory = dirname(file.path);
  return testFiles
    .map((test) => ({
      test,
      score: (test.path.includes(stem) ? 2 : 0) + (test.path.startsWith(directory) ? 1 : 0),
    }))
    .filter(({ score }) => score > 0)
    .sort((a, b) => b.score - a.score || a.test.path.localeCompare(b.test.path))
    .slice(0, MAX_RELATED_TESTS)
    .map(({ test }) => compactTest(test, stem));
}

function compactTest(test: SourceFile, sourceStem: string): SourceFile {
  const lines = test.content.split("\n");
  const selected = new Set<number>();
  const stem = sourceStem.toLowerCase();

  for (let index = 0; index < lines.length; index++) {
    const line = lines[index].toLowerCase();
    if (
      line.includes(stem) ||
      line.includes("describe(") ||
      line.includes("describe.") ||
      line.includes("test(") ||
      line.includes("test.") ||
      line.includes("it(") ||
      line.includes("it.")
    ) {
      for (let nearby = Math.max(0, index - 2); nearby <= Math.min(lines.length - 1, index + 2); nearby++) {
        selected.add(nearby);
      }
    }
  }

  let content = [...selected]
    .sort((a, b) => a - b)
    .map((index) => lines[index])
    .join("\n");
  if (content.length === 0) content = test.content;
  if (content.length > MAX_TEST_SNIPPET_CHARS) {
    const side = Math.floor((MAX_TEST_SNIPPET_CHARS - 7) / 2);
    content = content.slice(0, side) + "\n...\n" + content.slice(-side);
  }

  return { path: test.path, content };
}

src/review/judgments.ts

// Change-review judgments. Every call is narrow and receives patch evidence.
import { choice, noul, score, TypeSafeClient } from "@typesafe-ai/sdk";
import {
  BLOCKING_SEVERITY,
  type Dimension,
  dimensions,
  mechanisms,
  MIN_LOCATION_CONFIDENCE,
  owners,
  reviewPriorityRubric,
  ROUTE_SEVERITY,
  severityRubric,
} from "../domain/config.ts";
import { parseHunks } from "../domain/patch.ts";
import type {
  ChangedFile,
  FileProfile,
  Finding,
  Screening,
  Signal,
} from "../domain/types.ts";

const client = new TypeSafeClient();

const changeTypes = {
  behavior: "Adds or changes runtime behavior",
  interface: "Changes an exported API, type, protocol, or data shape",
  infrastructure: "Changes execution, scheduling, build, or operational plumbing",
  observability: "Changes events, logging, monitoring, or diagnostics",
  refactor: "Restructures implementation without intending behavior changes",
  routine: "A small routine change that fits none of the other categories",
};

export async function screenFile(
  file: ChangedFile,
  changedTests: ChangedFile[],
): Promise<Screening<ChangedFile>> {
  const response = await client.systemOne({
    state: { file, changedTests },
    questions: {
      correctness: noul(
        {
          question: "Does file.patch directly support that this change likely introduces incorrect runtime behavior?",
          inspect: "file.patch",
          focus: "Concrete behavior, state, data-flow, or async errors introduced by added or modified lines",
          ignore: ["Style preferences", "Naming concerns", "Unsupported speculation"],
        },
        {
          true: {
            what: "The patch contains a realistic path to a wrong runtime result",
            examples: ["A condition now handles the opposite case", "A value is written to the wrong field"],
          },
          false: {
            what: "The patch is correct, non-behavioral, or lacks direct evidence of a bug",
            examples: ["Formatting only", "A refactor that preserves data flow"],
          },
        },
      ),
      security: noul(
        {
          question: "Does file.patch directly support that this change introduces or weakens a security boundary?",
          inspect: "file.patch",
          focus: "Authorization, injection, secret exposure, trust boundaries, and unsafe defaults",
        },
        {
          true: {
            what: "The patch creates a concrete path around a security control or into an unsafe sink",
            examples: ["An authorization check is removed", "Untrusted input reaches command execution"],
          },
          false: {
            what: "No security boundary is weakened by the patch",
            not_for: "Code that merely uses security-related names",
          },
        },
      ),
      reliability: noul(
        {
          question: "Does file.patch directly support that this change can crash, race, leak, deadlock, or recover poorly?",
          inspect: "file.patch",
          focus: "Realistic resource, concurrency, cancellation, and failure paths",
        },
        {
          true: {
            what: "A changed path can lose work, leak resources, hang, crash, or leave inconsistent state",
            examples: ["Cleanup is skipped after failure", "Concurrent work updates shared state unsafely"],
          },
          false: { what: "The patch preserves safe lifecycle and failure handling" },
        },
      ),
      compatibility: noul(
        {
          question: "Does file.patch directly support that this change can break an existing caller, format, protocol, or public behavior?",
          inspect: "file.patch",
          focus: "Externally observed contracts rather than internal implementation details",
        },
        {
          true: {
            what: "An existing consumer can fail because a contract changed without a safe migration",
            examples: ["A required field is removed", "A persisted value changes meaning"],
          },
          false: { what: "The changed contract remains compatible or is entirely internal" },
        },
      ),
      testGap: noul(
        {
          question: "Does file.patch change important behavior without adequate targeted evidence in changedTests?",
          compare: ["file.patch", "changedTests"],
          focus: "New branches, boundaries, failure paths, and component interactions",
        },
        {
          true: {
            what: "Important changed behavior has no targeted changed test",
            examples: ["A new failure branch has no assertion", "A protocol change lacks a compatibility test"],
          },
          false: {
            what: "Changed tests exercise the important behavior, or the patch is non-behavioral",
            examples: ["A focused regression test covers the branch", "Documentation-only change"],
          },
        },
      ),
    },
  });

  return {
    file,
    probabilities: {
      correctness: response.answers.correctness.noul,
      security: response.answers.security.noul,
      reliability: response.answers.reliability.noul,
      compatibility: response.answers.compatibility.noul,
      testGap: response.answers.testGap.noul,
    },
  };
}

export async function profileFile(
  file: ChangedFile,
  screeningProbabilities: Record<Dimension, number>,
): Promise<FileProfile> {
  const response = await client.systemOne({
    state: { file, screeningProbabilities },
    questions: {
      category: choice(
        { question: "Which category best describes file.patch?", focus: "Primary purpose of the change" },
        changeTypes,
      ),
      reviewPriority: score(
        "Rate how closely a human should review file.patch, considering the code and screeningProbabilities.",
        [...reviewPriorityRubric],
      ),
    },
  });

  return {
    file: file.path,
    category: response.answers.category.choice,
    categoryConfidence: response.answers.category.confidence,
    reviewPriority: response.answers.reviewPriority.score,
    reviewPriorityConfidence: response.answers.reviewPriority.confidence,
  };
}

export async function locateSignal(
  signal: Signal<ChangedFile>,
): Promise<Finding<ChangedFile> | null> {
  const hunks = parseHunks(signal.file.patch);
  if (hunks.length === 0) return null;

  const suspectedConcern = {
    dimension: signal.dimension,
    definition: dimensions[signal.dimension],
  };
  const location = await client.systemOne({
    state: {
      file: signal.file.path,
      suspectedConcern: { ...suspectedConcern, screeningProbability: signal.probability },
      candidateHunks: hunks,
    },
    questions: {
      evidence: choice(
        {
          question: "Which candidate hunk provides the strongest direct evidence for suspectedConcern?",
          fallback: "Select noMatch when no hunk provides sufficient evidence",
        },
        {
          ...Object.fromEntries(
            hunks.map((hunk) => [hunk.id, "Candidate beginning at changed-file line " + hunk.startLine]),
          ),
          noMatch: "No candidate hunk directly supports the suspected concern",
        },
      ),
    },
  });

  const selected = location.answers.evidence;
  if (selected.choice === "noMatch" || selected.confidence < MIN_LOCATION_CONFIDENCE) return null;
  const hunk = hunks.find((candidate) => candidate.id === selected.choice);
  if (!hunk) return null;

  const classification = await client.systemOne({
    state: { file: signal.file.path, suspectedConcern, selectedEvidence: hunk },
    questions: {
      mechanism: choice(
        "Which mechanism best describes the suspected concern supported by selectedEvidence?",
        mechanisms[signal.dimension],
      ),
    },
  });
  const mechanism = classification.answers.mechanism;
  if (mechanism.choice === "noIssue") return null;

  const impact = await client.systemOne({
    state: { file: signal.file.path, suspectedConcern, selectedEvidence: hunk },
    questions: {
      severity: score(
        "Assuming selectedEvidence exhibits suspectedConcern, rate the likely production impact.",
        [...severityRubric],
      ),
    },
  });

  const severity = impact.answers.severity;
  let owner: string | null = null;
  let ownerConfidence: number | null = null;
  if (severity.score >= ROUTE_SEVERITY) {
    const routing = await client.systemOne({
      state: {
        file: signal.file.path,
        concern: {
          dimension: signal.dimension,
          mechanism: mechanism.choice,
          severity: severity.score,
        },
        selectedEvidence: hunk,
      },
      questions: {
        owner: choice("Which reviewer is best suited to investigate this concern?", owners),
      },
    });
    owner = routing.answers.owner.choice;
    ownerConfidence = routing.answers.owner.confidence;
  }

  return {
    ...signal,
    line: hunk.startLine,
    locationConfidence: selected.confidence,
    mechanism: mechanism.choice,
    mechanismConfidence: mechanism.confidence,
    severity: severity.score,
    severityConfidence: severity.confidence,
    owner,
    ownerConfidence,
    action: severity.score >= BLOCKING_SEVERITY ? "request_changes" : "comment",
  };
}