scoreHarnessTriage

RouteKit — Jev reads the task, a policy engine picks the model

jev.directorylast month1 likemain@b972d4b

Model routing for agents, with the judgment and the decision deliberately kept apart.

The premise is that asking a full LLM which LLM should handle a request is both slow and circular. RouteKit uses Jev as a lightweight decision layer that does one job — read the incoming request and Score its task requirements — and then hands off.

Agent request -> Jev: task requirements
              -> Eligibility: capabilities / context / constraints
              -> Policy engine: objective + model profiles
              -> Selected model

Jev never names a model. Eligibility filtering and the policy engine do that, from the scored requirements plus model profiles and whatever objective is configured — capability, cost or latency. That split is what keeps the routing table editable: adding a model or changing a cost ceiling is a profile change, not a re-prompt, and the judgment about the task stays valid across it.

Source (6 files)

src/decision/jev/jev-normalizer.ts

import { RoutingSignals } from '../../types/signals';
import { ResultFor } from '@typesafe-ai/sdk';
import { routingQuestions } from './jev-questions';

type JevResponses = {
  [K in keyof typeof routingQuestions]: ResultFor<typeof routingQuestions[K]>;
};

/**
 * Normalizes a 0-4 scale score to a 0.0-1.0 range.
 */
function normalizeScore(rawScore: number): number {
  return Math.max(0, Math.min(1, rawScore / 4.0));
}

/**
 * Normalizes raw Jev responses into standard 0.0-1.0 RoutingSignals and computes overall confidence.
 */
export function normalizeJevResponses(answers: JevResponses): { signals: RoutingSignals; confidence: number } {
  const signals = {
    difficulty: normalizeScore(answers.difficulty.score),
    reasoningNeed: normalizeScore(answers.reasoningNeed.score),
    ambiguity: normalizeScore(answers.ambiguity.score),
    toolComplexity: normalizeScore(answers.toolComplexity.score),
  };

  const confidence = 
    (answers.difficulty.confidence +
     answers.reasoningNeed.confidence +
     answers.ambiguity.confidence +
     answers.toolComplexity.confidence) / 4.0;

  return { signals, confidence };
}

src/decision/jev/jev-provider.ts

import { TypeSafeClient } from '@typesafe-ai/sdk';
import { DecisionProvider } from '../decision-provider';
import { RoutingRequest } from '../../types/routing';
import { RoutingSignals } from '../../types/signals';
import { routingQuestions } from './jev-questions';
import { normalizeJevResponses } from './jev-normalizer';

export class JevDecisionProvider implements DecisionProvider {
  private client: TypeSafeClient;

  /**
   * Initializes the JevDecisionProvider with an authenticated TypeSafeClient.
   * @param client The pre-configured TypeSafeClient instance.
   */
  constructor(client: TypeSafeClient) {
    this.client = client;
  }

  async evaluate(request: RoutingRequest): Promise<{ signals: RoutingSignals; confidence: number }> {
    const result = await this.client.systemOne({
      state: {
        task: request.task,
        contextTokens: request.contextTokens,
        toolsAvailable: request.tools ?? [],
        requiresVision: request.requiresVision ?? false,
        requiresStructuredOutput: request.requiresStructuredOutput ?? false,
      },
      questions: routingQuestions,
    });

    return normalizeJevResponses(result.answers);
  }
}

src/decision/jev/jev-questions.ts

import { score, Questions } from '@typesafe-ai/sdk';

/**
 * The core routing questions evaluated by Jev.
 */
export const routingQuestions = {
  difficulty: score(
    'Estimate how difficult the task is for an agent to complete correctly.',
    ['trivial', 'easy', 'moderate', 'hard', 'very hard']
  ),

  reasoningNeed: score(
    'Estimate how much deep/multi-step reasoning is required.',
    ['almost none', 'low', 'moderate', 'high', 'very high']
  ),

  ambiguity: score(
    'Estimate how much interpretation is required before the agent can confidently act.',
    ['unambiguous', 'mostly clear', 'moderately ambiguous', 'highly ambiguous', 'very ambiguous']
  ),

  toolComplexity: score(
    'Estimate how complex the expected tool interaction is.',
    [
      'no tools / trivial action',
      'simple tool usage',
      'several straightforward tool calls',
      'complex dependent workflow',
      'highly stateful/error-prone workflow',
    ]
  ),
} satisfies Questions;

src/index.ts

// Types
export type { RoutingRequest } from './types/routing';
export type { RoutingSignals } from './types/signals';
export type { RoutingDecision } from './types/decision';

// Models
export type { ModelProfile } from './models/model-profile';

// Policies
export type { RoutingObjective } from './policies/objective';
export type { RoutingPolicy } from './policies/policy';

// Decision Provider
export type { DecisionProvider } from './decision/decision-provider';
export { JevDecisionProvider } from './decision/jev/jev-provider';

// Router Core
export { ModelRouter } from './router/model-router';
export type { ModelRouterConfig } from './router/model-router';

src/router/model-router.ts

import { ModelProfile } from '../models/model-profile';
import { RoutingPolicy } from '../policies/policy';
import { RoutingObjective } from '../policies/objective';
import { DecisionProvider } from '../decision/decision-provider';
import { RoutingRequest } from '../types/routing';
import { RoutingDecision } from '../types/decision';
import { PolicyEngine } from './policy-engine';
import { filterEligibleModels } from './eligibility';
import { TypeSafeClient } from '@typesafe-ai/sdk';
import { JevDecisionProvider } from '../decision/jev/jev-provider';

export type DecisionProviderConfig = 
  | { type: 'jev' }
  | DecisionProvider;

export interface ModelRouterConfig {
  models: ModelProfile[];
  decisionProvider: DecisionProviderConfig;
  objective: RoutingObjective;
  fallback?: {
    strategy: 'configured-model';
    model: string;
  };
  constraints?: {
    maxInputCostPerMillion?: number;
    maxOutputCostPerMillion?: number;
    maxLatencyMs?: number;
  };
}

export class ModelRouter {
  private models: ModelProfile[];
  private decisionProvider: DecisionProvider;
  private policy: RoutingPolicy;
  private policyEngine: PolicyEngine;

  constructor(config: ModelRouterConfig) {
    if (!config.models || config.models.length === 0) {
      throw new Error('ModelRouter must be configured with at least one model.');
    }
    
    this.models = config.models;
    
    if ('type' in config.decisionProvider && config.decisionProvider.type === 'jev') {
      this.decisionProvider = new JevDecisionProvider(new TypeSafeClient());
    } else {
      this.decisionProvider = config.decisionProvider as DecisionProvider;
    }
    
    this.policy = {
      objective: config.objective,
    };
    if (config.fallback) this.policy.fallback = config.fallback;
    if (config.constraints) this.policy.constraints = config.constraints;
    
    this.policyEngine = new PolicyEngine();
  }

  /**
   * Routes a request to the best-fit model according to capabilities and configured policy.
   * 
   * @param request The task and its constraints
   * @returns A RoutingDecision containing the selected model and metadata
   */
  public async route(request: RoutingRequest): Promise<RoutingDecision> {
    const startTime = Date.now();

    try {
      // 1. Get task capability signals from Jev / DecisionProvider
      const { signals, confidence } = await this.decisionProvider.evaluate(request);

      // 2. Filter out models that lack hard capabilities
      const eligibleModels = filterEligibleModels(this.models, request);

      if (eligibleModels.length === 0) {
        return this.handleFailure(request, 'No models eligible for requested constraints.', startTime);
      }

      // 3. Select best model based on objectives
      const { selectedModel, requirementScore } = this.policyEngine.selectBestModel(
        eligibleModels,
        signals,
        confidence,
        this.policy
      );

      if (!selectedModel) {
        return this.handleFailure(request, 'Policy engine failed to select a model.', startTime);
      }

      const routingLatencyMs = Date.now() - startTime;

      return {
        model: selectedModel.id,
        signals,
        requirementScore,
        confidence,
        eligibleModels: eligibleModels.map(m => m.id),
        metadata: {
          routingLatencyMs
        }
      };
    } catch (error: any) {
      // If decision provider times out or throws, handle failure cleanly
      return this.handleFailure(request, error.message || 'Unknown routing failure.', startTime);
    }
  }

  /**
  * Falls back gracefully if routing fails (e.g., Jev API failure, no eligible models).
   */
  private handleFailure(request: RoutingRequest, reason: string, startTime: number): RoutingDecision {
    const routingLatencyMs = Date.now() - startTime;
    
    if (this.policy.fallback && this.policy.fallback.strategy === 'configured-model') {
      const fallbackModel = this.policy.fallback.model;
      const configuredFallback = this.models.find(model => model.id === fallbackModel);

      if (!configuredFallback) {
        throw new Error(`Routing failed and configured fallback model '${fallbackModel}' is not configured. Reason: ${reason}`);
      }

      if (filterEligibleModels([configuredFallback], request).length === 0) {
        throw new Error(`Routing failed and configured fallback model '${fallbackModel}' cannot satisfy the request. Reason: ${reason}`);
      }

      const constraints = this.policy.constraints;
      if (
        (constraints?.maxInputCostPerMillion !== undefined && configuredFallback.pricing.inputPerMillion > constraints.maxInputCostPerMillion) ||
        (constraints?.maxOutputCostPerMillion !== undefined && configuredFallback.pricing.outputPerMillion > constraints.maxOutputCostPerMillion) ||
        (constraints?.maxLatencyMs !== undefined && (configuredFallback.latency?.expectedMs ?? 1000) > constraints.maxLatencyMs)
      ) {
        throw new Error(`Routing failed and configured fallback model '${fallbackModel}' violates policy constraints. Reason: ${reason}`);
      }
      
      // We produce a synthetic decision that indicates complete fallback
      return {
        model: fallbackModel,
        signals: {
          difficulty: 1.0,
          reasoningNeed: 1.0,
          ambiguity: 1.0,
          toolComplexity: 1.0,
        },
        requirementScore: 1.0,
        confidence: 0.0,
        eligibleModels: [fallbackModel],
        metadata: {
          routingLatencyMs,
          error: reason,
          fallback: true,
        }
      };
    }

    // Fail closed if no fallback strategy is defined
    throw new Error(`Routing failed and no fallback strategy is configured. Reason: ${reason}`);
  }
}

src/router/policy-engine.ts

import { ModelProfile } from '../models/model-profile';
import { RoutingPolicy } from '../policies/policy';
import { RoutingSignals } from '../types/signals';
import {
  computeCapabilityMatch,
  computeRequirementScore,
  getCapabilityRequirements,
  meetsCapabilityRequirements,
} from './scoring';

export class PolicyEngine {
  /**
   * Selects the best model according to the policy and objectives.
   * 
   * @param eligibleModels Models that have already passed deterministic filters.
  * @param signals The signals extracted from the request by Jev.
   * @param policy The routing policy defining objectives and constraints.
   * @returns The selected ModelProfile, or null if no valid model is found.
   */
  public selectBestModel(
    eligibleModels: ModelProfile[],
    signals: RoutingSignals,
    confidence: number,
    policy: RoutingPolicy
  ): { selectedModel: ModelProfile | null; requirementScore: number } {
    
    // Apply budget/latency constraints first
    const constrainedModels = eligibleModels.filter((model) => {
      if (policy.constraints) {
        if (policy.constraints.maxInputCostPerMillion !== undefined && model.pricing.inputPerMillion > policy.constraints.maxInputCostPerMillion) return false;
        if (policy.constraints.maxOutputCostPerMillion !== undefined && model.pricing.outputPerMillion > policy.constraints.maxOutputCostPerMillion) return false;
        if (policy.constraints.maxLatencyMs !== undefined && (model.latency?.expectedMs ?? 1000) > policy.constraints.maxLatencyMs) return false;
      }
      return true;
    });

    if (constrainedModels.length === 0) {
      return { selectedModel: null, requirementScore: 0 };
    }

    const rawRequirementScore = computeRequirementScore(signals, policy.signalWeights);
    const safeRequirementScore = Math.min(1, rawRequirementScore + Math.max(0, 0.8 - confidence) * 0.5);
    const capabilityRequirements = getCapabilityRequirements(signals, confidence);

    // Match each model to the dimensions that this task actually requires.
    const modelCapabilityScores = new Map<string, number>();
    for (const model of constrainedModels) {
      const capScore = computeCapabilityMatch(model.capabilities, capabilityRequirements);
      modelCapabilityScores.set(model.id, capScore);
    }

    const sufficientModels = constrainedModels.filter(m => 
      meetsCapabilityRequirements(m.capabilities, capabilityRequirements)
    );

    if (sufficientModels.length === 0) {
      return { selectedModel: null, requirementScore: safeRequirementScore };
    }

    const candidateModels = sufficientModels;

    const objective = policy.objective;
    
    // Sort candidates according to the objective
    candidateModels.sort((a, b) => {
      switch (objective.type) {
        case 'cost': {
          // Minimize cost: cheapest first
          const costA = a.pricing.inputPerMillion + a.pricing.outputPerMillion;
          const costB = b.pricing.inputPerMillion + b.pricing.outputPerMillion;
          return costA - costB;
        }
        case 'latency': {
          // Minimize latency: fastest first
          return (a.latency?.expectedMs ?? 1000) - (b.latency?.expectedMs ?? 1000);
        }
        case 'quality': {
          // Maximize quality: highest capability first
          const capA = modelCapabilityScores.get(a.id) || 0;
          const capB = modelCapabilityScores.get(b.id) || 0;
          return capB - capA; 
        }
        case 'balanced': {
          // Maximize a composite score combining cost, latency, and quality
          
          // Normalized quality (0-1)
          const capA = modelCapabilityScores.get(a.id) || 0;
          const capB = modelCapabilityScores.get(b.id) || 0;

          // Normalized cost (inverse: lower is better, arbitrary scaling for ranking)
          // E.g., cost of $1 / M is better than $10 / M. We use 1 / (cost + 1)
          const costScoreA = 1.0 / (a.pricing.inputPerMillion + a.pricing.outputPerMillion + 1);
          const costScoreB = 1.0 / (b.pricing.inputPerMillion + b.pricing.outputPerMillion + 1);

          // Normalized latency (inverse: lower is better)
          const latScoreA = 1000.0 / ((a.latency?.expectedMs ?? 1000) + 1);
          const latScoreB = 1000.0 / ((b.latency?.expectedMs ?? 1000) + 1);

          const scoreA = 
            (capA * objective.qualityWeight) + 
            (costScoreA * objective.costWeight) + 
            (latScoreA * objective.latencyWeight);

          const scoreB = 
            (capB * objective.qualityWeight) + 
            (costScoreB * objective.costWeight) + 
            (latScoreB * objective.latencyWeight);

          return scoreB - scoreA;
        }
      }
    });

    return { 
      selectedModel: candidateModels[0], 
      requirementScore: safeRequirementScore 
    };
  }
}