Qualitative query

import { defineConfig } from 'eslint/config';
import obsidianmd from 'eslint-plugin-obsidianmd';

export default defineConfig([
  { ignores: ['dist/**', 'release/**', 'node_modules/**', 'scripts/**', 'tests/**', '*.mjs'] },
  ...obsidianmd.configs.recommended,
  {
    languageOptions: { parserOptions: { projectService: true } },
    rules: {
      "obsidianmd/ui/sentence-case": ["warn", {
        brands: ["Jev", "TypeSafe", "Qualitative Query", "Obsidian"],
        // Preserve the credential prefix and the established UI spelling for block IDs.
        ignoreRegex: ["^ts_$", "^.*\\bIDs\\b.*$"],
      }],
    },
  },
]);

/** Bundle for Obsidian's eval command. Reads only this plugin and the requested notes. */
import { parseQuery, resolveContext } from "../src/query";
import { TFile, type App } from "obsidian";
import type { QueryResult, QuerySpec } from "../src/types";
declare const app: App;
declare const require: (name:string)=>any;

(globalThis as any).captureQualitativePipeline = async (options:{queryPath:string;bakedPath?:string;output:string}) => {
  const plugin=(app as any).plugins.plugins['qualitative-query'];
  if(!plugin)throw Error('Enable Qualitative Query first.');
  const file=app.vault.getAbstractFileByPath(options.queryPath);
  if(!(file instanceof TFile))throw Error('Query note missing.');
  const queryText=await app.vault.read(file);
  if((queryText.match(/```(?:qq|qualitative-query)\b/g)||[]).length!==1)throw Error('Capture requires one query block.');
  const spec=parseQuery(file.path,queryText,plugin.settings.queryFolder);
  spec.contextPaths=[...resolveContext(app,file,spec.contextPaths)];
  await plugin.index.whenReady();
  const startedAt=new Date().toISOString();
  const candidates=await plugin.retrieve(spec.question,plugin.engine.getBlocks(),plugin.settings.candidateLimit);
  // Scoring criteria stay unchanged. Capture every score, not just the display slice.
  const result:QueryResult=await plugin.engine.run({...spec,limit:1000},plugin.settings.candidateLimit,1000,0);
  if(result.selection!=='jev')throw Error('Jev scoring did not complete; no successful capture recorded.');
  const threshold=spec.threshold??plugin.settings.threshold;
  const selected=result.judgements.filter(j=>j.score>=threshold).slice(0,spec.limit??plugin.settings.resultLimit);
  let baked:any=null;
  if(options.bakedPath){
    const bakedFile=app.vault.getAbstractFileByPath(options.bakedPath);
    if(!(bakedFile instanceof TFile))throw Error('Baked note missing.');
    const text=await app.vault.read(bakedFile);
    const refs=[];
    for(const match of text.matchAll(/!\[\[([^\]#]+)#\^([^\]|]+)(?:\|[^\]]+)?\]\]/g)){
      const source=app.metadataCache.getFirstLinkpathDest(match[1],bakedFile.path);
      const block=source&&app.metadataCache.getFileCache(source)?.blocks?.[match[2]];
      const sourceText=source?await app.vault.read(source):'';
      refs.push({link:match[0],path:source?.path,id:match[2],resolved:!!block,
        position:block?.position,text:block?sourceText.slice(block.position.start.offset,block.position.end.offset):null,
        surrounding:block?sourceText.split('\n').slice(Math.max(0,block.position.start.line-3),block.position.end.line+4).join('\n'):null});
    }
    baked={path:bakedFile.path,text,refs};
  }
  const data={startedAt,finishedAt:new Date().toISOString(),vault:app.vault.getName(),pluginVersion:plugin.manifest.version,
    model:plugin.settings.model,queryPath:file.path,queryText,spec,candidateLimit:plugin.settings.candidateLimit,
    corpusBlocks:plugin.engine.getBlocks().length,embeddingWarning:plugin.embeddingWarning,
    candidates:candidates.map((c:any,i:number)=>({id:c.id,rank:i+1,path:c.path,text:c.text,retrievalScore:c.retrievalScore})),
    evaluatedCandidates:result.candidates.length,warning:result.warning,judgements:result.judgements,selected,baked};
  require('node:fs').writeFileSync(options.output,JSON.stringify(data,null,2));
  return {output:options.output,scored:result.judgements.length,selected:selected.length};
};

const fs = require('node:fs');
const vm = require('node:vm');
const assert = require('node:assert/strict');

class Element {
  constructor() { this.children = []; this.text = ''; this.isConnected = true; this.classes = new Set(); }
  empty() { this.children = []; this.text = ''; }
  addClass(...names) { names.forEach(n => this.classes.add(n)); }
  removeClass(...names) { names.forEach(n => this.classes.delete(n)); }
  createEl(tag, options = {}) { const child = new Element(); child.tag = tag; child.text = options.text || ''; if(options.cls) child.addClass(options.cls); this.children.push(child); return child; }
  createDiv(options = {}) { return this.createEl('div', typeof options === 'string' ? {cls:options} : options); }
  createSpan(options) { return this.createEl('span', options); }
  setText(text) { this.text = text; }
  setAttribute() {}
  addEventListener() {}
  prepend(child) { this.children.unshift(child); }
  set textContent(text) { this.text = text; }
  appendChild(child) { this.children.push(child); return child; }
  remove() { this.isConnected = false; }
  closest() { return null; }
  querySelector() { return null; }
  querySelectorAll() { return []; }
  get textContent() { return this.text + this.children.map(c => c.textContent).join(' '); }
}
class Component {
  constructor() { this.children = []; this.cleanups = []; }
  addChild(child) { this.children.push(child); child.onload?.(); return child; }
  removeChild(child) { child.unload(); this.children = this.children.filter(c=>c!==child); }
  register(fn) { this.cleanups.push(fn); }
  registerDomEvent() {}
  load() { this.onload?.(); }
  unload() { this.onunload?.(); this.children.forEach(c=>c.unload()); this.cleanups.forEach(f=>f()); }
}
class MarkdownRenderChild extends Component { constructor(el) { super(); this.containerEl=el; } }
class TFile { constructor(path) { this.path=path; this.extension='md'; this.basename=path.split('/').pop().replace(/\.md$/,''); this.stat={mtime:1}; } }
const source = new TFile('Sources/Conflict.md');
const query = new TFile('Queries/Conflict.md');
const contents = new Map([[source.path,'# Conflict\n\nA conflict is a clash of incompatible goals.'],[query.path,'# Conflict\n\n```qualitative-query\nWhat defines a conflict?\n```']]);
const callbacks = new Map();
const app = {
  secretStorage:{getSecret(){return '';},setSecret(){}},
  vault:{configDir:".custom-config",getMarkdownFiles(){return [source,query];},cachedRead:async f=>contents.get(f.path),read:async f=>contents.get(f.path),getAbstractFileByPath:p=>[source,query].find(f=>f.path===p),on:(event,fn)=>{callbacks.set(event,fn);return {};},offref(){}},
  metadataCache:{getFirstLinkpathDest:()=>null,on:()=>({}),offref(){}},
  workspace:{iterateAllLeaves(){},on:()=>({}),offref(){},onLayoutReady(fn){fn();},openLinkText:async()=>{},getLeavesOfType:()=>[]}
};
class Plugin extends Component {
  constructor(){super();this.app=app;this.processors={};this.postprocessors=[];}
  async loadData(){return {candidateLimit:1000,provider:'embedded',model:'kev-0.5b',endpoint:'http://127.0.0.1:8009',embeddedModelDir:'runtime/kev'};}
  async saveData(){}
  registerMarkdownCodeBlockProcessor(name,fn){this.processors[name]=fn;}
  registerMarkdownPostProcessor(fn){this.postprocessors.push(fn);}
  registerEvent(){}
  registerInterval(id){this.register(()=>clearInterval(id));}
  addSettingTab(){}
  addCommand(){}
  addRibbonIcon(){}
}
class MarkdownView {}
const requests=[];
let allowRequests=false;
const obsidian={Modal:class{},Plugin,Component,MarkdownRenderChild,TFile,MarkdownView,PluginSettingTab:class{},Setting:class{},Notice:class{},normalizePath:p=>p.replace(/\\/g,'/'),MarkdownRenderer:{render:async(app,text,el)=>{el.createDiv({text});}},requestUrl:async(options)=>{
  assert.ok(allowRequests,'Unexpected network call without credentials');
  requests.push(options);
  const body=JSON.parse(options.body); const answers={};
  for(const [id,q] of Object.entries(body.questions)) answers[id]=q.type==='choice'?{type:'choice',choice:'definition',probabilities:{definition:1,condition:0,distinction:0,other:0},confidence:1}:{type:'noul',noul:.9};
  return {status:200,json:{model:body.model,answers,usage:{input_tokens:20,output_tokens:0}},headers:{}};
},parseYaml:()=>({})};
const moduleObject={exports:{}};
vm.runInNewContext(fs.readFileSync(process.argv[2],'utf8'),{module:moduleObject,exports:moduleObject.exports,require:(name)=>{assert.equal(name,'obsidian');return obsidian;},console,setTimeout,clearTimeout,setInterval,clearInterval,window:{setTimeout,clearTimeout,setInterval,clearInterval},createEl:()=>new Element(),createDiv:()=>new Element(),document:{createElement:()=>{throw new Error("Use Obsidian createEl helpers");}},URL,AbortController,TextEncoder,performance,crypto:require('node:crypto').webcrypto});
(async()=>{
  const plugin=new moduleObject.exports.default();
  await plugin.onload();
  assert.equal(plugin.settings.candidateLimit,1000);
  const el=new Element(); const children=[];
  const ctx={sourcePath:query.path,docId:'smoke',addChild:c=>{children.push(c);c.load();},getSectionInfo:()=>({lineStart:0,lineEnd:0,text:contents.get(query.path)})};
  await plugin.processors['qualitative-query']('What defines a conflict?',el,ctx);
  await new Promise(r=>setTimeout(r,60));
  assert.match(el.textContent,/key|provider|configure/i);
  assert.equal(requests.length,0);
  // A store install has no sibling worker file. Exercise the worker carried in main.js.
  const workerSource = await plugin.retrieval.semantic.assets.readWorker();
  assert.ok(workerSource.length > 1000);
  const responses = [];
  const workerContext = vm.createContext({ArrayBuffer,TextEncoder,TextDecoder,postMessage:message=>responses.push(message)});
  workerContext.self = workerContext;
  vm.runInContext(workerSource, workerContext);
  const weights = fs.readFileSync('models/embeddings/model.safetensors');
  workerContext.onmessage({data:{id:1,type:'load',weights:weights.buffer.slice(weights.byteOffset,weights.byteOffset+weights.byteLength),tokenizer:JSON.parse(fs.readFileSync('models/embeddings/tokenizer.json','utf8')),config:JSON.parse(fs.readFileSync('models/embeddings/tokenizer_config.json','utf8'))}});
  workerContext.onmessage({data:{id:2,type:'update',rows:[{id:'conflict',text:'Mediation resolves disputes.'},{id:'cake',text:'A chocolate cake recipe.'}]}});
  workerContext.onmessage({data:{id:3,type:'search',question:'resolving interpersonal conflict',limit:2}});
  assert.ok(responses.every(message=>!message.error),JSON.stringify(responses));
  assert.equal(responses.at(-1).result[0].id,'conflict');
  assert.match(el.textContent,/incompatible goals/);
  assert.match(el.textContent,/Local match/);
  allowRequests=true;
  plugin.settings.apiKey='synthetic-smoke-key';
  plugin.rebuildClient();
  const connectedEl=new Element();
  await plugin.processors['qualitative-query']('What defines a conflict?',connectedEl,ctx);
  await new Promise(r=>setTimeout(r,100));
  assert.match(connectedEl.textContent,/incompatible goals/);
  assert.ok(requests.length>0);
  assert.equal(requests.at(-1).url,'https://api.typesafe.ai/v1/systemone');
  contents.set(source.path,'# Conflict\n\nA conflict is a revised definition of incompatible goals.');
  source.stat.mtime++;
  callbacks.get('modify')(source);
  await new Promise(r=>setTimeout(r,700));
  assert.match(connectedEl.textContent,/revised definition/);
  assert.equal(plugin.settings.provider,undefined);
  assert.equal(plugin.settings.endpoint,undefined);
  assert.equal(plugin.settings.embeddedModelDir,undefined);
  assert.equal(plugin.settings.model,'jev-1.13.0');
  assert.ok(requests.every(request=>request.url==='https://api.typesafe.ai/v1/systemone'));
  // A modify timer must not reintroduce a source renamed to a non-Markdown file.
  callbacks.get('modify')(source);
  const previousPath = source.path;
  source.path = 'Sources/Conflict.txt'; source.extension = 'txt';
  callbacks.get('rename')(source, previousPath);
  await new Promise(r=>setTimeout(r,700));
  assert.equal(plugin.index.blocks.length,0);
  assert.doesNotMatch(connectedEl.textContent,/revised definition/);
  assert.equal(plugin.indexError,'');
  plugin.unload(); children.forEach(c=>c.unload());
  console.log('Bundled plugin smoke passed: startup, no-key guard, hosted rendering, live source refresh, legacy-settings migration, Jev-only transport, unload. Embedded worker executes from main.js with real model weights. All transport mocked.');
})().catch(error=>{console.error(error);process.exitCode=1;});

import { Modal, Notice, Setting, type App } from "obsidian";
import { createUniqueNote } from "./notes";

export const PRESETS: Record<string,{name:string;instructions:string;yes:string;no:string}> = {
  relevant:{name:"Relevant passages",instructions:"Select passages that directly answer the question with substantive content.",yes:"The passage contributes an answer, explanation, or specific example.",no:"It only mentions the topic or is unrelated."},
  definitions:{name:"Definitions",instructions:"Select passages that define the subject or explain its boundaries.",yes:"It states what the subject means or a condition that distinguishes it.",no:"It merely mentions the subject or gives an example without explaining its meaning."},
  distinctions:{name:"Distinctions",instructions:"Select passages that explain the distinction asked about.",yes:"It directly contrasts the concepts or gives an example demonstrating the difference.",no:"It discusses only one concept without explaining the distinction."},
  practices:{name:"Concrete practices",instructions:"Select passages describing an actionable practice relevant to the question.",yes:"It describes what someone does and why or what changes as a result.",no:"It only praises an ideal or names a practice without describing it."},
  evidence:{name:"Evidence",instructions:"Select passages providing specific evidence relevant to the question.",yes:"It describes an observation, study, event, or concrete example supporting a claim.",no:"It only asserts a claim, uses a metaphor, or mentions evidence without describing it."},
};
export function queryMarkdown(question:string,preset:string,include:string,exclude:string):string {
  const p=PRESETS[preset];
  if(!p || !question.trim()) throw new Error("Enter a question and choose what to look for.");
  const line=(s:string)=>s.replace(/[\r\n]+/g," ").replace(/`/g,"'").trim();
  // Fixed templates; the builder does not generate prose or infer hidden context.
  return `# ${line(question)}\n\n\`\`\`qualitative-query\nquestion: ${line(question)}\nmode: generic\ninstructions: ${p.instructions}\ntrue: ${line(include)||p.yes}\nfalse: ${line(exclude)||p.no}\nlimit: 6\nthreshold: 0.5\nadjacent: 0\n\`\`\`\n`;
}
export class QueryBuilder extends Modal {
  constructor(app:App,private readonly folder:string){super(app);}
  onOpen():void {
    const el=this.contentEl;el.empty();el.createEl("h2",{text:"Ask your vault"});
    el.createEl("p",{text:"Find passages that answer your question. Save them in a note to connect their sources."});
    let question="",preset="relevant",include="",exclude="";
    new Setting(el).setName("Question").addTextArea(t=>t.setPlaceholder("How does solitude differ from loneliness?").onChange(v=>question=v));
    new Setting(el).setName("Look for").addDropdown(d=>{for(const [id,p] of Object.entries(PRESETS))d.addOption(id,p.name);d.onChange(v=>{preset=v;defaults.setText(`${PRESETS[v].yes} Excludes: ${PRESETS[v].no}`);});});
    const defaults=el.createEl("p",{text:`${PRESETS.relevant.yes} Excludes: ${PRESETS.relevant.no}`});
    const details=el.createEl("details");details.createEl("summary",{text:"Refine matches (optional)"});
    new Setting(details).setName("Include when").setDesc("What must a passage explain or show? Leave blank to use the default.").addTextArea(t=>t.onChange(v=>include=v));
    new Setting(details).setName("Exclude when").setDesc("What should be left out? Leave blank to use the default.").addTextArea(t=>t.onChange(v=>exclude=v));
    el.createEl("p",{text:"Saves your question and finds up to 6 passages. Results update as your notes change. With a Jev key, scoring sends passages to TypeSafe."});
    new Setting(el).addButton(b=>b.setButtonText("Save question and search").setCta().onClick(async()=>{
      b.setDisabled(true);
      try {
        const text=queryMarkdown(question,preset,include,exclude);
        const file=await createUniqueNote(this.app,this.folder,question,text);
        this.close();await this.app.workspace.getLeaf(false).openFile(file,{state:{mode:"preview"}});
      }catch(error){new Notice(error instanceof Error?error.message:String(error));b.setDisabled(false);}
    }));
  }
  onClose():void{this.contentEl.empty();}
}

import { requestUrl, type DataAdapter } from "obsidian";
export const EMBEDDING_REVISION = "bf8b056651a2c21b8d2565580b8569da283cab23";
const FILES = {
  "model.safetensors": {
    "bytes": 30236760,
    "sha256": "f65d0f325faadc1e121c319e2faa41170d3fa07d8c89abd48ca5358d9a223de2"
  },
  "tokenizer.json": {
    "bytes": 683666,
    "sha256": "e67e803f624fb4d67dea1c730d06e1067e1b14d830e2c2202569e3ef0f70bb50"
  },
  "tokenizer_config.json": {
    "bytes": 1431,
    "sha256": "6725995e3ab3039857ff5bd99178a7cdf42863abb04449e7bb31feb1f55fe567"
  }
} as const;

/** Explicit download only. Requests contain no vault text or Jev credentials. */
export async function downloadEmbeddings(adapter: DataAdapter, directory: string): Promise<void> {
  if (!await adapter.exists(`${directory}/embeddings`)) await adapter.mkdir(`${directory}/embeddings`);
  for (const [name, expected] of Object.entries(FILES)) {
    const response = await requestUrl({ url: `https://huggingface.co/minishlab/potion-base-8M/resolve/${EMBEDDING_REVISION}/${name}`, method: "GET", throw: false });
    if (response.status !== 200 || response.arrayBuffer.byteLength !== expected.bytes) throw new Error(`Embedding download failed: ${name}.`);
    const digest = await crypto.subtle.digest("SHA-256", response.arrayBuffer);
    const hash = Array.from(new Uint8Array(digest), byte => byte.toString(16).padStart(2, "0")).join("");
    if (hash !== expected.sha256) throw new Error(`Embedding download checksum failed: ${name}.`);
    const target = `${directory}/embeddings/${name}`;
    await adapter.writeBinary(`${target}.download`, response.arrayBuffer);
    await adapter.rename(`${target}.download`, target);
  }
}

import { checkSignal, SharedWork } from "./work";
import type { RetrievalResult } from "./retrieval";
import type { ScoreCache } from "./score-cache";
import { MAX_CANDIDATES } from "./limits";
import { shortlist } from "./search";
import { JevClient } from "./jev";
import type { Block, Candidate, Contribution, Judgement, QueryResult, QuerySpec, QueryStats } from "./types";

const MAX_PASSAGE_CHARS = 24_000;
const MAX_CONTEXT_CHARS = 48_000;
const MAX_CACHE_ENTRIES = 512;
const MAX_CONCURRENCY = 16;


type ContextReader = (path: string) => Block[] | Promise<Block[]>;
type WarningResult = QueryResult & { warning?: string };

class QueryRunSuperseded extends Error {
  constructor() { super("This query was superseded by a newer query."); }
}

interface QueueItem {
  generation: number;
  current: () => boolean;
  resolve: (release: () => void) => void;
  reject: (error: unknown) => void;
}

class ConcurrencyGate {
  private running = 0;
  private limit = MAX_CONCURRENCY;
  private lastThrottle = -Infinity;
  private successes = 0;

  throttle(): void {
    const now = Date.now();
    if (now - this.lastThrottle >= 1000) this.limit = Math.max(1, Math.floor(this.limit / 2));
    this.lastThrottle = now; this.successes = 0;
  }

  success(): void {
    if (Date.now() - this.lastThrottle >= 5000 && ++this.successes >= 32) {
      this.limit = Math.min(MAX_CONCURRENCY, this.limit + 1); this.successes = 0;
      this.pump();
    }
  }
  private queue: QueueItem[] = [];

  acquire(generation: number, current: () => boolean): Promise<() => void> {
    if (!current()) return Promise.reject(new QueryRunSuperseded());
    return new Promise((resolve, reject) => {
      this.queue.push({ generation, current, resolve, reject });
      this.pump();
    });
  }

  cancelStale(generation: number): void {
    const retained: QueueItem[] = [];
    for (const item of this.queue) {
      if (item.generation < generation || !item.current()) item.reject(new QueryRunSuperseded());
      else retained.push(item);
    }
    this.queue = retained;
  }

  private pump(): void {
    while (this.running < this.limit && this.queue.length) {
      const item = this.queue.shift()!;
      if (!item.current()) { item.reject(new QueryRunSuperseded()); continue; }
      this.running++;
      let released = false;
      item.resolve(() => {
        if (released) return;
        released = true;
        this.running--;
        this.pump();
      });
    }
  }
}

function stable(value: unknown): string {
  if (value === null || typeof value !== "object") return JSON.stringify(value);
  if (Array.isArray(value)) return `[${value.map(stable).join(",")}]`;
  return `{${Object.keys(value as Record<string, unknown>).sort().map((key) => `${JSON.stringify(key)}:${stable((value as Record<string, unknown>)[key])}`).join(",")}}`;
}

function span(block: Block): number { return Math.max(0, block.lineEnd - block.lineStart); }

/** Remove parent/child duplicates while retaining the most complete passage. */
function dedupeRanges<T extends Block>(blocks: T[]): T[] {
  const selected: T[] = [];
  const ordered = [...blocks].sort((a, b) => a.path.localeCompare(b.path) || a.lineStart - b.lineStart || a.lineEnd - b.lineEnd);
  for (const block of ordered) {
    // Source-order sorting means only the last selected range can overlap.
    const existing = selected[selected.length - 1];
    if (!existing || existing.path !== block.path || existing.lineEnd < block.lineStart) { selected.push(block); continue; }
    if (span(block) > span(existing)) selected[selected.length - 1] = block;
  }
  return selected.sort((a, b) => a.path.localeCompare(b.path) || a.lineStart - b.lineStart || a.id.localeCompare(b.id));
}

function dedupeCandidates(candidates: Candidate[]): Candidate[] {
  const selected: Candidate[] = [];
  const ranges = new Map<string, Candidate[]>();
  // Score first so a broad ancestor cannot hide a better focused child.
  const ranked = [...candidates].sort((a, b) => (b.retrievalScore ?? b.lexicalScore) - (a.retrievalScore ?? a.lexicalScore) || span(a) - span(b) || a.id.localeCompare(b.id));
  for (const candidate of ranked) {
    const file = ranges.get(candidate.path) ?? [];
    let low = 0, high = file.length;
    while (low < high) { const mid = (low + high) >>> 1; if (file[mid].lineStart < candidate.lineStart) low = mid + 1; else high = mid; }
    if ((low > 0 && file[low - 1].lineEnd >= candidate.lineStart) || (low < file.length && file[low].lineStart <= candidate.lineEnd)) continue;
    file.splice(low, 0, candidate);
    ranges.set(candidate.path, file);
    selected.push(candidate);
  }
  return selected;
}

function displayText(block: Block): string { return block.renderText ?? block.text; }
function modelText(block: Block): string {
  const heading = block.headingPath.length ? `[Headings: ${block.headingPath.join(" > ")}]\n` : "";
  return `${heading}${displayText(block)}`;
}

export class QueryEngine {
  private cache = new Map<string, Judgement>();
  private inFlight = new Map<string, SharedWork<Judgement>>();
  private runs = new Set<AbortController>();
  private generation = 0;
  private disposed = false;
  private readonly gate = new ConcurrencyGate();

  constructor(
    private readonly client: Pick<JevClient, "rank"> | null,
    private readonly getBlocks: () => Block[],
    private readonly getSourceBlocks: ContextReader = () => [],
    /** Include provider/model identity when an engine is reused across clients. */
    private readonly cacheNamespace = "",
    private readonly persistentCache?: Pick<ScoreCache, "get" | "set"> & Partial<Pick<ScoreCache, "getMany">>,
    private readonly retrieve: (question: string, blocks: Block[], limit: number, signal?: AbortSignal) => RetrievalResult | Promise<RetrievalResult> = (question, blocks, limit) => ({ candidates: shortlist(question, blocks, limit), truncated: shortlist(question, blocks, limit + 1).length > limit }),
  ) {}

  clearCache(): void { this.cache.clear(); }

  invalidate(): void {
    this.generation++;
    for (const run of this.runs) run.abort();
    this.gate.cancelStale(this.generation);
  }

  dispose(): void { this.disposed = true; this.invalidate(); this.cache.clear(); this.inFlight.clear(); }

  async run(spec: QuerySpec, candidateLimit: number, limit: number, threshold: number, onProgress?: (completed: number, total: number) => void, onPartial?: (result: QueryResult) => void, signal?: AbortSignal): Promise<QueryResult> {
    const controller = new AbortController();
    const abort = () => controller.abort();
    signal?.addEventListener("abort", abort, { once: true });
    if (signal?.aborted) abort();
    this.runs.add(controller);
    try { return await this.runOwned(spec, candidateLimit, limit, threshold, onProgress, onPartial, controller.signal); }
    finally { controller.abort(); this.runs.delete(controller); signal?.removeEventListener("abort", abort); }
  }

  private async runOwned(spec: QuerySpec, candidateLimit: number, limit: number, threshold: number, onProgress: ((completed: number, total: number) => void) | undefined, onPartial: ((result: QueryResult) => void) | undefined, signal: AbortSignal): Promise<QueryResult> {
    checkSignal(signal);
    if (this.disposed) return { status: "error", candidates: [], judgements: [], error: "The query engine is unavailable." };
    const generation = this.generation;
    const corpus = this.getBlocks();
    let context: string;
    try { context = await this.readContext(spec.contextPaths); }
    catch (error) { return { status: "error", candidates: [], judgements: [], error: error instanceof Error ? error.message : String(error) }; }
    this.assertCurrent(generation, signal);
    if (context.length > MAX_CONTEXT_CHARS) return { status: "error", candidates: [], judgements: [], error: `Selected context is ${context.length.toLocaleString()} characters; the limit is ${MAX_CONTEXT_CHARS.toLocaleString()}. Choose specific blocks or sections, or remove context notes.` };
    const retrieval = await this.retrieve(spec.question, corpus, Math.min(MAX_CANDIDATES, Math.max(1, candidateLimit)), signal);
    const candidates = dedupeCandidates(retrieval.candidates);
    const stats: QueryStats = { searchable: corpus.length, shortlisted: retrieval.candidates.length,
      overlapRemoved: retrieval.candidates.length - candidates.length, windowLimit: Math.min(MAX_CANDIDATES, Math.max(1, candidateLimit)),
      truncated: retrieval.truncated ?? false, checked: 0, cached: 0, shared: 0, requested: 0, retries: 0,
      skipped: 0, passed: 0, contextChars: context.length, threshold };
    const warning = (message?: string) => [retrieval.warning, message].filter(Boolean).join(" ") || undefined;
    this.assertCurrent(generation, signal);
    if (!candidates.length) return { status: "empty", candidates, judgements: [], stats, selection: this.client ? "jev" : "local", warning: warning() };
    if (!this.client) {
      const ranked = [...candidates].sort((a, b) => (b.retrievalScore ?? b.lexicalScore) - (a.retrievalScore ?? a.lexicalScore) || a.id.localeCompare(b.id));
      return { status: "ready", selection: "local", candidates, stats,
        warning: warning("Local matches. Add a Jev API key in settings to select and order passages by how well they answer your question."),
        judgements: ranked.slice(0, spec.limit ?? limit).map(candidate => ({ candidate, score: candidate.retrievalScore ?? candidate.lexicalScore, contribution: "other", scores: {} })),
      };
    }

    const judgements: Judgement[] = [];
    const belowThreshold: Judgement[] = [];
    const accept = (value: Judgement) => { stats.checked++; if (value.score >= threshold) { judgements.push(value); stats.passed++; } else belowThreshold.push(value); };
    const rejected = () => [...belowThreshold].sort((a, b) => b.score - a.score || a.candidate.id.localeCompare(b.candidate.id));
    const skipped = candidates.filter((candidate) => displayText(candidate).length > MAX_PASSAGE_CHARS);
    stats.skipped = skipped.length;
    const judgeable = candidates.filter((candidate) => displayText(candidate).length <= MAX_PASSAGE_CHARS);
    const order: Record<Contribution, number> = { definition: 0, condition: 1, distinction: 2, other: 3 };
    const selection = () => [...judgements].sort((a, b) => spec.criteria.mode === "default"
      ? order[a.contribution] - order[b.contribution] || b.score - a.score || a.candidate.id.localeCompare(b.candidate.id)
      : b.score - a.score || a.candidate.id.localeCompare(b.candidate.id)).slice(0, spec.limit ?? limit);
    let completed = 0;
    onProgress?.(completed, judgeable.length);
    const keys = judgeable.map(candidate => this.scoreKey(spec, candidate, context));
    const persisted = await this.readScores(keys);
    this.assertCurrent(generation, signal);
    // Preserve existing location-based entries; promote them lazily without rescoring.
    const missing = judgeable.filter((_, i) => !persisted.has(keys[i]) && !this.cache.has(keys[i]));
    const legacy = await this.readScores(missing.map(candidate => this.scoreKey(spec, candidate, context, true)));
    this.assertCurrent(generation, signal);
    for (const candidate of missing) {
      const value = legacy.get(this.scoreKey(spec, candidate, context, true));
      if (value) {
        const key = this.scoreKey(spec, candidate, context);
        persisted.set(key, value);
        await this.persistentCache?.set(key, value);
      }
    }
    this.assertCurrent(generation, signal);
    let lastPartial = -Infinity;
    const publish = () => {
      if (Date.now() - lastPartial >= 250 || completed === judgeable.length) {
        lastPartial = Date.now();
        onPartial?.({ status: "ready", selection: "jev", candidates, judgements: selection(), belowThreshold: rejected(), stats: { ...stats }, warning: warning("Still checking passages. Results may change.") });
      }
    };
    const pending = judgeable.filter(candidate => {
      const key = this.scoreKey(spec, candidate, context);
      const value = this.cache.get(key) ?? persisted.get(key);
      if (!value) return true;
      stats.cached++; accept({ ...value, candidate });
      completed++; return false;
    });
    if (completed) { onProgress?.(completed, judgeable.length); publish(); }
    await Promise.all(pending.map(async (candidate) => {
      try {
        const value = await this.judge(spec, candidate, context, generation, stats, signal);
        this.assertCurrent(generation, signal);
        accept(value);
      } catch (error) {
        if (error && typeof error === "object" && "code" in error && error.code === "INPUT_TOO_LARGE") { skipped.push(candidate); stats.skipped++; }
        else throw error;
      }
      this.assertCurrent(generation, signal);
      onProgress?.(++completed, judgeable.length);
      publish();
    }));
    this.assertCurrent(generation, signal);
    if (!judgements.length) {
      const result: WarningResult = { status: "empty", candidates, selection: "jev", judgements: [], belowThreshold: rejected(), stats: { ...stats }, error: skipped.length ? "No checked passages met the minimum score. Broaden the question or lower its threshold." : "No passages met the minimum score. Broaden the question or lower its threshold." };
      if (skipped.length) result.warning = `${skipped.length} passage${skipped.length === 1 ? " was" : "s were"} skipped because they exceed the model input limit.`;
      result.warning = warning(result.warning);
      return result;
    }
    const result: WarningResult = { status: "ready", selection: "jev", candidates, judgements: selection(), belowThreshold: rejected(), stats: { ...stats } };
    if (skipped.length) result.warning = `${skipped.length} passage${skipped.length === 1 ? " was" : "s were"} skipped because they exceed the model input limit.`;
    result.warning = warning(result.warning);
    return result;
  }

  private async readContext(paths: string[]): Promise<string> {
    const blocks: Block[] = [];
    for (const path of paths) {
      const pathBlocks = (await this.getSourceBlocks(path)).filter((block) => block.path === path.split("#")[0]);
      for (const block of dedupeRanges(pathBlocks)) blocks.push(block);
    }
    return blocks.map(modelText).join("\n\n");
  }

  private scoreKey(spec: QuerySpec, candidate: Candidate, context: string, legacy = false): string {
    const text = modelText(candidate);
    return stable({ namespace: this.cacheNamespace, question: spec.question, criteria: spec.criteria, context,
      ...(legacy ? { candidate: { id: candidate.id, path: candidate.path, lineStart: candidate.lineStart, lineEnd: candidate.lineEnd, headingPath: candidate.headingPath, text } }
        : { passage: text, keyVersion: 2 }) });
  }

  private async readScores(keys: string[]): Promise<Map<string, Omit<Judgement, "candidate">>> {
    if (!this.persistentCache || !keys.length) return new Map();
    if (this.persistentCache.getMany) return this.persistentCache.getMany(keys);
    const values = new Map<string, Omit<Judgement, "candidate">>();
    await Promise.all(keys.map(async key => { const value = await this.persistentCache!.get(key); if (value) values.set(key, value); }));
    return values;
  }

  private async judge(spec: QuerySpec, candidate: Candidate, context: string, generation: number, stats: QueryStats, signal: AbortSignal): Promise<Judgement> {
    this.assertCurrent(generation, signal);
    const passage = modelText(candidate);
    const key = this.scoreKey(spec, candidate, context);
    const cached = this.cache.get(key);
    if (cached) {
      stats.cached++;
      this.cache.delete(key); this.cache.set(key, cached);
      return { ...cached, candidate };
    }
    const existing = this.inFlight.get(key);
    if (existing) { stats.shared++; return { ...(await existing.join(signal)), candidate }; }
    const request = new SharedWork<Judgement>(async sharedSignal => {
      const prune = () => this.gate.cancelStale(this.generation);
      sharedSignal.addEventListener("abort", prune, { once: true });
      let release: (() => void) | undefined;
      try {
        release = await this.gate.acquire(generation, () => !this.disposed && generation === this.generation && !sharedSignal.aborted);
        this.assertCurrent(generation, sharedSignal);
        stats.requested++;
        const response = await this.client!.rank(spec.question, passage, spec.criteria, context, () => this.gate.throttle(), () => { stats.retries++; }, sharedSignal);
        this.gate.success();
        release();
        this.assertCurrent(generation, sharedSignal);
        const judgement = this.validateResponse(response, spec, candidate);
        this.cache.set(key, judgement);
        if (this.persistentCache) {
          const { candidate: _candidate, ...score } = judgement;
          await this.persistentCache.set(key, score);
          this.assertCurrent(generation, sharedSignal);
        }
        while (this.cache.size > MAX_CACHE_ENTRIES) this.cache.delete(this.cache.keys().next().value!);
        return judgement;
      } finally { release?.(); sharedSignal.removeEventListener("abort", prune); }
    }, () => { if (this.inFlight.get(key) === request) this.inFlight.delete(key); });
    this.inFlight.set(key, request);
    return { ...(await request.join(signal)), candidate };
  }

  private validateResponse(response: unknown, spec: QuerySpec, candidate: Candidate): Judgement {
    if (!response || typeof response !== "object" || !("answers" in response) || !response.answers || typeof response.answers !== "object") throw new Error("The decision model returned an invalid response: answers are missing.");
    const answers = response.answers as Record<string, { noul?: unknown; choice?: unknown }>;
    const scores: Record<string, number> = {};
    const score = (id: string): number => {
      const value = answers[id]?.noul;
      if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) throw new Error(`The decision model returned an invalid score for “${id}”.`);
      scores[id] = value;
      return value;
    };
    let resultScore: number;
    let contribution: Contribution = "other";
    if (spec.criteria.mode === "generic") resultScore = score("relevant");
    else {
      const values = [score("definition"), score("condition"), score("distinction")];
      const choice = answers.contribution?.choice;
      if (choice !== "definition" && choice !== "condition" && choice !== "distinction" && choice !== "other") throw new Error("The decision model returned an invalid contribution choice.");
      contribution = choice;
      resultScore = Math.max(...values);
    }
    return { candidate, score: resultScore, contribution, scores };
  }

  private assertCurrent(generation: number, signal?: AbortSignal): void {
    checkSignal(signal);
    if (this.disposed || generation !== this.generation) throw new QueryRunSuperseded();
  }
}

import { checkSignal } from "./work";
import { requestUrl } from "obsidian";
import type { JevResponse, QueryCriteria } from "./types";

export interface JevTransport { post(url: string, headers: Record<string, string>, body: unknown, signal?: AbortSignal): Promise<{ status: number; json: unknown }> }
const defaultTransport: JevTransport = { post: async (url, headers, body) => { const response = await requestUrl({ url, method: "POST", headers, body: JSON.stringify(body), throw: false }); return { status: response.status, json: response.json }; } };

const ENDPOINT = "https://api.typesafe.ai/v1/systemone";
export class JevClient {
  constructor(private readonly apiKey: string, private readonly model: string, private readonly transport: JevTransport = defaultTransport, private readonly sleep: (ms: number) => Promise<void> = (ms) => new Promise((resolve) => window.setTimeout(resolve, ms))) {}

  async rank(question: string, candidate: string, criteria: QueryCriteria, context = "", onThrottle?: () => void, onRetry?: () => void, signal?: AbortSignal): Promise<JevResponse> {
    checkSignal(signal);
    const headers: Record<string, string> = { "Content-Type": "application/json" };
    if (this.apiKey.trim()) headers.Authorization = `Bearer ${this.apiKey}`;
    else throw new Error("TypeSafe API key is not configured.");
    const questions: Record<string, unknown> = {};
    if (criteria.mode === "generic") {
      questions.relevant = { type: "noul", instructions: criteria.instructions ?? `Does this passage answer the question: “${question}”?`, criteria: { true: criteria.true ?? "The passage directly addresses the question with substantive content.", false: criteria.false ?? "The passage only mentions the topic or is unrelated." } };
    } else {
      questions.definition = { type: "noul", instructions: `For the question “${question}”, does this passage directly define the subject?`, criteria: { true: "It states what the subject is or means, rather than merely mentioning it.", false: "It mentions, illustrates, or uses the subject without defining it." } };
      questions.condition = { type: "noul", instructions: `For the question “${question}”, does this passage state a condition that makes the subject apply?`, criteria: { true: "It states a qualifying condition, test, boundary, or necessary feature.", false: "It does not give a condition or boundary for the subject." } };
      questions.distinction = { type: "noul", instructions: `For the question “${question}”, does this passage distinguish the subject from a related concept?`, criteria: { true: "It contrasts, separates, or clarifies the subject against another concept.", false: "It does not distinguish the subject from a related concept." } };
      questions.contribution = { type: "choice", instructions: `Which contribution does this passage make toward answering “${question}”?`, criteria: { definition: "Directly defines the subject.", condition: "States qualifying conditions, tests, or boundaries.", distinction: "Distinguishes the subject from related concepts.", other: "None of those; it is another kind of relevant contribution." } };
    }
    const body = { model: this.model, state: { question, candidate_passage: candidate, context_notes: context }, questions };
    let attempt = 0;
    while (true) {
      checkSignal(signal);
      const response = await this.transport.post(ENDPOINT, headers, body, signal);
      checkSignal(signal);
      if (response.status >= 200 && response.status < 300) return response.json as JevResponse;
      if (response.status === 429 || response.status === 529) onThrottle?.();
      if ((response.status === 429 || response.status === 529) && attempt < 3) { onRetry?.(); await this.sleep(250 * 2 ** attempt); attempt++; continue; }
      if (response.status === 401 || response.status === 403) throw new Error(`Jev authentication failed (${response.status}). Check the TypeSafe API key.`);
      if (response.status === 422) throw new Error("Jev rejected the request (422). Check the model and typed question payload.");
      if (response.status === 429) throw new Error("Jev rate limit persisted after retries (429).");
      if (response.status === 529) throw new Error("Jev service remained unavailable after retries (529).");
      throw new Error(`Jev request failed (${response.status}).`);
    }
  }
}

import { applyInclusions, inclusionFor, updateInclusions } from "./manual-selection";
import { readContextBlocks } from "./context";
import { parseObject } from "./validation";
declare const EMBEDDING_WORKER_SOURCE: string;
import { QueryBuilder } from "./builder";
import { bakeNote, BAKED_FOLDER, PendingSave } from "./bake";
import { Notice, Component, MarkdownRenderChild, Plugin, TFile, type MarkdownPostProcessorContext, type TAbstractFile } from "obsidian";
import { EmbeddingIndex } from "./embeddings";
import { downloadEmbeddings } from "./embedding-assets";
import { LocalRetrieval } from "./retrieval";
import { ScoreCache } from "./score-cache";
import { VaultIndex } from "./indexer";
import { parseQuery, resolveContext } from "./query";
import { JevClient } from "./jev";
import { QueryEngine } from "./engine";
import { renderLoading, renderResult } from "./render";
import { expandBlock } from "./markdown";
import { DEFAULT_SETTINGS, TYPESAFE_ENDPOINT, SettingsTab, normalizeSettings, type Settings } from "./settings";
import type { Block, QueryResult, QuerySpec } from "./types";
const SCORING_VERSION = "jev-v1";

const QUERY_FENCE = /^ {0,3}(?:`{3,}|~{3,})(?:qualitative-query|qq)\s*$/m;

/** Each rendered query owns its subscriptions, async generation, and Markdown children. */
class QueryView extends MarkdownRenderChild {
  private version = 0;
  private run?: AbortController;
  private stopped = false;
  private saving = false;
  private content?: Component;

  constructor(
    private readonly plugin: QualitativeQueryPlugin,
    el: HTMLElement,
    private readonly sourcePath: string,
    private readonly blockSource?: string,
    private readonly autoKey?: string,
  ) { super(el); }

  onload(): void {
    this.plugin.views.add(this);
    void this.refresh();
  }

  invalidate(): void { if (!this.saving) { this.version++; this.run?.abort(); } }

  onunload(): void {
    this.stopped = true;
    this.run?.abort();
    this.invalidate();
    this.plugin.views.delete(this);
    if (this.autoKey && this.plugin.autoViews.get(this.autoKey) === this) this.plugin.autoViews.delete(this.autoKey);
    this.containerEl.empty();
  }

  async refresh(): Promise<void> {
    if (this.stopped || this.saving) return;
    this.run?.abort();
    const run = this.run = new AbortController();
    const version = ++this.version;
    const current = () => !this.stopped && version === this.version && !this.plugin.stopped;
    if (this.content) { this.removeChild(this.content); this.content = undefined; }
    renderLoading(this.containerEl);
    let spec: QuerySpec = { question: "", folder: this.plugin.settings.queryFolder, contextPaths: [], criteria: { mode: "generic" } };
    try {
      const file = this.plugin.app.vault.getAbstractFileByPath(this.sourcePath);
      if (!(file instanceof TFile)) throw new Error("The query note no longer exists. Open its new location or close this view.");
      const text = await this.plugin.app.vault.cachedRead(file);
      if (!current()) return;
      // A note with explicit query blocks does not also get an automatic view.
      if (this.blockSource === undefined && QUERY_FENCE.test(text)) { this.containerEl.empty(); return; }
      spec = parseQuery(file.path, text, this.plugin.settings.queryFolder, this.blockSource);
      spec.contextPaths = [...resolveContext(this.plugin.app, file, spec.contextPaths)];
      await this.plugin.index.whenReady();
      if (!current()) return;
      if (this.plugin.indexError) throw new Error(this.plugin.indexError);
      if (this.plugin.clientError) throw new Error(this.plugin.clientError);
      let result: QueryResult;
      {
        let completed = 0;
        let total = 0;
        const started = Date.now();
        let rendering = Promise.resolve();
        let partialVersion = 0;
        const showProgress = () => {
          if (!current()) return;
          let status = this.containerEl.querySelector<HTMLElement>(".qq-progress");
          if (!status) {
            status = createDiv(); status.className = "qq-status qq-progress";
            this.containerEl.prepend(status);
          }
          status.textContent = this.plugin.client ? `Jev is checking passages: ${completed}/${total} · ${Math.floor((Date.now() - started) / 1000)}s` : "Finding passages locally…";
        };
        const timer = window.setInterval(showProgress, 1000);
        try {
          result = await this.plugin.engine.run(spec, this.plugin.settings.candidateLimit, this.plugin.settings.resultLimit, spec.threshold ?? this.plugin.settings.threshold,
            (done, count) => { completed = done; total = count; showProgress(); },
            partial => {
              const revision = ++partialVersion;
              rendering = rendering.then(async () => {
                const valid = () => current() && revision === partialVersion;
                if (!valid()) return;
                await this.display(partial, spec, valid);
                showProgress();
              }).catch(error => console.warn("Qualitative Query: partial rendering failed", error));
            }, run.signal);
        } finally {
          window.clearInterval(timer);
          partialVersion++;
          await rendering;
        }
      }
      if (!current()) return;
      await this.display(result, spec, current, true);
    } catch (error) {
      if (!current()) return;
      await this.display({ status: "error", candidates: [], judgements: [], error: error instanceof Error ? error.message : String(error) }, spec, current);
    }
  }

  private async display(result: QueryResult, spec: QuerySpec, current: () => boolean, allowBake = false): Promise<void> {
    const baseResult = result;
    const queryFile = this.plugin.app.vault.getAbstractFileByPath(this.sourcePath);
    const manual = allowBake && queryFile instanceof TFile
      ? await applyInclusions(await this.plugin.app.vault.read(queryFile), spec, result, this.plugin.index.blocks)
      : { result, missing: [] };
    result = manual.result;
    const content = new Component();
    this.addChild(content);
    const stage = createDiv();
    stage.addClass("qq-view");
    try {
      await renderResult(this.plugin.app, stage, result, spec, content, (candidate, adjacent) =>
        expandBlock(candidate, this.plugin.index.blocksForPath(candidate.path), adjacent),
        allowBake && queryFile instanceof TFile ? async (judgement, include) => {
          if (!current()) return;
          const entry = await inclusionFor(judgement);
          if (!current()) return;
          this.saving = true;
          try { await this.plugin.app.vault.process(queryFile, text => updateInclusions(text, spec, entry, include)); }
          finally { this.saving = false; }
          if (current()) await this.display(baseResult, spec, current, true);
        } : undefined);
      for (const missing of manual.missing) {
        const warning = stage.createDiv({ cls: "qq-warning", text: missing.message });
        const remove = warning.createEl("button", { text: "Remove inclusion" });
        content.registerDomEvent(remove, "click", async () => {
          if (!current() || !(queryFile instanceof TFile) || remove.disabled) return;
          remove.disabled = true; this.saving = true;
          try {
            await this.plugin.app.vault.process(queryFile, text => updateInclusions(text, spec, missing.entry, false));
            this.saving = false;
            if (current()) await this.display(baseResult, spec, current, true);
          } catch (error) { this.saving = false; remove.disabled = false; new Notice(error instanceof Error ? error.message : String(error)); }
        });
      }
      if (allowBake && result.status === "ready" && result.judgements.length) {
        const tools = createDiv();
        tools.className = "qq-bake-actions";
        const button = tools.createEl("button", { text: "Save passages" });
        tools.createEl("small", { text: " Creates a note with these passages linked to their sources. Adds missing block IDs to source notes. Nearby context is not saved." });
        const saveStatus = tools.createEl("small", { text: "" });
        const controller = new AbortController();
        content.register(() => controller.abort());
        let save = () => bakeNote(this.plugin.app, result, spec, this.sourcePath, {
          signal: controller.signal,
          onWait: (pending, ms) => saveStatus.setText(` Waiting for ${pending} source links · timeout in ${Math.ceil(ms / 1000)}s.`),
        });
        content.registerDomEvent(button, "click", async () => {
          if (!current() || controller.signal.aborted) return;
          button.disabled = true;
          this.saving = true;
          try {
            const file = await save();
            this.saving = false;
            saveStatus.setText(" Saved.");
            new Notice("Passages saved. Open a source note’s backlinks to see the connection.");
            await this.plugin.app.workspace.getLeaf(true).openFile(file, { state: { mode: "preview" } });
          } catch (error) {
            if (controller.signal.aborted) return;
            if (error instanceof PendingSave) { save = error.retry; button.setText("Retry save"); }
            else this.saving = false;
            saveStatus.setText(error instanceof Error ? error.message : String(error)); button.disabled = false;
          }
        });
        stage.prepend(tools);
      }
      if (!current()) { this.removeChild(content); return; }
      if (this.content) this.removeChild(this.content);
      this.content = content;
      this.containerEl.empty();
      this.containerEl.appendChild(stage);
    } catch (error) { this.removeChild(content); throw error; }
  }
}

export default class QualitativeQueryPlugin extends Plugin {
  settings: Settings = { ...DEFAULT_SETTINGS };
  index!: VaultIndex;
  engine!: QueryEngine;
  client: Pick<JevClient, "rank"> | null = null;
  clientError = "";
  indexError = "";
  stopped = false;
  readonly views = new Set<QueryView>();
  readonly autoViews = new Map<string, QueryView>();
  private refreshTimer?: number;
  private fileTimers = new Map<string, number>();
  private exclusionKey = "";
  private scoreCache?: ScoreCache;
  private retrieval!: LocalRetrieval;
  private corpusSource?: Block[];
  private corpus: Block[] = [];

  async onload(): Promise<void> {
    await this.loadSettings();
    this.retrieval = new LocalRetrieval(() => new EmbeddingIndex({
      read: name => this.app.vault.adapter.readBinary(`${this.pluginDirectory()}/embeddings/${name}`),
      readWorker: async () => EMBEDDING_WORKER_SOURCE,
    }), () => downloadEmbeddings(this.app.vault.adapter, this.pluginDirectory()));
    this.addCommand({ id: "build-query", name: "Ask your vault", callback: () => new QueryBuilder(this.app, this.settings.queryFolder).open() });
    this.addRibbonIcon("search", "Ask your vault", () => new QueryBuilder(this.app, this.settings.queryFolder).open());
    const directory = this.manifest?.dir ?? `${this.app.vault.configDir}/plugins/qualitative-query`;
    const cachePath = `${directory}/scores-v1.json`;
    const adapter = this.app.vault.adapter;
    if (adapter) {
      let reported = false;
      const onError = (error: unknown) => {
        console.warn("Qualitative Query: score cache unavailable", error);
        if (!reported) { reported = true; new Notice("Qualitative Query could not persist some scores. Check available disk space and restart Obsidian."); }
      };
      try {
        const identityPath = `${directory}/score-cache-id.json`;
        let identity: string;
        if (await adapter.exists(identityPath)) {
          const storedIdentity = parseObject(await adapter.read(identityPath)).id;
          if (typeof storedIdentity !== "string" || !/^[a-f0-9-]{36}$/.test(storedIdentity)) throw new Error("Invalid score cache identity file.");
          identity = storedIdentity;
        } else {
          identity = crypto.randomUUID();
          await adapter.write(identityPath, JSON.stringify({ id: identity }));
        }
        this.scoreCache = new ScoreCache({ read: async () => await adapter.exists(cachePath) ? adapter.read(cachePath) : null },
          { databaseName: `qualitative-query-scores-v2-${identity}`, onError });
      } catch (error) { onError(error); }
    }
    this.index = new VaultIndex(this.app.vault, () => this.exclusions());
    this.exclusionKey = JSON.stringify(this.exclusions());
    this.rebuildClient();
    void this.scan();
    for (const language of ["qualitative-query", "qq"]) {
      this.registerMarkdownCodeBlockProcessor(language, (source, el, ctx) => {
        if (el.closest(".qq-source-content")) { el.createEl("pre", { text: source }); return; }
        this.mount(el, ctx, source);
      });
    }
    this.registerMarkdownPostProcessor((el, ctx) => {
      if (el.closest(".qq-view") || !this.inQueryFolder(ctx.sourcePath)) return;
      const key = `${ctx.docId}:${ctx.sourcePath}`;
      if (this.autoViews.has(key)) return;
      if (el.querySelector(".qq-view, .block-language-qualitative-query, .block-language-qq")) return;
      const view = this.mount(el.createDiv({ cls: "qq-auto-view" }), ctx, undefined, key);
      this.autoViews.set(key, view);
    });
    this.registerEvent(this.app.vault.on("create", file => this.scheduleIndex(file)));
    this.registerEvent(this.app.vault.on("modify", file => this.scheduleIndex(file)));
    this.registerEvent(this.app.vault.on("delete", file => {
      this.invalidateViews();
      this.index.remove(file.path);
      if (!(file instanceof TFile)) void this.scan();
      else this.scheduleQueries();
    }));
    this.registerEvent(this.app.vault.on("rename", (file, oldPath) => {
      window.clearTimeout(this.fileTimers.get(oldPath));
      this.fileTimers.delete(oldPath);
      this.index.remove(oldPath);
      if (file instanceof TFile && file.extension !== "md") { this.invalidateViews(); this.scheduleQueries(); }
      else this.scheduleIndex(file);
    }));
    this.addSettingTab(new SettingsTab(this.app, this));
    this.addCommand({ id: "reindex-vault", name: "Reindex vault", callback: () => void this.scan() });
    this.addCommand({ id: "refresh-queries", name: "Refresh open questions", callback: () => this.scheduleQueries() });
  }

  private mount(el: HTMLElement, ctx: MarkdownPostProcessorContext, source?: string, autoKey?: string): QueryView {
    const view = new QueryView(this, el, ctx.sourcePath, source, autoKey);
    ctx.addChild(view);
    return view;
  }

  async loadSettings(): Promise<void> {
    this.settings = normalizeSettings(await this.loadData());
    this.settings.apiKey = this.app.secretStorage.getSecret("qualitative-query-api-key") ?? "";
  }

  async saveSettings(): Promise<void> {
    const { apiKey, ...stored } = this.settings;
    this.app.secretStorage.setSecret("qualitative-query-api-key", apiKey);
    await this.saveData(stored);
    const exclusions = JSON.stringify(this.exclusions());
    if (exclusions !== this.exclusionKey) {
      this.exclusionKey = exclusions;
      await this.scan();
    } else this.scheduleQueries();
  }

  rebuildClient(): void {
    this.invalidateViews();
    this.engine?.dispose();
    this.clientError = "";
    try {
      this.client = this.settings.apiKey.trim() ? new JevClient(this.settings.apiKey, this.settings.model) : null;
    } catch (error) { this.client = null; this.clientError = error instanceof Error ? error.message : String(error); }
    if (this.index) this.engine = new QueryEngine(this.client, () => this.searchableBlocks(), path => readContextBlocks(this.app, path), `${SCORING_VERSION}:${TYPESAFE_ENDPOINT}:${this.settings.model}`, this.scoreCache, (question, blocks, limit, signal) => this.retrieval.search(question, blocks, limit, signal));
    this.scheduleQueries();
  }

  private searchableBlocks(): Block[] {
    const source = this.index.blocks;
    if (source !== this.corpusSource) {
      this.corpusSource = source;
      this.corpus = source.filter(block => !QUERY_FENCE.test(block.text));
    }
    return this.corpus;
  }

  private pluginDirectory(): string { return this.manifest?.dir ?? `${this.app.vault.configDir}/plugins/qualitative-query`; }

  embeddingStatus(): string { return this.retrieval.status; }

  restartLocalSearch(): void {
    this.retrieval.restart(); this.invalidateViews(); this.scheduleQueries();
  }

  async restoreEmbeddings(): Promise<void> {
    await this.retrieval.restore();
    if (this.stopped) return;
    this.invalidateViews(); this.scheduleQueries();
  }

  private exclusions(): string[] {
    return [...this.settings.excludedFolders.split(",").map(value => value.trim()).filter(Boolean), this.app.vault.configDir, this.settings.queryFolder, BAKED_FOLDER];
  }

  private inQueryFolder(path: string): boolean {
    const folder = this.settings.queryFolder.replace(/^\/+|\/+$/g, "");
    return Boolean(folder && path.startsWith(`${folder}/`));
  }

  private invalidateViews(): void {
    this.engine?.invalidate();
    for (const view of this.views) view.invalidate();
  }

  private async scan(): Promise<void> {
    if (this.stopped) return;
    this.invalidateViews();
    this.indexError = "";
    try { await this.index.initialScan(); }
    catch { this.indexError = "Some vault files could not be indexed. Run Reindex vault after checking file access."; }
    if (!this.stopped) this.scheduleQueries();
  }

  private scheduleIndex(file: TAbstractFile): void {
    if (this.stopped) return;
    if (file instanceof TFile && file.extension !== "md") return;
    this.invalidateViews();
    if (!(file instanceof TFile)) { void this.scan(); return; }
    if (file.extension !== "md") { this.scheduleQueries(); return; }
    const path = file.path;
    window.clearTimeout(this.fileTimers.get(path));
    const timer = window.setTimeout(() => {
      this.fileTimers.delete(path);
      void this.index.indexFile(file).catch(() => { this.indexError = `Could not index ${path}. Run Reindex vault.`; }).finally(() => this.scheduleQueries());
    }, 250);
    this.fileTimers.set(path, timer);
  }

  private scheduleQueries(): void {
    if (this.stopped) return;
    window.clearTimeout(this.refreshTimer);
    this.refreshTimer = window.setTimeout(() => {
      for (const view of this.views) void view.refresh();
    }, 300);
  }

  onunload(): void {
    this.stopped = true;
    window.clearTimeout(this.refreshTimer);
    for (const timer of this.fileTimers.values()) window.clearTimeout(timer);
    this.fileTimers.clear();
    this.engine?.dispose();
    this.retrieval?.dispose();
    this.index?.dispose();
    this.corpusSource = undefined; this.corpus = [];
    void this.scoreCache?.close();
    for (const view of [...this.views]) view.unload();
    this.autoViews.clear();
  }
}