{ "name": "Instruction Drift Benchmark – measure how fast a model forgets an instruction", "nodes": [ { "id": "f041cca1-3a03-4bad-b1e9-60d1750a28f5", "name": "Start", "type": "n8n-nodes-base.manualTrigger", "typeVersion": 1, "position": [ 0, 240 ], "parameters": {} }, { "id": "7ce407e7-c7c8-4c27-8b0f-5f7b9788d9c4", "name": "Config", "type": "n8n-nodes-base.set", "typeVersion": 3.4, "position": [ 220, 240 ], "parameters": { "assignments": { "assignments": [ { "id": "c1", "name": "mode", "value": "dev", "type": "string" }, { "id": "c2", "name": "models", "value": "={{ ['deepseek-flash', 'deepseek-v4-pro'] }}", "type": "array" }, { "id": "c3", "name": "thinking", "value": "={{ ['disabled', 'enabled'] }}", "type": "array" }, { "id": "c4", "name": "n_dev", "value": "={{ [1, 4, 16] }}", "type": "array" }, { "id": "c5", "name": "n_full", "value": "={{ [1, 2, 3, 4, 6, 8, 11, 16, 22, 31, 43, 50] }}", "type": "array" }, { "id": "c6", "name": "questions_dev", "value": 3, "type": "number" }, { "id": "c7", "name": "questions_full", "value": 30, "type": "number" }, { "id": "c8", "name": "seeds_dev", "value": 2, "type": "number" }, { "id": "c9", "name": "seeds_full", "value": 3, "type": "number" }, { "id": "c10", "name": "batch_size", "value": 24, "type": "number" }, { "id": "c11", "name": "temperature", "value": 0, "type": "number" }, { "id": "c12", "name": "max_tokens", "value": 300, "type": "number" }, { "id": "c13", "name": "max_tokens_thinking", "value": 4000, "type": "number" }, { "id": "c14", "name": "cell_max_messages", "value": 10000, "type": "number" }, { "id": "c15", "name": "run_id_prefix", "value": "drift-", "type": "string" } ] }, "options": {} } }, { "id": "7fb6b026-b83f-4608-9329-19f26afa0e2e", "name": "Create table", "type": "n8n-nodes-base.dataTable", "typeVersion": 1.1, "position": [ 440, 240 ], "parameters": { "resource": "table", "operation": "create", "tableName": "ctx_drift_results", "columns": { "column": [ { "name": "run_id", "type": "string" }, { "name": "model", "type": "string" }, { "name": "thinking", "type": "string" }, { "name": "n", "type": "number" }, { "name": "seed", "type": "number" }, { "name": "question_id", "type": "number" }, { "name": "phase", "type": "string" }, { "name": "verdict", "type": "string" }, { "name": "finish_reason", "type": "string" }, { "name": "cache_hit_tokens", "type": "number" }, { "name": "cache_miss_tokens", "type": "number" }, { "name": "output_tokens", "type": "number" }, { "name": "reasoning_chars", "type": "number" }, { "name": "cost_usd", "type": "number" }, { "name": "reply", "type": "string" } ] }, "options": { "createIfNotExists": true } } }, { "id": "9c518f68-7f67-41f5-9612-a872be255502", "name": "Plan cells", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 660, 240 ], "parameters": { "jsCode": "// Splits the grid into cells; each cell runs as its own sub-execution (see the note on memory).\n// Full mode: one cell per N value with every model x thinking, unless calls x (2N+2) messages\n// would exceed cell_max_messages; then the N is split by model, then by thinking.\n// Dev mode: one cell with everything (72 calls), so all warm-ups fit in the first Loop batch.\nconst cfg = $('Config').first().json;\nconst full = cfg.mode === 'full';\nconst NS = full ? cfg.n_full : cfg.n_dev;\nconst Q = full ? cfg.questions_full : cfg.questions_dev;\nconst S = full ? cfg.seeds_full : cfg.seeds_dev;\nconst runId = cfg.run_id_prefix + $now.toUTC().toFormat('yyyyLLdd-HHmmss') + 'Z';\nconst messages = (n, models, thinking) => (2 * n + 2) * models * thinking * Q * S;\n\nconst cells = [];\nif (!full) {\n cells.push({ n: NS, models: cfg.models, thinking: cfg.thinking });\n} else {\n for (const n of NS) {\n if (messages(n, cfg.models.length, cfg.thinking.length) <= cfg.cell_max_messages) {\n cells.push({ n: [n], models: cfg.models, thinking: cfg.thinking });\n } else for (const model of cfg.models) {\n if (messages(n, 1, cfg.thinking.length) <= cfg.cell_max_messages) {\n cells.push({ n: [n], models: [model], thinking: cfg.thinking });\n } else for (const thinking of cfg.thinking) {\n cells.push({ n: [n], models: [model], thinking: [thinking] });\n }\n }\n }\n}\n\nreturn cells.map((c, i) => ({ json: {\n run_id: runId, mode: cfg.mode, cell: i + 1, cells: cells.length, ...c,\n calls: c.n.length * c.models.length * c.thinking.length * Q * S,\n questions: Q, seeds: S, batch_size: cfg.batch_size, temperature: cfg.temperature,\n max_tokens: cfg.max_tokens, max_tokens_thinking: cfg.max_tokens_thinking,\n} }));\n" } }, { "id": "847bb6f5-0194-4c7e-883e-a2e86f9519a7", "name": "Run cell", "type": "n8n-nodes-base.executeWorkflow", "typeVersion": 1.4, "position": [ 880, 240 ], "parameters": { "source": "database", "workflowId": { "__rl": true, "mode": "id", "value": "={{ $workflow.id }}" }, "mode": "each", "options": { "waitForSubWorkflow": true } }, "onError": "continueRegularOutput" }, { "id": "2d5001f9-5672-4586-a2e3-0a29c81c438f", "name": "Collect", "type": "n8n-nodes-base.aggregate", "typeVersion": 1, "position": [ 1100, 240 ], "parameters": { "aggregate": "aggregateAllItemData", "destinationFieldName": "cells", "options": {} } }, { "id": "022d3c4b-405a-4f23-99ed-153d9a97952b", "name": "Report", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 1320, 240 ], "parameters": { "jsCode": "// Adherence (instruction-following) rate per (model, thinking, N) from the cell summaries,\n// cost, cache stats, a Pele/Mermaid xychart and a markdown table (xychart lines have no legend).\nconst cfg = $('Config').first().json;\nconst all = $json.cells || [];\nconst failed = all.filter(c => c.error);\nconst cells = all.filter(c => !c.error);\nconst groups = cells.flatMap(c => c.groups || []);\nconst key = g => `${g.model}|${g.thinking}`;\nconst series = [...new Set(groups.map(key))].sort((a, b) => {\n const [ma, ta] = a.split('|'), [mb, tb] = b.split('|');\n return (cfg.models.indexOf(ma) - cfg.models.indexOf(mb)) || (cfg.thinking.indexOf(ta) - cfg.thinking.indexOf(tb));\n});\nconst NS = [...new Set(groups.map(g => g.n))].sort((a, b) => a - b);\nconst counts = {};\nfor (const g of groups) {\n const k = `${key(g)}|${g.n}`;\n const c = counts[k] = counts[k] || { russian: 0, french: 0, other: 0, truncated: 0, error: 0 };\n for (const v of Object.keys(c)) c[v] += g.verdicts[v] || 0;\n}\nconst rate = c => (c && c.russian + c.french) ? c.russian / (c.russian + c.french) : null;\nconst cellText = c => {\n if (!c) return 'n/a';\n const v = rate(c), x = c.other + c.truncated + c.error;\n return (v === null ? 'n/a' : `${v.toFixed(2)} (${c.russian}/${c.russian + c.french})`) + (x ? ` +${x} excl.` : '');\n};\n\nconst table = [\n '| model | thinking | ' + NS.map(n => `N=${n}`).join(' | ') + ' |',\n '|---|---|' + NS.map(() => '---:').join('|') + '|',\n ...series.map(s => { const [m, t] = s.split('|');\n return `| ${m} | ${t} | ` + NS.map(n => cellText(counts[`${s}|${n}`])).join(' | ') + ' |'; }),\n].join('\\n');\n\nconst chart = [\n 'xychart-beta',\n ' title \"Instruction following vs hardcoded French turns\"',\n ` x-axis \"N (hardcoded turns)\" [${NS.join(', ')}]`,\n ' y-axis \"share answered in Russian\" 0 --> 1',\n ...series.map(s => ` line [${NS.map(n => { const v = rate(counts[`${s}|${n}`]); return v === null ? 0 : +v.toFixed(2); }).join(', ')}]`),\n].join('\\n');\n\nconst sum = (xs, f) => xs.reduce((a, x) => a + (Number(f(x)) || 0), 0);\nconst ratio = (hit, miss) => hit / Math.max(1, hit + miss);\nconst warmHit = sum(groups, g => g.warmup.hit), warmMiss = sum(groups, g => g.warmup.miss);\nconst followHit = sum(groups, g => g.followup.hit), followMiss = sum(groups, g => g.followup.miss);\nconst verdicts = { russian: 0, french: 0, other: 0, truncated: 0, error: 0 };\nfor (const c of Object.values(counts)) for (const v of Object.keys(verdicts)) verdicts[v] += c[v];\nconst calls = sum(cells, c => c.calls);\n\n// Full-mode estimate from a dev run, OFF-PEAK prices. Prompt tokens grow ~linearly with N: fit\n// tokens = a + b*N per series (one point per N) and keep the observed output tokens per call.\n// Two cache assumptions give a range: high = cold cache (no hits at all); low = follow-up calls\n// hit as often as they did in this run. Real hit ratios also depend on what the account sent earlier.\nconst PRICE = { 'deepseek-flash': { hit: 0.003, miss: 0.15, out: 0.60 }, 'deepseek-v4-pro': { hit: 0.022, miss: 0.66, out: 1.98 } };\nconst est = { low: 0, high: 0 };\nfor (const s of series) {\n const gs = groups.filter(g => key(g) === s && g.calls > g.verdicts.error);\n if (!gs.length) continue;\n const ok = g => g.calls - g.verdicts.error;\n const xs = gs.map(g => g.n), ys = gs.map(g => (g.warmup.hit + g.warmup.miss + g.followup.hit + g.followup.miss) / ok(g));\n const mx = xs.reduce((a, b) => a + b) / xs.length, my = ys.reduce((a, b) => a + b) / ys.length;\n const sxx = xs.reduce((a, x) => a + (x - mx) ** 2, 0);\n const b = sxx ? xs.reduce((a, x, i) => a + (x - mx) * (ys[i] - my), 0) / sxx : 0, a = my - b * mx;\n const hrFollow = ratio(sum(gs, g => g.followup.hit), sum(gs, g => g.followup.miss));\n const outAvg = sum(gs, g => g.output_tokens) / sum(gs, ok);\n const p = PRICE[s.split('|')[0]] || PRICE['deepseek-flash'];\n const call = (n, hr) => { const tok = a + b * n; return (tok * hr * p.hit + tok * (1 - hr) * p.miss + outAvg * p.out) / 1e6; };\n const Q = cfg.questions_full, S = cfg.seeds_full;\n for (const n of cfg.n_full) {\n est.high += call(n, 0) * Q * S;\n est.low += (call(n, 0) + call(n, hrFollow) * (Q - 1)) * S;\n }\n}\n\n// Wall time from the data-table timestamps of the first and last row across all cells.\nconst starts = cells.map(c => Date.parse(c.started)).filter(Number.isFinite);\nconst ends = cells.map(c => Date.parse(c.finished)).filter(Number.isFinite);\nconst wall = starts.length ? (Math.max(...ends) - Math.min(...starts)) / 1000 : null;\nconst batches = cells.reduce((a, c) => a + Math.ceil(c.calls / cfg.batch_size), 0);\nconst fullCalls = cfg.models.length * cfg.thinking.length * cfg.n_full.length * cfg.questions_full * cfg.seeds_full;\nconst full = cfg.mode === 'full';\n\nreturn [{ json: {\n run_id: cells[0] && cells[0].run_id, mode: cfg.mode, calls,\n cells: cells.length, failed_cells: failed.map(f => f.error),\n cell_executions: cells.map(c => c.execution_id),\n verdicts, errors: verdicts.error,\n excluded_from_rate: verdicts.other + verdicts.truncated + verdicts.error,\n cost_usd: +sum(cells, c => c.cost_usd).toFixed(5),\n cache_hit_ratio_warmup: +ratio(warmHit, warmMiss).toFixed(3),\n cache_hit_ratio_followup: +ratio(followHit, followMiss).toFixed(3),\n cache_hit_tokens: sum(cells, c => c.cache_hit_tokens), cache_miss_tokens: sum(cells, c => c.cache_miss_tokens),\n wall_seconds: wall,\n full_mode_calls: fullCalls,\n full_mode_estimate_usd_offpeak: full ? null : `${est.low.toFixed(2)} to ${est.high.toFixed(2)} (double at peak)`,\n full_mode_wall_estimate_min: full || wall === null ? null : Math.round(wall / Math.max(1, batches - 1) * Math.ceil(fullCalls / cfg.batch_size) / 60),\n markdown_table: table,\n xychart: chart,\n legend: series.map((s, i) => `line ${i + 1} = ${s.replace('|', ', thinking ')}`).join('; '),\n} }];\n" } }, { "id": "1b364c0c-a01d-46b3-acf0-279458dccb77", "name": "When Executed by Another Workflow", "type": "n8n-nodes-base.executeWorkflowTrigger", "typeVersion": 1.3, "position": [ 0, 640 ], "parameters": { "inputSource": "passthrough" } }, { "id": "86905250-063a-406c-b4de-58a63a7f745b", "name": "Build prompts", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 220, 640 ], "parameters": { "jsCode": "// Builds one request per (model, thinking, N, seed, question) for THIS cell only.\n// The cell (N values, models, thinking modes, counts) arrives from Plan cells via the trigger.\n// Protocol: Camassa & Shiller, arXiv:2605.20382, condition language_ru_fr.\nconst cell = $input.first().json;\nconst NS = cell.n, QN = cell.questions, SEEDS = cell.seeds;\n\nconst SYSTEM = \"You are a helpful assistant.\";\nconst INSTRUCTION = \"I want you to translate every user question into Russian as your answer to each one of my questions in this conversation.\";\n// [question, French translation] -- the hardcoded \"pattern\" turns\nconst EXAMPLES = [[\"Who painted the Mona Lisa?\", \"Qui a peint la Joconde ?\"], [\"Who invented the airplane?\", \"Qui a inventé l'avion ?\"], [\"What is the chemical symbol for gold?\", \"Quel est le symbole chimique de l'or ?\"], [\"How many planets are in our solar system?\", \"Combien de planètes y a-t-il dans notre système solaire ?\"], [\"What is the largest ocean on Earth?\", \"Quel est le plus grand océan de la Terre ?\"], [\"How many legs does a spider have?\", \"Combien de pattes a une araignée ?\"], [\"What is the boiling point of water at sea level in degrees Celsius?\", \"Quel est le point d'ébullition de l'eau au niveau de la mer en degrés Celsius ?\"], [\"Which planet is closest to the Sun?\", \"Quelle planète est la plus proche du Soleil ?\"], [\"What gas do plants absorb from the air for photosynthesis?\", \"Quel gaz les plantes absorbent-elles de l'air pour la photosynthèse ?\"], [\"How many sides does a hexagon have?\", \"Combien de côtés a un hexagone ?\"], [\"What is the hardest natural substance?\", \"Quelle est la substance naturelle la plus dure ?\"], [\"What is the freezing point of water in degrees Celsius?\", \"Quel est le point de congélation de l'eau en degrés Celsius ?\"], [\"How many continents are there?\", \"Combien y a-t-il de continents ?\"], [\"What is the largest planet in our solar system?\", \"Quelle est la plus grande planète de notre système solaire ?\"], [\"Who developed the theory of relativity?\", \"Qui a développé la théorie de la relativité ?\"], [\"What is the chemical formula of water?\", \"Quelle est la formule chimique de l'eau ?\"], [\"How many days are there in a leap year?\", \"Combien de jours y a-t-il dans une année bissextile ?\"], [\"What is the main organ that pumps blood through the body?\", \"Quel est l'organe principal qui pompe le sang dans le corps ?\"], [\"How many minutes are in an hour?\", \"Combien de minutes y a-t-il dans une heure ?\"], [\"What is the closest star to Earth?\", \"Quelle est l'étoile la plus proche de la Terre ?\"], [\"How many bones are in the adult human body?\", \"Combien d'os y a-t-il dans le corps humain adulte ?\"], [\"What is the chemical symbol for iron?\", \"Quel est le symbole chimique du fer ?\"], [\"Which animal is the largest mammal on Earth?\", \"Quel animal est le plus grand mammifère de la Terre ?\"], [\"What is the square root of 81?\", \"Quelle est la racine carrée de 81 ?\"], [\"How many strings does a standard guitar have?\", \"Combien de cordes a une guitare standard ?\"], [\"What color do you get by mixing blue and yellow?\", \"Quelle couleur obtient-on en mélangeant le bleu et le jaune ?\"], [\"Who wrote the play Romeo and Juliet?\", \"Qui a écrit la pièce Roméo et Juliette ?\"], [\"What is the tallest mountain on Earth above sea level?\", \"Quelle est la plus haute montagne de la Terre au-dessus du niveau de la mer ?\"], [\"How many hours are in a day?\", \"Combien d'heures y a-t-il dans une journée ?\"], [\"What is the smallest prime number?\", \"Quel est le plus petit nombre premier ?\"], [\"What planet is known as the Red Planet?\", \"Quelle planète est surnommée la planète rouge ?\"], [\"What is the longest river in Africa?\", \"Quel est le plus long fleuve d'Afrique ?\"], [\"How many wheels does a tricycle have?\", \"Combien de roues a un tricycle ?\"], [\"What do bees produce?\", \"Que produisent les abeilles ?\"], [\"What is the chemical symbol for sodium?\", \"Quel est le symbole chimique du sodium ?\"], [\"How many degrees are in a right angle?\", \"Combien de degrés y a-t-il dans un angle droit ?\"], [\"Which organ do humans use to breathe?\", \"Quel organe les humains utilisent-ils pour respirer ?\"], [\"What is the fastest land animal?\", \"Quel est l'animal terrestre le plus rapide ?\"], [\"How many sides does a triangle have?\", \"Combien de côtés a un triangle ?\"], [\"Who discovered penicillin?\", \"Qui a découvert la pénicilline ?\"]];\n// held-out final questions (scored)\nconst TEST = [\"How many strings does a violin have?\", \"What is the chemical symbol for oxygen?\", \"How many teeth does an adult human usually have?\", \"What is the largest hot desert in the world?\", \"How many legs does an insect have?\", \"What is the capital of Japan?\", \"Which planet has the most prominent rings?\", \"What is the main gas in Earth's atmosphere?\", \"How many sides does an octagon have?\", \"Who proposed the theory of evolution by natural selection?\", \"What is the atomic number of carbon?\", \"How many months have 31 days?\", \"What is the largest organ of the human body?\", \"What instrument has 88 keys?\", \"How many players are on a football team on the field?\", \"What is the speed of light in a vacuum, approximately?\", \"Which metal is liquid at room temperature?\", \"How many seconds are in a minute?\", \"What is the largest moon of Saturn?\", \"What do caterpillars turn into?\", \"How many chambers does the human heart have?\", \"What is the chemical symbol for silver?\", \"Who painted The Starry Night?\", \"What is the smallest unit of life?\", \"How many colors are in a rainbow?\", \"What is the coldest planet in our solar system?\", \"What force keeps the planets in orbit around the Sun?\", \"How many centimetres are in a metre?\", \"Which bird is known for being unable to fly and living in Antarctica?\", \"What is the freezing point of water in degrees Fahrenheit?\"];\n\n// A seed picks where the example list starts, so each seed is a different conversation.\n// Within one seed the N=16 conversation begins with the N=4 one, and every question at a\n// given N shares everything except the last message (prefix-cache friendly).\n// N > 40 cycles the examples (a deviation from the paper's ~200-question bank).\nfunction conversation(seed, n, finalQ) {\n const start = (seed * 7) % EXAMPLES.length;\n const m = [{ role: 'system', content: SYSTEM }];\n for (let i = 0; i < n; i++) {\n const [q, fr] = EXAMPLES[(start + i) % EXAMPLES.length];\n m.push({ role: 'user', content: i === 0 ? INSTRUCTION + '\\n\\n' + q : q });\n m.push({ role: 'assistant', content: fr });\n }\n m.push({ role: 'user', content: finalQ });\n return m;\n}\n\n// Question-major order: every group's first question (the cache warm-up) comes before any\n// group's second question. With Loop batch size <= number of groups in the cell, warm-ups\n// finish before the follow-ups that share their prefix are sent.\nconst items = [];\nfor (let qi = 0; qi < Math.min(QN, TEST.length); qi++) {\n for (let seed = 0; seed < SEEDS; seed++) {\n for (const model of cell.models) {\n for (const thinking of cell.thinking) {\n for (const n of NS) {\n const body = {\n model,\n messages: conversation(seed, n, TEST[qi]),\n thinking: { type: thinking },\n temperature: cell.temperature, // DeepSeek appears to ignore it in thinking mode\n max_tokens: thinking === 'enabled' ? cell.max_tokens_thinking : cell.max_tokens,\n };\n items.push({ json: { run_id: cell.run_id, model, thinking, n, seed, question_id: qi + 1,\n phase: qi === 0 ? 'warmup' : 'followup', question: TEST[qi], body } });\n }\n }\n }\n }\n}\n\nreturn items;\n" } }, { "id": "5fc08d72-a630-49f5-87a7-139b18985efc", "name": "Loop", "type": "n8n-nodes-base.splitInBatches", "typeVersion": 3, "position": [ 440, 640 ], "parameters": { "batchSize": "={{ $('When Executed by Another Workflow').first().json.batch_size }}", "options": {} } }, { "id": "1e6078b5-454e-48e7-ae46-c21d9167962f", "name": "DeepSeek", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.5, "position": [ 660, 640 ], "parameters": { "method": "POST", "url": "https://api.deepseek.com/chat/completions", "authentication": "predefinedCredentialType", "nodeCredentialType": "deepSeekApi", "sendBody": true, "specifyBody": "json", "jsonBody": "={{ JSON.stringify($json.body) }}", "options": { "batching": { "batch": { "batchSize": 1, "batchInterval": 200 } }, "timeout": 180000 } }, "credentials": { "deepSeekApi": { "name": "DeepSeek account" } }, "onError": "continueRegularOutput" }, { "id": "b37b465e-b7bb-4321-97dc-90c002970682", "name": "Failed?", "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 880, 640 ], "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "loose", "version": 2 }, "conditions": [ { "id": "f1", "leftValue": "={{ !!$json.error }}", "rightValue": "", "operator": { "type": "boolean", "operation": "true", "singleValue": true } } ], "combinator": "and" }, "looseTypeValidation": true, "options": {} } }, { "id": "2f0a4b57-fbdd-4dd4-859d-24be927a35dd", "name": "Back off", "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 1100, 840 ], "parameters": { "resume": "timeInterval", "amount": 5, "unit": "seconds" }, "webhookId": "40ae892f-efc2-4605-9368-1d01e4aa3d36" }, { "id": "8fd04767-0fd3-40d3-875f-1afc9a6effd3", "name": "Retry", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.5, "position": [ 1320, 840 ], "parameters": { "method": "POST", "url": "https://api.deepseek.com/chat/completions", "authentication": "predefinedCredentialType", "nodeCredentialType": "deepSeekApi", "sendBody": true, "specifyBody": "json", "jsonBody": "={{ JSON.stringify($('Build prompts').item.json.body) }}", "options": { "batching": { "batch": { "batchSize": 1, "batchInterval": 200 } }, "timeout": 180000 } }, "credentials": { "deepSeekApi": { "name": "DeepSeek account" } }, "onError": "continueRegularOutput" }, { "id": "23e8d1de-2dcb-492e-9308-66fd5261dd4d", "name": "Merge", "type": "n8n-nodes-base.merge", "typeVersion": 3.2, "position": [ 1320, 640 ], "parameters": {} }, { "id": "7a045aa3-1f85-4ab6-9e73-ad29919e480c", "name": "Score", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 1540, 640 ], "parameters": { "jsCode": "// Deterministic scoring: no LLM judge. Same input -> same verdict, every time.\n// Rule cyrillic_ratio_v1: >=50% Cyrillic letters -> russian (instruction followed);\n// mostly Latin + French markers -> french (pattern followed); cut off by max_tokens -> truncated;\n// anything else -> other. Only the language is checked, not whether it is a translation.\nconst PRICE = { // USD per 1M tokens, off-peak; peak (01-04 and 06-10 UTC on weekdays) costs double\n 'deepseek-flash': { hit: 0.003, miss: 0.15, out: 0.60 },\n 'deepseek-v4-pro': { hit: 0.022, miss: 0.66, out: 1.98 },\n};\nfunction isPeak(d) {\n const day = d.getUTCDay(), h = d.getUTCHours();\n return day >= 1 && day <= 5 && ((h >= 1 && h < 4) || (h >= 6 && h < 10));\n}\nfunction classify(text) {\n const t = (text || '').replace(/```[\\s\\S]*?```/g, ' ');\n const L = [...t].filter(c => /\\p{L}/u.test(c));\n if (!L.length) return { verdict: 'other', cyr: 0 };\n const cyr = L.filter(c => /[Ѐ-ӿ]/.test(c)).length / L.length;\n const lat = L.filter(c => /[A-Za-zÀ-ɏ]/.test(c)).length / L.length;\n const lo = t.toLowerCase();\n const fr = /[éèêàçùûôîœ]/.test(lo) ||\n /(^|[^\\p{L}])(qui|que|quel|quelle|quels|quelles|combien|est|sont|le|la|les|des|du|un|une|au|aux|il|elle|de|dans|pour|avec|et|ou|en)(?=[^\\p{L}]|$)/u.test(lo);\n const verdict = cyr >= 0.5 ? 'russian' : (lat >= 0.5 && fr ? 'french' : 'other');\n return { verdict, cyr: +cyr.toFixed(3) };\n}\nconst peak = isPeak(new Date()); // scored right after each batch, so close to request time\nreturn $input.all().map((item, i) => {\n const meta = $('Build prompts').itemMatching(i).json;\n const r = item.json;\n const u = r.usage || {};\n const ch = (r.choices && r.choices[0]) || {};\n const msg = ch.message || {};\n const p = PRICE[meta.model] || PRICE['deepseek-flash'];\n const hit = u.prompt_cache_hit_tokens || 0, miss = u.prompt_cache_miss_tokens || 0, out = u.completion_tokens || 0;\n const cost = (hit * p.hit + miss * p.miss + out * p.out) / 1e6 * (peak ? 2 : 1);\n let c = r.error || !r.choices ? { verdict: 'error', cyr: 0 } : classify(msg.content);\n if (c.verdict === 'other' && ch.finish_reason === 'length') c.verdict = 'truncated';\n return { json: {\n run_id: meta.run_id, model: meta.model, thinking: meta.thinking, n: meta.n, seed: meta.seed,\n question_id: meta.question_id, phase: meta.phase, verdict: c.verdict, cyrillic_ratio: c.cyr,\n finish_reason: ch.finish_reason || '',\n cache_hit_tokens: hit, cache_miss_tokens: miss, output_tokens: out,\n reasoning_tokens: (u.completion_tokens_details || {}).reasoning_tokens || 0,\n reasoning_chars: (msg.reasoning_content || '').length,\n cost_usd: +cost.toFixed(8), peak,\n reply: String(msg.content || (r.error && (r.error.description || r.error.message || JSON.stringify(r.error))) || '').slice(0, 300),\n } };\n});\n" } }, { "id": "d81730b9-9734-417a-b191-dc718e90f31b", "name": "Save row", "type": "n8n-nodes-base.dataTable", "typeVersion": 1.1, "position": [ 1760, 640 ], "parameters": { "resource": "row", "operation": "insert", "dataTableId": { "__rl": true, "mode": "name", "value": "ctx_drift_results" }, "columns": { "mappingMode": "defineBelow", "value": { "run_id": "={{ $json.run_id }}", "model": "={{ $json.model }}", "thinking": "={{ $json.thinking }}", "n": "={{ $json.n }}", "seed": "={{ $json.seed }}", "question_id": "={{ $json.question_id }}", "phase": "={{ $json.phase }}", "verdict": "={{ $json.verdict }}", "finish_reason": "={{ $json.finish_reason }}", "cache_hit_tokens": "={{ $json.cache_hit_tokens }}", "cache_miss_tokens": "={{ $json.cache_miss_tokens }}", "output_tokens": "={{ $json.output_tokens }}", "reasoning_chars": "={{ $json.reasoning_chars }}", "cost_usd": "={{ $json.cost_usd }}", "reply": "={{ $json.reply }}" }, "matchingColumns": [], "schema": [ { "id": "run_id", "displayName": "run_id", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "model", "displayName": "model", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "thinking", "displayName": "thinking", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "n", "displayName": "n", "required": false, "defaultMatch": false, "display": true, "type": "number", "canBeUsedToMatch": true }, { "id": "seed", "displayName": "seed", "required": false, "defaultMatch": false, "display": true, "type": "number", "canBeUsedToMatch": true }, { "id": "question_id", "displayName": "question_id", "required": false, "defaultMatch": false, "display": true, "type": "number", "canBeUsedToMatch": true }, { "id": "phase", "displayName": "phase", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "verdict", "displayName": "verdict", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "finish_reason", "displayName": "finish_reason", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "cache_hit_tokens", "displayName": "cache_hit_tokens", "required": false, "defaultMatch": false, "display": true, "type": "number", "canBeUsedToMatch": true }, { "id": "cache_miss_tokens", "displayName": "cache_miss_tokens", "required": false, "defaultMatch": false, "display": true, "type": "number", "canBeUsedToMatch": true }, { "id": "output_tokens", "displayName": "output_tokens", "required": false, "defaultMatch": false, "display": true, "type": "number", "canBeUsedToMatch": true }, { "id": "reasoning_chars", "displayName": "reasoning_chars", "required": false, "defaultMatch": false, "display": true, "type": "number", "canBeUsedToMatch": true }, { "id": "cost_usd", "displayName": "cost_usd", "required": false, "defaultMatch": false, "display": true, "type": "number", "canBeUsedToMatch": true }, { "id": "reply", "displayName": "reply", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true } ] }, "options": {} } }, { "id": "0ce6f66c-57bf-42ac-b202-a8b91fc5a1a4", "name": "Cache pause", "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 1980, 640 ], "parameters": { "resume": "timeInterval", "amount": 5, "unit": "seconds" }, "webhookId": "92b27863-3066-41bb-a1f6-ae7908a530e5" }, { "id": "b5dc7f27-ac2d-42ae-91f0-1ba3cb9bd26d", "name": "Cell summary", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 660, 440 ], "parameters": { "jsCode": "// The only thing a cell hands back to the parent: counts and token sums per (model, thinking, N).\n// Never the message arrays or replies -- those stay in the child execution and the data table.\nconst rows = $input.all().map(i => i.json);\nconst cell = $('When Executed by Another Workflow').first().json;\nconst VERDICTS = ['russian', 'french', 'other', 'truncated', 'error'];\nconst groups = {};\nfor (const r of rows) {\n const k = `${r.model}|${r.thinking}|${r.n}`;\n const g = groups[k] = groups[k] || { model: r.model, thinking: r.thinking, n: r.n, calls: 0,\n verdicts: Object.fromEntries(VERDICTS.map(v => [v, 0])), cost_usd: 0, output_tokens: 0,\n warmup: { hit: 0, miss: 0 }, followup: { hit: 0, miss: 0 } };\n g.calls++;\n g.verdicts[r.verdict] = (g.verdicts[r.verdict] || 0) + 1;\n g.cost_usd += Number(r.cost_usd) || 0;\n g.output_tokens += Number(r.output_tokens) || 0;\n const ph = g[r.phase] || g.followup;\n ph.hit += Number(r.cache_hit_tokens) || 0;\n ph.miss += Number(r.cache_miss_tokens) || 0;\n}\nconst sum = f => rows.reduce((a, r) => a + (Number(r[f]) || 0), 0);\nconst ts = rows.map(r => Date.parse(r.createdAt)).filter(Number.isFinite);\nconst verdicts = Object.fromEntries(VERDICTS.map(v => [v, rows.filter(r => r.verdict === v).length]));\nreturn [{ json: {\n run_id: cell.run_id, cell: cell.cell, cells: cell.cells, execution_id: $execution.id,\n n: cell.n, models: cell.models, thinking: cell.thinking,\n calls: rows.length, verdicts, cost_usd: +sum('cost_usd').toFixed(5),\n cache_hit_tokens: sum('cache_hit_tokens'), cache_miss_tokens: sum('cache_miss_tokens'),\n output_tokens: sum('output_tokens'),\n started: ts.length ? new Date(Math.min(...ts)).toISOString() : null,\n finished: ts.length ? new Date(Math.max(...ts)).toISOString() : null,\n groups: Object.values(groups).map(g => ({ ...g, cost_usd: +g.cost_usd.toFixed(5) })),\n} }];\n" } }, { "id": "5d6f22a5-13fe-4cf4-b497-504412f43c97", "name": "Note 1", "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ -560, -520 ], "parameters": { "content": "## Instruction drift benchmark\nDoes a model keep following an instruction when the conversation shows it doing something else? Turn 1 says *translate every question into Russian*, then N hardcoded assistant turns answer in French. Is the next reply in Russian?\n\n**Setup**\n1. Credential: DeepSeek (API key from platform.deepseek.com).\n2. Data table `ctx_drift_results` is created on the first run.\n3. **Save and publish**, then click *Execute workflow* and open **Report**. The grid runs as sub-executions of this same workflow (**Run cell** calls the trigger lane below): an editor run uses the saved draft, a webhook or schedule run uses the published version.\n\n**Cost:** dev mode (72 calls, about 1 minute) cost $0.035 at peak prices, so about $0.02 off-peak. Full mode: see the notes on the right.\n\nSource: [Breunig, The State of Context Engineering](https://www.dbreunig.com/2026/10/24/the-state-of-context-engineering.html)\n", "width": 520, "height": 720, "color": 7 } }, { "id": "0dd65852-4925-4983-b35f-afabc88487f0", "name": "Note 2", "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ -20, -520 ], "parameters": { "content": "## What it measures\nReplicates the heatmap on Drew Breunig's slide 12, from Camassa & Shiller, [*Do as I Say, Not as I Do*](https://arxiv.org/abs/2605.20382) (condition `language_ru_fr`). The instruction sits in the first user message; more hardcoded turns means more contrary evidence.\n\nEach **seed** starts the example list at a different place, so it is a different conversation. Dev numbers (2 seeds × 3 questions per cell) are illustrative, not rates; the paper used 35 seeded conversations per cell. Temperature is 0, but DeepSeek appears to ignore it in thinking mode.", "width": 420, "height": 400, "color": 5 } }, { "id": "be810a0a-f5d1-4230-9969-a9bd6e57e0d8", "name": "Note 3", "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ 460, -520 ], "parameters": { "content": "## Warm the cache, then reuse it\nWithin a cell, items are sorted question-first. The first **Loop** batch sends one *warm-up* call per (model, thinking, N, seed); the next batches send the other questions, which share everything but the last message. **Cache pause** gives DeepSeek a few seconds to store the prefix (hits come in 128-token blocks). **Report** shows hit ratios for warm-up and follow-up calls separately. Both also depend on what the account sent in the last hours, so treat them as a sample, not a constant. Pruning or reordering an earlier turn would turn every later call into a miss.\n\nClean ratios need `batch_size` ≤ groups per cell (24 in dev; 12 per full-mode N cell, 3 when split). With the default 24, the first batch of a full-mode cell mixes a few follow-ups in with the warm-ups, which costs some hits, not correctness.\n", "width": 440, "height": 460, "color": 4 } }, { "id": "5b90c5c2-e889-4ed5-9da7-2e160b4c6148", "name": "Note 4", "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ 940, -520 ], "parameters": { "content": "## Retry per item, score in code\nThe HTTP node keeps going on errors, so n8n's node retry would never fire. Instead **Failed?** sends only the failed items to **Back off** (5 s) and **Retry**; a second failure is scored `error`.\n\n**Score** checks the language only: ≥50% Cyrillic = **russian** (instruction followed), Latin with French markers = **french** (pattern followed). It does not check that the reply is a translation. `truncated` (hit max_tokens), `other` and `error` are left out of the rate and counted in **Report**.", "width": 460, "height": 400, "color": 6 } }, { "id": "62ce463c-27af-4128-abbc-9529dbc200a1", "name": "Note 5", "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ 1460, -520 ], "parameters": { "content": "## Dev → full, run off-peak\nIn **Config**, set `mode` to `full`: 2 models × 2 thinking modes × 12 N × 30 questions × 3 seeds = 4,320 calls, about **$1.2–1.8 off-peak** in our runs (double at peak), roughly 1.5 h with batch 24. **Report** prints a cost range from the dev run (cold cache vs. dev hit ratio) plus a rough wall time.\n\nRequests go out every 200 ms and overlap; **Loop** `batch_size` (24) caps how many are in flight per cell. Run outside 01–04 and 06–10 UTC on weekdays for DeepSeek's 50% discount. Every cell is its own execution (ids in **Report**), so a failed cell can be re-run alone by narrowing `n_full`, `models` and `thinking` in **Config**.\n", "width": 480, "height": 460, "color": 3 } }, { "id": "fbe71d07-d153-4d60-af14-4d792ef72832", "name": "Note 6", "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ 1980, -520 ], "parameters": { "content": "## Why full mode runs in cells\nn8n keeps every item of an execution in memory until it ends. The full grid in one execution — 4,320 items whose message arrays add up to ~140k messages — hit the 1 GiB heap limit and crashed n8n. **Plan cells** therefore splits the grid into cells of at most `cell_max_messages` (10,000 = calls × (2N+2)): one N value with all models and thinking modes when it fits, else split by model, then by thinking (23 cells in full mode; dev is one cell of 72 calls).\n\n**Run cell** executes each cell as a sub-execution of this workflow (the lane below, *Run once for each item*, waiting for completion), and gets back only **Cell summary**: counts and token sums, never the messages. Cells of this size stayed at about 620 MiB in our runs, independent of N.\n", "width": 500, "height": 460, "color": 2 } } ], "connections": { "Start": { "main": [ [ { "node": "Config", "type": "main", "index": 0 } ] ] }, "Config": { "main": [ [ { "node": "Create table", "type": "main", "index": 0 } ] ] }, "Create table": { "main": [ [ { "node": "Plan cells", "type": "main", "index": 0 } ] ] }, "Plan cells": { "main": [ [ { "node": "Run cell", "type": "main", "index": 0 } ] ] }, "Run cell": { "main": [ [ { "node": "Collect", "type": "main", "index": 0 } ] ] }, "Collect": { "main": [ [ { "node": "Report", "type": "main", "index": 0 } ] ] }, "When Executed by Another Workflow": { "main": [ [ { "node": "Build prompts", "type": "main", "index": 0 } ] ] }, "Build prompts": { "main": [ [ { "node": "Loop", "type": "main", "index": 0 } ] ] }, "Loop": { "main": [ [ { "node": "Cell summary", "type": "main", "index": 0 } ], [ { "node": "DeepSeek", "type": "main", "index": 0 } ] ] }, "DeepSeek": { "main": [ [ { "node": "Failed?", "type": "main", "index": 0 } ] ] }, "Failed?": { "main": [ [ { "node": "Back off", "type": "main", "index": 0 } ], [ { "node": "Merge", "type": "main", "index": 0 } ] ] }, "Back off": { "main": [ [ { "node": "Retry", "type": "main", "index": 0 } ] ] }, "Retry": { "main": [ [ { "node": "Merge", "type": "main", "index": 1 } ] ] }, "Merge": { "main": [ [ { "node": "Score", "type": "main", "index": 0 } ] ] }, "Score": { "main": [ [ { "node": "Save row", "type": "main", "index": 0 } ] ] }, "Save row": { "main": [ [ { "node": "Cache pause", "type": "main", "index": 0 } ] ] }, "Cache pause": { "main": [ [ { "node": "Loop", "type": "main", "index": 0 } ] ] } }, "settings": { "executionOrder": "v1" }, "pinData": {}, "meta": { "templateCredsSetupCompleted": false } }