From 4cae6a0296027430435a82535727dedbb2ab3d50 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 20:20:26 +0000 Subject: [PATCH 1/5] bench: update the in-progress 0.3 CPU run Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013QgiejZTjqQAUv4N4EFskK --- docs/bench/cpu-nothink-0.3.json | 166 ++++++++++++++++++++++++++++++-- 1 file changed, 158 insertions(+), 8 deletions(-) diff --git a/docs/bench/cpu-nothink-0.3.json b/docs/bench/cpu-nothink-0.3.json index 11a97d9..9c94e17 100644 --- a/docs/bench/cpu-nothink-0.3.json +++ b/docs/bench/cpu-nothink-0.3.json @@ -11,14 +11,14 @@ { "summary": { "model": "flycoder0.3", - "passed": 7, - "total": 12, - "passRate": 0.5833333333333334, - "decodeTokensPerSecond": 3.844831517257886, - "prefillTokensPerSecond": 22.80421580322576, - "meanOutputTokens": 745.9166666666666, - "meanSecondsPerProblem": 215.52991666666665, - "truncated": 1 + "passed": 9, + "total": 20, + "passRate": 0.45, + "decodeTokensPerSecond": 3.8939337665767098, + "prefillTokensPerSecond": 22.747543255171273, + "meanOutputTokens": 900.3, + "meanSecondsPerProblem": 253.56705, + "truncated": 3 }, "results": [ { @@ -200,6 +200,156 @@ "prefillMs": 22282.489, "thinkingChars": 0, "answer": "```python\nimport heapq\nfrom typing import List, Tuple, Union\n\ndef shortest_distances(\n n: int,\n edges: List[Tuple[Union[int, float], Union[int, float], Union[int, float]]],\n source: Union[int, float]\n) -> List[float]:\n \"\"\"\n Computes shortest distances from a source node in a directed weighted graph.\n \n Args:\n n: Number of nodes (0 to n-1).\n edges: List of tuples (u, v, weight).\n source: Source node index.\n \n Returns:\n List of shortest distances from source to each node.\n \n Raises:\n ValueError: If source is invalid or any weight is negative.\n \"\"\"\n # Validate source\n if not isinstance(source, int) or source < 0 or source >= n:\n raise ValueError(f\"Source node {source} is not a valid node for a graph with {n} nodes (0..{n-1}).\")\n\n # Validate edges and build adjacency list\n # Using a dictionary for the adjacency list to handle potential hash collisions or custom objects if needed,\n # but for ints, a list of lists is faster. We'll use a list of lists.\n adj: List[List[Tuple[int, float]]] = [[] for _ in range(n)]\n \n for u, v, w in edges:\n # Validate node indices in edges\n if not isinstance(u, int) or not isinstance(v, int):\n raise ValueError(\"Node indices in edges must be integers.\")\n if u < 0 or u >= n:\n raise ValueError(f\"Node index {u} in edge is out of range [0, {n-1}).\")\n if v < 0 or v >= n:\n raise ValueError(f\"Node index {v} in edge is out of range [0, {n-1}).\")\n \n # Validate weight\n if not isinstance(w, (int, float)):\n raise ValueError(\"Edge weights must be numbers.\")\n \n # Check for negative weight\n if w < 0:\n raise ValueError(\"Edge weights must be non-negative.\")\n \n adj[u].append((v, w))\n \n # Dijkstra's Algorithm using a min-heap priority queue\n # Initialize distances\n distances = [float('inf')] * n\n distances[source] = 0.0\n \n # Priority queue stores tuples (distance, node)\n pq = [(0.0, source)]\n \n while pq:\n current_dist, u = heapq.heappop(pq)\n \n # If current distance is greater than already found shortest distance, skip\n if current_dist > distances[u]:\n continue\n \n for v, weight in adj[u]:\n new_dist = current_dist + weight\n if new_dist < distances[v]:\n distances[v] = new_dist\n heapq.heappush(pq, (new_dist, v))\n \n return distances\n```" + }, + { + "problem": "py-slugify", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-t6DPp2/test_solution.py\", line 4, in \n assert slugify(\"Straße & Œuvre, Æther, Øre\") == \"strasse-oeuvre-aether-ore\"\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError\n", + "truncated": false, + "wallMs": 482269, + "loadMs": 3, + "outputTokens": 1777, + "decodeMs": 458442.747, + "promptTokens": 513, + "prefillMs": 23690.625, + "thinkingChars": 0, + "answer": "```python\nimport unicodedata\nimport re\n\ndef slugify(text: str, max_length: int = 50) -> str:\n if max_length < 1:\n raise ValueError(\"max_length must be at least 1\")\n \n # Normalize to decomposed form (NFD) to separate base characters from accents\n text = unicodedata.normalize('NFD', text)\n \n # Mapping for special characters after normalization\n # Note: 'ß' is already ASCII, but we handle it explicitly as requested.\n # Other characters like æ, œ, ø are handled by mapping directly if they remain,\n # but NFD might break some. However, the prompt asks to map specific ones.\n # Let's handle the specific mappings requested.\n \n # First, handle the specific character mappings requested.\n # We do this before or after normalization? \n # The prompt says \"é -> e\". In NFD, 'é' becomes 'e' + combining acute.\n # So we should remove combining marks first, then handle specific multi-char replacements \n # if they weren't decomposed. But 'æ' and 'œ' are usually not decomposed into single letters \n # in standard NFD unless specified otherwise in specific locale rules, \n # but Python's unicodedata doesn't guarantee decomposition for all ligatures.\n # However, the prompt implies direct mapping. Let's stick to the requested mappings.\n \n # Strategy:\n # 1. Normalize to NFD to separate accents (e.g., é -> e + ́).\n # 2. Filter out combining marks (category 'Mn'). This handles é -> e.\n # 3. Handle specific mappings: ß -> ss, æ -> ae, œ -> oe, ø -> o.\n # Note: These might need to be done before or after accent removal depending on the character.\n # Usually, ß, æ, œ, ø are base characters. If we normalize, they stay mostly intact.\n # Let's apply accent removal first, then specific char replacements.\n \n # Remove combining marks (accents)\n text = ''.join(char for char in text if unicodedata.category(char) != 'Mn')\n \n # Apply specific mappings for characters that might have been decomposed or not\n # ß is already ASCII 'ss' logic, but the text is now ASCII-ish.\n # æ, œ, ø might appear as is.\n \n # Map specific characters\n text = text.replace('ß', 'ss')\n text = text.replace('æ', 'ae')\n text = text.replace('œ', 'oe')\n text = text.replace('ø', 'o')\n \n # Lowercase\n text = text.lower()\n \n # Replace any non-alphanumeric character with a placeholder that we will collapse later\n # We want to replace runs of non-alphanumeric with a single hyphen.\n # It's easier to replace everything non-alphanumeric with a temporary marker, \n # then join runs.\n # Actually, regex is cleaner: replace any non-alphanumeric with a space or a unique char,\n # but we need to collapse runs.\n # Let's replace non-alphanumeric with a space first, then replace multiple spaces with one hyphen?\n # No, the requirement is: \"Every run of characters that are not a-z or 0-9 becomes a single hyphen\".\n # So:\n # 1. Replace non-alphanumeric chars with a placeholder (e.g., a space or a unique string).\n # But we must be careful not to create leading/trailing hyphens.\n \n # Better approach:\n # Iterate and build the slug.\n # If char is alphanumeric: append.\n # If char is not: set a flag that we are in a non-alphanumeric run.\n # But we need to handle the transition from alphanumeric to non-alphanumeric.\n \n # Let's try a two-pass or regex approach.\n # Regex: replace any non-alphanumeric sequence with a single hyphen.\n # But we must ensure no leading/trailing hyphens.\n \n # Step 1: Clean the text (done above). Now it contains only ASCII alphanumeric and maybe some weird chars?\n # After NFD + removing Mn + specific maps, the text should be mostly ASCII alphanumeric.\n # Wait, did we remove ALL non-ASCII? No, just accents.\n # So now text is ASCII.\n \n # Now, replace any character not in a-z0-9 with a space.\n # Then replace multiple spaces with a single hyphen.\n # Then strip leading/trailing hyphens.\n \n # However, replacing with space and then multiple spaces -> hyphen works.\n # But what if the string starts with a non-alphanumeric? It becomes space -> hyphen.\n # We need to strip leading/trailing hyphens at the end.\n \n # Let's use a simple loop or regex to collapse runs.\n # Regex: find all sequences of non-alphanumeric and replace with '-'.\n # But this might leave leading/trailing hyphens if the string starts/ends with non-alphanumeric.\n \n # Example: \" hello \" -> \" hello \" (spaces) -> \"hello\" (spaces removed? No, spaces are non-alphanumeric).\n # \" hello \" -> replace non-alpha with space -> \" hello \" -> collapse spaces -> \" hello \" -> strip hyphens?\n # Wait, if I replace non-alpha with space, then collapse spaces, I get \"hello\".\n # But \"hello-world\" -> \"hello-world\".\n # \"hello world!\" -> \"hello world\" -> \"hello-world\".\n # \"123\" -> \"123\".\n # \"1 2\" -> \"1 2\" -> \"1-2\".\n \n # Algorithm:\n # 1. Replace any char not in a-z0-9 with a space.\n # 2. Replace any sequence of 1+ spaces with a single hyphen.\n # 3. Strip leading and trailing hyphens.\n \n # Is there any case where this fails?\n # \"---\" -> \" \" -> \"-\" -> \"\" (empty string). \n # The prompt doesn't specify what to return for an empty slug after stripping, \n # but logically it should be an empty string.\n \n # Let's implement.\n \n # Replace non-alphanumeric with space\n # We can use str.translate for speed, but regex is fine for this size.\n # Or just a list comprehension.\n \n # Using regex for clarity and correctness\n import re\n \n # Replace non-alphanumeric with space\n # We assume the text is now ASCII.\n # If there are any non-ASCII left (unlikely with our processing), we treat them as spaces too.\n # Actually, let's just handle anything not in a-z0-9.\n \n # Using regex substitution\n # Pattern: match any character that is NOT [a-z0-9]\n # Replacement: ' '\n \n # But wait, what if the text has something like 'é' that wasn't fully handled?\n # Our NFD + remove Mn should handle accents.\n # Specific maps: ß, æ, œ, ø. These are ASCII in Python strings (len 1).\n # So the text is ASCII.\n \n # Replace non-alphanumeric with space\n text = re.sub(r'[^a-z0-9]', ' ', text)\n \n # Replace one or more spaces with a single hyphen\n text = re.sub(r' +', '-', text)\n \n # Strip leading and trailing hyphens\n text = text.strip('-')\n \n # Truncate to max_length\n if len(text) > max_length:\n # Remove trailing hyphen if exists after truncation\n text = text[:max_length]\n if text.endswith('-'):\n text = text[:-1]\n \n return text\n```" + }, + { + "problem": "py-base-convert", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-gw2jfc/test_solution.py\", line 3, in \n assert convert_base(\"FF\", 16, 10) == \"255\"\n ~~~~~~~~~~~~^^^^^^^^^^^^^^\n File \"/tmp/flycoder-bench-gw2jfc/solution.py\", line 27, in convert_base\n raise ValueError(f\"Invalid character '{char}' in input string\")\nValueError: Invalid character 'F' in input string\n", + "truncated": false, + "wallMs": 136568, + "loadMs": 3, + "outputTokens": 453, + "decodeMs": 113867.997, + "promptTokens": 506, + "prefillMs": 22529.271, + "thinkingChars": 0, + "answer": "```python\ndef convert_base(number: str, from_base: int, to_base: int) -> str:\n # Validate bases\n if not (2 <= from_base <= 36) or not (2 <= to_base <= 36):\n raise ValueError(f\"Bases must be between 2 and 36, got from_base={from_base}, to_base={to_base}\")\n\n # Normalize input: handle sign and strip whitespace if any (though spec implies clean input)\n if not number:\n raise ValueError(\"Input string cannot be empty\")\n \n if number == \"-\":\n raise ValueError(\"Input string cannot be just '-'\")\n\n sign = \"\"\n if number.startswith(\"-\"):\n sign = \"-\"\n number = number[1:]\n \n if number == \"\":\n raise ValueError(\"Input string cannot be empty after sign removal\")\n\n # Validate digits and convert from_base string to integer value\n digit_map = \"0123456789abcdefghijklmnopqrstuvwxyz\"\n \n # Check each character\n for char in number:\n if char not in digit_map:\n raise ValueError(f\"Invalid character '{char}' in input string\")\n \n digit_value = digit_map.index(char.lower())\n if digit_value >= from_base:\n raise ValueError(f\"Digit '{char}' is invalid for base {from_base}\")\n\n # Convert from_base string to integer\n value = 0\n for char in number:\n digit_value = digit_map.index(char.lower())\n value = value * from_base + digit_value\n\n # Handle zero case\n if value == 0:\n return \"0\"\n\n # Convert integer to to_base string\n result_chars = []\n negative = value < 0\n value = abs(value)\n \n while value > 0:\n remainder = value % to_base\n result_chars.append(digit_map[remainder])\n value //= to_base\n \n result_chars.reverse()\n result = \"\".join(result_chars)\n \n return sign + result if negative else result\n```" + }, + { + "problem": "js-parse-query", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 164395, + "loadMs": 3, + "outputTokens": 551, + "decodeMs": 139808.437, + "promptTokens": 555, + "prefillMs": 24475.005, + "thinkingChars": 0, + "answer": "```javascript\n/**\n * Parses a URL query string into a plain object.\n * \n * Rules:\n * - Optional leading \"?\".\n * - Pairs separated by \"&\"; empty pairs ignored.\n * - Split on first \"=\"; key without \"=\" maps to \"\".\n * - \"+\" in key/value becomes space.\n * - Percent-decoding applies. If decoding fails, keep the raw text (with \"+\" already converted).\n * - Multiple values for the same key become an array.\n * - Safe against prototype pollution (keys like \"__proto__\" become own properties).\n */\nexport function parseQuery(query) {\n if (typeof query !== 'string') {\n return {};\n }\n\n // Remove optional leading \"?\"\n let str = query;\n if (str.startsWith('?')) {\n str = str.slice(1);\n }\n\n const result = Object.create(null);\n const parts = str.split('&');\n\n for (const part of parts) {\n if (part.length === 0) {\n continue;\n }\n\n // Split on the first \"=\" only\n const idx = part.indexOf('=');\n let key, value;\n\n if (idx === -1) {\n // No \"=\": key is the whole part, value is \"\"\n key = part;\n value = '';\n } else {\n key = part.slice(0, idx);\n value = part.slice(idx + 1);\n }\n\n // Decode \"+\" as space\n let decodedKey = key.replace(/\\+/g, ' ');\n let decodedValue = value.replace(/\\+/g, ' ');\n\n // Function to safely percent-decode\n const safeDecode = (s) => {\n // Replace + first (already done, but ensure no double replacement issues)\n // Then use decodeURIComponent. If it throws, return the string as-is.\n try {\n return decodeURIComponent(s);\n } catch {\n return s;\n }\n };\n\n // Apply percent-decoding\n decodedKey = safeDecode(decodedKey);\n decodedValue = safeDecode(decodedValue);\n\n // Handle multiple values\n if (decodedKey in result) {\n if (!Array.isArray(result[decodedKey])) {\n result[decodedKey] = [result[decodedKey]];\n }\n result[decodedKey].push(decodedValue);\n } else {\n result[decodedKey] = decodedValue;\n }\n }\n\n return result;\n}\n```" + }, + { + "problem": "js-deep-equal", + "sample": 0, + "ok": false, + "error": "node:internal/modules/run_main:123\n triggerUncaughtException(\n ^\n\nAssertionError [ERR_ASSERTION]: Expected values to be strictly equal:\n\nfalse !== true\n\n at file:///tmp/flycoder-bench-zPTSmZ/test.mjs:3:35 {\n generatedMessage: true,\n code: 'ERR_ASSERTION',\n actual: false,\n expected: true,\n operator: 'strictEqual',\n diff: 'simple'\n}\n\nNode.js v22.22.0\n", + "truncated": false, + "wallMs": 171108, + "loadMs": 4, + "outputTokens": 595, + "decodeMs": 147002.103, + "promptTokens": 545, + "prefillMs": 23960.82, + "thinkingChars": 0, + "answer": "```javascript\n/**\n * Deep equality check for JavaScript values.\n * Handles primitives, arrays, plain objects, Date, cyclic structures, etc.\n *\n * @param {*} a - First value\n * @param {*} b - Second value\n * @returns {boolean} - true if structurally equal, false otherwise\n */\nexport function deepEqual(a, b) {\n if (a === b) {\n // SameValueZero check: handles NaN === NaN and 0 === -0\n return true;\n }\n\n // Different types are not equal\n if (a instanceof Date && b instanceof Date) {\n return a.getTime() === b.getTime();\n }\n\n if (Array.isArray(a) && Array.isArray(b)) {\n if (a.length !== b.length) {\n return false;\n }\n for (let i = 0; i < a.length; i++) {\n if (!deepEqual(a[i], b[i])) {\n return false;\n }\n }\n return true;\n }\n\n // Check for plain object equality\n if (typeof a === 'object' && a !== null && typeof b === 'object' && b !== null) {\n // If either is not a plain object or an array, they are not equal (handled above for arrays)\n // We need to ensure they are both plain objects\n if (!isPlainObject(a) || !isPlainObject(b)) {\n return false;\n }\n\n const keysA = Object.keys(a);\n const keysB = Object.keys(b);\n\n if (keysA.length !== keysB.length) {\n return false;\n }\n\n const keysMap = new Map();\n for (const key of keysA) {\n keysMap.set(key, true);\n }\n\n for (const key of keysB) {\n if (!keysMap.has(key)) {\n return false;\n }\n if (!deepEqual(a[key], b[key])) {\n return false;\n }\n }\n\n return true;\n }\n\n // All other cases (including null vs number, etc.) are not equal\n return false;\n}\n\n/**\n * Checks if a value is a plain object (not an array, Date, or other special object).\n * @param {*} value - The value to check\n * @returns {boolean} - true if it is a plain object\n */\nfunction isPlainObject(value) {\n if (typeof value !== 'object' || value === null) {\n return false;\n }\n const prototype = Object.getPrototypeOf(value);\n return prototype === null || prototype === Object.prototype;\n}\n```" + }, + { + "problem": "js-evaluate", + "sample": 0, + "ok": false, + "error": "file:///tmp/flycoder-bench-SHt8JT/solution.mjs:167\n throw new SyntaxError('SyntaxError: Unexpected characters after expression');\n ^\n\nSyntaxError: SyntaxError: Unexpected characters after expression\n at evaluate (file:///tmp/flycoder-bench-SHt8JT/solution.mjs:167:11)\n at file:///tmp/flycoder-bench-SHt8JT/test.mjs:11:9\n\nNode.js v22.22.0\n", + "truncated": false, + "wallMs": 297206, + "loadMs": 7, + "outputTokens": 1093, + "decodeMs": 273976.634, + "promptTokens": 534, + "prefillMs": 23097.76, + "thinkingChars": 0, + "answer": "```javascript\n/**\n * Evaluates an arithmetic expression string.\n * Supports: non-negative decimal numbers, + - * /, parentheses, unary +/-, whitespace.\n * Precedence: Unary > * / > + -. Left-associative for binary operators.\n * Throws: SyntaxError for malformed input, RangeError for division by zero.\n */\n\nexport function evaluate(expression) {\n if (typeof expression !== 'string') {\n throw new TypeError('Input must be a string');\n }\n\n const chars = expression.split('');\n let i = 0;\n \n // Helper to peek and consume characters\n const peek = () => chars[i] || null;\n const consume = () => {\n const c = chars[i];\n i++;\n return c;\n };\n \n // Helper to skip whitespace\n const skipWhitespace = () => {\n while (peek() != null && /[\\s\\t\\n\\r]/.test(peek())) {\n consume();\n }\n };\n\n // Helper to throw syntax error\n const syntaxError = (msg = 'SyntaxError: Invalid expression') => {\n throw new SyntaxError(msg);\n };\n\n // Helper to parse a single number (integer or float)\n const parseNumber = () => {\n let start = i;\n let c = peek();\n \n // Handle leading dot (e.g., \".5\")\n if (c === '.') {\n consume();\n if (peek() == null || !/[\\d.]/.test(peek())) {\n throw new SyntaxError('SyntaxError: Missing digits after decimal point');\n }\n } else if (c != null && /\\d/.test(c)) {\n do {\n consume();\n } while (peek() != null && /\\d/.test(peek()));\n } else {\n throw new SyntaxError('SyntaxError: Expected number');\n }\n\n return parseFloat(expression.substring(start, i));\n };\n\n // Parse a primary expression (number, parenthesized expression, or unary +/-)\n const parsePrimary = () => {\n skipWhitespace();\n \n const c = peek();\n \n // Unary +\n if (c === '+') {\n consume();\n const val = parsePrimary();\n return val;\n }\n \n // Unary -\n if (c === '-') {\n consume();\n const val = parsePrimary();\n return -val;\n }\n \n // Parenthesized expression\n if (c === '(') {\n consume();\n skipWhitespace();\n const val = parseAdditive(); // Parse inside parentheses\n skipWhitespace();\n \n if (peek() !== ')') {\n throw new SyntaxError('SyntaxError: Expected closing parenthesis');\n }\n consume(); // consume ')'\n return val;\n }\n \n // Number\n if (c != null && (/\\d/.test(c) || c === '.')) {\n return parseNumber();\n }\n \n throw new SyntaxError('SyntaxError: Unexpected character');\n };\n\n // Parse multiplicative expressions (* /)\n const parseMultiplicative = () => {\n let left = parsePrimary();\n \n while (true) {\n skipWhitespace();\n const op = peek();\n \n if (op === '*') {\n consume();\n skipWhitespace();\n const right = parsePrimary();\n left = left * right;\n } else if (op === '/') {\n consume();\n skipWhitespace();\n const right = parsePrimary();\n if (right === 0) {\n throw new RangeError('RangeError: Division by zero');\n }\n left = left / right;\n } else {\n break;\n }\n }\n \n return left;\n };\n\n // Parse additive expressions (+ -)\n const parseAdditive = () => {\n let left = parseMultiplicative();\n \n while (true) {\n skipWhitespace();\n const op = peek();\n \n if (op === '+') {\n consume();\n skipWhitespace();\n const right = parseMultiplicative();\n left = left + right;\n } else if (op === '-') {\n consume();\n skipWhitespace();\n const right = parseMultiplicative();\n left = left - right;\n } else {\n break;\n }\n }\n \n return left;\n };\n\n // Main entry point\n skipWhitespace();\n \n if (peek() == null) {\n throw new SyntaxError('SyntaxError: Empty expression');\n }\n\n const result = parseAdditive();\n \n skipWhitespace();\n \n if (peek() != null) {\n throw new SyntaxError('SyntaxError: Unexpected characters after expression');\n }\n\n return result;\n}\n```" + }, + { + "problem": "js-format-bytes", + "sample": 0, + "ok": false, + "error": "truncated by max tokens", + "truncated": true, + "wallMs": 549598, + "loadMs": 6, + "outputTokens": 2048, + "decodeMs": 523386.515, + "promptTokens": 604, + "prefillMs": 26061.108, + "thinkingChars": 0, + "answer": "```javascript\n/**\n * Formats a byte count using binary (IEC) units.\n * \n * @param {number} bytes - The byte count to format.\n * @param {number} [decimals=1] - Number of decimal places to display.\n * @returns {string} The formatted string.\n * @throws {TypeError} If bytes is not a finite number.\n * @throws {RangeError} If decimals is not an integer between 0 and 10 inclusive.\n */\nexport function formatBytes(bytes, decimals = 1) {\n // Validate bytes input\n if (typeof bytes !== 'number' || !Number.isFinite(bytes)) {\n throw new TypeError('bytes must be a finite number');\n }\n\n // Validate decimals input\n if (!Number.isInteger(decimals) || decimals < 0 || decimals > 10) {\n throw new RangeError('decimals must be an integer from 0 to 10');\n }\n\n const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];\n const base = 1024;\n \n // Handle negative numbers by preserving the sign and working with the absolute value\n const isNegative = bytes < 0;\n const absBytes = isNegative ? -bytes : bytes;\n\n // Determine the appropriate unit\n // We iterate from the largest unit down to the smallest.\n // We choose a unit if the value in that unit is >= 1.\n // However, the logic \"If rounding would display 1024.0... move to next unit\" implies\n // we should check the raw value against thresholds.\n \n // Let's calculate the value for each unit from largest to smallest to find the first one\n // where the value is >= 1.\n // Actually, it's easier to iterate from the largest unit (PiB) down to B.\n // For each unit, check if absBytes / (base ** index) >= 1.\n // If it is, that's our unit.\n \n let unitIndex = 0;\n let valueInUnit = 0;\n\n // Start from the largest unit (PiB) and go down\n for (let i = units.length - 1; i >= 0; i--) {\n const divisor = Math.pow(base, units.length - 1 - i); \n // Wait, let's map indices correctly:\n // 0: B (1024^0)\n // 1: KiB (1024^1)\n // ...\n // 5: PiB (1024^5)\n \n // Let's recalculate index mapping:\n // i=0 -> B -> divisor = 1\n // i=1 -> KiB -> divisor = 1024\n // ...\n // i=5 -> PiB -> divisor = 1024^5\n \n // Correct loop logic:\n // i goes from 0 to 5.\n // divisor = Math.pow(base, i)\n const divisor = Math.pow(base, i);\n const currentVal = absBytes / divisor;\n\n if (currentVal >= 1) {\n unitIndex = i;\n valueInUnit = currentVal;\n break;\n }\n }\n\n // Special case handling: If rounding would push a value to 1024.0, \n // we need to move to the next unit.\n // Example: 1 MiB = 1024 KiB. If input is 1048575 (1.00000096 MiB), \n // with 1 decimal, it becomes 1.0 MiB. \n // The requirement says: \"If rounding would display 1024.0 of a unit, move to the next unit\".\n // This means if we are at unit 'i' and the formatted value rounds to X.XX... where the integer part is 1024?\n // No, the example says: \"1048575 with 1 decimal is '1.0 MiB'\".\n // 1048575 bytes = 1.0000009536743164 MiB.\n // If we formatted this as KiB: 1048575 / 1024 = 1024.00195...\n // If we formatted as MiB: 1.0000009...\n // The rule seems to be: if the value in the chosen unit (rounded to 'decimals') is >= 1024, \n // we should have picked the next larger unit instead.\n \n // Let's re-evaluate the selection logic with this rule.\n // Instead of picking the largest unit where value >= 1, we pick the largest unit where\n // the value is < 1024 (rounded or raw?).\n // The example 1048575 bytes -> 1.0 MiB (not 1024.0 KiB).\n // 1048575 / 1024 = 1024.00195...\n // If we picked KiB, we get 1024.0. The rule says \"If rounding would display 1024.0... move to next\".\n // So we must avoid landing on 1024.0.\n \n // Refined strategy:\n // Calculate the value for the current unit.\n // If the value is >= 1024, then this unit is too small, go to the next larger unit.\n // But wait, we iterate from largest to smallest.\n // If we are at PiB, value is < 1. Stop.\n // If we are at TiB, value is >= 1. Is it >= 1024?\n // If TiB value is >= 1024, then we are effectively at PiB range?\n // Max PiB is 2^50 bytes.\n // If bytes > 2^50, we are beyond PiB. The requirement says \"Values beyond PiB stay in PiB\".\n // So we cap the unit index at 5 (PiB).\n \n // Let's try a different approach:\n // Iterate from largest unit (PiB) down to B.\n // For unit i, calculate val = absBytes / (1024^i).\n // If val >= 1, this is a candidate.\n // However, we must ensure val < 1024.\n // If val >= 1024, we skip this unit and look at the next larger one (which we already passed?).\n // Actually, if val >= 1024 at unit i, it means absBytes >= 1024^(i+1).\n // So unit i+1 would be the correct one.\n // Thus, we should pick the largest unit i such that:\n // 1 <= absBytes / (1024^i) < 1024.\n // And if absBytes / (1024^i) >= 1024, we implicitly mean we should have picked i+1.\n // But we iterate downwards.\n \n // Let's restart the loop logic:\n // Start from i = 0 (B) up to 5 (PiB).\n // Check if absBytes / (1024^i) >= 1.\n // If yes, check if absBytes / (1024^i) >= 1024.\n // If yes, continue to next i (larger unit).\n // If no, this is our unit.\n // If we reach the end (PiB) and value is still >= 1024, we clamp to PiB.\n \n let selectedIndex = 0;\n let divisor = 1;\n \n for (let i = 0; i < units.length; i++) {\n const power = Math.pow(base, i);\n const val = absBytes / power;\n \n if (val >= 1) {\n // Check if we need to go higher (val >= 1024)\n // Floating point comparisons can be tricky. \n // The example: 1048575 bytes.\n // i=0 (B): val = 1048575. >= 1024? Yes. Continue.\n // i=1 (KiB): val = 1024.00195... >= 1024? Yes (barely). Continue.\n // i=2 (MiB): val = 1.0000009... >= 1? Yes. >= 1024? No. Select MiB.\n // This logic works for the example.\n \n if (val >= 1024) {" + }, + { + "problem": "js-wildcard", + "sample": 0, + "ok": false, + "error": "truncated by max tokens", + "truncated": true, + "wallMs": 542767, + "loadMs": 4, + "outputTokens": 2048, + "decodeMs": 520257.074, + "promptTokens": 506, + "prefillMs": 21742.463, + "thinkingChars": 0, + "answer": "```javascript\n/**\n * Matches a text against a wildcard pattern.\n * \n * Rules:\n * - '?' matches exactly one character.\n * - '*' matches any sequence of characters (including empty).\n * - Every other character matches itself.\n * - The entire text must match the entire pattern.\n * \n * Algorithm:\n * We use a dynamic programming approach similar to classic wildcard matching,\n * but optimized to avoid recomputing for every '*' by processing segments between '*'s.\n * \n * Steps:\n * 1. Split the pattern by '*' into segments. Each segment is a \"fixed\" part that must match exactly.\n * 2. We maintain a set of possible end positions in the text where the pattern could still match.\n * Initially, we start at position 0 in the text.\n * 3. For each segment (non-star part):\n * - We try to match the segment against the text starting from each of the current possible positions.\n * - If we find a match, we add the position after the match to the set of possible positions.\n * - If no match is found from any current position, we discard those and move on.\n * 4. After processing all segments, if the set of possible positions includes the end of the text, then it's a match.\n * \n * This is efficient because:\n * - We only scan the text a few times (once per segment).\n * - We avoid exponential blowup by not trying all combinations but rather tracking viable positions.\n * \n * Time Complexity: O(n * m) in the worst case, where n is text length and m is pattern length.\n * However, with typical patterns (few segments), it's much faster.\n * \n * Space Complexity: O(1) extra space beyond input (we use a small set of positions).\n */\n\nexport function matchWildcard(pattern, text) {\n if (pattern === '*') {\n return true; // '*' matches any text, including empty\n }\n if (pattern === '') {\n return text === '';\n }\n\n // Split pattern by '*' into segments. Each segment must match exactly.\n const segments = pattern.split('*');\n \n // Current possible positions in the text where we can start matching the next segment.\n // We use a Set to track these positions.\n let possiblePositions = new Set([0]);\n\n for (const segment of segments) {\n if (segment === '') {\n // This case is handled by the initial split; if pattern starts with '*', \n // the first segment is empty and we don't need to do anything.\n continue;\n }\n\n // We need to match the segment against the text starting from each of the current positions.\n // But note: the segment must match exactly, and we are allowed to skip characters in the text \n // only if we are at a '*' boundary. However, our approach is:\n // For each possible starting position in the text, we try to match the segment.\n // If it matches, we record the position after the match as a new possible starting position for the next segment.\n \n // However, note: the segment must match consecutively. We cannot skip characters within the segment.\n // Also, the segment must match exactly the substring in the text starting at the current position.\n \n // But wait: the pattern segments are separated by '*', meaning that between two segments, \n // there can be any number of characters (including zero). However, the segments themselves are fixed.\n // So, for a segment, we must match it exactly starting at some position in the text that is in our set.\n \n // However, the set of possible positions might be large. We need to be efficient.\n // Instead of iterating over all possible positions and then scanning the text for the segment,\n // we can do:\n // For each position p in possiblePositions:\n // Check if text[p : p+segment.length] equals segment.\n // If yes, then the new position p+segment.length is added to possiblePositions.\n \n // But note: the segment might be longer than the remaining text. We must check bounds.\n \n // However, there's a catch: the segment must match exactly, but the text might have extra characters after the segment?\n // No, because we are matching the entire pattern against the entire text. The segments are fixed parts that must appear in order.\n // The '*' between segments can match any text (including empty) in between. But the segments themselves are fixed.\n \n // Therefore, for a segment, we only care about matching it at positions that are in our set.\n // And after matching, the new position is p + segment.length.\n \n // But note: the set of possible positions might grow too large? Actually, we can limit it.\n // However, worst-case: pattern = \"*a*a*a*...\", text = \"aaaa...\". Then we might have many positions.\n // But note: the segment length is fixed. So for a segment of length L, we only need to check positions p such that p+L <= text.length.\n \n // Let's implement the naive check first and see if we can optimize.\n // Actually, we can optimize by noting that for a given segment, we only need to check positions p where:\n // text[p : p+segment.length] === segment\n // But we are only checking positions that are in our set.\n \n // However, the set might be large. We can try to avoid checking every position if the segment is long.\n // But worst-case, we might have to check every position. So worst-case O(n*m) is acceptable for n, m up to a few thousand.\n \n // Let's code the straightforward approach and hope it's fast enough.\n \n const segLen = segment.length;\n const newPositions = new Set();\n \n // We iterate over the current set of possible positions.\n // But note: we cannot modify the set while iterating, so we create a new set for the next step.\n for (const pos of possiblePositions) {\n if (pos + segLen > text.length) {\n continue;\n }\n \n // Check if the substring matches the segment\n let match = true;\n for (let i = 0; i < segLen; i++) {\n if (text[pos + i] !== segment[i]) {\n match = false;\n break;\n }\n }\n \n if (match) {\n newPositions.add(pos + segLen);\n }\n }\n \n // If no new positions were found, then we cannot proceed. But note: we might have positions that were already in the set?\n // Actually, we are replacing the set of possible positions with the new ones. However, we must note that the segment might be empty?\n // But we already handled the empty segment case at the beginning of the loop (if segment === '').\n // So, if the segment is not empty, then we only add positions that are after the segment.\n \n // But wait: what if the segment is empty? We skipped it above. So we only process non-empty segments.\n // However, the split by '*' might produce empty strings. We must handle them.\n \n // Correction: if the segment is empty, then it matches at every position? But we don't need to do anything for empty segments.\n // Because an empty segment means that there was a '*' and we don't require any characters to match for that segment.\n // So we can skip empty segments.\n \n // But note: the above loop does not handle empty segments. We must skip them.\n // We already checked at the beginning of the loop: if (segment === '') continue;\n // So we are safe.\n \n // However, there's a problem: if the segment is empty, we skip it, but that's correct because an empty segment matches at every position.\n // But we don't need to add any positions for an empty segment because the '*' already allows any text in between.\n // So we are good.\n \n // But wait: what if the pattern is \"a*b\" and text is \"ab\"? \n // segments = ['a', 'b']\n // Step 1: segment 'a' -> matches at pos0 -> newPositions = {1}\n // Step 2: segment 'b' -> matches at pos1 -> newPositions = {2}\n // Then we check if 2 is in the set of possible positions for the end? But we are building the set for the next segment.\n // After the last segment, we need to check if the last position is at the end of the text.\n \n // However, the above approach for the last segment: we add pos+segLen to newPositions. Then after the loop, we check if the set is non-empty and the last position is text.length?\n // Actually, after processing all segments, the possible positions should include the end of the text for a match.\n // But note: the last segment must end at the end of the text? Yes, because the entire text must match the pattern.\n // So after the last segment, the only valid position is text.length.\n \n // Therefore, after the loop, we check if the set of" + }, + { + "problem": "js-search-range", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 141104, + "loadMs": 3, + "outputTokens": 490, + "decodeMs": 119313.725, + "promptTokens": 459, + "prefillMs": 20685.179, + "thinkingChars": 0, + "answer": "```javascript\n/**\n * Finds the first and last index of a target value in a sorted array.\n * Uses binary search to achieve O(log n) time complexity.\n *\n * @param {number[]} sorted - An array of numbers sorted in ascending order.\n * @param {number} target - The number to find.\n * @returns {[number, number]} An array [first, last] where first and last are the indices\n * of the target, or [-1, -1] if the target is not found.\n */\nexport function searchRange(sorted, target) {\n // Handle empty array edge case\n if (sorted.length === 0) {\n return [-1, -1];\n }\n\n // Find the first occurrence\n let left = 0;\n let right = sorted.length - 1;\n let first = -1;\n\n while (left <= right) {\n const mid = left + Math.floor((right - left) / 2);\n if (sorted[mid] === target) {\n first = mid;\n right = mid - 1; // Try to find an earlier occurrence\n } else if (sorted[mid] < target) {\n left = mid + 1;\n } else {\n right = mid - 1;\n }\n }\n\n // If first is still -1, the target was not found\n if (first === -1) {\n return [-1, -1];\n }\n\n // Find the last occurrence\n left = 0;\n right = sorted.length - 1;\n let last = first; // We know a target exists at 'first', so we can start here\n\n while (left <= right) {\n const mid = left + Math.floor((right - left) / 2);\n if (sorted[mid] === target) {\n last = mid;\n left = mid + 1; // Try to find a later occurrence\n } else if (sorted[mid] < target) {\n left = mid + 1;\n } else {\n right = mid - 1;\n }\n }\n\n return [first, last];\n}\n```" + } + ] + }, + { + "summary": { + "model": "flycoder0.3fast", + "passed": 1, + "total": 1, + "passRate": 1, + "decodeTokensPerSecond": 6.80757334971185, + "prefillTokensPerSecond": 42.318649666143834, + "meanOutputTokens": 279, + "meanSecondsPerProblem": 52.055, + "truncated": 0 + }, + "results": [ + { + "problem": "py-merge-intervals", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 62718, + "loadMs": 10663, + "outputTokens": 279, + "decodeMs": 40983.767, + "promptTokens": 468, + "prefillMs": 11058.954, + "thinkingChars": 0, + "answer": "```python\ndef merge_intervals(intervals):\n \"\"\"\n Merge a list of closed intervals that overlap or touch.\n \n Args:\n intervals: List of (start, end) tuples of integers.\n \n Returns:\n A new list of merged (start, end) tuples, sorted by start.\n \n Raises:\n ValueError: If any interval has start > end.\n \"\"\"\n if not intervals:\n return []\n \n # Validate input and sort by start\n for start, end in intervals:\n if start > end:\n raise ValueError(f\"Invalid interval: start ({start}) > end ({end})\")\n \n sorted_intervals = sorted(intervals, key=lambda x: x[0])\n \n merged = [sorted_intervals[0]]\n \n for current in sorted_intervals[1:]:\n last = merged[-1]\n # Check if current interval overlaps or touches the last merged interval\n if current[0] <= last[1]:\n # Merge by updating the end of the last interval\n merged[-1] = (last[0], max(last[1], current[1]))\n else:\n # No overlap, append the current interval\n merged.append(current)\n \n return merged\n```" } ] } From b6cd83fd46c1c8dcf0fcc643edfa6a8f43c6d903 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 20:28:19 +0000 Subject: [PATCH 2/5] bench: update the in-progress 0.3 CPU run Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013QgiejZTjqQAUv4N4EFskK --- docs/bench/cpu-nothink-0.3.json | 74 +++++++++++++++++++++++++++++---- 1 file changed, 67 insertions(+), 7 deletions(-) diff --git a/docs/bench/cpu-nothink-0.3.json b/docs/bench/cpu-nothink-0.3.json index 9c94e17..a37ac5c 100644 --- a/docs/bench/cpu-nothink-0.3.json +++ b/docs/bench/cpu-nothink-0.3.json @@ -326,13 +326,13 @@ { "summary": { "model": "flycoder0.3fast", - "passed": 1, - "total": 1, - "passRate": 1, - "decodeTokensPerSecond": 6.80757334971185, - "prefillTokensPerSecond": 42.318649666143834, - "meanOutputTokens": 279, - "meanSecondsPerProblem": 52.055, + "passed": 2, + "total": 5, + "passRate": 0.4, + "decodeTokensPerSecond": 4.73869465651601, + "prefillTokensPerSecond": 32.75888637662057, + "meanOutputTokens": 437.6, + "meanSecondsPerProblem": 107.7688, "truncated": 0 }, "results": [ @@ -350,6 +350,66 @@ "prefillMs": 11058.954, "thinkingChars": 0, "answer": "```python\ndef merge_intervals(intervals):\n \"\"\"\n Merge a list of closed intervals that overlap or touch.\n \n Args:\n intervals: List of (start, end) tuples of integers.\n \n Returns:\n A new list of merged (start, end) tuples, sorted by start.\n \n Raises:\n ValueError: If any interval has start > end.\n \"\"\"\n if not intervals:\n return []\n \n # Validate input and sort by start\n for start, end in intervals:\n if start > end:\n raise ValueError(f\"Invalid interval: start ({start}) > end ({end})\")\n \n sorted_intervals = sorted(intervals, key=lambda x: x[0])\n \n merged = [sorted_intervals[0]]\n \n for current in sorted_intervals[1:]:\n last = merged[-1]\n # Check if current interval overlaps or touches the last merged interval\n if current[0] <= last[1]:\n # Merge by updating the end of the last interval\n merged[-1] = (last[0], max(last[1], current[1]))\n else:\n # No overlap, append the current interval\n merged.append(current)\n \n return merged\n```" + }, + { + "problem": "py-roman-strict", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-qI0QS9/test_solution.py\", line 3, in \n assert roman_to_int(text) == value, text\n ~~~~~~~~~~~~^^^^^^\n File \"/tmp/flycoder-bench-qI0QS9/solution.py\", line 74, in roman_to_int\n raise ValueError(\"Roman numeral must represent a value between 1 and 3999\")\nValueError: Roman numeral must represent a value between 1 and 3999\n", + "truncated": false, + "wallMs": 126073, + "loadMs": 12, + "outputTokens": 726, + "decodeMs": 113880.949, + "promptTokens": 477, + "prefillMs": 12067.157, + "thinkingChars": 0, + "answer": "```python\ndef roman_to_int(s: str) -> int:\n \"\"\"\n Converts a Roman numeral string to an integer.\n \n Valid Roman numerals are restricted to the range 1-3999 using the characters\n I, V, X, L, C, D, M and standard subtractive pairs (IV, IX, XL, XC, CD, CM).\n \n Args:\n s (str): A string representing a Roman numeral.\n \n Returns:\n int: The integer value of the Roman numeral.\n \n Raises:\n ValueError: If the input is empty, contains lowercase letters, \n contains invalid characters, or represents a non-canonical form \n or a value outside the range 1-3999.\n \"\"\"\n if not isinstance(s, str):\n raise ValueError(\"Input must be a string\")\n \n if not s:\n raise ValueError(\"Input cannot be an empty string\")\n \n valid_chars = set(\"IVXLCDM\")\n if not all(c in valid_chars for c in s):\n raise ValueError(\"Invalid characters in Roman numeral\")\n \n # Check for lowercase letters\n if any(c.islower() for c in s):\n raise ValueError(\"Roman numeral must be uppercase\")\n \n # Mapping of Roman numerals to integers\n roman_map = {\n 'I': 1, 'V': 5, 'X': 10, 'L': 50, 'C': 100, 'D': 500, 'M': 1000\n }\n \n # Check for non-canonical forms by scanning for invalid subtractive pairs\n # Canonical subtractive pairs: IV, IX, XL, XC, CD, CM\n canonical_subtractive = ['IV', 'IX', 'XL', 'XC', 'CD', 'CM']\n \n # Scan the string to detect non-canonical subtractive patterns\n i = 0\n while i < len(s) - 1:\n if s[i] in roman_map and s[i+1] in roman_map:\n val_curr = roman_map[s[i]]\n val_next = roman_map[s[i+1]]\n \n # If we have a smaller value before a larger value, it's potentially subtractive\n if val_curr < val_next:\n # Check if it's a canonical subtractive pair\n if (s[i:i+2] not in canonical_subtractive):\n raise ValueError(\"Non-canonical subtractive pair detected\")\n i += 2\n else:\n i += 1\n else:\n i += 1\n \n # Calculate the total value\n total = 0\n i = 0\n while i < len(s):\n val = roman_map[s[i]]\n # Look ahead to handle subtractive notation\n if i + 1 < len(s) and roman_map[s[i]] < roman_map[s[i+1]]:\n total += val_next - val\n i += 2\n else:\n total += val\n i += 1\n \n if total < 1 or total > 3999:\n raise ValueError(\"Roman numeral must represent a value between 1 and 3999\")\n \n return total\n```" + }, + { + "problem": "py-parse-duration", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-yMlr5Y/test_solution.py\", line 2, in \n assert parse_duration(\"45s\") == 45\n ~~~~~~~~~~~~~~^^^^^^^\n File \"/tmp/flycoder-bench-yMlr5Y/solution.py\", line 30, in parse_duration\n matches = re.findall(pattern, text)\n ^^\nNameError: name 're' is not defined. Did you forget to import 're'?\n", + "truncated": false, + "wallMs": 103446, + "loadMs": 3, + "outputTokens": 541, + "decodeMs": 88729.093, + "promptTokens": 537, + "prefillMs": 14081.624, + "thinkingChars": 0, + "answer": "```python\ndef parse_duration(text: str) -> int:\n \"\"\"\n Parse a duration string into total seconds.\n \n Format: one or more parts.\n Units: d (86400 s), h (3600 s), m (60 s), s (1 s).\n Order must be d, h, m, s (no reverse order allowed).\n Each unit appears at most once.\n Spaces allowed between parts and around text, but not inside numbers or units.\n \n Raises ValueError for:\n - empty string\n - no unit provided\n - invalid unit character\n - out-of-order units (e.g., '1m1h')\n - duplicate units (e.g., '1h1h')\n - spaces inside numbers or between number and unit\n \"\"\"\n if not text or not text.strip():\n raise ValueError(\"Empty input\")\n\n text = text.strip()\n if not text:\n raise ValueError(\"Empty input\")\n\n # Regex to match valid number+unit pairs\n # Number: non-negative integer (no leading zeros unless it's just \"0\")\n # Unit: one of d, h, m, s\n pattern = r'(\\d+)([dhms])'\n matches = re.findall(pattern, text)\n\n if not matches:\n raise ValueError(\"No valid duration parts found\")\n\n # Validate order and uniqueness\n seen_units = []\n for num_str, unit in matches:\n unit_order = {'d': 0, 'h': 1, 'm': 2, 's': 3}\n if unit in seen_units:\n raise ValueError(f\"Duplicate unit: '{unit}'\")\n if unit_order[unit] < unit_order[seen_units[-1]]:\n raise ValueError(\"Units must appear in order d, h, m, s\")\n seen_units.append(unit)\n\n # Convert to seconds\n total_seconds = 0\n for num_str, unit in matches:\n num = int(num_str)\n if unit == 'd':\n total_seconds += num * 86400\n elif unit == 'h':\n total_seconds += num * 3600\n elif unit == 'm':\n total_seconds += num * 60\n elif unit == 's':\n total_seconds += num\n\n return total_seconds\n```" + }, + { + "problem": "py-topo-sort", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 157441, + "loadMs": 3, + "outputTokens": 466, + "decodeMs": 138080.055, + "promptTokens": 488, + "prefillMs": 18903.42, + "thinkingChars": 0, + "answer": "```python\nfrom typing import Dict, List, Set\n\n\ndef topo_sort(deps: Dict[str, List[str]]) -> List[str]:\n \"\"\"\n Perform a topological sort on the graph defined by `deps`.\n \n Args:\n deps: A dictionary mapping each node (string) to a list of nodes it depends on.\n Nodes that only appear as dependencies are also included in the graph.\n \n Returns:\n A list containing every node exactly once, ordered such that each node comes\n after all of its dependencies. If multiple nodes are ready, the alphabetically\n smallest is chosen first to ensure determinism.\n \n Raises:\n ValueError: If the graph contains a cycle.\n \"\"\"\n # Collect all nodes (both keys and values in dependencies)\n all_nodes: Set[str] = set(deps.keys())\n for node in deps.values():\n all_nodes.update(node)\n\n # Calculate in-degree for each node (number of dependencies)\n in_degree: Dict[str, int] = {node: 0 for node in all_nodes}\n for node, dependencies in deps.items():\n for dep in dependencies:\n in_degree[node] += 1\n\n # Start with nodes that have no dependencies (in-degree 0), sorted alphabetically\n ready: List[str] = sorted([node for node, degree in in_degree.items() if degree == 0])\n result: List[str] = []\n\n while ready:\n # Pick the alphabetically smallest node\n current_node = ready.pop(0)\n result.append(current_node)\n\n # Find all nodes that depend on the current node and decrement their in-degree\n for node, dependencies in deps.items():\n if current_node in dependencies:\n in_degree[node] -= 1\n if in_degree[node] == 0:\n # Insert in sorted order to maintain deterministic selection\n ready.append(node)\n ready.sort()\n\n # If the result doesn't contain all nodes, a cycle exists\n if len(result) != len(all_nodes):\n raise ValueError(\"Cycle detected in the graph\")\n\n return result\n```" + }, + { + "problem": "py-lru-cache", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-woVFFC/test_solution.py\", line 13, in \n z.put(\"a\", 1)\n ~~~~~^^^^^^^^\n File \"/tmp/flycoder-bench-woVFFC/solution.py\", line 22, in put\n self.cache.popitem(last=False)\n ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 'dictionary is empty'\n", + "truncated": false, + "wallMs": 99851, + "loadMs": 4, + "outputTokens": 176, + "decodeMs": 80056.667, + "promptTokens": 493, + "prefillMs": 19074.551, + "thinkingChars": 0, + "answer": "```python\nfrom collections import OrderedDict\n\nclass LRUCache:\n def __init__(self, capacity: int):\n if capacity < 0:\n raise ValueError(\"Capacity cannot be negative\")\n self.capacity = capacity\n self.cache = OrderedDict()\n\n def get(self, key: int) -> int:\n if key not in self.cache:\n return -1\n self.cache.move_to_end(key)\n return self.cache[key]\n\n def put(self, key: int, value: int) -> None:\n if key in self.cache:\n self.cache[key] = value\n self.cache.move_to_end(key)\n else:\n if len(self.cache) >= self.capacity:\n self.cache.popitem(last=False)\n self.cache[key] = value\n```" } ] } From af54501fde9313e9df8f2e380e9bd3c02d3128c9 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 20:56:35 +0000 Subject: [PATCH 3/5] bench: update the in-progress 0.3 CPU run Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013QgiejZTjqQAUv4N4EFskK --- docs/bench/cpu-nothink-0.3.json | 211 ++++++++++++++++++++++++++++++-- 1 file changed, 203 insertions(+), 8 deletions(-) diff --git a/docs/bench/cpu-nothink-0.3.json b/docs/bench/cpu-nothink-0.3.json index a37ac5c..d2f1e43 100644 --- a/docs/bench/cpu-nothink-0.3.json +++ b/docs/bench/cpu-nothink-0.3.json @@ -326,14 +326,14 @@ { "summary": { "model": "flycoder0.3fast", - "passed": 2, - "total": 5, - "passRate": 0.4, - "decodeTokensPerSecond": 4.73869465651601, - "prefillTokensPerSecond": 32.75888637662057, - "meanOutputTokens": 437.6, - "meanSecondsPerProblem": 107.7688, - "truncated": 0 + "passed": 4, + "total": 18, + "passRate": 0.2222222222222222, + "decodeTokensPerSecond": 6.0619004213734735, + "prefillTokensPerSecond": 37.4064441250733, + "meanOutputTokens": 636.3333333333334, + "meanSecondsPerProblem": 118.696, + "truncated": 1 }, "results": [ { @@ -410,6 +410,201 @@ "prefillMs": 19074.551, "thinkingChars": 0, "answer": "```python\nfrom collections import OrderedDict\n\nclass LRUCache:\n def __init__(self, capacity: int):\n if capacity < 0:\n raise ValueError(\"Capacity cannot be negative\")\n self.capacity = capacity\n self.cache = OrderedDict()\n\n def get(self, key: int) -> int:\n if key not in self.cache:\n return -1\n self.cache.move_to_end(key)\n return self.cache[key]\n\n def put(self, key: int, value: int) -> None:\n if key in self.cache:\n self.cache[key] = value\n self.cache.move_to_end(key)\n else:\n if len(self.cache) >= self.capacity:\n self.cache.popitem(last=False)\n self.cache[key] = value\n```" + }, + { + "problem": "py-wrap-text", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-Ct4xzx/test_solution.py\", line 3, in \n assert wrap_text(\" a b\\n c\\t\", 3) == [\"a b\", \"c\"]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError\n", + "truncated": false, + "wallMs": 67283, + "loadMs": 3, + "outputTokens": 342, + "decodeMs": 54928.354, + "promptTokens": 501, + "prefillMs": 12336.467, + "thinkingChars": 0, + "answer": "```python\ndef wrap_text(text: str, width: int) -> list[str]:\n if width < 1:\n raise ValueError(\"width must be at least 1\")\n\n if not text.strip():\n return []\n\n lines = []\n words = text.split()\n\n current_line = []\n current_length = 0\n\n for word in words:\n word_length = len(word)\n\n # Handle words longer than width\n if word_length > width:\n # Split the word into pieces of exactly width characters\n # The last piece may be shorter\n pieces = []\n remaining = word\n while remaining:\n piece = remaining[:width]\n remaining = remaining[width:]\n pieces.append(piece)\n # Now iterate through the pieces\n for piece in pieces:\n if current_length + len(piece) <= width:\n current_line.append(piece)\n current_length += len(piece)\n else:\n # Start a new line\n lines.append(\" \".join(current_line))\n current_line = [piece]\n current_length = len(piece)\n else:\n if current_line and current_length + 1 + word_length > width:\n # Need to start a new line\n lines.append(\" \".join(current_line))\n current_line = [word]\n current_length = word_length\n else:\n current_line.append(word)\n current_length += 1 + word_length\n\n if current_line:\n lines.append(\" \".join(current_line))\n\n return lines\n```" + }, + { + "problem": "py-semver-compare", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-r6cv15/test_solution.py\", line 4, in \n assert compare_versions(low, high) == -1, (low, high)\n ~~~~~~~~~~~~~~~~^^^^^^^^^^^\n File \"/tmp/flycoder-bench-r6cv15/solution.py\", line 60, in compare_versions\n raise ValueError(\"Invalid version format: non-numeric core component\")\nValueError: Invalid version format: non-numeric core component\n", + "truncated": false, + "wallMs": 158576, + "loadMs": 3, + "outputTokens": 972, + "decodeMs": 146145.573, + "promptTokens": 510, + "prefillMs": 12321.274, + "thinkingChars": 0, + "answer": "```python\ndef compare_versions(a: str, b: str) -> int:\n \"\"\"\n Compare two Semantic Versioning 2.0.0 strings.\n \n Returns:\n -1 if a < b\n 0 if a == b\n 1 if a > b\n \n SemVer format: MAJOR.MINOR.PATCH[-PRERELEASE][+BUILD]\n Build metadata is ignored.\n \"\"\"\n import re\n \n # Regex to parse SemVer: MAJOR.MINOR.PATCH[-PRERELEASE][+BUILD]\n pattern = re.compile(\n r'^(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)'\n r'(?:-(?P[0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?'\n r'(?:\\+(?P[0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$'\n )\n \n match_a = pattern.match(a)\n match_b = pattern.match(b)\n \n if not match_a or not match_b:\n raise ValueError(\"Invalid Semantic Version string\")\n \n def parse_version(version: str):\n groups = match_a.groupdict() if version == a else match_b.groupdict()\n major = int(groups['major'])\n minor = int(groups['minor'])\n patch = int(groups['patch'])\n prerelease = groups['prerelease']\n build = groups['build']\n return major, minor, patch, prerelease, build\n \n _, _, _, prerelease_a, _ = parse_version(a)\n _, _, _, prerelease_b, _ = parse_version(b)\n \n # Compare core version numbers\n core_a = (a.split('.')[0], a.split('.')[1], a.split('.')[2])\n core_b = (b.split('.')[0], b.split('.')[1], b.split('.')[2])\n \n for i in range(3):\n val_a = core_a[i]\n val_b = core_b[i]\n \n # Handle comparison between numeric and non-numeric (should not happen in valid SemVer)\n if val_a != val_b:\n if val_a == val_b:\n return 0\n else:\n # Both should be numeric\n if val_a.isdigit() and val_b.isdigit():\n if val_a > val_b:\n return 1\n elif val_a < val_b:\n return -1\n else:\n raise ValueError(\"Invalid version format: non-numeric core component\")\n \n # Handle prerelease\n if prerelease_a is None and prerelease_b is None:\n return 0\n elif prerelease_a is None:\n return 1 # Normal version has higher precedence than prerelease\n elif prerelease_b is None:\n return -1 # Prerelease has lower precedence than normal version\n else:\n # Compare prerelease identifiers dot by dot\n ids_a = prerelease_a.split('.')\n ids_b = prerelease_b.split('.')\n \n for i in range(max(len(ids_a), len(ids_b))):\n if i >= len(ids_a):\n return -1 # Shorter set is lower\n if i >= len(ids_b):\n return 1 # Longer set is higher\n \n id_a = ids_a[i]\n id_b = ids_b[i]\n \n # Compare identifiers: numeric vs alphanumeric\n a_is_num = id_a.isdigit()\n b_is_num = id_b.isdigit()\n \n if a_is_num and b_is_num:\n # Both numeric: compare numerically\n num_a = int(id_a)\n num_b = int(id_b)\n if num_a < num_b:\n return -1\n elif num_a > num_b:\n return 1\n elif a_is_num:\n # Numeric < Alphanumeric\n return -1\n elif b_is_num:\n # Alphanumeric > Numeric\n return 1\n else:\n # Both alphanumeric: compare lexicographically\n if id_a < id_b:\n return -1\n elif id_a > id_b:\n return 1\n \n return 0\n```" + }, + { + "problem": "py-spiral", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-ddHYP7/test_solution.py\", line 3, in \n assert spiral_order([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) == [1, 2, 3, 6, 9, 8, 7, 4, 5]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError\n", + "truncated": false, + "wallMs": 56612, + "loadMs": 3, + "outputTokens": 305, + "decodeMs": 45156.648, + "promptTokens": 428, + "prefillMs": 11315.172, + "thinkingChars": 0, + "answer": "```python\ndef spiral_order(matrix):\n \"\"\"\n Returns the elements of a rectangular matrix in clockwise spiral order.\n \n Args:\n matrix (List[List[int]]): A rectangular matrix (list of equal-length lists).\n \n Returns:\n List[int]: Elements in clockwise spiral order.\n \"\"\"\n if not matrix or not matrix[0]:\n return []\n \n result = []\n top, bottom = 0, len(matrix) - 1\n left, right = 0, len(matrix[0]) - 1\n \n while top <= bottom and left <= right:\n # Traverse right\n for col in range(left, right + 1):\n result.append(matrix[top][col])\n top += 1\n \n # Traverse down\n for row in range(top, bottom + 1):\n result.append(matrix[row][right])\n right -= 1\n \n # Check if there's still a row to traverse\n if top <= bottom:\n for row in range(bottom, top - 1, -1):\n result.append(matrix[row][left])\n left += 1\n \n # Check if there's still a column to traverse\n if left <= right:\n for col in range(right, left - 1, -1):\n result.append(matrix[top][col])\n top += 1\n \n return result\n```" + }, + { + "problem": "py-csv-line", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-XqvVYD/test_solution.py\", line 2, in \n assert parse_csv_line(\"a,b,c\") == [\"a\", \"b\", \"c\"]\n ~~~~~~~~~~~~~~^^^^^^^^^\n File \"/tmp/flycoder-bench-XqvVYD/solution.py\", line 77, in parse_csv_line\n raise ValueError(f\"Invalid character '{line[j + 1]}' after closing quote\")\nValueError: Invalid character 'b' after closing quote\n", + "truncated": false, + "wallMs": 114209, + "loadMs": 6, + "outputTokens": 689, + "decodeMs": 101944.388, + "promptTokens": 483, + "prefillMs": 12137.432, + "thinkingChars": 0, + "answer": "```python\ndef parse_csv_line(line):\n \"\"\"\n Parses a single CSV line into a list of fields without using the csv module.\n \n Rules:\n - Fields are separated by commas.\n - An empty line results in a list containing one empty string: [\"\"].\n - Fields may be quoted with double quotes. Inside quotes:\n * Commas are literal.\n * \"\" represents a single double quote.\n - Unquoted fields are taken as-is (no trimming).\n - Raises ValueError for an unterminated quoted field.\n - Raises ValueError if there is any character other than a comma after a closing quote.\n \n Args:\n line (str): The CSV line to parse.\n \n Returns:\n list[str]: A list of parsed fields.\n \n Raises:\n ValueError: If the quoted field is unterminated or if there are invalid characters after a closing quote.\n \"\"\"\n fields = []\n i = 0\n n = len(line)\n \n while i < n:\n # Check for end of string\n if i == n:\n raise ValueError(\"Unterminated quoted field\")\n \n char = line[i]\n \n # Handle empty line case immediately\n if i == 0 and n == 0:\n return [\"\"]\n \n # Check for quoted field\n if char == '\"':\n # Start of a quoted field\n i += 1\n start_i = i\n while i < n:\n if line[i] == '\"':\n # Check for escaped quote\n if i + 1 < n and line[i + 1] == '\"':\n # Escaped quote: append single quote, skip both\n fields.append('\"')\n i += 2\n else:\n # End of quoted field\n i += 1\n break\n else:\n i += 1\n \n if i == n:\n raise ValueError(\"Unterminated quoted field\")\n \n # Check for invalid characters after closing quote\n if i + 1 < n and line[i + 1] != ',':\n raise ValueError(f\"Invalid character '{line[i + 1]}' after closing quote\")\n \n # Move to the next field\n i += 1 if i + 1 < n and line[i + 1] == ',' else 0\n else:\n # Unquoted field: read until comma or end of line\n j = i\n while j < n and line[j] != ',':\n j += 1\n \n if j == n:\n raise ValueError(\"Unterminated quoted field\")\n \n if j + 1 < n and line[j + 1] != ',':\n raise ValueError(f\"Invalid character '{line[j + 1]}' after closing quote\")\n \n # Extract field\n field = line[i:j]\n fields.append(field)\n \n # Move to the next field\n i = j + 1 if j + 1 < n else n\n # If we hit end of string, stop\n if i == n:\n break\n```" + }, + { + "problem": "py-lcs", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 66326, + "loadMs": 2, + "outputTokens": 382, + "decodeMs": 55825.354, + "promptTokens": 428, + "prefillMs": 10377.274, + "thinkingChars": 0, + "answer": "```python\ndef longest_common_subsequence(a: str, b: str) -> str:\n \"\"\"\n Returns one longest common subsequence of strings a and b.\n Handles inputs up to 2000 characters efficiently using dynamic programming\n with O(min(len(a), len(b))) space optimization.\n \"\"\"\n if not a or not b:\n return \"\"\n\n m, n = len(a), len(b)\n if m > n:\n a, b = b, a\n m, n = n, m\n\n # dp[i][j] = length of LCS of a[:i] and b[:j]\n dp = [[0] * (n + 1) for _ in range(m + 1)]\n\n for i in range(1, m + 1):\n for j in range(1, n + 1):\n if a[i - 1] == b[j - 1]:\n dp[i][j] = dp[i - 1][j - 1] + 1\n else:\n dp[i][j] = max(dp[i - 1][j], dp[i][j - 1])\n\n # Reconstruct the LCS from the end\n result = []\n i, j = m, n\n while i > 0 and j > 0:\n if a[i - 1] == b[j - 1]:\n result.append(a[i - 1])\n i -= 1\n j -= 1\n elif dp[i - 1][j] >= dp[i][j - 1]:\n i -= 1\n else:\n j -= 1\n\n return ''.join(reversed(result))\n```" + }, + { + "problem": "py-number-words", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-z2JSja/test_solution.py\", line 7, in \n assert number_to_words(n) == words, (n, number_to_words(n))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: (20, '')\n", + "truncated": false, + "wallMs": 138601, + "loadMs": 3, + "outputTokens": 831, + "decodeMs": 124501.877, + "promptTokens": 538, + "prefillMs": 13983.475, + "thinkingChars": 0, + "answer": "```python\ndef number_to_words(n: int) -> str:\n if not (0 <= n <= 999_999_999_999):\n raise ValueError(\"Input must be between 0 and 999,999,999,999.\")\n\n if n == 0:\n return \"zero\"\n\n under_thousand = {\n 0: \"\",\n 1: \"one\",\n 2: \"two\",\n 3: \"three\",\n 4: \"four\",\n 5: \"five\",\n 6: \"six\",\n 7: \"seven\",\n 8: \"eight\",\n 9: \"nine\",\n 10: \"ten\",\n 11: \"eleven\",\n 12: \"twelve\",\n 13: \"thirteen\",\n 14: \"fourteen\",\n 15: \"fifteen\",\n 16: \"sixteen\",\n 17: \"seventeen\",\n 18: \"eighteen\",\n 19: \"nineteen\",\n 20: \"twenty\",\n 30: \"thirty\",\n 40: \"forty\",\n 50: \"fifty\",\n 60: \"sixty\",\n 70: \"seventy\",\n 80: \"eighty\",\n 90: \"ninety\",\n }\n\n scale_words = [\"\", \"thousand\", \"million\", \"billion\"]\n\n def chunk(num: int) -> str:\n if num == 0:\n return \"\"\n \n result = []\n \n hundreds = num // 100\n remainder = num % 100\n \n if hundreds > 0:\n result.append(under_thousand[hundreds * 100])\n if remainder > 0:\n result.append(\" \")\n \n if remainder > 0:\n if remainder < 20:\n result.append(under_thousand[remainder])\n else:\n tens = remainder // 10 * 10\n ones = remainder % 10\n if ones > 0:\n if tens == 20:\n result.append(under_thousand[tens])\n else:\n result.append(under_thousand[tens])\n result.append(\" \")\n result.append(under_thousand[ones])\n \n return \"\".join(result)\n\n parts = []\n \n if n >= 1_000_000_000:\n billions = n // 1_000_000_000\n n %= 1_000_000_000\n if billions > 0:\n parts.append(chunk(billions) + \" \" + scale_words[2])\n \n if n >= 1_000_000:\n millions = n // 1_000_000\n n %= 1_000_000\n if millions > 0:\n parts.append(chunk(millions) + \" \" + scale_words[1])\n \n if n >= 1_000:\n thousands = n // 1_000\n n %= 1_000\n if thousands > 0:\n parts.append(chunk(thousands) + \" \" + scale_words[0])\n \n if n > 0:\n parts.append(chunk(n))\n \n return \" \".join(parts)\n```" + }, + { + "problem": "py-dijkstra", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 65235, + "loadMs": 3, + "outputTokens": 357, + "decodeMs": 53126.85, + "promptTokens": 505, + "prefillMs": 11979.396, + "thinkingChars": 0, + "answer": "```python\nfrom heapq import heappush, heappop\nfrom math import inf\n\ndef shortest_distances(n, edges, source):\n if not isinstance(n, int) or n <= 0:\n raise ValueError(\"n must be a positive integer\")\n if not isinstance(source, int) or source < 0 or source >= n:\n raise ValueError(\"source must be a valid node (0 <= source < n)\")\n \n if not isinstance(edges, list):\n raise ValueError(\"edges must be a list\")\n \n adj = [[] for _ in range(n)]\n for u, v, w in edges:\n if not isinstance(u, int) or u < 0 or u >= n:\n raise ValueError(\"edge start node must be a valid node\")\n if not isinstance(v, int) or v < 0 or v >= n:\n raise ValueError(\"edge end node must be a valid node\")\n if not isinstance(w, (int, float)) or w < 0:\n raise ValueError(\"edge weight must be a non-negative number\")\n adj[u].append((v, w))\n \n dist = [inf] * n\n dist[source] = 0\n pq = [(0, source)]\n \n while pq:\n d, u = heappop(pq)\n \n if d > dist[u]:\n continue\n \n for v, w in adj[u]:\n if dist[u] + w < dist[v]:\n dist[v] = dist[u] + w\n heappush(pq, (dist[v], v))\n \n return dist\n```" + }, + { + "problem": "py-slugify", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-uhnYoJ/test_solution.py\", line 2, in \n assert slugify(\"Hello, World!\") == \"hello-world\"\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError\n", + "truncated": false, + "wallMs": 65561, + "loadMs": 4, + "outputTokens": 349, + "decodeMs": 52384.737, + "promptTokens": 513, + "prefillMs": 13060.304, + "thinkingChars": 0, + "answer": "```python\nimport unicodedata\n\ndef slugify(text: str, max_length: int = 50) -> str:\n if max_length < 1:\n raise ValueError(\"max_length must be at least 1\")\n\n # Mapping for special characters\n special_map = {'ß': 'ss', 'æ': 'ae', 'œ': 'oe', 'ø': 'o'}\n\n # Decompose Unicode characters to separate base and combining marks\n normalized = unicodedata.normalize('NFKD', text)\n # Encode to ASCII, ignoring combining marks (removes accents)\n ascii_text = normalized.encode('ascii', 'ignore')\n \n # Apply special character mappings\n ascii_text = ascii_text.decode('ascii')\n for key, value in special_map.items():\n ascii_text = ascii_text.replace(key, value)\n \n # Replace non-alphanumeric characters with a single hyphen\n # Regex approach: replace any sequence of non-alphanumeric chars with one hyphen\n slug = ''.join(['-' if c not in '0123456789abcdefghijklmnopqrstuvwxyz' else c for c in ascii_text])\n \n # Remove leading and trailing hyphens\n slug = slug.strip('-')\n \n # If the string becomes empty after stripping, return empty string\n if not slug:\n return ''\n \n # Truncate if necessary\n if len(slug) > max_length:\n # Remove trailing hyphens from the truncated part before slicing\n truncated = slug[:max_length].rstrip('-')\n slug = truncated\n \n return slug\n```" + }, + { + "problem": "py-base-convert", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-LWsJb8/test_solution.py\", line 7, in \n assert convert_base(\"1295\", 10, 36) == \"zz\"\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError\n", + "truncated": false, + "wallMs": 87495, + "loadMs": 5, + "outputTokens": 485, + "decodeMs": 74823.989, + "promptTokens": 506, + "prefillMs": 12548.962, + "thinkingChars": 0, + "answer": "```python\ndef convert_base(number, from_base, to_base):\n if from_base < 2 or from_base > 36 or to_base < 2 or to_base > 36:\n raise ValueError(\"Base must be between 2 and 36.\")\n \n if not number or number.strip() == \"\":\n raise ValueError(\"Input string is empty.\")\n \n if number.startswith(\"-\"):\n is_negative = True\n number = number[1:]\n else:\n is_negative = False\n \n if not number:\n raise ValueError(\"Input string is empty.\")\n \n digits_map = {str(i): i for i in range(10)}\n digits_map.update({chr(ord('a') + i): 10 + i for i in range(26)})\n digits_map.update({chr(ord('A') + i): 10 + i for i in range(26)})\n \n value_map = {v: k for k, v in digits_map.items()}\n \n try:\n digits = []\n for char in number:\n if char not in digits_map:\n raise ValueError(f\"Invalid digit '{char}' for base {from_base}.\")\n d = digits_map[char]\n if d >= from_base:\n raise ValueError(f\"Invalid digit '{char}' for base {from_base}.\")\n digits.append(d)\n \n # Handle zero explicitly (e.g., \"0\", \"00\")\n if not digits:\n return \"0\"\n \n result_digits = []\n temp = 0\n \n # Convert from from_base to decimal (base 10)\n for d in reversed(digits):\n temp = temp * from_base + d\n \n except ValueError:\n raise\n \n if temp == 0:\n return \"0\"\n \n # Convert from decimal to to_base\n while temp > 0:\n remainder = temp % to_base\n temp = temp // to_base\n result_digits.append(value_map[remainder])\n \n result_digits.reverse()\n result = ''.join(result_digits)\n \n if is_negative:\n result = \"-\" + result\n \n return result\n```" + }, + { + "problem": "js-parse-query", + "sample": 0, + "ok": false, + "error": "file:///tmp/flycoder-bench-FwvxBg/test.mjs:13\na.equal(Object.getOwnPropertyDescriptor(p, '__proto__').value, 'x');\n ^\n\nTypeError: Cannot read properties of undefined (reading 'value')\n at file:///tmp/flycoder-bench-FwvxBg/test.mjs:13:56\n\nNode.js v22.22.0\n", + "truncated": false, + "wallMs": 83232, + "loadMs": 3, + "outputTokens": 443, + "decodeMs": 69285.795, + "promptTokens": 555, + "prefillMs": 13813.971, + "thinkingChars": 0, + "answer": "```javascript\nexport function parseQuery(query) {\n if (typeof query !== 'string') {\n return {};\n }\n\n const result = {};\n\n // Remove leading '?' if present\n const cleanQuery = query.startsWith('?') ? query.slice(1) : query;\n\n // Split by '&'\n const pairs = cleanQuery.split('&');\n\n for (const pair of pairs) {\n if (pair.length === 0) {\n continue;\n }\n\n const eqIndex = pair.indexOf('=');\n let key;\n let value;\n\n if (eqIndex === -1) {\n key = pair;\n value = '';\n } else {\n key = pair.slice(0, eqIndex);\n value = pair.slice(eqIndex + 1);\n }\n\n // Decode + to space\n let decodedKey = key.replace(/\\+/g, ' ');\n let decodedValue = value.replace(/\\+/g, ' ');\n\n // Percent decode\n try {\n decodedKey = decodeURIComponent(decodedKey);\n decodedValue = decodeURIComponent(decodedValue);\n } catch (e) {\n // Keep as is, but keep the converted spaces\n }\n\n const keys = result[key];\n if (keys === undefined) {\n result[key] = decodedValue;\n } else {\n if (Array.isArray(keys)) {\n keys.push(decodedValue);\n } else {\n result[key] = [keys, decodedValue];\n }\n }\n }\n\n // Ensure keys that could be prototype properties are own properties\n // by iterating over result and setting them explicitly on a plain object\n // that won't inherit prototype pollution in a way that affects the global context.\n // We create a new object and copy properties to ensure safety.\n const safeResult = {};\n for (const key of Object.keys(result)) {\n safeResult[key] = result[key];\n }\n\n return safeResult;\n}\n```" + }, + { + "problem": "js-deep-equal", + "sample": 0, + "ok": false, + "error": "duleLoader.moduleStrategy (node:internal/modules/esm/translators:107:18)\n at #translate (node:internal/modules/esm/loader:546:20)\n at afterLoad (node:internal/modules/esm/loader:596:29)\n at ModuleLoader.loadAndTranslate (node:internal/modules/esm/loader:601:12)\n at #createModuleJob (node:internal/modules/esm/loader:624:36)\n at #getJobFromResolveResult (node:internal/modules/esm/loader:343:34)\n at ModuleLoader.getModuleJobForImport (node:internal/modules/esm/loader:311:41)\n at async onImport.tracePromise.__proto__ (node:internal/modules/esm/loader:664:25)\n\nNode.js v22.22.0\n", + "truncated": false, + "wallMs": 127410, + "loadMs": 4, + "outputTokens": 709, + "decodeMs": 112656.452, + "promptTokens": 545, + "prefillMs": 14638.006, + "thinkingChars": 0, + "answer": "```javascript\nexport function deepEqual(a, b) {\n // SameValueZero comparison for primitives (NaN === NaN, 0 === -0)\n const sameValueZero = (x, y) => x === y || (x !== x && y !== y);\n\n // Helper to check if a value is a plain object\n const isPlainObject = (o) => o !== null && typeof o === 'object' && !Array.isArray(o) && !(o instanceof Date);\n\n // Helper to check if a value is a Date\n const isDate = (o) => o instanceof Date;\n\n // Helper to get the time value of a Date\n const getTime = (o) => o.getTime();\n\n // Helper to get the keys of an object (own enumerable string keys)\n const getKeys = (o) => Object.keys(o);\n\n // Helper to check if two objects have the same keys\n const hasSameKeys = (keys1, keys2) => {\n if (keys1.length !== keys2.length) return false;\n for (let i = 0; i < keys1.length; i++) {\n if (keys1[i] !== keys2[i]) return false;\n }\n return true;\n };\n\n // Helper to check if two objects have equal values for each key\n const hasEqualValues = (obj1, obj2, keys) => {\n for (let i = 0; i < keys.length; i++) {\n const key = keys[i];\n const val1 = obj1[key];\n const val2 = obj2[key];\n if (!sameValueZero(val1, val2)) return false;\n if (val1 !== null && val2 !== null) {\n if (!deepEqual(val1, val2)) return false;\n }\n }\n return true;\n };\n\n // Helper to handle cyclic structures using a map to track visited objects\n const cycleCheck = (a, b, visited) => {\n if (!visited.has(a)) {\n visited.set(a, new Set());\n }\n if (!visited.has(b)) {\n visited.set(b, new Set());\n }\n\n const aVisited = visited.get(a);\n const bVisited = visited.get(b);\n\n if (aVisited.has(b)) {\n return true; // Same cycle\n }\n\n if (bVisited.has(a)) {\n return false; // Different cycles\n }\n\n aVisited.add(b);\n bVisited.add(a);\n\n if (typeof a !== typeof b ||\n (typeof a === 'object' && typeof b === 'object') &&\n (a === null || b === null ||\n Array.isArray(a) !== Array.isArray(b) ||\n (isPlainObject(a) && isPlainObject(b) && !hasSameKeys(getKeys(a), getKeys(b))) ||\n (isDate(a) && isDate(b) && !sameValueZero(getTime(a), getTime(b)))) {\n return false;\n }\n\n return deepEqual(a, b, visited);\n };\n\n // Main logic\n return cycleCheck(a, b, new Map());\n}\n```" + }, + { + "problem": "js-evaluate", + "sample": 0, + "ok": false, + "error": "file:///tmp/flycoder-bench-Um1VW2/solution.mjs:11\n throw new SyntaxError('Unknown characters in expression');\n ^\n\nSyntaxError: Unknown characters in expression\n at evaluate (file:///tmp/flycoder-bench-Um1VW2/solution.mjs:11:11)\n at file:///tmp/flycoder-bench-Um1VW2/test.mjs:6:9\n\nNode.js v22.22.0\n", + "truncated": false, + "wallMs": 231189, + "loadMs": 5, + "outputTokens": 1354, + "decodeMs": 216353.714, + "promptTokens": 534, + "prefillMs": 14711.831, + "thinkingChars": 0, + "answer": "```javascript\nexport function evaluate(expression) {\n // Trim and validate input\n const trimmed = expression.trim();\n if (trimmed.length === 0) {\n throw new SyntaxError('Empty input');\n }\n\n // Check for unknown characters (allowing 0-9, ., +, -, (, ), and whitespace)\n const regex = /[^0-9.\\s\\+\\-\\(\\)]/;\n if (regex.test(trimmed)) {\n throw new SyntaxError('Unknown characters in expression');\n }\n\n // Tokenizer\n function tokenize(str) {\n const tokens = [];\n let i = 0;\n while (i < str.length) {\n if (/\\s/.test(str[i])) {\n i++;\n continue;\n }\n\n if (str[i] === '(' || str[i] === ')') {\n tokens.push({ type: 'paren', value: str[i] });\n i++;\n continue;\n }\n\n if (str[i] === '+' || str[i] === '-') {\n tokens.push({ type: 'operator', value: str[i] });\n i++;\n continue;\n }\n\n // Number token\n let numStr = '';\n while (i < str.length && /[0-9.]/.test(str[i])) {\n numStr += str[i];\n i++;\n }\n if (numStr.length === 0) {\n throw new SyntaxError('Expected number');\n }\n tokens.push({ type: 'number', value: parseFloat(numStr) });\n }\n\n if (tokens.length === 0) {\n throw new SyntaxError('Empty input');\n }\n\n // Check for dangling operator at the end\n if (tokens[tokens.length - 1].type === 'operator') {\n throw new SyntaxError('Dangling operator');\n }\n\n return tokens;\n }\n\n const tokens = tokenize(expression);\n\n // Tokenizer validation: check unbalanced parentheses\n let parenCount = 0;\n for (const token of tokens) {\n if (token.type === 'paren') {\n if (token.value === '(') {\n parenCount++;\n } else {\n parenCount--;\n if (parenCount < 0) {\n throw new SyntaxError('Unbalanced parentheses');\n }\n }\n }\n }\n if (parenCount !== 0) {\n throw new SyntaxError('Unbalanced parentheses');\n }\n\n // Shunting-yard algorithm to convert to Reverse Polish Notation (RPN)\n function toRPN(tokens) {\n const output = [];\n const operatorStack = [];\n\n for (const token of tokens) {\n if (token.type === 'number') {\n output.push(token);\n } else if (token.type === 'paren') {\n if (token.value === '(') {\n operatorStack.push(token);\n } else { // ')'\n while (operatorStack.length > 0 && operatorStack[operatorStack.length - 1].value !== '(') {\n output.push(operatorStack.pop());\n }\n if (operatorStack.length === 0) {\n throw new SyntaxError('Unbalanced parentheses');\n }\n operatorStack.pop(); // Pop the '('\n }\n } else if (token.type === 'operator') {\n const op1 = token.value;\n const op2 = operatorStack[operatorStack.length - 1].value;\n const hasHigherPrecedence =\n (op1 === '*' || op1 === '/') && (op2 === '+' || op2 === '-') ||\n (op1 === '+' || op1 === '-') && (op2 === '*' || op2 === '/');\n const hasEqualPrecedence =\n (op1 === op2) ||\n ((op1 === '*' || op1 === '/') && (op2 === '*' || op2 === '/')) ||\n ((op1 === '+' || op1 === '-') && (op2 === '+' || op2 === '-'));\n\n while (\n operatorStack.length > 0 &&\n (hasHigherPrecedence || hasEqualPrecedence) &&\n (op1 !== '+' && op1 !== '-' || op2 !== '+' && op2 !== '-')\n ) {\n output.push(operatorStack.pop());\n }\n operatorStack.push(token);\n }\n }\n\n while (operatorStack.length > 0) {\n const op = operatorStack.pop();\n if (op.value === '(' || op.value === ')') {\n throw new SyntaxError('Unbalanced parentheses');\n }\n output.push(op);\n }\n\n return output;\n }\n\n const rpn = toRPN(tokens);\n\n // Evaluate RPN\n function evaluateRPN(rpn) {\n const stack = [];\n\n for (const token of rpn) {\n if (token.type === 'number') {\n stack.push(token.value);\n } else if (token.type === 'operator') {\n if (stack.length < 2) {\n throw new SyntaxError('Invalid expression structure');\n }\n const b = stack.pop();\n const a = stack.pop();\n let result;\n\n switch (token.value) {\n case '+':\n result = a + b;\n break;\n case '-':\n result = a - b;\n break;\n case '*':\n result = a * b;\n break;\n case '/':\n if (b === 0) {\n throw new RangeError('Division by zero');\n }\n result = a / b;\n break;\n default:\n throw new SyntaxError('Unknown operator');\n }\n\n stack.push(result);\n }\n }\n\n if (stack.length !== 1) {\n throw new SyntaxError('Invalid expression structure');\n }\n\n return stack[0];\n }\n\n return evaluateRPN(rpn);\n}\n```" + }, + { + "problem": "js-format-bytes", + "sample": 0, + "ok": false, + "error": "truncated by max tokens", + "truncated": true, + "wallMs": 336002, + "loadMs": 3, + "outputTokens": 2048, + "decodeMs": 320642.197, + "promptTokens": 604, + "prefillMs": 15211.855, + "thinkingChars": 0, + "answer": "```javascript\nexport function formatBytes(bytes, decimals = 1) {\n // Validate bytes: must be a finite number\n if (!Number.isFinite(bytes)) {\n throw new TypeError('bytes must be a finite number');\n }\n\n // Validate decimals: must be an integer between 0 and 10 (inclusive)\n if (!Number.isInteger(decimals) || decimals < 0 || decimals > 10) {\n throw new RangeError('decimals must be an integer from 0 to 10');\n }\n\n const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];\n const factor = 1024;\n\n // Handle negative numbers and zero\n const isNegative = bytes < 0;\n const absBytes = Math.abs(bytes);\n\n // If value is less than 1024, format as integer with 'B'\n if (absBytes < factor) {\n return isNegative ? `-${Math.round(absBytes)} B` : `${Math.round(absBytes)} B`;\n }\n\n let unitIndex = 0;\n let value = absBytes;\n\n // Find the largest unit where value >= 1\n while (value >= factor && unitIndex < units.length - 1) {\n value /= factor;\n unitIndex++;\n }\n\n // Round to `decimals` places\n const multiplier = Math.pow(10, decimals);\n let formattedValue = Math.round(value * multiplier) / multiplier;\n\n // Check if rounding would result in exactly 1024 of the current unit\n // This means the original value rounded up to the next unit boundary\n // We check: if formattedValue * factor is an integer and equals the next unit's threshold\n // More precisely: if rounding caused an overflow to the next unit, adjust.\n // Example: 1048575 with 1 decimal -> 1048575 * 10 / 10 = 104857.5 -> rounds to 104857.5? \n // Wait, let's re-read: \"If rounding would display 1024.0 of a unit, move to the next unit instead\"\n // This means if the rounded value is exactly 1024, it should be treated as 1.0 of the next unit.\n // But 1024.0 of KiB is 1.0 MiB. So we check if formattedValue is exactly 1024^n.\n // Actually, the condition is: if the rounded value is a power of 1024 (i.e., 1024, 1024^2, etc.), \n // it should be represented in the next unit.\n\n // Let's interpret: \"1048575 with 1 decimal is '1.0 MiB'\"\n // 1048575 / 1024 = 1024.995... -> rounds to 1025.0 KiB? That's not 1.0 MiB.\n // Wait, 1048575 * 10 / 10 = 104857.5 -> rounded? \n // Let's recalculate: 1048575 / 1024 = 1024.9951171875\n // With 1 decimal: round(1024.9951171875 * 10) / 10 = round(10249.951171875) / 10 = 10250 / 10 = 1025.0 KiB\n // But the example says it should be \"1.0 MiB\".\n // Ah, I see: 1048575 is just below 1048576 (which is 1024^2). \n // 1048576 / 1024 = 1024. So 1048575 is 1024.995... KiB.\n // The example says \"1048575 with 1 decimal is '1.0 MiB'\".\n // This implies that if the rounded value is very close to the next unit boundary, we should round up to the next unit.\n // Specifically, if the rounded value is 1024.0 (exactly), then it should be 1.0 of the next unit.\n // But 1048575 rounds to 1025.0 KiB, not 1024.0.\n // Let me re-read: \"If rounding would display 1024.0 of a unit, move to the next unit instead\"\n // Perhaps it means: if the value before rounding is such that rounding it to `decimals` places gives exactly 1024.0,\n // then we should use the next unit.\n // But 1048575 / 1024 = 1024.995... which rounds to 1025.0, not 1024.0.\n // What if the input was 1048576? Then 1048576 / 1024 = 1024.0 -> should be 1.0 MiB.\n // So the condition is: if the rounded value is exactly 1024.0, 1024^2.0, etc., then use the next unit.\n // But the example says 1048575 -> 1.0 MiB. \n // 1048575 is 1024.995... KiB. If we round to 1 decimal, we get 1025.0 KiB. \n // But the example says it should be 1.0 MiB. \n // This suggests that if the value is within 0.5 of the next unit boundary, we round up to the next unit.\n // Or perhaps: if the value is >= 1024.0 in the current unit, it should be represented in the next unit?\n // No, because 1024.0 KiB is exactly 1.0 MiB, so we don't want to show 1024.0 KiB.\n // The rule is: if the value is >= 1024.0 in the current unit, it should be shown in the next unit.\n // But then 1024.0 KiB would be shown as 1.0 MiB, which is correct.\n // However, the example says 1048575 -> 1.0 MiB. \n // 1048575 is less than 1048576 (which is 1024^2). \n // 1048575 / 1024 = 1024.995... which is greater than 1024.0.\n // So if the value is >= 1024.0 in the current unit, we move to the next unit.\n // But wait, what about 1024.0 exactly? Then it should be 1.0 MiB.\n // So the condition is: if value >= 1024.0, then use the next unit.\n // But then 1024.0 KiB would be shown as 1.0 MiB, which is correct.\n // And 1048575 KiB would be shown as 1024.995... KiB, which rounds to 1025.0 KiB, but the example says 1.0 MiB.\n // I think I misunderstood the example. Let me re-read: \"1048575 with 1 decimal is '1.0 MiB'\"\n // 1048575 is 1024.995... KiB. If we round to 1 decimal, we get 1025.0 KiB.\n // But the example says it should be 1.0 MiB.\n // This suggests that if the value is >= 1024.0 in the current unit, we should show it in the next unit.\n // So 1048575 KiB is >= 1024.0 KiB, so we show it as 1.0 MiB.\n // But 1025.0 KiB is also >= 1024.0 KiB, so we should show it as 1.0 MiB? No, 1025.0 KiB is 1.009765625 MiB.\n // I think the condition is" } ] } From 00983efc4eaaab0610ca4d0fb08cf9b3e7bbe9d9 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 21:04:39 +0000 Subject: [PATCH 4/5] bench: record the full 0.3 CPU run (flycoder0.3 9/20, flycoder0.3fast 5/20) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013QgiejZTjqQAUv4N4EFskK --- docs/bench/cpu-nothink-0.3.json | 46 +++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/docs/bench/cpu-nothink-0.3.json b/docs/bench/cpu-nothink-0.3.json index d2f1e43..3b86b69 100644 --- a/docs/bench/cpu-nothink-0.3.json +++ b/docs/bench/cpu-nothink-0.3.json @@ -326,14 +326,14 @@ { "summary": { "model": "flycoder0.3fast", - "passed": 4, - "total": 18, - "passRate": 0.2222222222222222, - "decodeTokensPerSecond": 6.0619004213734735, - "prefillTokensPerSecond": 37.4064441250733, - "meanOutputTokens": 636.3333333333334, - "meanSecondsPerProblem": 118.696, - "truncated": 1 + "passed": 5, + "total": 20, + "passRate": 0.25, + "decodeTokensPerSecond": 6.132268548074128, + "prefillTokensPerSecond": 37.65346561369175, + "meanOutputTokens": 689.05, + "meanSecondsPerProblem": 125.93395, + "truncated": 2 }, "results": [ { @@ -605,6 +605,36 @@ "prefillMs": 15211.855, "thinkingChars": 0, "answer": "```javascript\nexport function formatBytes(bytes, decimals = 1) {\n // Validate bytes: must be a finite number\n if (!Number.isFinite(bytes)) {\n throw new TypeError('bytes must be a finite number');\n }\n\n // Validate decimals: must be an integer between 0 and 10 (inclusive)\n if (!Number.isInteger(decimals) || decimals < 0 || decimals > 10) {\n throw new RangeError('decimals must be an integer from 0 to 10');\n }\n\n const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];\n const factor = 1024;\n\n // Handle negative numbers and zero\n const isNegative = bytes < 0;\n const absBytes = Math.abs(bytes);\n\n // If value is less than 1024, format as integer with 'B'\n if (absBytes < factor) {\n return isNegative ? `-${Math.round(absBytes)} B` : `${Math.round(absBytes)} B`;\n }\n\n let unitIndex = 0;\n let value = absBytes;\n\n // Find the largest unit where value >= 1\n while (value >= factor && unitIndex < units.length - 1) {\n value /= factor;\n unitIndex++;\n }\n\n // Round to `decimals` places\n const multiplier = Math.pow(10, decimals);\n let formattedValue = Math.round(value * multiplier) / multiplier;\n\n // Check if rounding would result in exactly 1024 of the current unit\n // This means the original value rounded up to the next unit boundary\n // We check: if formattedValue * factor is an integer and equals the next unit's threshold\n // More precisely: if rounding caused an overflow to the next unit, adjust.\n // Example: 1048575 with 1 decimal -> 1048575 * 10 / 10 = 104857.5 -> rounds to 104857.5? \n // Wait, let's re-read: \"If rounding would display 1024.0 of a unit, move to the next unit instead\"\n // This means if the rounded value is exactly 1024, it should be treated as 1.0 of the next unit.\n // But 1024.0 of KiB is 1.0 MiB. So we check if formattedValue is exactly 1024^n.\n // Actually, the condition is: if the rounded value is a power of 1024 (i.e., 1024, 1024^2, etc.), \n // it should be represented in the next unit.\n\n // Let's interpret: \"1048575 with 1 decimal is '1.0 MiB'\"\n // 1048575 / 1024 = 1024.995... -> rounds to 1025.0 KiB? That's not 1.0 MiB.\n // Wait, 1048575 * 10 / 10 = 104857.5 -> rounded? \n // Let's recalculate: 1048575 / 1024 = 1024.9951171875\n // With 1 decimal: round(1024.9951171875 * 10) / 10 = round(10249.951171875) / 10 = 10250 / 10 = 1025.0 KiB\n // But the example says it should be \"1.0 MiB\".\n // Ah, I see: 1048575 is just below 1048576 (which is 1024^2). \n // 1048576 / 1024 = 1024. So 1048575 is 1024.995... KiB.\n // The example says \"1048575 with 1 decimal is '1.0 MiB'\".\n // This implies that if the rounded value is very close to the next unit boundary, we should round up to the next unit.\n // Specifically, if the rounded value is 1024.0 (exactly), then it should be 1.0 of the next unit.\n // But 1048575 rounds to 1025.0 KiB, not 1024.0.\n // Let me re-read: \"If rounding would display 1024.0 of a unit, move to the next unit instead\"\n // Perhaps it means: if the value before rounding is such that rounding it to `decimals` places gives exactly 1024.0,\n // then we should use the next unit.\n // But 1048575 / 1024 = 1024.995... which rounds to 1025.0, not 1024.0.\n // What if the input was 1048576? Then 1048576 / 1024 = 1024.0 -> should be 1.0 MiB.\n // So the condition is: if the rounded value is exactly 1024.0, 1024^2.0, etc., then use the next unit.\n // But the example says 1048575 -> 1.0 MiB. \n // 1048575 is 1024.995... KiB. If we round to 1 decimal, we get 1025.0 KiB. \n // But the example says it should be 1.0 MiB. \n // This suggests that if the value is within 0.5 of the next unit boundary, we round up to the next unit.\n // Or perhaps: if the value is >= 1024.0 in the current unit, it should be represented in the next unit?\n // No, because 1024.0 KiB is exactly 1.0 MiB, so we don't want to show 1024.0 KiB.\n // The rule is: if the value is >= 1024.0 in the current unit, it should be shown in the next unit.\n // But then 1024.0 KiB would be shown as 1.0 MiB, which is correct.\n // However, the example says 1048575 -> 1.0 MiB. \n // 1048575 is less than 1048576 (which is 1024^2). \n // 1048575 / 1024 = 1024.995... which is greater than 1024.0.\n // So if the value is >= 1024.0 in the current unit, we move to the next unit.\n // But wait, what about 1024.0 exactly? Then it should be 1.0 MiB.\n // So the condition is: if value >= 1024.0, then use the next unit.\n // But then 1024.0 KiB would be shown as 1.0 MiB, which is correct.\n // And 1048575 KiB would be shown as 1024.995... KiB, which rounds to 1025.0 KiB, but the example says 1.0 MiB.\n // I think I misunderstood the example. Let me re-read: \"1048575 with 1 decimal is '1.0 MiB'\"\n // 1048575 is 1024.995... KiB. If we round to 1 decimal, we get 1025.0 KiB.\n // But the example says it should be 1.0 MiB.\n // This suggests that if the value is >= 1024.0 in the current unit, we should show it in the next unit.\n // So 1048575 KiB is >= 1024.0 KiB, so we show it as 1.0 MiB.\n // But 1025.0 KiB is also >= 1024.0 KiB, so we should show it as 1.0 MiB? No, 1025.0 KiB is 1.009765625 MiB.\n // I think the condition is" + }, + { + "problem": "js-wildcard", + "sample": 0, + "ok": false, + "error": "truncated by max tokens", + "truncated": true, + "wallMs": 328751, + "loadMs": 6, + "outputTokens": 2048, + "decodeMs": 315857.047, + "promptTokens": 506, + "prefillMs": 12720.302, + "thinkingChars": 0, + "answer": "```javascript\n/**\n * Matches a wildcard pattern against a text.\n * - \"?\" matches exactly one character.\n * - \"*\" matches any sequence of characters (including empty).\n * - No escaping, no character classes.\n * - Fast on adversarial inputs (O(n) or O(n*m) with small constant).\n *\n * @param {string} pattern - The pattern to match.\n * @param {string} text - The text to match against.\n * @returns {boolean} True if the text matches the pattern, false otherwise.\n */\nexport function matchWildcard(pattern, text) {\n if (typeof pattern !== 'string' || typeof text !== 'string') {\n throw new TypeError('Both arguments must be strings');\n }\n\n const pLen = pattern.length;\n const tLen = text.length;\n\n // If pattern is empty, text must be empty\n if (pLen === 0) {\n return tLen === 0;\n }\n\n // If text is empty, pattern must be empty or end with '*'\n if (tLen === 0) {\n return pLen === 0 || pattern[pLen - 1] === '*';\n }\n\n // Use dynamic programming: dp[i][j] indicates if pattern[0..i-1] matches text[0..j-1]\n // To optimize space, we can use two rows, but for readability and given constraints (few thousand chars),\n // a 2D array is acceptable. However, let's optimize for the worst-case scenario.\n // The naive DP is O(pLen * tLen). For 2000 * 2000 = 4e6 ops, it's fine.\n // But we can optimize the inner loop to stop early if possible.\n\n // dp[i][j] = true if pattern[0..i-1] matches text[0..j-1]\n // We only need the previous row to compute the current row.\n const prev = new Array(tLen + 1).fill(false);\n prev[0] = true; // Empty pattern matches empty text\n\n for (let i = 0; i < pLen; i++) {\n const curr = new Array(tLen + 1).fill(false);\n const pChar = pattern[i];\n\n // If current pattern char is '*', it can match zero or more characters.\n // If pattern[i] is '*', then pattern[0..i-1] must match text[0..j-1] or text[0..j-2]...\n // Actually, if pattern[i] is '*', then dp[i][j] = dp[i-1][j] || dp[i][j-1] (if j>0)\n // Wait, the recurrence for '*' is:\n // dp[i][j] = dp[i-1][j] (match 0 chars) || dp[i][j-1] (match 1 char, then '*' matches rest)\n // But since '*' matches any sequence, dp[i][j] = dp[i-1][j] || dp[i][j-1] is correct ONLY if we consider\n // that the '*' at position i can extend the match from j-1.\n // However, if the previous char was also '*', we can optimize.\n // Let's stick to the standard recurrence:\n // If pattern[i] == '*':\n // dp[i][j] = dp[i-1][j] || (j > 0 && dp[i][j-1])\n // Else:\n // dp[i][j] = dp[i-1][j-1] && (text[j-1] == pattern[i] || pattern[i] == '?')\n\n // But wait, the above recurrence for '*' is slightly off because '*' can match multiple characters.\n // Correct recurrence for '*':\n // dp[i][j] = dp[i-1][j] || (j > 0 && dp[i][j-1])\n // This is because:\n // 1. '*' matches 0 characters: then pattern[0..i-1] must match text[0..j-1] -> dp[i-1][j]\n // 2. '*' matches 1 character (text[j-1]): then pattern[0..i-1] must match text[0..j-2] AND '*' matches text[j-1]\n // But since '*' can match more, if dp[i][j-1] is true, it means pattern[0..i-1] (with '*' at end) matches text[0..j-2].\n // So yes, dp[i][j] = dp[i-1][j] || (j > 0 && dp[i][j-1]) is correct.\n\n if (pChar === '*') {\n curr[0] = false; // '*' cannot match empty text if there's a '*' at the end of pattern (unless handled by prev)\n // Wait, if pattern is \"*\", it matches empty text. So dp[1][0] should be true.\n // The recurrence: dp[i][j] = dp[i-1][j] || (j > 0 && dp[i][j-1])\n // For i=1 (pattern \"*\"), j=0: dp[1][0] = dp[0][0] || (false) = true. Correct.\n for (let j = 1; j <= tLen; j++) {\n curr[j] = prev[j] || curr[j - 1];\n }\n } else {\n curr[0] = false; // Non-star char cannot match empty text\n for (let j = 1; j <= tLen; j++) {\n // If previous pattern char matches current text char\n const matches = (pattern[i - 1] === text[j - 1] || pattern[i - 1] === '?');\n // If previous pattern char was '*', then dp[i-1][j] already handles the '*' matching zero or more,\n // but we need to be careful.\n // Actually, the recurrence is:\n // If pattern[i] is not '*', then:\n // dp[i][j] = dp[i-1][j-1] && (text[j-1] == pattern[i] || pattern[i] === '?')\n // But what if pattern[i-1] was '*'?\n // Example: pattern = \"*a\", text = \"ba\"\n // i=1 ('*'), j=1 ('b'): dp[1][1] = dp[0][1] || dp[1][0] = false || false = false.\n // i=2 ('a'), j=1 ('b'): dp[2][1] = dp[1][0] && ('b'=='a' or '?') = false && ... = false.\n // i=2 ('a'), j=2 ('a'): dp[2][2] = dp[1][1] && ('a'=='a' or '?') = false && true = false.\n // Wait, dp[1][1] is false, but pattern \"*a\" should match \"ba\".\n // The issue is that dp[i][j] should be true if pattern[0..i-1] matches text[0..j-1].\n // For pattern \"*a\", text \"ba\":\n // i=1 ('*'), j=1 ('b'): dp[1][1] = dp[0][1] || dp[1][0] = false || false = false.\n // But pattern \"*\" matches \"b\", so dp[1][1] should be true.\n // Ah, the recurrence for '*' is correct, but my manual trace was wrong.\n // dp[0][0] = true.\n // i=1 ('*'), j=1 ('b'): dp[1][1] = dp[0][1] (false) || dp[1][0] (true) = true. Correct.\n // i=2 ('a'), j=2 ('a'): dp[2][2] = dp[1][1] (true) && ('a'=='a') = true. Correct.\n \n // So the recurrence is:\n // If pattern[i] == '*':\n // curr[j] = prev[j] || curr[j-1]\n // Else:\n // curr[j] = prev[j-1] && (text[j-1] === pattern[i] || pattern[i] === '?')\n \n // Wait, there's a corner case: if pattern[i] is not '*', but pattern[i-1] was '*',\n // then dp[i-1][j] already covers the case where '*' matches zero or more, including the current char.\n // So we don't need to check pattern[i-1] for the non-star case.\n // The recurrence is standard.\n\n if (pattern[i - 1] === '*' || pattern[i - 1] === '?') {\n // If the previous char was '*' or '?', then dp[i-1][j] already considers matching\n // the current text char with the previous pattern char (if it was '?') or\n // the '*' extended its match (if it was '*')." + }, + { + "problem": "js-search-range", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 53414, + "loadMs": 8, + "outputTokens": 279, + "decodeMs": 41928.812, + "promptTokens": 459, + "prefillMs": 11309.899, + "thinkingChars": 0, + "answer": "```javascript\nexport function searchRange(sorted, target) {\n let left = -1;\n let right = -1;\n\n // Find first occurrence\n let idx = binarySearch(sorted, target, true);\n if (idx !== -1) {\n left = idx;\n }\n\n // Find last occurrence\n idx = binarySearch(sorted, target, false);\n if (idx !== -1) {\n right = idx;\n }\n\n return [left, right];\n}\n\nfunction binarySearch(arr, target, findFirst) {\n let low = 0;\n let high = arr.length - 1;\n let result = -1;\n\n while (low <= high) {\n const mid = Math.floor((low + high) / 2);\n\n if (arr[mid] === target) {\n result = mid;\n if (findFirst) {\n high = mid - 1;\n } else {\n low = mid + 1;\n }\n } else if (arr[mid] < target) {\n low = mid + 1;\n } else {\n high = mid - 1;\n }\n }\n\n return result;\n}\n```" } ] } From d7ed237e67f1d24e4d82e682e02271c00247e09d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 20:13:18 +0000 Subject: [PATCH 5/5] fix(pro): rebuild flycoder0.3pro on the prompt that measured best A/B on the same Qwen3.5 9B base (20 bench problems, thinking off, one attempt each): - FlyCoder 0.3 prompt: 9/20 - same prompt without the final-check line: 8/20 (keeps the line) - first pro recipe, six-step workflow plus a worked example: 6/20 flycoder0.3pro now uses the shared prompt plus the coding-agent rules, with Qwen3.8 27B, 64K context and multi-token prediction. The README and Documentation.md get the real 0.3 bench numbers and the comparison. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013QgiejZTjqQAUv4N4EFskK --- Documentation.md | 47 +- Modelfile.pro | 73 +-- docs/bench/cpu-nothink-0.3-prompts.json | 630 ++++++++++++++++++++++++ docs/ollama/flycoder0.3pro.md | 2 +- readme.md | 12 +- tests/modelfile.test.mjs | 47 +- 6 files changed, 704 insertions(+), 107 deletions(-) create mode 100644 docs/bench/cpu-nothink-0.3-prompts.json diff --git a/Documentation.md b/Documentation.md index ea01baf..e8fca46 100644 --- a/Documentation.md +++ b/Documentation.md @@ -117,7 +117,7 @@ Take the largest variant your memory allows: - **`flycoder0.3`** (Qwen3.5 9B, 16 GB Macs): the default. With its 64K context it uses about 9 GB in memory (measured: 8.8 GB). - **`flycoder0.3fast`** (Qwen3.5 4B, 8 GB Macs): about twice as fast per token, a lower pass rate on hard problems. -- **`flycoder0.3pro`** (Qwen3.8 27B, 32 GB Macs or more): the latest Qwen generation with its own FlyCoder recipe, for hard bugs, algorithms and coding agents (section 6.5). +- **`flycoder0.3pro`** (Qwen3.8 27B, 32 GB Macs or more): the latest Qwen generation, set up for hard bugs, algorithms and coding agents (section 6.5). Every Modelfile declares the Ollama version it needs (`REQUIRES`): an older Ollama names the version to install instead of failing to load the model. @@ -280,14 +280,14 @@ Replace it for a session with `/set system`, or with a `system` message in the A ### 6.5 What makes `flycoder0.3pro` different -`flycoder0.3pro` is not Qwen3.8 with a new name. Its Modelfile gives it: +`flycoder0.3pro` is not Qwen3.8 with a new name. On top of the base, its Modelfile sets: -- **Its own workflow** in the system prompt, in six steps: contract (exact names, inputs, errors; one question only if an ambiguity changes the code), plan (at most five lines for multi-file work), code, tests that prove the code, a check of the tricky cases before answering, and a one-line report on how to run the tests. -- **Agent rules**: read files before editing them, change as little as the task needs, run the project's tests and report their real output, never run a destructive command without the user's agreement. -- **A worked example** (`MESSAGE`): one request and the expected answer (assumption, code, pytest tests, run command), so the model sees the format instead of only reading about it. `npm test` runs the example's code against its own test cases. -- **Multi-token prediction** (`draft_num_predict 4`) and 64K tokens of context. +- **FlyCoder's system prompt** (section 6.4), the one that scored best in the prompt comparison of section 7, with the base model named. +- **Rules for coding agents**: read files before editing them, change as little as the task needs, run the project's tests and report their real output, never claim a result it did not see, never run a destructive command without the user's agreement. +- **Qwen's coding sampling** (temperature 0.6 instead of the library's 1.0), **64K tokens of context** (instead of Ollama's 4,096 default under 24 GB) and **multi-token prediction** (`draft_num_predict 4`). +- **`REQUIRES 0.32.12`**: an older Ollama names the version to install. -The weights are Qwen3.8's: FlyCoder does not retrain them. The recipe was measured on the 9B base, because the test server cannot hold the 27B (section 7). +A first pro recipe had a six-step workflow (contract, plan, code, tests, check, report) and a worked example conversation. It was compared on the 9B base, since the test server cannot hold the 27B: it solved 6 of the 20 bench problems, against 9 for FlyCoder's prompt. It was dropped. FlyCoder does not retrain the weights. ### 6.6 Ollama server settings @@ -323,7 +323,32 @@ npm run bench -- --models flycoder0.3,flycoder0.3fast The bench runs the code written by the models on your machine, in a temporary folder, with a time limit. On macOS, `sandbox-exec` blocks network access and writes outside that folder. -The 0.3 reference run (`docs/bench/cpu-nothink-0.3.json`) is being recorded. The 0.2 run stays in [docs/bench/cpu-nothink.json](docs/bench/cpu-nothink.json). +Reference results, recorded on a 4-core Linux server **without a GPU** (Xeon at 2.8 GHz, GGUF Q4_K_M weights, thinking off, one attempt per problem, at most 2,048 tokens per answer). On a Mac, the absolute speeds are much higher; only the comparisons between profiles matter. + +| Profile | Base | Passed | Generation | Tokens per answer | Time per problem | +|---|---|---|---|---|---| +| `flycoder0.3` | Qwen3.5 9B | 9/20 | 3.9 tok/s | 900 | 254 s | +| `flycoder0.3fast` | Qwen3.5 4B | 5/20 | 6.1 tok/s | 689 | 126 s | +| 0.2 beta, for comparison | Gemma 4 12B | 14/20 | 3.4 tok/s | 739 | 235 s | +| 0.2 beta fast, for comparison | Qwen3.5 4B | 5/20 | 6.5 tok/s | 537 | 89 s | + +Results: [docs/bench/cpu-nothink-0.3.json](docs/bench/cpu-nothink-0.3.json); the 0.2 rows come from [docs/bench/cpu-nothink.json](docs/bench/cpu-nothink.json), measured on a slower Xeon (2.1 GHz), so their speeds compare only roughly. + +What these numbers say: + +- **Gemma 4 12B solved more problems** (14 against 9). It was replaced because it did not run on every 16 GB Mac; `flycoder0.3` runs on any Ollama 0.30 or later and leaves room for a 64K context. +- **3 of the 9B's 11 failures are cut-off answers**: with thinking off, the model reasons in code comments until it hits the 2,048-token cap. With thinking on (the default of `ollama run`), that reasoning goes to the thinking channel instead. The other 8 failures are wrong logic. +- **`flycoder0.3fast` matches 0.2's fast variant** (5/20), with a context twice as large. + +Prompt comparison on the same 9B base and settings, recorded in [docs/bench/cpu-nothink-0.3-prompts.json](docs/bench/cpu-nothink-0.3-prompts.json): + +| System prompt | Passed | Cut-off answers | Tokens per answer | +|---|---|---|---| +| FlyCoder 0.3 (shipped) | 9/20 | 3 | 900 | +| The same, without the final-check line | 8/20 | 3 | 890 | +| First pro recipe: six-step workflow and a worked example | 6/20 | 2 | 772 | + +With one attempt per problem, a gap of one problem is noise; the gap to the first pro recipe is why it was dropped. ## 8. Publishing on ollama.com @@ -395,7 +420,7 @@ The tests check: - that the Modelfiles start from the right bases, with the expected settings and an identical system prompt apart from the base name; - that the installer, the publishing script and the workflow agree with the Modelfiles and the version; - that each ollama.com page shows the right name, size, memory and context; -- that every Modelfile declares its minimum Ollama version, and that the code of the pro example passes its own test cases; +- that every Modelfile declares its minimum Ollama version, and that `flycoder0.3pro` keeps every shared rule plus the agent rules; - FlyBrain's rules, the fallback to the large model when the micro-model fails, conversation memory, the scheduler, and the server against a fake Ollama. CI runs them on Linux and on macOS, where the reference solutions run inside the sandbox. @@ -413,8 +438,8 @@ Add an object to `bench/problems.mjs` with `id` (`py-` or `js-` prefix), `langua - The weights are not retrained: FlyCoder is a profile of existing models. - The bench has 20 problems and one attempt per problem: it separates profiles without replacing a public benchmark. - The reference measurements come from a Linux server without a GPU; speed on a Mac has not been measured for this version. -- `flycoder0.3pro` was not run by the bench: its 27B base needs more memory than the test server has. Its recipe was measured on the 9B base instead. -- The pro example conversation is sent with every conversation: about 600 tokens of the 64K context. +- `flycoder0.3pro` was not run by the bench: its 27B base needs more memory than the test server has. Its prompt choices were measured on the 9B base instead. +- The bench runs with thinking off and one attempt per problem; quality with thinking on (the default) is not measured yet. - FlyBrain does not lower the memory peak of hard requests, and switching experts costs a few seconds. ## 12. Version history diff --git a/Modelfile.pro b/Modelfile.pro index d55287d..9e94ce1 100644 --- a/Modelfile.pro +++ b/Modelfile.pro @@ -18,66 +18,19 @@ PARAMETER repeat_penalty 1 # Multi-token prediction: the built-in draft layer proposes 4 tokens that the model checks in one pass. PARAMETER draft_num_predict 4 -SYSTEM """You are FlyCoder 0.3 pro, the strongest FlyCoder, a coding model running locally on the user's computer through Ollama. You are built on Qwen3.8 27B by the Qwen team at Alibaba, configured by the FlyCoder project for hard coding work and coding agents. - -Work in this order: -1. Contract: pin down the exact names, signatures, inputs, outputs and errors. If an ambiguity would change the code, ask one short question; otherwise state your assumption in one line and go on. -2. Plan: for anything larger than one function, give a plan of at most five lines: the files you touch and the approach. -3. Code: complete, runnable code with every import. No placeholders, no "TODO", no "rest of the code here". Keep the user's names, language, export style and the project's conventions. Prefer the standard library; never invent functions, options or packages. -4. Tests: write the tests that prove the code, with the project's test framework (otherwise pytest or node:test), covering the edge cases you handled: empty input, boundaries, invalid arguments, Unicode, large inputs. -5. Check: before answering, trace the tricky cases through your code by hand and fix what you find. Give one final version, never an abandoned first attempt. -6. Report: one line on how to run the tests, and any risk or limit the user must know. - -Choose the simplest correct algorithm and avoid quadratic work on large inputs. Do not write insecure code: no hard-coded secrets, parameterized SQL queries, validated untrusted input, no shell injection. When changing existing code, show only the changed parts unless the user asks for the whole file. +SYSTEM """You are FlyCoder 0.3 pro, a coding model running locally on the user's computer through Ollama. You are built on Qwen3.8 27B by the Qwen team at Alibaba, configured by the FlyCoder project for hard coding work and coding agents. + +Write code the way a careful senior engineer would: +- Follow the request exactly: keep the names, signatures, file names, language and export style the user gives. +- Deliver complete, runnable code with every import it needs. No placeholders, no "TODO", no "rest of the code here". +- Handle edge cases on purpose: empty input, missing values, boundaries, invalid arguments (raise or return exactly as specified), Unicode. +- Prefer the standard library and the existing style of the project. Never invent functions, options or packages; if you are not sure an API exists, say so. +- Choose the simplest correct algorithm with suitable complexity, and avoid quadratic work on large inputs. +- Before answering, check your code against the request and the edge cases once more, and fix what you find. Give one final version, never an abandoned first attempt. +- When changing existing code, show only the changed parts unless the user asks for the whole file. +- Never claim that you ran code or tests unless a tool actually ran them. When useful, give a short command or test the user can run. +- Do not write insecure code: no hard-coded secrets, parameterized SQL queries, validated untrusted input, no shell injection. As an agent with tools: read the relevant files before editing them, change as little code as the task needs, run the project's tests when you can and report their real output. Never claim a result you did not see. Never run a destructive command (deleting files, force-pushing, dropping data) without the user's explicit agreement. -Format: each file goes in one fenced code block with a language tag, preceded by its path. Keep explanations short and write them in the user's language.""" - -# One worked exchange that shows the expected shape of an answer: contract, code, tests, how to run them. -MESSAGE user """Write a Python function is_valid_ipv4(text) that returns True for a valid dotted-quad IPv4 address and False otherwise.""" -MESSAGE assistant """Assumption: four decimal parts from 0 to 255 separated by dots, ASCII digits only, no leading zeros ("01.2.3.4" is rejected because some tools read it as octal), no surrounding spaces; a non-string raises TypeError. - -`ipv4.py` -```python -def is_valid_ipv4(text: str) -> bool: - '''Return True if text is a dotted-quad IPv4 address such as "192.168.0.1".''' - if not isinstance(text, str): - raise TypeError("text must be a string") - parts = text.split(".") - if len(parts) != 4: - return False - for part in parts: - # ASCII digits only (str.isdigit also accepts other scripts), no sign, no leading zero. - if not part.isascii() or not part.isdigit() or (len(part) > 1 and part[0] == "0"): - return False - if int(part) > 255: - return False - return True -``` - -`test_ipv4.py` -```python -import pytest - -from ipv4 import is_valid_ipv4 - - -@pytest.mark.parametrize("text", ["0.0.0.0", "192.168.0.1", "255.255.255.255", "10.0.0.10", "1.2.3.4"]) -def test_valid(text): - assert is_valid_ipv4(text) - - -@pytest.mark.parametrize("text", [ - "", "1.2.3", "1.2.3.4.5", "256.1.1.1", "01.2.3.4", "1.2.3.-4", "1.2.3.4 ", "a.b.c.d", "1..2.3", "١.2.3.4", -]) -def test_invalid(text): - assert not is_valid_ipv4(text) - - -def test_not_a_string(): - with pytest.raises(TypeError): - is_valid_ipv4(1234) -``` - -Run: `python -m pytest test_ipv4.py`. Limit: IPv6 and CIDR suffixes such as "/24" are rejected; use the standard `ipaddress` module if you need them.""" +Format: each file goes in one fenced code block with a language tag, preceded by its path when there are several files. Keep explanations short and write them in the user's language.""" diff --git a/docs/bench/cpu-nothink-0.3-prompts.json b/docs/bench/cpu-nothink-0.3-prompts.json new file mode 100644 index 0000000..2069d8c --- /dev/null +++ b/docs/bench/cpu-nothink-0.3-prompts.json @@ -0,0 +1,630 @@ +{ + "note": "A/B of system prompts on the same Qwen3.5 9B base, one problem per run, merged. flycoder0.3-nocheck: the 0.3 prompt without the final-check line. flycoder-pro-recipe9b: the first flycoder0.3pro recipe (six-step workflow and a worked example) on the 9B base.", + "think": "off", + "maxTokens": 2048, + "models": [ + { + "summary": { + "model": "flycoder0.3-nocheck", + "passed": 8, + "total": 20, + "truncated": 3, + "meanOutputTokens": 889.85 + }, + "results": [ + { + "problem": "py-merge-intervals", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-cMHyg7/test_solution.py\", line 3, in \n assert merge_intervals(data) == [(1, 4), (5, 6)]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError\n", + "truncated": false, + "wallMs": 185645, + "loadMs": 89504, + "outputTokens": 318, + "decodeMs": 78344.155, + "promptTokens": 433, + "prefillMs": 17773.059, + "thinkingChars": 0, + "answer": "```python\nfrom typing import List, Tuple\n\ndef merge_intervals(intervals: List[Tuple[int, int]]) -> List[Tuple[int, int]]:\n \"\"\"\n Merges a list of closed intervals.\n \n Intervals that overlap or touch (one ends exactly where the next starts) are merged.\n Returns a new list of (start, end) tuples sorted by start.\n Raises ValueError if any interval has start > end.\n Does not mutate the input.\n \"\"\"\n if not intervals:\n return []\n \n # Validate inputs and sort them by start time\n for start, end in intervals:\n if start > end:\n raise ValueError(f\"Invalid interval: start ({start}) > end ({end})\")\n \n # Sort intervals based on start time\n sorted_intervals = sorted(intervals, key=lambda x: x[0])\n \n merged = []\n \n for current_start, current_end in sorted_intervals:\n if not merged:\n merged.append([current_start, current_end])\n else:\n last_start, last_end = merged[-1]\n \n # Check for overlap or touching intervals\n if current_start <= last_end:\n # Merge intervals\n merged[-1] = [last_start, max(last_end, current_end)]\n else:\n # No overlap, add the new interval\n merged.append([current_start, current_end])\n \n return merged\n```" + }, + { + "problem": "py-roman-strict", + "sample": 0, + "ok": false, + "error": "truncated by max tokens", + "truncated": true, + "wallMs": 558803, + "loadMs": 30541, + "outputTokens": 2048, + "decodeMs": 509480.678, + "promptTokens": 442, + "prefillMs": 18765.474, + "thinkingChars": 0, + "answer": "```python\ndef roman_to_int(s: str) -> int:\n \"\"\"\n Convert a Roman numeral string to an integer.\n\n Valid input:\n - Non-empty string containing only uppercase canonical Roman characters (I, V, X, L, C, D, M).\n - Must follow standard subtractive notation (IV, IX, XL, XC, CD, CM) and not contain\n invalid repetitions like IIII, VV, or invalid subtractive pairs like IC, XM.\n\n Raises:\n ValueError: If the string is empty, contains lowercase/unknown characters, or is not a canonical form.\n \"\"\"\n # Mapping of Roman symbols to integer values\n values = {\n 'I': 1,\n 'V': 5,\n 'X': 10,\n 'L': 50,\n 'C': 100,\n 'D': 500,\n 'M': 1000,\n }\n\n # Validate characters and ensure uppercase\n if not s:\n raise ValueError(\"Empty string is not a valid Roman numeral.\")\n \n for char in s:\n if char not in values:\n raise ValueError(f\"Invalid character '{char}' in Roman numeral.\")\n\n # Pre-check for known invalid patterns to catch non-canonical forms early.\n # However, the main logic below will also detect invalid forms naturally.\n \n total = 0\n n = len(s)\n \n for i, char in enumerate(s):\n val = values[char]\n \n # Look ahead to the next character if it exists\n if i + 1 < n:\n next_val = values[s[i + 1]]\n \n # If the current value is less than the next, it's a subtractive pair.\n # In a canonical form, the subtraction only happens for specific pairs.\n # But more importantly, we must ensure we don't subtract if the next character\n # is not exactly double the current value's \"step\" relative to the next standard.\n # Actually, the standard rule is:\n # I can precede V and X.\n # X can precede L and C.\n # C can precede D and M.\n # We can't have IIII, VV, etc.\n \n if val < next_val:\n # This is a subtractive case.\n # We subtract the current value.\n total -= val\n else:\n total += val\n else:\n # Last character\n total += val\n\n # Now we must validate that the resulting integer and the path taken were canonical.\n # However, simply summing/subtracting isn't enough to catch things like \"IC\" or \"XM\"\n # if we just rely on the subtraction logic, because \"IC\" would be interpreted as:\n # I (1) then C (100) -> 1 < 100 -> -1 + 100 = 99.\n # But \"IC\" is invalid.\n # We need to check validity of subtractive pairs explicitly.\n \n # Valid subtractive pairs: IV, IX, XL, XC, CD, CM\n valid_subtractive = {\"IV\", \"IX\", \"XL\", \"XC\", \"CD\", \"CM\"}\n \n # Check if every subtractive pair in the string is valid.\n # We can reconstruct the expected string based on our calculation or just check substrings.\n # Reconstructing is safer.\n \n expected_s = []\n current_val = 0\n \n for i, char in enumerate(s):\n val = values[char]\n if i + 1 < n and val < values[s[i + 1]]:\n # It's a subtractive pair. Check if it's valid.\n pair = char + s[i + 1]\n if pair not in valid_subtractive:\n raise ValueError(f\"Non-canonical subtractive pair '{pair}'.\")\n current_val += 0 # Skip next char in reconstruction logic below, handled by loop\n else:\n current_val += val\n \n # Wait, the reconstruction logic above is flawed because I'm not building the string.\n # Let's just iterate and build the expected string based on the subtraction logic.\n \n reconstructed_s = []\n for i, char in enumerate(s):\n val = values[char]\n if i + 1 < n and val < values[s[i + 1]]:\n pair = char + s[i + 1]\n if pair not in valid_subtractive:\n raise ValueError(f\"Non-canonical subtractive pair '{pair}'.\")\n # We consume the next character\n reconstructed_s.append(pair)\n i += 1 # Skip next char\n else:\n reconstructed_s.append(char)\n \n # Note: The loop variable 'i' increment in the for loop is handled by Python.\n # My manual i += 1 inside the loop doesn't affect the for loop's iteration.\n # I need to implement this check differently.\n \n # Correct approach for validation:\n # 1. Calculate the integer value assuming standard rules.\n # 2. Check if the string matches the canonical representation of that integer.\n # Or, strictly check the subtractive pairs during the scan.\n \n # Let's restart the logic cleanly.\n \n # Step 1: Validate characters (already done).\n # Step 2: Validate subtractive pairs.\n \n i = 0\n while i < len(s):\n curr_val = values[s[i]]\n if i + 1 < len(s):\n next_val = values[s[i + 1]]\n if curr_val < next_val:\n # It is a subtractive pair\n pair = s[i:i+2]\n if pair not in valid_subtractive:\n raise ValueError(f\"Non-canonical subtractive pair '{pair}'.\")\n i += 2\n continue\n i += 1\n \n # Step 3: Check for invalid repetitions.\n # A canonical Roman numeral cannot have a symbol repeated 4 times (e.g., IIII, VVVV).\n # Also, a symbol cannot be repeated consecutively if it's part of a subtractive pair logic incorrectly.\n # Actually, the rule is:\n # - I, X, C, M can be repeated up to 3 times (II, XX, CCC, MMM).\n # - V, L, D can never be repeated.\n # - Subtractive pairs are single usage.\n \n # We can simply check the calculated integer against a canonical generator?\n # No, that's complex.\n # Let's check the structure directly.\n \n # Check for V, L, D repetition\n for char in ['V', 'L', 'D']:\n if char in s:\n count = s.count(char)\n if count > 1:\n # Check if they are adjacent\n indices = [i for i, c in enumerate(s) if c == char]\n for j in range(len(indices) - 1):\n if indices[j+1] == indices[j] + 1:\n raise ValueError(f\"Invalid repetition of '{char}'.\")\n \n # Check for I, X, C, M repetition > 3\n for char in ['I', 'X', 'C', 'M']:\n count = 0\n current_run = 0\n for c in s:\n if c == char:\n current_run += 1\n if current_run > 3:\n raise ValueError(f\"Invalid repetition of '{char}'.\")\n else:\n current_run = 0\n \n # Check for invalid patterns like \"IC\", \"XM\" etc.\n # We already checked valid_subtractive.\n # But what about \"IIIC\"? \n # III C -> III (3) + C (100). Not subtractive.\n # IIIC is valid? No.\n # Standard: III is 3. C is 100. III C is 103.\n # Is \"IIIC\" valid? Yes, 99 is IC? No, 99 is XCIX.\n # 103 is CIII.\n # So \"IIIC\" is not standard.\n # Wait, the input \"IIIC\" means 103?\n # Canonical for 103 is CIII.\n # So \"IIIC\" is invalid because the smaller symbols must be to the left of the larger one for subtraction,\n # or to the right for addition.\n # If we have I (1) then C (100), it's subtractive.\n # If we have I (1) then I (1) then I (1) then C (100), it's 103.\n # But the canonical form for 103 is CIII.\n # So we must ensure the result is the canonical form.\n \n # How to ensure canonical form?\n # Generate the canonical string for the calculated integer and compare.\n \n def int_to_roman(num):\n val_map = [\n (1" + }, + { + "problem": "py-parse-duration", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-7ppw6r/test_solution.py\", line 2, in \n assert parse_duration(\"45s\") == 45\n ~~~~~~~~~~~~~~^^^^^^^\n File \"/tmp/flycoder-bench-7ppw6r/solution.py\", line 42, in parse_duration\n raise ValueError(f\"Invalid part: '{part}'\")\nValueError: Invalid part: '45s'\n", + "truncated": false, + "wallMs": 268929, + "loadMs": 56977, + "outputTokens": 788, + "decodeMs": 190829.443, + "promptTokens": 502, + "prefillMs": 21103.86, + "thinkingChars": 0, + "answer": "```python\nimport re\n\ndef parse_duration(text: str) -> int:\n \"\"\"\n Parse a duration string and return the total duration in whole seconds.\n \n Valid format: one or more parts.\n Units: d (86400s), h (3600s), m (60s), s (1s).\n Order must be strictly d, h, m, s.\n Spaces are allowed between parts but not inside numbers.\n \n Raises ValueError for invalid input.\n \"\"\"\n text = text.strip()\n if not text:\n raise ValueError(\"Input cannot be empty\")\n \n # Split by whitespace to handle optional spaces between parts\n parts = text.split()\n \n if not parts:\n raise ValueError(\"No valid parts found\")\n \n # Define unit values\n unit_values = {\n 'd': 86400,\n 'h': 3600,\n 'm': 60,\n 's': 1\n }\n \n parsed_units = []\n parsed_values = []\n \n for part in parts:\n # Check if the part ends with a valid unit\n if part not in unit_values:\n # Check if it's just a number without a unit (invalid)\n if part.isdigit():\n raise ValueError(f\"Invalid part: '{part}' - missing unit\")\n else:\n raise ValueError(f\"Invalid part: '{part}'\")\n \n number_str = part[:-1]\n unit = part[-1]\n \n # Validate number (non-negative integer)\n if not number_str or not number_str.isdigit():\n raise ValueError(f\"Invalid number format in part: '{part}'\")\n \n number = int(number_str)\n \n # Check for duplicate units\n if unit in parsed_units:\n raise ValueError(f\"Duplicate unit '{unit}' found\")\n \n parsed_units.append(unit)\n parsed_values.append((number, unit_values[unit]))\n \n # Validate order: must be d, h, m, s strictly\n expected_order = ['d', 'h', 'm', 's']\n parsed_unit_order = [u for n, u in parsed_values]\n \n if parsed_unit_order != parsed_unit_order:\n # This check is redundant since we just appended them, \n # but we need to ensure no gaps or wrong order logic is violated.\n # Actually, the requirement is \"units must appear in the order d, h, m, s\".\n # This means if we have d and h, d must come before h.\n # If we have only h and s, h must come before s.\n # So we just check that the sequence of units is non-decreasing according to the order d,h,m,s.\n pass\n \n # Map units to their order index\n unit_order_map = {'d': 0, 'h': 1, 'm': 2, 's': 3}\n unit_indices = [unit_order_map[u] for n, u in parsed_values]\n \n # Check if the indices are in strictly non-decreasing order\n for i in range(1, len(unit_indices)):\n if unit_indices[i] < unit_indices[i-1]:\n raise ValueError(f\"Units must appear in order d, h, m, s. Found invalid order: {parsed_unit_order}\")\n \n # Calculate total seconds\n total_seconds = sum(n * v for n, v in parsed_values)\n \n return total_seconds\n```" + }, + { + "problem": "py-topo-sort", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-3TXOnN/test_solution.py\", line 3, in \n assert topo_sort({\"app\": [\"lib\", \"utils\"], \"lib\": [\"utils\"]}) == [\"utils\", \"lib\", \"app\"]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError\n", + "truncated": false, + "wallMs": 390975, + "loadMs": 33308, + "outputTokens": 1374, + "decodeMs": 338191.75, + "promptTokens": 453, + "prefillMs": 19459.627, + "thinkingChars": 0, + "answer": "```python\nfrom collections import deque\n\ndef topo_sort(deps):\n \"\"\"\n Perform topological sort on a graph defined by dependencies.\n \n Args:\n deps (dict): A dictionary where each key is a node (string) and the value is a list \n of nodes that the key depends on.\n \n Returns:\n list: A list of all nodes in topological order. If multiple nodes are ready, \n the alphabetically smallest is chosen first.\n \n Raises:\n ValueError: If the graph contains a cycle.\n \"\"\"\n # Determine all nodes in the graph: keys are explicit nodes, values are dependencies.\n # Dependencies might not be keys in deps if they have no outgoing dependencies.\n all_nodes = set(deps.keys())\n for dependencies in deps.values():\n all_nodes.update(dependencies)\n \n # Build the adjacency list for Kahn's algorithm.\n # We need: for each node, list of nodes that depend on it (reverse graph).\n # And the in-degree of each node.\n in_degree = {node: 0 for node in all_nodes}\n reverse_adj = {node: [] for node in all_nodes}\n \n for node, dependencies in deps.items():\n for dep in dependencies:\n # node depends on dep, meaning dep -> node in the dependency graph\n # We track in-degree of 'node'\n in_degree[node] += 1\n # Reverse graph: dep points to node\n reverse_adj[dep].append(node)\n \n # Initialize queue with nodes having zero in-degree, sorted alphabetically\n queue = deque(sorted([node for node in all_nodes if in_degree[node] == 0]))\n \n result = []\n \n while queue:\n # Pop the alphabetically smallest node\n current = queue.popleft()\n result.append(current)\n \n # Process neighbors in the reverse graph\n neighbors = reverse_adj[current]\n for neighbor in neighbors:\n in_degree[neighbor] -= 1\n if in_degree[neighbor] == 0:\n # Insert maintaining sorted order for deterministic behavior\n # Since we process neighbors one by one, we can insert in sorted position\n # or append and sort later. Given the small number of neighbors per step,\n # inserting in sorted order keeps the queue sorted.\n # However, deque doesn't support insert at index efficiently.\n # Instead, we can collect all zero-degree nodes and resort.\n # But to keep it simple and efficient enough:\n # We will maintain the invariant that the deque is sorted.\n # When a node becomes zero degree, we insert it in the correct position.\n # Since it's a deque, we can't easily insert in middle. \n # Alternative: use a sorted list or heap.\n # Given the requirement for determinism and simplicity, let's use a min-heap logic.\n # But the prompt asks for a function, and standard library allows heapq.\n # Let's switch to a heap approach for the queue to ensure O(log N) insertions.\n pass\n \n # Re-evaluating the queue strategy:\n # Using a list and keeping it sorted, or using heapq.\n # Let's use heapq for the zero-in-degree nodes to maintain sorted order efficiently.\n pass\n\n # Refactoring to use heapq for the queue\n import heapq\n \n zero_degree_heap = []\n for node in all_nodes:\n if in_degree[node] == 0:\n heapq.heappush(zero_degree_heap, node)\n \n zero_degree_heap = [] # Re-initialize\n for node in all_nodes:\n if in_degree[node] == 0:\n heapq.heappush(zero_degree_heap, node)\n \n # Reset logic with heap\n zero_degree_heap = []\n for node in all_nodes:\n if in_degree[node] == 0:\n heapq.heappush(zero_degree_heap, node)\n \n final_result = []\n \n while zero_degree_heap:\n current = heapq.heappop(zero_degree_heap)\n final_result.append(current)\n \n for neighbor in reverse_adj[current]:\n in_degree[neighbor] -= 1\n if in_degree[neighbor] == 0:\n heapq.heappush(zero_degree_heap, neighbor)\n \n if len(final_result) != len(all_nodes):\n raise ValueError(\"Cycle detected in the graph\")\n \n return final_result\n```\n\nWait, I made a mistake in the thought process above by re-writing the code partially and introducing confusion in the block. Let me write the clean, final version in one go.\n\n```python\nimport heapq\nfrom collections import defaultdict\n\ndef topo_sort(deps):\n \"\"\"\n Perform topological sort on a graph defined by dependencies.\n \n Args:\n deps (dict): A dictionary where each key is a node (string) and the value is a list \n of nodes that the key depends on.\n \n Returns:\n list: A list of all nodes in topological order. If multiple nodes are ready, \n the alphabetically smallest is chosen first.\n \n Raises:\n ValueError: If the graph contains a cycle.\n \"\"\"\n all_nodes = set(deps.keys())\n for dependencies in deps.values():\n all_nodes.update(dependencies)\n \n in_degree = {node: 0 for node in all_nodes}\n reverse_adj = {node: [] for node in all_nodes}\n \n for node, dependencies in deps.items():\n for dep in dependencies:\n in_degree[node] += 1\n reverse_adj[dep].append(node)\n \n heap = []\n for node in all_nodes:\n if in_degree[node] == 0:\n heapq.heappush(heap, node)\n \n result = []\n while heap:\n current = heapq.heappop(heap)\n result.append(current)\n \n for neighbor in reverse_adj[current]:\n in_degree[neighbor] -= 1\n if in_degree[neighbor] == 0:\n heapq.heappush(heap, neighbor)\n \n if len(result) != len(all_nodes):\n raise ValueError(\"Cycle detected in the graph\")\n \n return result\n```" + }, + { + "problem": "py-lru-cache", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 129759, + "loadMs": 33073, + "outputTokens": 322, + "decodeMs": 77883.743, + "promptTokens": 458, + "prefillMs": 18786.421, + "thinkingChars": 0, + "answer": "```python\nfrom collections import OrderedDict\n\n\nclass LRUCache:\n \"\"\"\n A Least Recently Used (LRU) Cache implementation with fixed capacity.\n Uses OrderedDict to maintain insertion order and achieve O(1) operations.\n \"\"\"\n\n def __init__(self, capacity: int):\n if capacity < 0:\n raise ValueError(\"Capacity cannot be negative.\")\n self.capacity = capacity\n # OrderedDict maintains insertion order; we access items to update their position.\n self.cache = OrderedDict()\n\n def get(self, key: int) -> int:\n if key not in self.cache:\n return -1\n \n # Move the accessed key to the end (most recently used)\n self.cache.move_to_end(key)\n return self.cache[key]\n\n def put(self, key: int, value: int) -> None:\n if key in self.cache:\n # Update the value and move the key to the end (most recently used)\n self.cache[key] = value\n self.cache.move_to_end(key)\n else:\n if self.capacity == 0:\n # If capacity is 0, we cannot store anything, so we do nothing\n return\n \n # Insert the new key-value pair\n self.cache[key] = value\n \n # If the cache exceeds its capacity, remove the least recently used item (first item)\n if len(self.cache) > self.capacity:\n self.cache.popitem(last=False)\n```" + }, + { + "problem": "py-wrap-text", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 152665, + "loadMs": 31052, + "outputTokens": 420, + "decodeMs": 101929.046, + "promptTokens": 466, + "prefillMs": 19671.219, + "thinkingChars": 0, + "answer": "```python\ndef wrap_text(text, width):\n if width < 1:\n raise ValueError(\"width must be at least 1\")\n \n if not text or text.isspace():\n return []\n \n # Split text into words, handling any whitespace\n words = text.split()\n \n lines = []\n current_line = []\n current_length = 0\n \n for word in words:\n # Handle words longer than width by splitting them\n if len(word) > width:\n # Split the long word into pieces of exactly width characters\n # The last piece will naturally be shorter if needed\n for i in range(0, len(word), width):\n piece = word[i:i+width]\n \n # Try to add this piece to the current line\n if current_length == 0:\n # First piece on an empty line\n current_line.append(piece)\n current_length = len(piece)\n elif current_length + 1 + len(piece) <= width:\n # Fits on the current line\n current_line.append(piece)\n current_length += 1 + len(piece)\n else:\n # Need to start a new line\n lines.append(' '.join(current_line))\n current_line = [piece]\n current_length = len(piece)\n else:\n # Normal word handling\n if current_length == 0:\n current_line.append(word)\n current_length = len(word)\n elif current_length + 1 + len(word) <= width:\n current_line.append(word)\n current_length += 1 + len(word)\n else:\n lines.append(' '.join(current_line))\n current_line = [word]\n current_length = len(word)\n \n # Add the last line if there is any content\n if current_line:\n lines.append(' '.join(current_line))\n \n return lines\n```" + }, + { + "problem": "py-semver-compare", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-M7b2QJ/test_solution.py\", line 4, in \n assert compare_versions(low, high) == -1, (low, high)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: ('1.0.0-rc.1', '1.0.0')\n", + "truncated": false, + "wallMs": 261035, + "loadMs": 32048, + "outputTokens": 829, + "decodeMs": 208982.772, + "promptTokens": 475, + "prefillMs": 19986.585, + "thinkingChars": 0, + "answer": "```python\ndef compare_versions(a: str, b: str) -> int:\n \"\"\"\n Compare two Semantic Versioning 2.0.0 strings.\n \n Returns:\n -1 if a < b\n 0 if a == b\n 1 if a > b\n \n SemVer precedence rules:\n 1. Core (MAJOR.MINOR.PATCH) is compared numerically.\n 2. A pre-release version has lower precedence than a normal version.\n 3. Pre-release identifiers are compared dot by dot:\n - Numeric identifiers are compared as integers.\n - Alphanumeric identifiers are compared in ASCII order.\n - Numeric identifiers are lower precedence than alphanumeric ones.\n 4. A shorter set of pre-release identifiers is lower precedence if all preceding ones are equal.\n 5. Build metadata is ignored.\n \"\"\"\n def parse(version: str):\n # Extract core, prerelease, and build\n # Core is everything before the first hyphen (if any)\n # Prerelease is between the first hyphen and the second hyphen (if any)\n # Build is after the second hyphen (if any)\n \n parts = version.split('-', 2)\n core = parts[0]\n prerelease = parts[1] if len(parts) > 1 else None\n build = parts[2] if len(parts) > 2 else None\n \n # Parse core\n core_parts = core.split('.')\n major = int(core_parts[0])\n minor = int(core_parts[1])\n patch = int(core_parts[2])\n \n # Parse prerelease\n prerelease_identifiers = []\n if prerelease:\n identifiers = prerelease.split('.')\n for ident in identifiers:\n if ident.isdigit():\n prerelease_identifiers.append(('numeric', int(ident)))\n else:\n prerelease_identifiers.append(('alphanumeric', ident))\n \n return (major, minor, patch, prerelease_identifiers)\n\n parsed_a = parse(a)\n parsed_b = parse(b)\n\n # Compare core\n if parsed_a[:3] != parsed_b[:3]:\n if parsed_a[:3] < parsed_b[:3]:\n return -1\n else:\n return 1\n\n # Compare prerelease\n prerelease_a = parsed_a[3]\n prerelease_b = parsed_b[3]\n\n if prerelease_a is None and prerelease_b is None:\n return 0\n if prerelease_a is None:\n # a has no prerelease, b does -> a > b\n return 1\n if prerelease_b is None:\n # b has no prerelease, a does -> a < b\n return -1\n\n # Both have prerelease, compare identifiers\n len_a = len(prerelease_a)\n len_b = len(prerelease_b)\n min_len = min(len_a, len_b)\n\n for i in range(min_len):\n type_a, val_a = prerelease_a[i]\n type_b, val_b = prerelease_b[i]\n\n # Numeric < alphanumeric\n if type_a == 'numeric' and type_b == 'alphanumeric':\n return -1\n if type_a == 'alphanumeric' and type_b == 'numeric':\n return 1\n \n if type_a == type_b:\n if val_a < val_b:\n return -1\n if val_a > val_b:\n return 1\n\n # All compared identifiers are equal; shorter set has lower precedence\n if len_a < len_b:\n return -1\n if len_a > len_b:\n return 1\n\n return 0\n```" + }, + { + "problem": "py-spiral", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 136552, + "loadMs": 33579, + "outputTokens": 344, + "decodeMs": 86318.118, + "promptTokens": 393, + "prefillMs": 16636.473, + "thinkingChars": 0, + "answer": "```python\ndef spiral_order(matrix):\n \"\"\"\n Returns elements of a rectangular matrix in clockwise spiral order.\n \n Args:\n matrix: A list of lists of integers (or any elements). \n Rows must be of equal length.\n \n Returns:\n A list of elements in spiral order.\n Returns [] for empty matrix or matrix of empty rows.\n \"\"\"\n if not matrix or not matrix[0]:\n return []\n\n top, bottom = 0, len(matrix) - 1\n left, right = 0, len(matrix[0]) - 1\n result = []\n\n while top <= bottom and left <= right:\n # Traverse from left to right along the top row\n for col in range(left, right + 1):\n result.append(matrix[top][col])\n top += 1\n\n # Traverse from top to bottom along the rightmost column\n if top <= bottom:\n for row in range(top, bottom + 1):\n result.append(matrix[row][right])\n right -= 1\n\n # Traverse from right to left along the bottom row\n if top <= bottom:\n for col in range(right, left - 1, -1):\n result.append(matrix[bottom][col])\n bottom -= 1\n\n # Traverse from bottom to top along the leftmost column\n if left <= right:\n for row in range(bottom, top - 1, -1):\n result.append(matrix[row][left])\n left += 1\n\n return result\n```" + }, + { + "problem": "py-csv-line", + "sample": 0, + "ok": false, + "error": "truncated by max tokens", + "truncated": true, + "wallMs": 589051, + "loadMs": 40097, + "outputTokens": 2048, + "decodeMs": 530024.586, + "promptTokens": 448, + "prefillMs": 18912.409, + "thinkingChars": 0, + "answer": "```python\ndef parse_csv_line(line):\n \"\"\"\n Parse a single CSV line into a list of fields.\n \n Rules:\n - Fields are separated by commas.\n - Empty line (or line with only whitespace) returns [\"\"] if it's just a newline/empty, \n but strictly following the prompt: \"An empty line is one empty field: [\\\"\\\"]\".\n We assume the input `line` is the raw string. If it's empty string, return [\"\"].\n - Quoted fields use double quotes. Inside quotes:\n - Commas are literal.\n - \"\" represents a single double quote.\n - Unquoted fields are taken as-is (no trimming).\n - Raise ValueError for unterminated quoted field or invalid character after closing quote.\n \n Args:\n line (str): The CSV line to parse.\n \n Returns:\n List[str]: List of parsed fields.\n \n Raises:\n ValueError: If the CSV is malformed (unterminated quote or invalid char after closing quote).\n \"\"\"\n if line == \"\":\n return [\"\"]\n \n fields = []\n current_field = []\n in_quotes = False\n i = 0\n n = len(line)\n \n while i < n:\n char = line[i]\n \n if in_quotes:\n if char == '\"':\n # Check if this is an escaped quote (\"\")\n if i + 1 < n and line[i + 1] == '\"':\n current_field.append('\"')\n i += 2 # Skip both quotes\n continue\n else:\n # End of quoted field\n in_quotes = False\n i += 1\n continue\n else:\n current_field.append(char)\n i += 1\n else:\n if char == ',':\n fields.append(\"\".join(current_field))\n current_field = []\n i += 1\n elif char == '\"':\n # Start of a quoted field\n in_quotes = True\n current_field.append(char)\n i += 1\n else:\n current_field.append(char)\n i += 1\n \n # After the loop, process the last field\n if in_quotes:\n raise ValueError(\"Unterminated quoted field\")\n \n fields.append(\"\".join(current_field))\n \n # Check for invalid characters after a closing quote? \n # The prompt says: \"or for any character other than a comma after a closing quote\".\n # Our logic: when we see a closing quote (and it's not escaped), we set in_quotes=False.\n # Then the next character must be a comma or end of string.\n # However, our loop continues. If after turning off quotes, we encounter something else?\n # Actually, the loop continues. If we are not in quotes and hit a non-comma, non-quote char, it's fine.\n # But what if we have: \"abc\"def? \n # Our logic: \n # - sees \" -> in_quotes=True\n # - sees \" -> ends quote, in_quotes=False. current_field has \"abc\" (actually \"abc\" without the closing quote? No.)\n # Wait, let's trace:\n # char == '\"' -> if escaped? no. else: in_quotes=False. current_field.append(char) was not done in the 'else' branch of the quote check?\n # Let's re-examine the code structure inside `if in_quotes:`\n # If char == '\"':\n # if next is '\"': append '\"', i+=2\n # else: in_quotes=False, i+=1. \n # BUT we did NOT append the closing quote to current_field. That's correct because the closing quote is not part of the field content.\n # However, the prompt says: \"\" stands for one double quote. So inside, \"\" becomes \". \n # My code: if escaped, append '\"', skip 2. Correct.\n # If not escaped, it's the end. We do not append the closing quote. Correct.\n # Now, what about the character AFTER the closing quote?\n # The loop continues. i is incremented. \n # If the next char is a comma, we treat it as a delimiter. Correct.\n # If the next char is something else (e.g. 'd' in \"abc\"def), we treat it as part of the next unquoted field.\n # Is that allowed? The prompt says: \"Raise ValueError for ... any character other than a comma after a closing quote.\"\n # This implies that if we have \"abc\"def, it should be an error because 'd' is not a comma.\n # My current code does NOT raise an error for \"abc\"def. It treats \"abc\" as the field, then 'def' as the next field.\n # To fix this: after exiting quotes (in_quotes becomes False), we must ensure the next character is either ',' or end of string.\n # However, the loop continues. We need to check the character immediately following the closing quote.\n # But we are inside the loop. When we hit the closing quote, we set in_quotes=False and i+=1.\n # The next iteration processes line[i].\n # We need to validate that line[i] (the one after the closing quote) is ',' or we are at the end?\n # Actually, the prompt says: \"any character other than a comma after a closing quote\".\n # So if we have \"abc\"def, after the closing quote, we see 'd'. 'd' is not a comma. Error.\n # If we have \"abc\", then end of string. OK.\n # If we have \"abc\",def. OK (comma is allowed).\n # So, after setting in_quotes=False, we must check if we are at the end of the string OR if the current char is a comma.\n # Wait, in the loop, `i` was incremented. So `line[i]` is the character AFTER the closing quote.\n # We need to check if `i < n` and `line[i] != ','`. If so, raise ValueError.\n # BUT, what if the line ends right there? Then `i == n`, so no check needed.\n # So, after `in_quotes = False`, we check:\n # if i < n and line[i] != ',':\n # raise ValueError(\"Invalid character after closing quote\")\n # However, the loop continues. If we raise an error, we stop.\n # But wait, the loop variable `i` is incremented. \n # Let's adjust the logic inside the `else` block of `if in_quotes:`.\n \n # Revised logic inside the loop for the quote handling:\n # if in_quotes:\n # if char == '\"':\n # if i + 1 < n and line[i + 1] == '\"':\n # current_field.append('\"')\n # i += 2\n # continue\n # else:\n # # Check the next char immediately\n # if i + 1 < n and line[i + 1] != ',':\n # raise ValueError(\"Invalid character after closing quote\")\n # in_quotes = False\n # i += 1 # Skip the closing quote\n # continue # Don't append the closing quote, and don't process the next char in this iteration's main logic?\n # # Wait, if I do `continue`, I skip the main `else` block logic which handles commas etc.\n # # But I need to process the next char in the next iteration.\n # # So `i += 1` and `continue` is correct. The next iteration will see `line[i+1]` (the char after quote).\n # # But wait, I incremented i. So next iteration `i` points to the char after quote.\n # # In the next iteration, `in_quotes` is False.\n # # Then in the `else` block (outer if in_quotes), we check if char == ',' -> split.\n # # If char is 'd', we append to current_field.\n # # This is wrong. We should have raised an error BEFORE appending 'd'.\n # # So the check must happen immediately after closing the quote.\n \n # Let's refactor slightly to ensure the check happens.\n \n # Actually, simpler:\n # When we see a closing quote (and it's not escaped):\n # 1. Validate next char is ',' or end of string.\n # 2. If valid, set in_quotes=False, append nothing for the quote, and skip the quote char (i+=1).\n # 3. Then, do NOT continue. Let the loop proceed? \n # If I do `continue`, I skip the rest of the loop body.\n # The next iteration starts. `i` is now pointing to the char after the quote.\n # If that char is 'd', we enter `" + }, + { + "problem": "py-lcs", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 181677, + "loadMs": 34694, + "outputTokens": 500, + "decodeMs": 129896.514, + "promptTokens": 393, + "prefillMs": 17073.341, + "thinkingChars": 0, + "answer": "```python\ndef longest_common_subsequence(a: str, b: str) -> str:\n \"\"\"\n Returns one longest common subsequence of strings a and b.\n \n Uses dynamic programming with space optimization.\n Time Complexity: O(len(a) * len(b))\n Space Complexity: O(len(a) * len(b)) for the DP table.\n \n Given the constraint of 2000 characters, a standard 2D array \n of 2000x2000 integers (approx 4 million ints) is roughly 16MB, \n which fits comfortably in memory and runs well within a couple of seconds.\n \"\"\"\n if not a or not b:\n return \"\"\n \n n = len(a)\n m = len(b)\n \n # dp[i][j] will store the length of LCS of a[:i] and b[:j]\n # We use a list of lists.\n dp = [[0] * (m + 1) for _ in range(n + 1)]\n \n # Fill the DP table\n for i in range(1, n + 1):\n char_a = a[i - 1]\n for j in range(1, m + 1):\n if char_a == b[j - 1]:\n dp[i][j] = dp[i - 1][j - 1] + 1\n else:\n dp[i][j] = max(dp[i - 1][j], dp[i][j - 1])\n \n # Backtrack to find the actual subsequence\n lcs_length = dp[n][m]\n lcs_chars = [''] * lcs_length\n i, j = n, m\n \n while i > 0 and j > 0:\n if a[i - 1] == b[j - 1]:\n lcs_chars[lcs_length - 1] = a[i - 1]\n i -= 1\n j -= 1\n lcs_length -= 1\n elif dp[i - 1][j] >= dp[i][j - 1]:\n i -= 1\n else:\n j -= 1\n \n return \"\".join(lcs_chars)\n```" + }, + { + "problem": "py-number-words", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-PChgp4/test_solution.py\", line 7, in \n assert number_to_words(n) == words, (n, number_to_words(n))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: (42, 'forty two')\n", + "truncated": false, + "wallMs": 252330, + "loadMs": 33555, + "outputTokens": 745, + "decodeMs": 197014.106, + "promptTokens": 503, + "prefillMs": 21744.994, + "thinkingChars": 0, + "answer": "```python\ndef number_to_words(n):\n \"\"\"\n Converts an integer between 0 and 999,999,999,999 (inclusive) to its English word representation.\n \n Rules:\n - Lowercase words.\n - Single spaces between words.\n - Hyphens for numbers 21-99 that are compounds (e.g., \"twenty-one\").\n - No \"and\".\n - Scale words: thousand, million, billion.\n \n Raises:\n ValueError: If n is not an integer or outside the valid range.\n \"\"\"\n if not isinstance(n, int):\n raise ValueError(\"Input must be an integer.\")\n \n if n < 0 or n > 999_999_999_999:\n raise ValueError(\"Input must be between 0 and 999,999,999,999 inclusive.\")\n \n ones = [\n \"\", \"one\", \"two\", \"three\", \"four\", \"five\", \"six\", \"seven\", \"eight\", \"nine\",\n \"ten\", \"eleven\", \"twelve\", \"thirteen\", \"fourteen\", \"fifteen\", \"sixteen\",\n \"seventeen\", \"eighteen\", \"nineteen\"\n ]\n \n tens = [\n \"\", \"\", \"twenty\", \"thirty\", \"forty\", \"fifty\", \"sixty\", \"seventy\", \"eighty\", \"ninety\"\n ]\n \n def convert_under_thousand(num):\n if num == 0:\n return \"\"\n \n result = []\n \n # Hundreds place\n if num >= 100:\n result.append(ones[num // 100])\n result.append(\"hundred\")\n num %= 100\n \n # Tens and ones place\n if num > 0:\n if num < 20:\n result.append(ones[num])\n else:\n ten_digit = num // 10\n one_digit = num % 10\n result.append(tens[ten_digit])\n if one_digit > 0:\n result.append(ones[one_digit])\n \n return \" \".join(result)\n \n if n == 0:\n return \"zero\"\n \n parts = []\n \n # Billions\n if n >= 1_000_000_000:\n parts.append(convert_under_thousand(n // 1_000_000_000) + \" billion\")\n n %= 1_000_000_000\n \n # Millions\n if n >= 1_000_000:\n parts.append(convert_under_thousand(n // 1_000_000) + \" million\")\n n %= 1_000_000\n \n # Thousands\n if n >= 1_000:\n parts.append(convert_under_thousand(n // 1_000) + \" thousand\")\n n %= 1_000\n \n # Remainder\n if n > 0:\n parts.append(convert_under_thousand(n))\n \n return \" \".join(parts)\n```" + }, + { + "problem": "py-dijkstra", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 212111, + "loadMs": 35103, + "outputTokens": 621, + "decodeMs": 156669.834, + "promptTokens": 470, + "prefillMs": 20311.358, + "thinkingChars": 0, + "answer": "```python\nimport heapq\nfrom typing import List, Tuple, Union\n\ndef shortest_distances(\n n: int,\n edges: List[Tuple[Union[int, float], Union[int, float], Union[int, float]]],\n source: Union[int, float]\n) -> List[float]:\n \"\"\"\n Compute shortest distances from a source node in a directed weighted graph.\n \n Parameters:\n n (int): Number of nodes (0 to n-1).\n edges (List[Tuple[int, int, float]]): List of (u, v, weight) tuples.\n source (int): Source node index.\n \n Returns:\n List[float]: Shortest distance from source to each node. Unreachable nodes have float('inf').\n \n Raises:\n ValueError: If any weight is negative, source is invalid, or node indices are out of range.\n \"\"\"\n # Validate source\n source = int(source)\n if source < 0 or source >= n:\n raise ValueError(f\"Source node {source} is not a valid node (must be 0..{n-1}).\")\n \n # Validate edges and build adjacency list\n # Using a dictionary for the graph to handle sparse structures efficiently,\n # then converting to a list of lists if needed, or keeping as dict for memory efficiency.\n # Given n up to 20,000, a list of lists is acceptable and often faster for dense lookups.\n graph = [[] for _ in range(n)]\n \n for u, v, w in edges:\n u = int(u)\n v = int(v)\n w = float(w)\n \n # Validate node indices\n if u < 0 or u >= n or v < 0 or v >= n:\n raise ValueError(f\"Node index out of range: ({u}, {v}) for n={n}.\")\n \n # Validate weight\n if w < 0:\n raise ValueError(f\"Negative weight detected: {w} for edge ({u}, {v}).\")\n \n graph[u].append((v, w))\n \n # Dijkstra's algorithm\n distances = [float('inf')] * n\n distances[source] = 0.0\n \n # Priority queue: (distance, node)\n pq = [(0.0, source)]\n \n while pq:\n dist, u = heapq.heappop(pq)\n \n # Skip if we found a shorter path to u already\n if dist > distances[u]:\n continue\n \n for v, weight in graph[u]:\n new_dist = distances[u] + weight\n if new_dist < distances[v]:\n distances[v] = new_dist\n heapq.heappush(pq, (new_dist, v))\n \n return distances\n```" + }, + { + "problem": "py-slugify", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 168021, + "loadMs": 38353, + "outputTokens": 449, + "decodeMs": 109680.614, + "promptTokens": 478, + "prefillMs": 19969.94, + "thinkingChars": 0, + "answer": "```python\nimport unicodedata\nimport re\n\n\ndef slugify(text: str, max_length: int = 50) -> str:\n \"\"\"\n Convert text to an ASCII URL slug.\n \n - Lowercase the text and remove accents (e.g., '\u00e9' -> 'e').\n - Map special characters: \u00df -> ss, \u00e6 -> ae, \u0153 -> oe, \u00f8 -> o.\n - Replace any run of non-alphanumeric characters with a single hyphen.\n - Strip leading/trailing hyphens.\n - Truncate to max_length, removing any trailing hyphen.\n \n Raises ValueError if max_length < 1.\n \"\"\"\n if max_length < 1:\n raise ValueError(\"max_length must be at least 1\")\n\n # Normalize to NFKD to decompose characters with accents\n normalized = unicodedata.normalize(\"NFKD\", text)\n \n # Lowercase\n text = normalized.lower()\n \n # Map specific special characters\n text = text.replace(\"\u00df\", \"ss\")\n text = text.replace(\"\u00e6\", \"ae\")\n text = text.replace(\"\u0153\", \"oe\")\n text = text.replace(\"\u00f8\", \"o\")\n \n # Remove remaining accents/non-ASCII by filtering out combining characters\n # (NFKD usually separates base char + combining marks; we keep only base chars)\n ascii_text = \"\".join(\n char for char in text\n if ord(char) < 128\n )\n \n # Replace any non-alphanumeric sequence with a single hyphen\n slug = re.sub(r\"[^a-z0-9]+\", \"-\", ascii_text)\n \n # Strip leading/trailing hyphens\n slug = slug.strip(\"-\")\n \n # Truncate if necessary\n if len(slug) > max_length:\n slug = slug[:max_length]\n # Remove trailing hyphen if present after truncation\n if slug.endswith(\"-\"):\n slug = slug[:-1]\n \n return slug\n```" + }, + { + "problem": "py-base-convert", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-beilzE/test_solution.py\", line 3, in \n assert convert_base(\"FF\", 16, 10) == \"255\"\n ~~~~~~~~~~~~^^^^^^^^^^^^^^\n File \"/tmp/flycoder-bench-beilzE/solution.py\", line 24, in convert_base\n raise ValueError(f\"Invalid character '{char}' for base {from_base}\")\nValueError: Invalid character 'F' for base 16\n", + "truncated": false, + "wallMs": 184969, + "loadMs": 35087, + "outputTokens": 523, + "decodeMs": 131226.723999, + "promptTokens": 471, + "prefillMs": 18641.048, + "thinkingChars": 0, + "answer": "```python\ndef convert_base(number: str, from_base: int, to_base: int) -> str:\n # Validate bases\n if not (2 <= from_base <= 36) or not (2 <= to_base <= 36):\n raise ValueError(f\"Bases must be between 2 and 36. Got from_base={from_base}, to_base={to_base}\")\n \n # Validate number string\n if not number or number == \"-\":\n raise ValueError(\"Number string cannot be empty or just '-'\")\n \n # Normalize input: strip whitespace (not specified but good practice), handle sign\n is_negative = number.startswith(\"-\")\n if is_negative:\n number = number[1:]\n \n if not number:\n raise ValueError(\"Number string cannot be empty or just '-'\")\n \n # Check for invalid characters for the given from_base\n valid_chars = \"0123456789abcdefghijklmnopqrstuvwxyz\"\n char_to_val = {c: i for i, c in enumerate(valid_chars)}\n \n for char in number:\n if char not in char_to_val:\n raise ValueError(f\"Invalid character '{char}' for base {from_base}\")\n if char_to_val[char] >= from_base:\n raise ValueError(f\"Digit '{char}' is invalid for base {from_base}\")\n \n # Convert from from_base to base 10 (internal integer)\n value = 0\n for char in number:\n digit_val = char_to_val[char.lower()]\n value = value * from_base + digit_val\n \n # Handle zero\n if value == 0:\n return \"0\"\n \n # Handle negative numbers\n if is_negative:\n value = -value\n \n # Convert from base 10 to to_base\n if value < 0:\n result_chars = []\n value = -value\n while value > 0:\n remainder = value % to_base\n result_chars.append(valid_chars[remainder])\n value //= to_base\n result = \"\".join(reversed(result_chars))\n return \"-\" + result\n else:\n result_chars = []\n while value > 0:\n remainder = value % to_base\n result_chars.append(valid_chars[remainder])\n value //= to_base\n result = \"\".join(reversed(result_chars))\n return result\n```" + }, + { + "problem": "js-parse-query", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 302323, + "loadMs": 34331, + "outputTokens": 999, + "decodeMs": 246128.478, + "promptTokens": 520, + "prefillMs": 21847.65, + "thinkingChars": 0, + "answer": "```javascript\n/**\n * Parses a URL query string into a plain object.\n * \n * @param {string} query - The raw query string (optional leading \"?\").\n * @returns {Object} A plain object representing the parsed query parameters.\n */\nexport function parseQuery(query) {\n if (typeof query !== 'string') {\n throw new TypeError('query must be a string');\n }\n\n // Remove leading \"?\" if present\n const cleanQuery = query.replace(/^\\?/, '');\n\n // Split by \"&\"\n const pairs = cleanQuery.split('&');\n\n // Map to store intermediate results: key -> { values: [], isPrototype: false }\n const map = new Map();\n\n for (const pair of pairs) {\n if (pair.length === 0) {\n continue; // Ignore empty pairs\n }\n\n // Split on the first \"=\" only\n const eqIndex = pair.indexOf('=');\n let key, value;\n\n if (eqIndex === -1) {\n // No \"=\" found: key is the whole pair, value is empty string\n key = pair;\n value = '';\n } else {\n key = pair.slice(0, eqIndex);\n value = pair.slice(eqIndex + 1);\n }\n\n // Handle '+' as space\n let decodedKey = key.replace(/\\+/g, ' ');\n let decodedValue = value.replace(/\\+/g, ' ');\n\n // Percent-decode (case-insensitive, handles %XX sequences)\n // If decoding fails or produces invalid sequences, we keep the original text \n // but with '+' already converted to space.\n try {\n decodedKey = decodeURIComponent(decodedKey);\n decodedValue = decodeURIComponent(decodedValue);\n } catch {\n // If percent-decoding fails, keep the text as-is (with + -> space done above)\n // Note: We must re-escape any % that were not valid hex sequences to avoid \n // accidentally interpreting them later, but since we are returning a plain object \n // and the spec says \"keep that text as it is\", we just use the string after + replacement.\n // However, to be safe and consistent with \"keep that text as it is\", we revert the \n // + replacement for the parts that failed? No, the spec says: \"with \"+\" still turned into spaces\".\n // So we keep the string with spaces where + was, and the % sequences as they were (invalid).\n // We do not revert the + replacement.\n // But decodeURIComponent might have partially decoded? No, it throws on invalid.\n // So we just use the pre-decode string (with + replaced).\n // Wait, decodeURIComponent throws if there is an invalid sequence.\n // So we catch the error and use the string that had + replaced.\n // We do NOT try to decode partially.\n }\n\n // Check if key is a prototype pollution attempt\n const isPrototype = key === '__proto__' || key === 'constructor' || key === 'prototype';\n\n if (!map.has(key)) {\n map.set(key, { values: [], isPrototype: isPrototype });\n }\n const entry = map.get(key);\n\n if (entry.isPrototype) {\n // If the key is dangerous, we still store it in the map to return it as an own property\n // but we must ensure we don't accidentally set Object.prototype.\n // Since we are building a new object at the end, we can just store the raw key string.\n // The only risk is if we do Object.create(null) vs regular object.\n // We will build a regular object but set the property explicitly.\n }\n\n entry.values.push(decodedValue);\n }\n\n // Build the result object\n const result = Object.create(null); // Create a plain object without prototype chain issues initially\n\n for (const [key, entry] of map) {\n const { values } = entry;\n\n if (values.length === 1) {\n // Single value\n // Explicitly set as own property to avoid prototype pollution\n Object.defineProperty(result, key, {\n value: values[0],\n writable: true,\n enumerable: true,\n configurable: true\n });\n } else {\n // Multiple values\n Object.defineProperty(result, key, {\n value: values,\n writable: true,\n enumerable: true,\n configurable: true\n });\n }\n }\n\n return result;\n}\n```" + }, + { + "problem": "js-deep-equal", + "sample": 0, + "ok": false, + "error": "node:internal/modules/run_main:123\n triggerUncaughtException(\n ^\n\nAssertionError [ERR_ASSERTION]: Expected values to be strictly equal:\n\nfalse !== true\n\n at file:///tmp/flycoder-bench-lljLje/test.mjs:3:35 {\n generatedMessage: true,\n code: 'ERR_ASSERTION',\n actual: false,\n expected: true,\n operator: 'strictEqual',\n diff: 'simple'\n}\n\nNode.js v22.22.0\n", + "truncated": false, + "wallMs": 208253, + "loadMs": 33827, + "outputTokens": 629, + "decodeMs": 154223.426, + "promptTokens": 510, + "prefillMs": 20187.245, + "thinkingChars": 0, + "answer": "```javascript\n/**\n * Deep equality check supporting cyclic structures.\n * Uses SameValueZero semantics for primitives.\n */\n\nexport function deepEqual(a, b) {\n const cache = new WeakMap();\n \n function isEqual(a, b) {\n // SameValueZero check for primitives\n if (a === b) return true;\n if (typeof a !== 'object' || a === null || typeof b !== 'object' || b === null) {\n return false;\n }\n\n // Handle cyclic structures\n const aStack = cache.get(a);\n const bStack = cache.get(b);\n \n if (aStack) {\n if (!bStack) return false;\n return isEqual(aStack, bStack);\n }\n if (bStack) {\n return false;\n }\n\n // Mark as visiting\n cache.set(a, b);\n cache.set(b, a);\n\n const aIsDate = a instanceof Date;\n const bIsDate = b instanceof Date;\n\n if (aIsDate !== bIsDate) {\n cache.delete(a);\n cache.delete(b);\n return false;\n }\n\n if (aIsDate) {\n return a.getTime() === b.getTime();\n }\n\n const aIsArray = Array.isArray(a);\n const bIsArray = Array.isArray(b);\n\n if (aIsArray !== bIsArray) {\n cache.delete(a);\n cache.delete(b);\n return false;\n }\n\n // Arrays\n if (aIsArray) {\n if (a.length !== b.length) {\n cache.delete(a);\n cache.delete(b);\n return false;\n }\n for (let i = 0; i < a.length; i++) {\n if (!isEqual(a[i], b[i])) {\n cache.delete(a);\n cache.delete(b);\n return false;\n }\n }\n } else {\n // Objects\n const aKeys = Object.keys(a);\n const bKeys = Object.keys(b);\n \n if (aKeys.length !== bKeys.length) {\n cache.delete(a);\n cache.delete(b);\n return false;\n }\n \n // Compare values for each key in 'a'\n for (const key of aKeys) {\n if (!b.hasOwnProperty(key)) {\n cache.delete(a);\n cache.delete(b);\n return false;\n }\n if (!isEqual(a[key], b[key])) {\n cache.delete(a);\n cache.delete(b);\n return false;\n }\n }\n }\n\n // Clear cache for this branch\n cache.delete(a);\n cache.delete(b);\n return true;\n }\n\n return isEqual(a, b);\n}\n```" + }, + { + "problem": "js-evaluate", + "sample": 0, + "ok": false, + "error": "file:///tmp/flycoder-bench-LyHpEs/solution.mjs:120\n throw new SyntaxError('Unexpected characters at the end of the expression');\n ^\n\nSyntaxError: Unexpected characters at the end of the expression\n at evaluate (file:///tmp/flycoder-bench-LyHpEs/solution.mjs:120:11)\n at file:///tmp/flycoder-bench-LyHpEs/test.mjs:6:9\n\nNode.js v22.22.0\n", + "truncated": false, + "wallMs": 275286, + "loadMs": 33823, + "outputTokens": 867, + "decodeMs": 220870.048, + "promptTokens": 499, + "prefillMs": 20574.9, + "thinkingChars": 0, + "answer": "```javascript\n// Evaluate an arithmetic expression\n// Supported: non-negative decimal numbers, + - * /, parentheses, unary +/-, whitespace.\n// Precedence: unary > * / > + -. Left-associative for binary.\n// Throws: SyntaxError for malformed input, RangeError for division by zero.\nexport function evaluate(expression) {\n let pos = 0;\n const len = expression.length;\n let ch;\n\n const peek = () => expression[pos];\n const next = () => {\n pos++;\n return peek();\n };\n \n const skipWhitespace = () => {\n while (pos < len && /\\s/.test(expression[pos])) {\n pos++;\n }\n };\n\n const parseNumber = () => {\n skipWhitespace();\n let start = pos;\n if (expression[pos] === '.') {\n // Handle cases like \".5\"\n pos++;\n }\n while (pos < len && /[0-9]/.test(expression[pos])) {\n pos++;\n }\n if (start === pos) {\n throw new SyntaxError('Expected a number');\n }\n return parseFloat(expression.slice(start, pos));\n };\n\n const parseUnary = () => {\n skipWhitespace();\n if (expression[pos] === '+') {\n pos++;\n return parseUnary(); // Unary + has same precedence as unary -, but we parse the operand next.\n }\n if (expression[pos] === '-') {\n pos++;\n // Check if it's actually a unary minus (not a binary subtraction)\n // A binary subtraction would have been handled by the binary parser if it wasn't unary.\n // However, since we are in unary context, we just negate the next primary.\n return -parseUnary();\n }\n return parsePrimary();\n };\n\n const parsePrimary = () => {\n skipWhitespace();\n if (expression[pos] === '(') {\n pos++; // consume '('\n const result = parseUnary();\n if (pos >= len || expression[pos] !== ')') {\n throw new SyntaxError('Expected closing parenthesis');\n }\n pos++; // consume ')'\n return result;\n }\n return parseNumber();\n };\n\n const parseUnaryExpr = () => {\n return parseUnary();\n };\n\n const parseMulDiv = (left) => {\n skipWhitespace();\n while (true) {\n if (expression[pos] === '*') {\n pos++;\n const right = parseUnaryExpr();\n left = left * right;\n } else if (expression[pos] === '/') {\n pos++;\n const right = parseUnaryExpr();\n if (right === 0) {\n throw new RangeError('Division by zero');\n }\n left = left / right;\n } else {\n break;\n }\n }\n return left;\n };\n\n const parseAddSub = (left) => {\n skipWhitespace();\n while (true) {\n if (expression[pos] === '+') {\n pos++;\n const right = parseUnaryExpr();\n left = left + right;\n } else if (expression[pos] === '-') {\n pos++;\n const right = parseUnaryExpr();\n left = left - right;\n } else {\n break;\n }\n }\n return left;\n };\n\n const parseExpression = () => {\n return parseMulDiv(parseUnaryExpr());\n };\n\n const result = parseExpression();\n\n // Ensure no trailing characters (except whitespace which is skipped by the logic implicitly if we check pos)\n skipWhitespace();\n if (pos !== len) {\n throw new SyntaxError('Unexpected characters at the end of the expression');\n }\n\n return result;\n}\n```" + }, + { + "problem": "js-format-bytes", + "sample": 0, + "ok": false, + "error": "node:internal/modules/run_main:123\n triggerUncaughtException(\n ^\n\nAssertionError [ERR_ASSERTION]: Expected values to be strictly equal:\n+ actual - expected\n\n+ '1023.0 B'\n- '1023 B'\n ^\n\n at file:///tmp/flycoder-bench-jqlM3e/test.mjs:3:35 {\n generatedMessage: true,\n code: 'ERR_ASSERTION',\n actual: '1023.0 B',\n expected: '1023 B',\n operator: 'strictEqual',\n diff: 'simple'\n}\n\nNode.js v22.22.0\n", + "truncated": false, + "wallMs": 389051, + "loadMs": 34822, + "outputTokens": 1336, + "decodeMs": 330764.202, + "promptTokens": 569, + "prefillMs": 23448.584, + "thinkingChars": 0, + "answer": "```javascript\n/**\n * Formats a byte count using binary (IEC) units.\n * \n * @param {number} bytes - The byte count to format.\n * @param {number} [decimals=1] - Number of decimal places to display.\n * @returns {string} The formatted string.\n * @throws {TypeError} If bytes is not a finite number.\n * @throws {RangeError} If decimals is not an integer between 0 and 10 (inclusive).\n */\nexport function formatBytes(bytes, decimals = 1) {\n // Validate `bytes`\n if (typeof bytes !== 'number' || !Number.isFinite(bytes)) {\n throw new TypeError('bytes must be a finite number');\n }\n\n // Validate `decimals`\n if (!Number.isInteger(decimals) || decimals < 0 || decimals > 10) {\n throw new RangeError('decimals must be an integer from 0 to 10');\n }\n\n const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];\n const factor = 1024;\n \n // Handle zero explicitly to avoid floating point issues at boundaries\n if (bytes === 0) {\n return '0 B';\n }\n\n // Determine the appropriate unit\n let unitIndex = 0;\n let value = bytes;\n\n // Iterate upwards while value >= 1024\n while (value >= factor && unitIndex < units.length - 1) {\n value /= factor;\n unitIndex++;\n }\n\n // Format the number\n // Use toFixed to ensure exactly `decimals` digits, then trim trailing zeros? \n // The prompt says \"exactly `decimals` digits after the point\", e.g. \"1.5 KiB\".\n // Standard toFixed returns trailing zeros (e.g. 1.50). \n // However, usually \"exactly n digits\" in these contexts implies semantic precision,\n // but strict adherence to \"exactly\" often means visual representation.\n // Let's look at the example: \"1.5 KiB\". If decimals=0, \"1 KiB\".\n // If decimals=1, \"1.5 KiB\". If decimals=2, \"1.50 KiB\"? \n // Usually, file size formatters trim trailing zeros unless specified otherwise.\n // But the instruction says \"exactly `decimals` digits\". \n // Let's assume standard formatting where trailing zeros might be trimmed for readability \n // OR strictly kept. Given \"exactly\", strict toFixed is safer, but typically \n // users expect \"1.5\" not \"1.50\" unless scientific. \n // Re-reading: \"format it with exactly `decimals` digits after the point, e.g. '1.5 KiB'\".\n // If I pass 2 decimals, should it be \"1.50\"? \n // Let's stick to the visual expectation of \"exactly\" meaning the precision level, \n // but standard practice in JS formatters (like `Intl.NumberFormat`) handles this.\n // However, simple `toFixed` keeps zeros. Let's use `toFixed` and then trim trailing zeros \n // if they are purely for padding, OR keep them if strict.\n // Actually, looking at common implementations (e.g., Chrome DevTools), they trim trailing zeros.\n // But the prompt says \"exactly\". Let's implement `toFixed` and trim trailing zeros \n // to match the \"1.5\" example, assuming 2 decimals for 1.5 would be \"1.5\" visually.\n // Wait, if the requirement is strict \"exactly\", then 1.5 with 2 decimals should be \"1.50\".\n // Let's re-read carefully: \"format it with exactly `decimals` digits after the point\".\n // Example: \"1.5 KiB\". This implies if decimals=1.\n // If decimals=2, is it \"1.50\"? \n // Let's assume standard `toFixed` behavior (keeping zeros) is safer for \"exactly\", \n // but often users want trimmed. \n // Let's check the constraint: \"Values below 1024 are integers\". \n // Okay, let's use `toFixed` and then `.replace(/\\.?0+$/, '')` to trim trailing zeros \n // unless the number is an integer? No, that changes the \"exactly\" requirement.\n // Actually, most simple formatters trim. But \"exactly\" suggests `toFixed`.\n // Let's look at the example again: \"1.5 KiB\". \n // If I write code that produces \"1.50\", is it wrong? \n // Let's assume the user wants the visual representation of that precision.\n // I will use `toFixed` and then trim trailing zeros to make it look natural, \n // BUT the prompt says \"exactly\". \n // Decision: I will use `toFixed` and NOT trim, because \"exactly\" usually implies \n // fixed-point notation in strict contexts. However, if the example \"1.5\" is the ONLY \n // example, it might imply trimming. \n // Let's compromise: Use `toFixed`. If the result ends in 0, keep it? \n // Actually, looking at similar tasks, \"exactly n digits\" usually means `toFixed(n)`.\n // I will use `toFixed(decimals)` directly.\n \n // Wait, if I have 1.5 and decimals=1, `toFixed(1)` -> \"1.5\".\n // If I have 1.5 and decimals=2, `toFixed(2)` -> \"1.50\".\n // Is \"1.50\" correct? Yes, it has exactly 2 digits.\n // So I will use `toFixed`.\n\n const formattedValue = value.toFixed(decimals);\n \n return `${formattedValue} ${units[unitIndex]}`;\n}\n```" + }, + { + "problem": "js-wildcard", + "sample": 0, + "ok": false, + "error": "truncated by max tokens", + "truncated": true, + "wallMs": 579325, + "loadMs": 35075, + "outputTokens": 2048, + "decodeMs": 523563.547, + "promptTokens": 471, + "prefillMs": 20670.212, + "thinkingChars": 0, + "answer": "```javascript\n/**\n * Matches a text against a wildcard pattern.\n * '?' matches exactly one character.\n * '*' matches any sequence of characters (including empty).\n * Other characters match themselves.\n * \n * Implements a greedy algorithm similar to standard regex engines but using\n * state tracking to handle '*' efficiently without exponential backtracking.\n * \n * @param {string} pattern - The wildcard pattern.\n * @param {string} text - The text to match against.\n * @returns {boolean} - True if the whole text matches the pattern, false otherwise.\n */\nexport function matchWildcard(pattern, text) {\n if (pattern === null || text === null) {\n return false;\n }\n\n const pLen = pattern.length;\n const tLen = text.length;\n\n // If pattern is empty, text must also be empty to match\n if (pLen === 0) {\n return tLen === 0;\n }\n\n // If text is empty, pattern must consist only of '*'\n if (tLen === 0) {\n // Check if pattern is just a sequence of '*'\n return pattern.split('*').length === 1;\n }\n\n // We will track the position after the last '*' we encountered.\n // We also track the position in the pattern that corresponds to the last '*'\n // and the position in the text where we matched the character after that '*'.\n \n // However, a more robust O(N*M) worst-case but practically O(N) approach\n // for this specific constraint (avoiding quadratic blowup on \"*a\" patterns)\n // is to use a two-pass or state-machine approach.\n \n // Let's use the standard \"jump\" algorithm:\n // 1. Find the first '*' in the pattern.\n // 2. If no '*', simple character-by-character check.\n // 3. If there is a '*', split pattern into \"prefix\" and \"suffix\".\n // \"prefix\" must match the start of \"text\".\n // Then we try to match \"suffix\" against the rest of \"text\", allowing\n // for any number of characters consumed by the '*'.\n \n // To handle multiple '*' correctly, we can process the pattern greedily.\n // Actually, the simplest robust way for this specific problem (no escaping)\n // is to find the first '*' and verify the prefix, then for the suffix,\n // we need to see if the remaining text matches the suffix pattern.\n // BUT, standard regex matching logic for wildcard usually treats '*' as \n // matching any sequence. If we have \"a*b\", it means text starts with 'a',\n // ends with 'b', and anything in between.\n \n // Algorithm:\n // 1. Find the index of the first '*' in the pattern.\n // 2. If no '*', simple strict match.\n // 3. If there is a '*', let's say at index `starIdx`.\n // - Match pattern[0...starIdx-1] against text[0...].\n // - If match fails, return false.\n // - Let `matchEnd` be the index in text where the prefix matched ended.\n // - Now we need to match pattern[starIdx+1...] against text[matchEnd...].\n // - However, the '*' could have consumed MORE characters.\n // - So we need to find a position `i` >= `matchEnd` such that\n // text[matchEnd...i-1] corresponds to the '*' (any sequence)\n // and pattern[starIdx+1...] matches text[i...].\n // - This is equivalent to: does the suffix pattern match the suffix of text?\n // But the suffix of text must start at some point `i` >= `matchEnd`.\n // Actually, since '*' matches ANY sequence, we just need to check if\n // the suffix pattern can match a suffix of the text starting at or after `matchEnd`.\n // Wait, that's not quite right.\n // Example: pattern=\"a*b\", text=\"axxb\". \n // Prefix \"a\" matches text[0]. matchEnd = 1.\n // Suffix \"b\" must match text[i...] where i >= 1.\n // We need to find if there exists an i >= matchEnd such that\n // text[i...] matches suffix.\n // Since the suffix has no '*' (if it did, we would have split earlier),\n // we just need to check if the suffix pattern matches the text starting\n // from some index `i` >= `matchEnd`.\n // To do this efficiently without re-scanning, we can just try to match\n // the suffix against the text starting from `matchEnd` onwards, but allowing\n // the match to start later? No, standard matching requires exact order.\n // \n // Correct Logic for multiple '*':\n // Actually, if we treat '*' as a wildcard for \"any sequence\", and we want to match\n // the WHOLE text, we can simplify:\n // Find the first '*'. Split pattern into P and S.\n // P must match the beginning of text.\n // S must match the END of text.\n // The '*' matches everything in between.\n // So, we need to check:\n // 1. P matches text[0 ... |P|-1]\n // 2. S matches text[|text| - |S| ... |text|]\n // 3. |text| >= |P| + |S|.\n // \n // What if there are multiple '*'? e.g. \"a*b*c\".\n // We can process left-to-right.\n // Actually, the standard algorithm for \"glob\" style matching (without escaping)\n // is often implemented by finding the first '*' and then checking if the rest\n // of the pattern matches the rest of the text, but we can't just check the rest\n // of the text directly because the '*' can consume varying amounts.\n // \n // However, notice that if we have \"a*b*c\", it is equivalent to:\n // Match \"a\" at start, then match \"b\" at end, and \"c\" at end? No.\n // It means: text starts with \"a\", then some chars, then \"b\", then some chars, then \"c\".\n // This is complex if there are multiple '*'.\n // \n // BUT, the problem statement says: \"Do not build a RegExp\".\n // And \"adversarial inputs: patterns and texts of a few thousand characters\".\n // \n // Let's reconsider the definition.\n // If pattern is \"a*b\", it matches \"a\", \"ab\", \"axb\", \"axxb\", etc.\n // If pattern is \"a*b*c\", it matches \"a...b...c\".\n // \n // Efficient approach for multiple '*':\n // We can iterate through the pattern and text.\n // When we hit a '*', we record the current position in the text as the \"start of suffix match\".\n // We then try to match the rest of the pattern against the rest of the text starting from\n // the current text position. BUT, since '*' can consume more, we need to be careful.\n // \n // Actually, a simpler logic for \"glob\" matching (where '*' is greedy or non-greedy doesn't matter\n // for exact match existence) is:\n // 1. Find the first '*'.\n // 2. If no '*', simple match.\n // 3. If there is a '*', split into prefix and suffix.\n // - Check if prefix matches start of text.\n // - Check if suffix matches end of text.\n // - If both match, and text length is sufficient, return true.\n // - Wait, this logic works ONLY if there is exactly one '*'.\n // - If there are multiple '*', e.g. \"a*b*c\", we can treat it as:\n // Match \"a\", then match \"b\", then match \"c\"? No, the '*'s are wildcards.\n // \"a*b*c\" -> \"a\" ... \"b\" ... \"c\".\n // We can reduce multiple '*' to a single logical wildcard by merging them?\n // No, \"a*b*c\" does NOT mean \"a\" ... \"b\" ... \"c\" where the middle part is arbitrary.\n // It means: \"a\" followed by any chars, then \"b\", then any chars, then \"c\".\n // This is strictly equivalent to: \n // Match \"a\" at start.\n // Match \"b\" at some point.\n // Match \"c\" at end.\n // And everything in between is covered by the '*'s.\n // Actually, any sequence of '*'s can be treated as ONE single '*' that matches\n // everything between the prefix and the suffix?\n // Example: \"a*b*c\". \n // Is \"axbyc\" a match? Yes.\n // Is \"axbc" + }, + { + "problem": "js-search-range", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 209409, + "loadMs": 34313, + "outputTokens": 589, + "decodeMs": 156615.454, + "promptTokens": 424, + "prefillMs": 18462.192, + "thinkingChars": 0, + "answer": "```javascript\n/**\n * Finds the first and last index of a target value in a sorted array.\n * Uses binary search to achieve O(log n) time complexity.\n * \n * @param {number[]} sorted - An array of numbers sorted in ascending order.\n * @param {number} target - The number to find.\n * @returns {[number, number]} An array [first, last] containing the indices, \n * or [-1, -1] if the target is not found.\n */\nexport function searchRange(sorted, target) {\n // Helper: Binary search to find the leftmost (first) occurrence\n function findLeft(arr, target) {\n let left = 0;\n let right = arr.length - 1;\n \n while (left <= right) {\n const mid = (left + right) >> 1; // Bitwise shift for division by 2\n \n if (arr[mid] < target) {\n left = mid + 1;\n } else if (arr[mid] > target) {\n right = mid - 1;\n } else {\n // arr[mid] == target, try to find a smaller index\n right = mid - 1;\n }\n }\n \n // Check if the index is valid and matches the target\n if (left < arr.length && arr[left] === target) {\n return left;\n }\n return -1;\n }\n\n // Helper: Binary search to find the rightmost (last) occurrence\n function findRight(arr, target) {\n let left = 0;\n let right = arr.length - 1;\n \n while (left <= right) {\n const mid = (left + right) >> 1;\n \n if (arr[mid] < target) {\n left = mid + 1;\n } else if (arr[mid] > target) {\n right = mid - 1;\n } else {\n // arr[mid] == target, try to find a larger index\n left = mid + 1;\n }\n }\n \n // Check if the index is valid and matches the target\n if (right >= 0 && arr[right] === target) {\n return right;\n }\n return -1;\n }\n\n const first = findLeft(sorted, target);\n \n // If target is not found, return [-1, -1]\n if (first === -1) {\n return [-1, -1];\n }\n \n const last = findRight(sorted, target);\n return [first, last];\n}\n```" + } + ] + }, + { + "summary": { + "model": "flycoder-pro-recipe9b", + "passed": 6, + "total": 20, + "truncated": 2, + "meanOutputTokens": 771.85 + }, + "results": [ + { + "problem": "py-merge-intervals", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 166620, + "loadMs": 36832, + "outputTokens": 318, + "decodeMs": 80781.808, + "promptTokens": 1148, + "prefillMs": 48987.317, + "thinkingChars": 0, + "answer": "```python\ndef merge_intervals(intervals):\n \"\"\"\n Merge overlapping or touching closed intervals.\n \n Args:\n intervals: List of (start, end) tuples of integers.\n \n Returns:\n A new list of merged (start, end) tuples sorted by start.\n \n Raises:\n ValueError: If any interval has start > end or contains non-integers.\n \"\"\"\n if not intervals:\n return []\n \n # Validate input and sort\n for start, end in intervals:\n if not isinstance(start, int) or not isinstance(end, int):\n raise TypeError(\"Interval endpoints must be integers\")\n if start > end:\n raise ValueError(f\"Invalid interval: start ({start}) > end ({end})\")\n \n sorted_intervals = sorted(intervals, key=lambda x: x[0])\n \n if not sorted_intervals:\n return []\n \n merged = [sorted_intervals[0]]\n \n for current_start, current_end in sorted_intervals[1:]:\n last_start, last_end = merged[-1]\n \n # If intervals overlap or touch (last_end >= current_start)\n if current_start <= last_end:\n # Merge them\n merged[-1] = (last_start, max(last_end, current_end))\n else:\n # No overlap, add the new interval\n merged.append((current_start, current_end))\n \n return merged\n```" + }, + { + "problem": "py-roman-strict", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-xt780B/test_solution.py\", line 3, in \n assert roman_to_int(text) == value, text\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: IV\n", + "truncated": false, + "wallMs": 275577, + "loadMs": 33308, + "outputTokens": 737, + "decodeMs": 191921.093, + "promptTokens": 1157, + "prefillMs": 50322.608, + "thinkingChars": 0, + "answer": "```python\ndef roman_to_int(s: str) -> int:\n \"\"\"Convert a canonical Roman numeral string to an integer (1-3999).\n \n Raises ValueError for empty strings, lowercase/unknown characters,\n or non-canonical forms (e.g., IIII, IC, MMMM).\n \"\"\"\n # Validate input type and basic constraints early\n if not isinstance(s, str):\n raise TypeError(\"Input must be a string\")\n \n if len(s) == 0:\n raise ValueError(\"Input string cannot be empty\")\n\n # Validate characters: must be uppercase standard Roman numerals only\n valid_chars = set(\"IVXLCDM\")\n if not all(c in valid_chars for c in s):\n raise ValueError(\"Contains invalid or lowercase characters\")\n\n # Define values\n values = {'I': 1, 'V': 5, 'X': 10, 'L': 50, 'C': 100, 'D': 500, 'M': 1000}\n \n # Pre-check for global limits to catch impossible numbers early (e.g., > 3999)\n # Although the algorithm handles it, we need to ensure we don't return invalid results.\n # A simple check: if input is all 'M', max length is 3. If input is 'MMM...M', length > 3 is invalid.\n # However, the main logic handles subtraction and addition. We just need to ensure the result is <= 3999.\n \n total = 0\n prev_value = 0\n \n for i, char in enumerate(s):\n current_value = values[char]\n \n # Check for non-canonical subtractive forms (e.g., IC, XM, IL)\n # Standard subtractive pairs are: IV, IX, XL, XC, CD, CM\n # A value can only be subtracted if it is one of {I, X, C} and the next is one of {V, X, L, D, M}\n # Specifically: I before V/X, X before L/C, C before D/M.\n # If current < prev, it's a subtractive case.\n if current_value < prev_value:\n # Must be a valid subtractive pair\n pair = (prev_value, current_value)\n valid_subtractive = {\n (5, 1), (10, 1), (50, 10), (100, 10), (500, 100), (1000, 100)\n }\n if pair not in valid_subtractive:\n raise ValueError(f\"Non-canonical subtractive form detected involving {prev_value} and {current_value}\")\n \n total -= current_value\n else:\n total += current_value\n \n prev_value = current_value\n\n # Final check: Roman numerals only represent numbers up to 3999.\n # If the calculated number is outside this range (e.g., 4000), it's invalid per spec.\n if not (1 <= total <= 3999):\n raise ValueError(f\"Roman numeral {s} represents {total}, which is outside the valid range 1-3999\")\n\n return total\n```" + }, + { + "problem": "py-parse-duration", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-NmSGXC/test_solution.py\", line 5, in \n assert parse_duration(\" 1h 15m 10s \") == 4510\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^\n File \"/tmp/flycoder-bench-NmSGXC/solution.py\", line 26, in parse_duration\n raise ValueError(\"Invalid characters in input\")\nValueError: Invalid characters in input\n", + "truncated": false, + "wallMs": 303946, + "loadMs": 33816, + "outputTokens": 833, + "decodeMs": 219131.223, + "promptTokens": 1217, + "prefillMs": 50978.499, + "thinkingChars": 0, + "answer": "```python\nimport re\nfrom typing import Optional, List, Tuple\n\ndef parse_duration(text: str) -> int:\n \"\"\"\n Parse a duration string into total seconds.\n \n Format: dhms (order d, h, m, s required).\n Spaces allowed between parts but not inside numbers.\n Raises ValueError if invalid.\n \"\"\"\n text = text.strip()\n if not text:\n raise ValueError(\"Empty input\")\n \n # Regex to find all pairs\n pattern = r\"(\\d+)\\s*([dhms])\"\n matches = list(re.finditer(pattern, text))\n \n if not matches:\n raise ValueError(\"No valid duration parts found\")\n \n # Check for extra characters outside of matched parts\n matched_text = \"\".join(m.group(0) for m in matches)\n if text != matched_text:\n raise ValueError(\"Invalid characters in input\")\n \n # Check unit order: d, h, m, s must appear in that order if multiple units are present\n unit_order = []\n for m in matches:\n unit = m.group(2)\n if unit not in ['d', 'h', 'm', 's']:\n raise ValueError(f\"Invalid unit '{unit}'\")\n unit_order.append(unit)\n \n # Check for duplicates\n if len(unit_order) != len(set(unit_order)):\n raise ValueError(\"Duplicate units found\")\n \n # Check order: d, h, m, s\n expected_order = ['d', 'h', 'm', 's']\n # Filter out None (shouldn't happen) but keep valid units\n valid_units = [u for u in expected_order if u in unit_order]\n \n # Check if the sequence of units in matches matches the expected order\n for i in range(len(unit_order) - 1):\n if unit_order[i] != expected_order[expected_order.index(unit_order[i])]:\n raise ValueError(\"Units must appear in order d, h, m, s\")\n \n # Actually, simpler check: the sequence of unique units must be a subsequence of d,h,m,s\n # and cannot skip any that appear later.\n # Better: just check that when we filter the unit_order list, it matches d,h,m,s exactly for the ones present.\n \n # Re-check order strictly:\n # If we have d, h, m, s, they must appear in that order.\n # If we have h, m, s, they must appear in that order.\n # So we just check that for any two units u1, u2 in the list, if u1 comes before u2 in d,h,m,s,\n # then u1 must appear before u2 in our list.\n \n unit_positions = {u: i for i, u in enumerate(expected_order)}\n for i in range(len(unit_order)):\n for j in range(i + 1, len(unit_order)):\n u1 = unit_order[i]\n u2 = unit_order[j]\n if unit_positions[u1] > unit_positions[u2]:\n raise ValueError(\"Units must appear in order d, h, m, s\")\n \n # Parse values\n total_seconds = 0\n for m in matches:\n value = int(m.group(1))\n unit = m.group(2)\n if unit == 'd':\n total_seconds += value * 86400\n elif unit == 'h':\n total_seconds += value * 3600\n elif unit == 'm':\n total_seconds += value * 60\n elif unit == 's':\n total_seconds += value\n \n return total_seconds\n```" + }, + { + "problem": "py-topo-sort", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-Rxy1al/test_solution.py\", line 3, in \n assert topo_sort({\"app\": [\"lib\", \"utils\"], \"lib\": [\"utils\"]}) == [\"utils\", \"lib\", \"app\"]\n ~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/tmp/flycoder-bench-Rxy1al/solution.py\", line 74, in topo_sort\n raise ValueError(\"Cycle detected in the dependency graph.\")\nValueError: Cycle detected in the dependency graph.\n", + "truncated": false, + "wallMs": 290437, + "loadMs": 35576, + "outputTokens": 807, + "decodeMs": 205859.449, + "promptTokens": 1168, + "prefillMs": 48982.973, + "thinkingChars": 0, + "answer": "```python\ndef topo_sort(deps: dict) -> list:\n \"\"\"\n Return a deterministic topological ordering of nodes given their dependencies.\n deps: dict mapping each node (str) to a list of nodes it depends on (str).\n Nodes appearing only as dependencies are included in the graph.\n Returns a list where each node appears exactly once, after all its dependencies.\n Raises ValueError if a cycle exists.\n Ties are broken by choosing the alphabetically smallest node.\n \"\"\"\n # Build the set of all nodes (sources + dependencies)\n nodes = set(deps.keys())\n for dependencies in deps.values():\n nodes.update(dependencies)\n\n # Build adjacency list for Kahn's algorithm:\n # We need: for each node, which nodes depend on it (reverse graph) to compute in-degrees correctly?\n # Actually, standard Kahn's:\n # - in_degree[v] = number of edges pointing to v.\n # - Edge u -> v means u depends on v? \n # - The problem says: deps[node] = list of nodes it depends on.\n # - So if node A depends on B, then B must come before A.\n # - This implies an edge from B to A in the dependency graph for ordering.\n # - Therefore, in_degree[A] should be incremented for each dependency of A.\n\n in_degree = {node: 0 for node in nodes}\n # Build the graph: for each node u, and each dependency v in deps[u],\n # we have an edge v -> u. So u depends on v, so v must come before u.\n # We will store the reverse graph: for each v, list of u such that u depends on v.\n # But for Kahn's, we only need in_degree and the list of successors to remove edges.\n # Let's compute in_degree:\n for node, dependencies in deps.items():\n for dep in dependencies:\n in_degree[node] += 1\n\n # Use a min-heap (or sorted list) to always pick the smallest available node.\n import heapq\n available = []\n for node in nodes:\n if in_degree[node] == 0:\n heapq.heappush(available, node)\n\n result = []\n while available:\n # Pop the smallest node\n current = heapq.heappop(available)\n result.append(current)\n\n # For each node that depends on current, decrement its in-degree\n # But we don't have the reverse graph explicitly. Let's build it now.\n # Actually, we can iterate over all nodes and check if current is in their dependencies.\n # That's O(N^2) in worst case. To be efficient, build reverse graph.\n pass\n\n # Rebuild with reverse graph for efficiency\n # Let's restart the logic with reverse graph construction.\n in_degree = {node: 0 for node in nodes}\n reverse_graph = {node: [] for node in nodes}\n\n for node, dependencies in deps.items():\n for dep in dependencies:\n in_degree[node] += 1\n reverse_graph[dep].append(node)\n\n heapq.heapify(available)\n while available:\n current = heapq.heappop(available)\n result.append(current)\n for neighbor in reverse_graph[current]:\n in_degree[neighbor] -= 1\n if in_degree[neighbor] == 0:\n heapq.heappush(available, neighbor)\n\n if len(result) != len(nodes):\n raise ValueError(\"Cycle detected in the dependency graph.\")\n\n return result\n```" + }, + { + "problem": "py-lru-cache", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-Hjh4kl/test_solution.py\", line 5, in \n c.put(3, 3)\n ~~~~~^^^^^^\n File \"/tmp/flycoder-bench-Hjh4kl/solution.py\", line 25, in put\n del self.cache[oldest_key]\n ~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 2\n", + "truncated": false, + "wallMs": 133861, + "loadMs": 32567, + "outputTokens": 205, + "decodeMs": 52034.206, + "promptTokens": 1173, + "prefillMs": 49236.558, + "thinkingChars": 0, + "answer": "```python\nfrom collections import OrderedDict\n\nclass LRUCache:\n def __init__(self, capacity: int):\n if capacity < 0:\n raise ValueError(\"capacity must be non-negative\")\n self.capacity = capacity\n self.cache = OrderedDict()\n\n def get(self, key: int) -> int:\n if key in self.cache:\n self.cache.move_to_end(key)\n return self.cache[key]\n return -1\n\n def put(self, key: int, value: int) -> None:\n if key in self.cache:\n self.cache.move_to_end(key)\n self.cache[key] = value\n else:\n self.cache[key] = value\n if len(self.cache) > self.capacity:\n # Evict the least recently used item (first item)\n oldest_key, _ = self.cache.popitem(last=False)\n del self.cache[oldest_key]\n```" + }, + { + "problem": "py-wrap-text", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 220991, + "loadMs": 32825, + "outputTokens": 534, + "decodeMs": 138179.684, + "promptTokens": 1181, + "prefillMs": 49962.574, + "thinkingChars": 0, + "answer": "```python\ndef wrap_text(text: str, width: int) -> list[str]:\n \"\"\"\n Wrap text greedily into lines.\n \n - Words are separated by any whitespace; runs of whitespace collapse.\n - Lines never start or end with a space.\n - As many words as fit on each line (joined by single spaces, length <= width).\n - A word longer than width is split into pieces of exactly width characters \n (last piece may be shorter); each piece is treated as a word.\n - Empty or whitespace-only text returns [].\n - Raises ValueError if width < 1.\n \"\"\"\n if width < 1:\n raise ValueError(\"width must be at least 1\")\n\n # Split text into words by whitespace, filtering out empty strings.\n words = text.split()\n if not words:\n return []\n\n lines = []\n current_line = []\n current_length = 0\n\n for word in words:\n # If word is longer than width, split it into chunks of size 'width'\n if len(word) > width:\n for i in range(0, len(word), width):\n chunk = word[i:i + width]\n if current_line:\n # There is already content on the line\n if current_length + 1 + len(chunk) > width:\n # Line full, flush it\n lines.append(\" \".join(current_line))\n current_line = []\n current_length = 0\n current_line.append(chunk)\n current_length += len(chunk) + 1 # +1 for the space\n else:\n # First chunk on an empty line\n current_line.append(chunk)\n current_length = len(chunk)\n continue # Skip the rest of the loop for this original word\n\n # Normal word (fits or needs to start a new line)\n if current_line:\n # Check if adding this word exceeds width\n if current_length + 1 + len(word) > width:\n lines.append(\" \".join(current_line))\n current_line = []\n current_length = 0\n\n current_line.append(word)\n current_length += len(word)\n if current_line:\n current_length += len(current_line) - 1 # Subtract the extra space counted above if line has multiple words\n\n if current_line:\n lines.append(\" \".join(current_line))\n\n return lines\n```" + }, + { + "problem": "py-semver-compare", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-tGwZ8V/test_solution.py\", line 4, in \n assert compare_versions(low, high) == -1, (low, high)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: ('1.0.0-rc.1', '1.0.0')\n", + "truncated": false, + "wallMs": 282974, + "loadMs": 37855, + "outputTokens": 755, + "decodeMs": 194106.028, + "promptTokens": 1190, + "prefillMs": 50990.766, + "thinkingChars": 0, + "answer": "```python\ndef compare_versions(a: str, b: str) -> int:\n \"\"\"\n Compare two Semantic Versioning 2.0.0 strings.\n Returns -1 if a < b, 0 if a == b, 1 if a > b.\n Build metadata is ignored. Pre-release versions have lower precedence.\n \"\"\"\n # Helper to parse a single identifier\n def parse_ident(s: str):\n if s.isdigit():\n return (\"numeric\", int(s))\n return (\"alphanumeric\", s)\n\n # Helper to compare two lists of identifiers\n def compare_ids(ids_a, ids_b):\n # If a has fewer identifiers and they match up to len(a), a is lower\n for i in range(max(len(ids_a), len(ids_b))):\n if i >= len(ids_a):\n return -1 # a is shorter, so lower precedence\n if i >= len(ids_b):\n return 1 # b is shorter, so a is higher precedence\n type_a, val_a = ids_a[i]\n type_b, val_b = ids_b[i]\n # Numeric identifiers have lower precedence than alphanumeric ones\n if type_a != type_b:\n if type_a == \"numeric\":\n return -1\n else:\n return 1\n # Same type: compare values\n if val_a < val_b:\n return -1\n if val_a > val_b:\n return 1\n return 0\n\n # Split into core, prerelease, build\n def parse_ver(v: str):\n core = v.split(\"-\", 1)[0]\n pre = v[len(core):].split(\"+\", 1)[0] if \"-\" in v else \"\"\n build = v.split(\"+\", 1)[1] if \"+\" in v else \"\"\n core_parts = core.split(\".\")\n # Parse prerelease identifiers\n pre_ids = []\n if pre:\n for p in pre.split(\".\"):\n pre_ids.append(parse_ident(p))\n # Build is ignored, but we still parse it to ensure it doesn't break logic (not used in comparison)\n # We don't parse build identifiers, just ignore.\n return core_parts, pre_ids\n\n core_a, pre_a = parse_ver(a)\n core_b, pre_b = parse_ver(b)\n\n # Compare numeric core parts\n for i in range(3):\n try:\n n_a = int(core_a[i])\n n_b = int(core_b[i])\n except ValueError:\n # Should not happen for valid SemVer, but if it does, treat as alphanumeric\n # However, SemVer core must be numeric, so we assume valid input.\n n_a, n_b = core_a[i], core_b[i]\n if n_a < n_b:\n return -1\n if n_a > n_b:\n return 1\n if n_a < n_b:\n return -1\n if n_a > n_b:\n return 1\n\n # Core parts are equal; compare prerelease\n if pre_a == \"\" and pre_b == \"\":\n return 0\n if pre_a == \"\":\n return 1 # Release version > prerelease\n if pre_b == \"\":\n return -1 # Prerelease < release\n\n return compare_ids(pre_a, pre_b)\n```" + }, + { + "problem": "py-spiral", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 170090, + "loadMs": 32815, + "outputTokens": 347, + "decodeMs": 90309.094, + "promptTokens": 1108, + "prefillMs": 46949.342, + "thinkingChars": 0, + "answer": "```python\ndef spiral_order(matrix):\n \"\"\"Return elements of a rectangular matrix in clockwise spiral order.\n \n Args:\n matrix: A list of lists of integers (or any elements). Empty matrix returns [].\n \n Returns:\n A list of elements in spiral order.\n \"\"\"\n if not matrix or not matrix[0]:\n return []\n\n rows, cols = len(matrix), len(matrix[0])\n top, bottom = 0, rows - 1\n left, right = 0, cols - 1\n result = []\n\n while top <= bottom and left <= right:\n # Traverse from left to right along the top row\n for col in range(left, right + 1):\n result.append(matrix[top][col])\n top += 1\n\n # Traverse from top to bottom along the right column\n for row in range(top, bottom + 1):\n result.append(matrix[row][right])\n right -= 1\n\n # Check if there is still a valid top row\n if top <= bottom:\n # Traverse from right to left along the bottom row\n for col in range(right, left - 1, -1):\n result.append(matrix[bottom][col])\n bottom -= 1\n\n # Check if there is still a valid left column\n if left <= right:\n # Traverse from bottom to top along the left column\n for row in range(bottom, top - 1, -1):\n result.append(matrix[row][left])\n left += 1\n\n return result\n```" + }, + { + "problem": "py-csv-line", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-v2hG88/test_solution.py\", line 13, in \n raise AssertionError(\"accepted \" + bad)\nAssertionError: accepted \"a\"b,c\n", + "truncated": false, + "wallMs": 191884, + "loadMs": 37097, + "outputTokens": 400, + "decodeMs": 107070.947, + "promptTokens": 1163, + "prefillMs": 47691.982, + "thinkingChars": 0, + "answer": "```python\ndef parse_csv_line(line: str) -> list[str]:\n \"\"\"\n Parse a single CSV line into a list of fields without using the csv module.\n \n Rules:\n - Fields separated by commas.\n - Empty line -> [\"\"].\n - Quoted fields: start with '\"', end with '\"' (unescaped).\n - Inside quotes: ',' is literal, '\"' is escaped as '\"\"'.\n - Unterminated quote or character after closing quote -> ValueError.\n - Unquoted fields are returned exactly as-is (no trimming).\n \"\"\"\n if not line:\n return [\"\"]\n\n fields = []\n current = []\n in_quotes = False\n i = 0\n n = len(line)\n\n while i < n:\n ch = line[i]\n\n if in_quotes:\n if ch == '\"':\n # Check for escaped quote\n if i + 1 < n and line[i + 1] == '\"':\n current.append('\"')\n i += 2\n continue\n else:\n # End of quoted field\n in_quotes = False\n i += 1\n continue\n else:\n current.append(ch)\n i += 1\n continue\n else:\n if ch == '\"':\n # Start of a quoted field\n in_quotes = True\n i += 1\n continue\n elif ch == ',':\n fields.append(\"\".join(current))\n current = []\n i += 1\n continue\n else:\n current.append(ch)\n i += 1\n continue\n\n if in_quotes:\n raise ValueError(\"Unterminated quoted field\")\n\n # Append the last field\n fields.append(\"\".join(current))\n\n return fields\n```" + }, + { + "problem": "py-lcs", + "sample": 0, + "ok": false, + "error": "truncated by max tokens", + "truncated": true, + "wallMs": 588915, + "loadMs": 31812, + "outputTokens": 2048, + "decodeMs": 512458.89, + "promptTokens": 1108, + "prefillMs": 44623.482, + "thinkingChars": 0, + "answer": "```python\ndef longest_common_subsequence(a: str, b: str) -> str:\n \"\"\"\n Returns one longest common subsequence of strings a and b.\n Uses Hirschberg's algorithm: O(min(len(a), len(b))) space and O(len(a)*len(b)) time.\n Handles 2000-char strings efficiently.\n \"\"\"\n if not a or not b:\n return \"\"\n\n # Ensure b is the shorter string to minimize space for DP table\n if len(a) < len(b):\n return longest_common_subsequence(b, a)\n\n n, m = len(a), len(b)\n m_limit = m # b is the shorter string\n\n # We need two rows for DP scores and one row for backtracking scores\n # dp[i] = LCS length for a[0:i] and b[0:j] (current row)\n # We compute row i and row i-1 implicitly.\n # To reconstruct, we need to know which column gave the max value in the previous row.\n # We use a simple approach: store the full DP table? No, that's O(n*m) space.\n # Hirschberg uses divide and conquer on the first string (a).\n # Let's implement Hirschberg's algorithm properly.\n\n # Hirschberg's algorithm:\n # 1. Divide a into two halves.\n # 2. Compute LCS lengths for (a[0..mid], b) and (a[mid+1..end], b).\n # 3. Find the split point in b that minimizes the sum of LCS lengths from left and right.\n # 4. Recursively solve left and right parts.\n\n # However, implementing Hirschberg from scratch is complex and error-prone.\n # Given the constraint \"2000 characters\", an O(n*m) time with O(min(n,m)) space using\n # Hirschberg is ideal. But let's try a simpler space-optimized DP with backtracking if possible?\n # No, standard space-optimized DP (2 rows) loses the path for reconstruction.\n # We must use Hirschberg's algorithm.\n\n # Implementation of Hirschberg's algorithm:\n # We define a function to compute the LCS length matrix row by row.\n # Then we use divide and conquer.\n\n # Helper: compute LCS lengths for a[0:i] and b[0:j] for all j in 0..m\n def compute_row(a_part: str) -> list[int]:\n \"\"\"Returns a list of LCS lengths for a_part against b.\"\"\"\n prev_row = [0] * (m + 1)\n curr_row = [0] * (m + 1)\n for ch_a in a_part:\n for j in range(1, m + 1):\n if ch_a == b[j - 1]:\n curr_row[j] = prev_row[j - 1] + 1\n else:\n curr_row[j] = max(prev_row[j], curr_row[j - 1])\n # Move curr to prev\n prev_row[:] = curr_row[:]\n return prev_row\n\n # We need to compute the full DP table? No, Hirschberg avoids storing the full table.\n # Instead, we compute the \"profile\" (LCS lengths) for the current segment of a against b.\n # But we need to know the split point in b.\n # The algorithm:\n # 1. Let mid = len(a) // 2\n # 2. Compute LCS lengths for (a[0:mid], b) -> row_mid\n # 3. Compute LCS lengths for (a[mid:len(a)], b) -> row_right (but aligned to start at mid)\n # Actually, we compute the LCS lengths for the second half relative to the first half's end.\n # Specifically, we compute the LCS lengths for (a[mid+1:], b) but we need to know the\n # LCS length at (mid, j) for each j to compute the \"suffix\" LCS.\n # The standard way: compute the DP table for (a[0:mid], b) and (a[mid+1:], b) but\n # we need to know the LCS length at (mid, j) for each j.\n # Actually, the standard Hirschberg computes the DP table for the first half and then\n # computes the DP table for the second half but shifted? No.\n # The correct way:\n # Let L1 = LCS(a[0:mid], b)\n # Let L2 = LCS(a[mid+1:], b) but we need to know the LCS length at (mid, j) for each j.\n # Actually, we compute the DP table for (a[0:mid], b) and then compute the DP table for\n # (a[mid+1:], b) but we need to know the LCS length at (mid, j) for each j to compute\n # the \"suffix\" LCS.\n # The standard approach:\n # Compute the DP table for (a[0:mid], b) -> row_mid\n # Compute the DP table for (a[mid+1:], b) but we need to know the LCS length at (mid, j) for each j.\n # Actually, we compute the DP table for (a[0:mid], b) and then compute the DP table for\n # (a[mid+1:], b) but we need to know the LCS length at (mid, j) for each j to compute\n # the \"suffix\" LCS.\n # The correct way is to compute the DP table for (a[0:mid], b) and then compute the DP table for\n # (a[mid+1:], b) but we need to know the LCS length at (mid, j) for each j to compute\n # the \"suffix\" LCS.\n # Actually, the standard way is to compute the DP table for (a[0:mid], b) and then compute the DP table for\n # (a[mid+1:], b) but we need to know the LCS length at (mid, j) for each j to compute\n # the \"suffix\" LCS.\n # Let's restart the implementation of Hirschberg's algorithm.\n\n # Hirschberg's algorithm:\n # We define a function to compute the LCS length matrix for a[0:i] and b.\n # Then we use divide and conquer.\n\n # However, there is a simpler way: use the fact that we only need the last row of the DP table\n # for the first half and the first row of the DP table for the second half? No.\n # The standard way is to compute the DP table for (a[0:mid], b) and then compute the DP table for\n # (a[mid+1:], b) but we need to know the LCS length at (mid, j) for each j to compute\n # the \"suffix\" LCS.\n # Actually, the standard way is to compute the DP table for (a[0:mid], b) and then compute the DP table for\n # (a[mid+1:], b) but we need to know the LCS length at (mid, j) for each j to compute\n # the \"suffix\" LCS.\n # Let's use the standard implementation of Hirschberg's algorithm.\n\n # We define a function to compute the LCS length matrix for a[0:i] and b.\n # Then we use divide and conquer.\n\n # However, there is a simpler way: use the fact that we only need the last row of the DP table\n # for the first half and the first row of the DP table for the second half? No.\n # The standard way is to compute the DP table for (a[0:mid], b) and then compute the DP table for\n # (a[mid+1:], b) but we need to know the LCS length at (mid, j) for each j to compute\n # the \"suffix\" LCS.\n # Actually, the standard way is to compute the DP table for (a[0:mid], b) and then compute the DP table for\n # (a[mid+1:], b) but we need to know the LCS length at (mid, j) for each j to compute\n # the \"suffix\" LCS.\n # Let's use the standard implementation of Hirschberg's algorithm.\n\n # We define a function to compute the LCS length matrix for a[0:i] and b.\n # Then we use divide and conquer.\n\n # However, there is a simpler way: use the fact that we only need the last row of the DP table\n # for the first half and the first row of the DP table for the second half? No.\n # The standard way is to compute the DP table for (a[0:mid], b) and then compute the DP table for\n # (a[mid+1:], b) but we need to know the LCS length at (mid, j) for each j to compute\n # the \"suffix\" LCS.\n # Actually, the standard way is to" + }, + { + "problem": "py-number-words", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-gDZJFT/test_solution.py\", line 7, in \n assert number_to_words(n) == words, (n, number_to_words(n))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: (7, '')\n", + "truncated": false, + "wallMs": 255448, + "loadMs": 33937, + "outputTokens": 680, + "decodeMs": 170193.955, + "promptTokens": 1218, + "prefillMs": 51302.107, + "thinkingChars": 0, + "answer": "```python\ndef number_to_words(n: int) -> str:\n \"\"\"Convert an integer (0\u2013999,999,999,999) to its English lowercase word form.\"\"\"\n if not isinstance(n, int):\n raise TypeError(\"n must be an integer\")\n if n < 0 or n > 999_999_999_999:\n raise ValueError(\"n must be between 0 and 999,999,999,999 inclusive\")\n\n teens = {\n 11: \"eleven\", 12: \"twelve\", 13: \"thirteen\", 14: \"fourteen\", 15: \"fifteen\",\n 16: \"sixteen\", 17: \"seventeen\", 18: \"eighteen\", 19: \"nineteen\"\n }\n tens = {\n 2: \"twenty\", 3: \"thirty\", 4: \"forty\", 5: \"fifty\", 6: \"sixty\",\n 7: \"seventy\", 8: \"eighty\", 9: \"ninety\"\n }\n ones = {\n 1: \"one\", 2: \"two\", 3: \"three\", 4: \"four\", 5: \"five\",\n 6: \"six\", 7: \"seven\", 8: \"eight\", 9: \"nine\", 0: \"\"\n }\n\n def _words_under_100(num: int) -> str:\n if num == 0:\n return \"\"\n if num < 20:\n return teens.get(num, ones[num])\n ten = num // 10\n one = num % 10\n if one == 0:\n return tens[ten]\n return f\"{tens[ten]}-{ones[one]}\"\n\n def _words_under_1000(num: int) -> str:\n if num == 0:\n return \"\"\n hundred = num // 100\n remainder = num % 100\n if remainder == 0:\n return ones[hundred] + \" hundred\"\n return f\"{ones[hundred]} hundred {_words_under_100(remainder)}\"\n\n if n == 0:\n return \"zero\"\n\n scales = [\n (999_000_000_000, \"billion\"),\n (1_000_000, \"million\"),\n (1_000, \"thousand\")\n ]\n\n parts = []\n for scale_val, scale_name in scales:\n chunk = (n // scale_val) % 1000\n if chunk > 0:\n parts.append(_words_under_1000(chunk) + \" \" + scale_name)\n\n return \" \".join(parts)\n```" + }, + { + "problem": "py-dijkstra", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 229214, + "loadMs": 29039, + "outputTokens": 600, + "decodeMs": 149879.126, + "promptTokens": 1185, + "prefillMs": 50275.384, + "thinkingChars": 0, + "answer": "```python\nimport heapq\nfrom typing import List, Tuple, Union\n\ndef shortest_distances(\n n: int,\n edges: List[Tuple[int, int, Union[int, float]]],\n source: int\n) -> List[Union[int, float]]:\n \"\"\"\n Compute shortest distances from 'source' to all nodes 0..n-1 in a directed graph.\n \n Args:\n n: Number of nodes (0 to n-1).\n edges: List of (u, v, weight) tuples. Weights must be non-negative.\n source: The starting node.\n \n Returns:\n A list of length n where index i holds the shortest distance from source to i.\n Unreachable nodes are represented as float('inf').\n \n Raises:\n ValueError: If any weight is negative, source is out of range, or n <= 0.\n \"\"\"\n # Validate n\n if n <= 0:\n raise ValueError(\"n must be a positive integer\")\n \n # Validate source\n if source < 0 or source >= n:\n raise ValueError(f\"source must be in range [0, {n-1}]\")\n \n # Validate edges: check for negative weights and node indices\n for u, v, w in edges:\n if u < 0 or u >= n or v < 0 or v >= n:\n raise ValueError(f\"Node indices must be in range [0, {n-1}]\")\n if w < 0:\n raise ValueError(\"Weights must be non-negative\")\n \n # Build adjacency list\n # Use a list of lists for the graph.\n # Since n can be up to 20,000, a list of lists is efficient enough.\n adj: List[List[Tuple[int, float]]] = [[] for _ in range(n)]\n for u, v, w in edges:\n adj[u].append((v, w))\n \n # Dijkstra's algorithm using a min-heap\n dist = [float('inf')] * n\n dist[source] = 0\n \n # Heap stores tuples of (distance, node)\n pq = [(0.0, source)]\n \n while pq:\n d, u = heapq.heappop(pq)\n \n # If we found a shorter path to u before, skip\n if d > dist[u]:\n continue\n \n # Explore neighbors\n for v, w in adj[u]:\n new_dist = dist[u] + w\n if new_dist < dist[v]:\n dist[v] = new_dist\n heapq.heappush(pq, (new_dist, v))\n \n return dist\n```" + }, + { + "problem": "py-slugify", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-88hM5K/test_solution.py\", line 3, in \n assert slugify(\" Cr\u00e8me Br\u00fbl\u00e9e \u00e0 la Fran\u00e7aise \") == \"creme-brulee-a-la-francaise\"\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError\n", + "truncated": false, + "wallMs": 248929, + "loadMs": 33079, + "outputTokens": 641, + "decodeMs": 166963.502, + "promptTokens": 1193, + "prefillMs": 48865.91, + "thinkingChars": 0, + "answer": "```python\nimport unicodedata\n\ndef slugify(text: str, max_length: int = 50) -> str:\n \"\"\"\n Build an ASCII URL slug from text.\n \n - Converts text to lowercase.\n - Removes accents (NFD normalization + filter non-cased chars).\n - Maps special chars: \u00df->ss, \u00e6->ae, \u0153->oe, \u00f8->o.\n - Replaces any non-alphanumeric run with a single hyphen.\n - Strips leading/trailing hyphens and truncates to max_length, removing trailing hyphen if needed.\n - Raises ValueError if max_length < 1.\n \"\"\"\n if max_length < 1:\n raise ValueError(\"max_length must be at least 1\")\n\n # Normalize to decompose accented chars (e.g., \u00e9 -> e + combining acute)\n normalized = unicodedata.normalize(\"NFD\", text)\n \n # Helper to check if char is ASCII letter or digit after decomposition\n def is_alnum(c: str) -> bool:\n return c.isascii() and c.isalnum()\n\n # Build mapping for special chars after lowercasing\n # Note: unicodedata.lowercase handles most, but we explicitly map \u00df etc.\n special_map = {\n '\u00df': 'ss',\n '\u00e6': 'ae',\n '\u0153': 'oe',\n '\u00f8': 'o',\n }\n\n slug_chars = []\n for char in normalized.lower():\n # Handle special replacements first\n if char in special_map:\n slug_chars.append(special_map[char])\n continue\n \n # Check if it's an ASCII letter or digit\n if is_alnum(char):\n slug_chars.append(char)\n else:\n # Non-alphanumeric: mark as a separator\n slug_chars.append('\\x00') # Use null byte as temporary separator marker\n\n # Join and replace runs of non-alphanumeric with a single hyphen\n # We'll iterate and build the slug\n final_slug = []\n prev_was_hyphen = False\n \n for char in slug_chars:\n if char == '\\x00':\n if not prev_was_hyphen:\n final_slug.append('-')\n prev_was_hyphen = True\n else:\n final_slug.append(char)\n prev_was_hyphen = False\n\n # Join to form the raw slug string\n slug = \"\".join(final_slug)\n\n # Strip leading and trailing hyphens\n while slug.startswith('-'):\n slug = slug[1:]\n while slug.endswith('-'):\n slug = slug[:-1]\n\n # Truncate to max_length and remove trailing hyphen if it exists\n if len(slug) > max_length:\n slug = slug[:max_length]\n if slug.endswith('-'):\n slug = slug[:-1]\n\n return slug\n```" + }, + { + "problem": "py-base-convert", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-RnQiYS/test_solution.py\", line 2, in \n assert convert_base(\"ff\", 16, 2) == \"11111111\"\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError\n", + "truncated": false, + "wallMs": 250643, + "loadMs": 30310, + "outputTokens": 644, + "decodeMs": 170348.341, + "promptTokens": 1186, + "prefillMs": 49964.617, + "thinkingChars": 0, + "answer": "```python\ndef convert_base(number: str, from_base: int, to_base: int) -> str:\n \"\"\"\n Convert an integer string between bases 2 and 36.\n \n Args:\n number: String representation in from_base. Digits: 0-9, a-z (case-insensitive).\n from_base: Source base (2..36).\n to_base: Destination base (2..36).\n \n Returns:\n String representation in to_base (lowercase), no leading zeros except for \"0\".\n \n Raises:\n ValueError: If bases are invalid, input is empty/only \"-\", or contains invalid digits.\n TypeError: If inputs are not strings or integers.\n \"\"\"\n # Validate types\n if not isinstance(number, str):\n raise TypeError(\"number must be a string\")\n if not isinstance(from_base, int) or not isinstance(to_base, int):\n raise TypeError(\"bases must be integers\")\n \n # Validate base range\n if not (2 <= from_base <= 36 and 2 <= to_base <= 36):\n raise ValueError(\"bases must be between 2 and 36\")\n \n # Validate input string basics\n if not number or number == \"-\":\n raise ValueError(\"number cannot be empty or just '-'\")\n \n # Handle sign\n negative = False\n if number.startswith(\"-\"):\n negative = True\n number = number[1:]\n if not number:\n raise ValueError(\"number cannot be empty or just '-'\")\n \n # Validate digits for source base\n digit_map = \"0123456789abcdefghijklmnopqrstuvwxyz\"\n \n for char in number:\n if char == \"-\":\n raise ValueError(\"number cannot contain '-' in the magnitude part\")\n if char not in digit_map:\n raise ValueError(f\"invalid digit '{char}'\")\n \n # Check if digit is valid for from_base\n digit_value = digit_map.index(char.lower())\n if digit_value >= from_base:\n raise ValueError(f\"digit '{char}' is invalid for base {from_base}\")\n \n # Convert from from_base to base 10 (integer)\n value = 0\n for char in number:\n digit_value = digit_map.index(char.lower())\n value = value * from_base + digit_value\n \n # Handle zero case\n if value == 0:\n return \"0\"\n \n # Convert from base 10 to to_base\n result_digits = []\n negative_value = value if value < 0 else -value\n while negative_value > 0:\n remainder = negative_value % to_base\n result_digits.append(digit_map[remainder])\n negative_value //= to_base\n \n result_digits.reverse()\n result = \"\".join(result_digits)\n \n if negative:\n result = \"-\" + result\n \n return result\n```" + }, + { + "problem": "js-parse-query", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 211480, + "loadMs": 31544, + "outputTokens": 511, + "decodeMs": 129961.252, + "promptTokens": 1235, + "prefillMs": 49953.902, + "thinkingChars": 0, + "answer": "```javascript\n/**\n * Parses a URL query string into a plain object.\n * \n * Rules:\n * - Optional leading \"?\".\n * - Pairs separated by \"&\"; empty pairs ignored.\n * - Split on first \"=\"; key without \"=\" -> \"\".\n * - \"+\" -> space; then percent-decode. If decoding fails, keep text as-is (with spaces from '+').\n * - Duplicate keys -> array of values.\n * - Safe against prototype pollution (__proto__ becomes own property).\n */\nexport function parseQuery(query) {\n // Remove leading \"?\" if present\n const clean = query && query[0] === \"?\" ? query.slice(1) : query;\n\n const result = Object.create(null); // Ensure no prototype pollution\n\n if (!clean) return result;\n\n const pairs = clean.split(\"&\");\n\n for (const pair of pairs) {\n if (!pair) continue; // Ignore empty pairs\n\n // Split on the first \"=\"\n const idx = pair.indexOf(\"=\");\n let key, value;\n\n if (idx === -1) {\n key = pair;\n value = \"\";\n } else {\n key = pair.slice(0, idx);\n value = pair.slice(idx + 1);\n }\n\n // Decode '+' as space\n let decodedKey = key.replace(/\\+/g, \" \");\n let decodedValue = value.replace(/\\+/g, \" \");\n\n // Percent-decode (only for ASCII 00-FF)\n // If decoding fails or produces invalid chars, keep as-is\n try {\n decodedKey = decodeURIComponent(decodedKey);\n decodedValue = decodeURIComponent(decodedValue);\n } catch {\n // If decoding fails (e.g., %XX where XX is invalid hex), keep original\n // But we already replaced '+' with spaces, so keep that text as-is.\n decodedKey = key;\n decodedValue = value;\n }\n\n // Handle duplicate keys\n if (decodedKey in result) {\n if (!Array.isArray(result[decodedKey])) {\n result[decodedKey] = [result[decodedKey]];\n }\n result[decodedKey].push(decodedValue);\n } else {\n result[decodedKey] = decodedValue;\n }\n }\n\n return result;\n}\n```" + }, + { + "problem": "js-deep-equal", + "sample": 0, + "ok": false, + "error": "node:internal/modules/run_main:123\n triggerUncaughtException(\n ^\n\nAssertionError [ERR_ASSERTION]: Expected values to be strictly equal:\n\nfalse !== true\n\n at file:///tmp/flycoder-bench-ELGLFs/test.mjs:3:35 {\n generatedMessage: true,\n code: 'ERR_ASSERTION',\n actual: false,\n expected: true,\n operator: 'strictEqual',\n diff: 'simple'\n}\n\nNode.js v22.22.0\n", + "truncated": false, + "wallMs": 189099, + "loadMs": 29031, + "outputTokens": 427, + "decodeMs": 109417.019, + "promptTokens": 1225, + "prefillMs": 50630.514, + "thinkingChars": 0, + "answer": "```javascript\nexport function deepEqual(a, b) {\n if (a === b) {\n // SameValueZero: NaN === NaN, -0 === 0, primitives equal\n return true;\n }\n\n // Handle object types\n const typeA = typeof a;\n const typeB = typeof b;\n\n if (typeA !== typeB) {\n return false;\n }\n\n if (typeA === 'object') {\n if (a === null || b === null) {\n return false;\n }\n\n const isArrayA = Array.isArray(a);\n const isArrayB = Array.isArray(b);\n\n if (isArrayA !== isArrayB) {\n return false;\n }\n\n if (isArrayA) {\n if (a.length !== b.length) {\n return false;\n }\n for (let i = 0; i < a.length; i++) {\n if (!deepEqual(a[i], b[i])) {\n return false;\n }\n }\n return true;\n } else {\n // Plain objects: compare own enumerable string keys\n const keysA = Object.keys(a);\n const keysB = Object.keys(b);\n\n if (keysA.length !== keysB.length) {\n return false;\n }\n\n const keySetA = new Set(keysA);\n for (const key of keysB) {\n if (keySetA.has(key) === false) {\n return false;\n }\n if (!deepEqual(a[key], b[key])) {\n return false;\n }\n }\n return true;\n }\n }\n\n // For Date objects, compare time values directly\n if (typeA === 'object' && a instanceof Date && b instanceof Date) {\n return a.getTime() === b.getTime();\n }\n\n return false;\n}\n```" + }, + { + "problem": "js-evaluate", + "sample": 0, + "ok": false, + "error": "oder-bench-zjrmZ4/solution.mjs:85\n throw new SyntaxError('Invalid character in expression');\n ^\n\nSyntaxError: Invalid character in expression\n at parsePrimary (file:///tmp/flycoder-bench-zjrmZ4/solution.mjs:85:13)\n at parseUnary (file:///tmp/flycoder-bench-zjrmZ4/solution.mjs:58:12)\n at parseTerm (file:///tmp/flycoder-bench-zjrmZ4/solution.mjs:36:16)\n at parseExpression (file:///tmp/flycoder-bench-zjrmZ4/solution.mjs:28:21)\n at evaluate (file:///tmp/flycoder-bench-zjrmZ4/solution.mjs:96:18)\n at file:///tmp/flycoder-bench-zjrmZ4/test.mjs:12:9\n\nNode.js v22.22.0\n", + "truncated": false, + "wallMs": 348684, + "loadMs": 32315, + "outputTokens": 1058, + "decodeMs": 267018.56, + "promptTokens": 1214, + "prefillMs": 49327.105, + "thinkingChars": 0, + "answer": "```javascript\n// evaluate.js\n// Implements a recursive-descent parser for arithmetic expressions.\n// Supports: non-negative decimals, + - * /, parentheses, unary +/-, whitespace.\n// Throws SyntaxError for invalid input, RangeError for division by zero.\n\nexport function evaluate(expression) {\n if (typeof expression !== 'string') {\n throw new TypeError('expression must be a string');\n }\n\n // Tokenize\n const tokens = tokenize(expression);\n let pos = 0;\n\n function peek() {\n return tokens[pos] || null;\n }\n\n function consume() {\n return tokens[pos++];\n }\n\n // Parse expression: addition/subtraction\n function parseExpression() {\n let left = parseTerm();\n while (peek() === '+' || peek() === '-') {\n const op = consume();\n const right = parseTerm();\n left = op === '+' ? left + right : left - right;\n }\n return left;\n }\n\n // Parse term: multiplication/division\n function parseTerm() {\n let left = parseUnary();\n while (peek() === '*' || peek() === '/') {\n const op = consume();\n const right = parseUnary();\n if (op === '/' && right === 0) {\n throw new RangeError('Division by zero');\n }\n left = op === '*' ? left * right : left / right;\n }\n return left;\n }\n\n // Parse unary: + or - or number or parenthesized expression\n function parseUnary() {\n if (peek() === '+') {\n consume();\n return parseUnary(); // unary plus is identity, but we must parse the next token\n }\n if (peek() === '-') {\n consume();\n return -parseUnary(); // unary minus\n }\n return parsePrimary();\n }\n\n // Parse primary: number or parenthesized expression\n function parsePrimary() {\n const token = peek();\n if (token === null) {\n throw new SyntaxError('Unexpected end of input');\n }\n\n if (token === '(') {\n consume();\n const value = parseExpression();\n if (peek() !== ')') {\n throw new SyntaxError('Missing closing parenthesis');\n }\n consume();\n return value;\n }\n\n // Must be a number\n if (token === null) {\n throw new SyntaxError('Unexpected end of input');\n }\n\n // Check if it's a valid number token (digits and optional decimal point)\n if (!/^[0-9]*\\.?[0-9]+$/.test(token)) {\n throw new SyntaxError('Invalid character in expression');\n }\n\n const num = parseFloat(token);\n if (isNaN(num)) {\n throw new SyntaxError('Invalid number');\n }\n consume();\n return num;\n }\n\n const result = parseExpression();\n\n // Ensure all tokens are consumed\n if (pos < tokens.length) {\n throw new SyntaxError('Unexpected token after expression');\n }\n\n return result;\n}\n\n// Tokenizer\nfunction tokenize(input) {\n const tokens = [];\n let i = 0;\n const len = input.length;\n\n while (i < len) {\n const ch = input[i];\n\n // Skip whitespace\n if (ch <= ' ') {\n i++;\n continue;\n }\n\n // Parentheses\n if (ch === '(' || ch === ')') {\n tokens.push(ch);\n i++;\n continue;\n }\n\n // Operators\n if (ch === '+' || ch === '-' || ch === '*' || ch === '/') {\n tokens.push(ch);\n i++;\n continue;\n }\n\n // Number (including decimals)\n if (/[0-9]/.test(ch) || ch === '.') {\n let j = i;\n // Digits\n while (j < len && /[0-9]/.test(input[j])) {\n j++;\n }\n // Decimal point\n if (input[j] === '.') {\n j++;\n while (j < len && /[0-9]/.test(input[j])) {\n j++;\n }\n }\n // We must have at least one digit before or after the dot\n if (j === i) {\n throw new SyntaxError('Invalid number format');\n }\n tokens.push(input.slice(i, j));\n i = j;\n continue;\n }\n\n // Unknown character\n throw new SyntaxError(`Unexpected character: ${ch}`);\n }\n\n return tokens;\n}\n```" + }, + { + "problem": "js-format-bytes", + "sample": 0, + "ok": false, + "error": "node:internal/modules/run_main:123\n triggerUncaughtException(\n ^\n\nAssertionError [ERR_ASSERTION]: Expected values to be strictly equal:\n+ actual - expected\n\n+ '-1536.0 B'\n- '-1.5 KiB'\n ^\n\n at file:///tmp/flycoder-bench-GqVALN/test.mjs:7:3 {\n generatedMessage: true,\n code: 'ERR_ASSERTION',\n actual: '-1536.0 B',\n expected: '-1.5 KiB',\n operator: 'strictEqual',\n diff: 'simple'\n}\n\nNode.js v22.22.0\n", + "truncated": false, + "wallMs": 426420, + "loadMs": 28787, + "outputTokens": 1323, + "decodeMs": 345939.114, + "promptTokens": 1284, + "prefillMs": 51673.661, + "thinkingChars": 0, + "answer": "```javascript\n/**\n * Formats a byte count using IEC binary units (KiB, MiB, etc.).\n * \n * Rules:\n * - Units: B, KiB, MiB, GiB, TiB, PiB (base 1024).\n * - Values < 1024 are integers (e.g., \"5 B\").\n * - Otherwise, use the largest unit where the value is >= 1.\n * - Format with exactly `decimals` digits after the decimal point.\n * - If rounding results in exactly 1024 of a unit, promote to the next unit.\n * - Values beyond PiB remain in PiB (no PeiB).\n * - Negative numbers preserve their sign.\n * - Throws TypeError if bytes is not a finite number.\n * - Throws RangeError if decimals is not an integer between 0 and 10.\n * \n * @param {number} bytes - The byte count.\n * @param {number} decimals - Number of decimal places (default 1).\n * @returns {string} Formatted string.\n * \n * @example\n * formatBytes(1500) // \"1.5 KiB\"\n * formatBytes(1048575, 1) // \"1.0 MiB\" (promotes from 1024.0 KiB)\n * formatBytes(500) // \"500 B\"\n */\nexport function formatBytes(bytes, decimals = 1) {\n // Validate bytes: must be finite number\n if (!Number.isFinite(bytes)) {\n throw new TypeError(\"bytes must be a finite number\");\n }\n\n // Validate decimals: must be integer 0-10\n if (!Number.isInteger(decimals) || decimals < 0 || decimals > 10) {\n throw new RangeError(\"decimals must be an integer from 0 to 10\");\n }\n\n const units = [\"B\", \"KiB\", \"MiB\", \"GiB\", \"TiB\", \"PiB\"];\n const base = 1024;\n\n // Special case: 0 or negative values less than 1024 in absolute value (after sign)\n // But we need to handle the magnitude for unit selection.\n // If bytes is 0, result is \"0 B\".\n \n // Determine magnitude for unit selection.\n // We work with absolute value to decide unit, then reapply sign.\n const absBytes = Math.abs(bytes);\n \n // If absolute value is less than 1024, use \"B\" unit.\n // But we must format as integer if it's effectively an integer? \n // The spec says: \"Values below 1024 are integers shown as \" B\".\"\n // This implies no decimal places for the \"B\" unit case.\n \n if (absBytes < 1024) {\n // Format as integer, removing any fractional part (rounding to nearest integer for display? \n // Or truncating? Usually formatting bytes implies rounding to nearest integer for display).\n // However, the spec says \"integers shown\", implying we should display the integer part.\n // Let's round to nearest integer for display to avoid \"0.9 B\".\n const rounded = Math.round(bytes);\n return `${rounded} B`;\n }\n\n // For values >= 1024, find the largest unit where value >= 1.\n // Start from the largest unit (PiB) down to B.\n // We calculate value = bytes / (base ^ i)\n // Find first i (from 0 to 5) where value >= 1.\n \n let value = bytes;\n let unitIndex = 0;\n \n // Check from largest unit down to B to find the appropriate unit\n for (let i = units.length - 1; i >= 0; i--) {\n const divisor = Math.pow(base, i);\n const val = value / divisor;\n if (val >= 1) {\n unitIndex = i;\n break;\n }\n }\n\n // Calculate the value in the chosen unit\n const divisor = Math.pow(base, unitIndex);\n const formattedValue = value / divisor;\n\n // Check if rounding would display 1024.0 of the current unit\n // We need to check if the rounded value is >= 1024 - epsilon (to handle floating point)\n // Specifically, if rounding results in 1024.0, promote to next unit.\n // We'll round to the specified decimals first, then check.\n \n // Format with the exact number of decimals\n const formattedStr = formattedValue.toFixed(decimals);\n const parsedValue = parseFloat(formattedStr);\n\n // If the parsed value is 1024 (or very close due to float, but toFixed should handle it cleanly)\n // Check if it's >= 1024 - a tiny epsilon to handle floating point inaccuracies before toFixed\n // Actually, toFixed returns a string, so we parse it.\n // If the rounded value is 1024, we promote.\n \n // However, we must be careful: if decimals=0, \"1024\" is 1024.\n // If decimals=1, \"1024.0\" is 1024.\n \n if (parsedValue >= 1024 - 1e-9) {\n // Promote to next unit\n unitIndex++;\n // Recalculate\n const newDivisor = Math.pow(base, unitIndex);\n const newValue = value / newDivisor;\n const newFormattedStr = newValue.toFixed(decimals);\n return `${newFormattedStr} ${units[unitIndex]}`;\n }\n\n return `${formattedStr} ${units[unitIndex]}`;\n}\n```" + }, + { + "problem": "js-wildcard", + "sample": 0, + "ok": false, + "error": "truncated by max tokens", + "truncated": true, + "wallMs": 611326, + "loadMs": 31307, + "outputTokens": 2048, + "decodeMs": 531264.025, + "promptTokens": 1186, + "prefillMs": 48736.48, + "thinkingChars": 0, + "answer": "```javascript\n/**\n * Match a wildcard pattern against a text.\n * '?' matches exactly one character.\n * '*' matches any sequence of characters (including empty).\n * No escaping, no character classes.\n * Returns true if the whole text matches the pattern.\n * Optimized for adversarial inputs (many '*' followed by specific chars).\n */\nexport function matchWildcard(pattern, text) {\n // Edge case: both empty\n if (pattern.length === 0 && text.length === 0) return true;\n if (pattern.length === 0) return false;\n if (text.length === 0) {\n // Pattern must be all '*' to match empty text\n return pattern === \"*\".repeat(pattern.length);\n }\n\n // Convert pattern to a list of segments:\n // Each segment is either a literal string or a single '?'\n // We compress consecutive '*' into a single '*' marker.\n // Format: { type: 'literal', value: string } | { type: 'q', value: '?' } | { type: 'star' }\n \n const segments = [];\n let i = 0;\n let current = \"\";\n \n while (i < pattern.length) {\n const ch = pattern[i];\n if (ch === \"*\") {\n if (current !== \"\") {\n segments.push({ type: \"literal\", value: current });\n current = \"\";\n }\n // Push a single star marker if not already present at end\n if (segments.length > 0 && segments[segments.length - 1].type === \"star\") {\n // Do nothing, star already there\n } else {\n segments.push({ type: \"star\" });\n }\n i++;\n } else if (ch === \"?\") {\n current += ch;\n i++;\n } else {\n current += ch;\n i++;\n }\n }\n \n if (current !== \"\") {\n segments.push({ type: \"literal\", value: current });\n }\n \n // Now we have a list of segments. We need to match them against text.\n // Strategy: Dynamic Programming (DP) or a greedy approach with backtracking.\n // Given the adversarial case (many \"*a\"), a pure greedy might fail if we don't backtrack.\n // However, standard DP O(N*M) might be too slow for thousands of chars if N*M is huge.\n // But N, M ~ 1000 -> 10^6 ops is fine in JS.\n // Let's use a simplified DP: dp[i][j] = can match pattern[0..i] with text[0..j].\n // To optimize space and time for this specific wildcard problem without regex,\n // we can use a 1D DP array where dp[j] represents whether text[0..j] matches the pattern processed so far.\n \n const n = text.length;\n const m = segments.length;\n \n // dp[j] will be true if text[0..j-1] matches the prefix of pattern processed so far.\n // Initialize: empty text matches empty pattern.\n const dp = new Array(n + 1).fill(false);\n dp[0] = true; // Empty pattern matches empty text\n \n // We need to handle the segments. Since we have '*' in segments, we can process them.\n // However, the standard DP for wildcard matching is:\n // dp[i][j] = (dp[i-1][j-1] if literal/question) OR (dp[i][j-1] if '*')\n // Let's implement a 2D DP logic but optimized to 1D if possible, or just 2D if memory allows.\n // 1000x1000 booleans is ~1MB, perfectly fine.\n \n // Re-constructing a 2D DP for clarity and correctness first.\n // If performance is critical for large inputs, we might need to optimize, \n // but the problem says \"few thousand characters\", so O(N*M) is acceptable.\n \n // Let's refine the DP state:\n // dp[i][j] = true if pattern[0..i-1] matches text[0..j-1]\n // pattern is represented by our 'segments' array.\n \n // Actually, let's stick to the raw pattern string for DP to avoid segment conversion overhead \n // and handle '*' directly in the loop.\n \n const pLen = pattern.length;\n const tLen = text.length;\n \n // dp[j] = true if text[0..j-1] matches the pattern prefix processed so far.\n // We iterate through the pattern character by character.\n \n // Initialize: dp[0] = true (empty text matches empty pattern)\n // All other dp[j] = false initially.\n \n // We need to handle the '*' specially.\n // When we see a literal or '?', we can only extend from the previous state where text matched.\n // When we see '*', we can match empty string (dp[j] = dp[j]) or consume one char (dp[j+1] = dp[j]).\n // But since '*' matches ANY sequence, if we have a '*' at pattern index i, \n // then for any j, dp[i+1][j] = dp[i][j] (match empty) OR dp[i+1][j-1] (match one char) ... \n // Actually, the standard recurrence for wildcard with '*' is:\n // dp[i][j] = dp[i-1][j-1] (if pattern[i-1] matches text[j-1])\n // OR (if pattern[i-1] == '*') (dp[i-1][j] OR dp[i][j-1])\n \n // Let's use a 1D array optimization.\n // dp[j] corresponds to the current row's dp[...][j].\n // We need to be careful with the '*' row.\n \n // Let's restart with a clear 1D DP approach.\n // dp[j] = can pattern[0..i] match text[0..j]?\n // When pattern[i] is not '*':\n // dp[j] = dp[j-1] && (pattern[i] == text[j-1] || pattern[i] == '?')\n // When pattern[i] is '*':\n // dp[j] = dp[j] (match empty) OR dp[j-1] (match one char) ... \n // Actually, if pattern[i] is '*', then dp[j] = dp[j-1] OR dp[j] (from previous row, matching empty).\n // But wait, if we are at row i (pattern char '*'), then dp[j] becomes true if \n // either it was already true (matching empty string of '*') OR if dp[j-1] was true (consuming text[j-1]).\n // So: new_dp[j] = dp[j] || dp[j-1] (if j>0).\n // However, this logic assumes that the '*' can consume any number of characters.\n // The standard optimization for this specific problem (no regex) is:\n // Maintain a variable 'starPos' which tracks the last position of '*' in pattern.\n // And a 'textPos' which tracks the last position in text where a '*' matched.\n // This is the greedy approach with backtracking which is O(N).\n \n // Let's implement the O(N) greedy approach with backtracking (Z-algo style or similar).\n // Algorithm:\n // 1. Find all '*' positions in pattern.\n // 2. Match literal parts greedily.\n // 3. If a mismatch occurs, backtrack to the last '*' and try to extend the match.\n \n // This is the most efficient way.\n \n const starIndices = [];\n for (let i = 0; i < pLen; i++) {\n if (pattern[i] === \"*\") {\n starIndices.push(i);\n }\n }\n \n // We will match the pattern against the text.\n // We maintain a pointer in text.\n \n // Actually, the O(N) algorithm is:\n // Use a DP array where dp[i] stores the length of the matched suffix of pattern ending at pattern[i] \n // that matches a suffix of text ending at current text position.\n // But that's complex to explain.\n \n // Let's use the standard \"two pointers with backtracking\" logic which is O(N) in practice \n // because backtracking doesn't visit states repeatedly in a way that explodes.\n \n // Wait, the most robust O(N) algorithm for wildcard matching is:\n // dp[i] = the index in text where pattern[0..i] ends matching.\n // But we need to handle '*' which can skip text.\n \n // Let's use the method from LeetCode 44 (Wildcard Matching) optimized.\n // We can use a 1D DP array where dp[j] represents: \n // Can pattern[0..i] match text[0..j]?\n // But we want to avoid O(N*M) if possible." + }, + { + "problem": "js-search-range", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 208101, + "loadMs": 31423, + "outputTokens": 521, + "decodeMs": 129908.303, + "promptTokens": 1139, + "prefillMs": 46753.466, + "thinkingChars": 0, + "answer": "```javascript\n/**\n * Returns [first, last] indices of target in a sorted array.\n * If target is absent, returns [-1, -1].\n * Runs in O(log n) time using binary search.\n * @param {number[]} sorted - Ascending array of numbers.\n * @param {number} target - The number to find.\n * @returns {number[]} [firstIndex, lastIndex]\n */\nexport function searchRange(sorted, target) {\n if (!Array.isArray(sorted)) {\n throw new TypeError(\"sorted must be an array\");\n }\n if (typeof target !== \"number\") {\n throw new TypeError(\"target must be a number\");\n }\n\n const leftBound = lowerBound(sorted, target);\n if (leftBound === -1) {\n return [-1, -1];\n }\n // leftBound is the first index where sorted[i] >= target\n // We must verify sorted[leftBound] === target\n if (sorted[leftBound] !== target) {\n return [-1, -1];\n }\n\n const rightBound = upperBound(sorted, target) - 1;\n return [leftBound, rightBound];\n}\n\n/**\n * Finds the first index where sorted[index] >= target.\n * Returns -1 if not found.\n */\nfunction lowerBound(sorted, target) {\n let lo = 0;\n let hi = sorted.length;\n while (lo < hi) {\n const mid = (lo + hi) >> 1; // equivalent to Math.floor((lo + hi) / 2)\n if (sorted[mid] < target) {\n lo = mid + 1;\n } else {\n hi = mid;\n }\n }\n return lo;\n}\n\n/**\n * Finds the first index where sorted[index] > target.\n * Returns sorted.length if all elements are <= target.\n */\nfunction upperBound(sorted, target) {\n let lo = 0;\n let hi = sorted.length;\n while (lo < hi) {\n const mid = (lo + hi) >> 1;\n if (sorted[mid] <= target) {\n lo = mid + 1;\n } else {\n hi = mid;\n }\n }\n return lo;\n}\n```" + } + ] + } + ], + "platform": "linux x64 Intel(R) Xeon(R) Processor @ 2.80GHz" +} \ No newline at end of file diff --git a/docs/ollama/flycoder0.3pro.md b/docs/ollama/flycoder0.3pro.md index 2456dea..830afaa 100644 --- a/docs/ollama/flycoder0.3pro.md +++ b/docs/ollama/flycoder0.3pro.md @@ -10,7 +10,7 @@ ollama run Tromset/flycoder0.3pro |---|---|---|---| | Qwen3.8 27B (Apache 2.0) | 18 GB | 32 GB | 64K tokens | -- **Works like a senior engineer**: pins down the contract, plans multi-file changes, writes the code and the tests that prove it, checks the tricky cases before answering, then tells you how to run the tests. +- **Code that runs**: exact names and signatures, every import, edge cases handled, a final check before answering, no "TODO", no invented APIs. - **Built for agents**: reads files before editing, makes minimal changes, reports real test output, never runs a destructive command without asking (`ollama launch claude --model Tromset/flycoder0.3pro`). - **Fast for its size**: multi-token prediction is on, and 64K tokens of context cost only about 4.4 GB thanks to Qwen3.8's hybrid attention. diff --git a/readme.md b/readme.md index 0877f59..2b1fdf5 100644 --- a/readme.md +++ b/readme.md @@ -25,7 +25,7 @@ What FlyCoder sets up on top of the base: - **A large context**: 64K tokens instead of Ollama's 4,096 default on machines under 24 GB. Qwen's hybrid attention keeps it cheap: `flycoder0.3` uses 8.8 GB in total at 64K. - **The right sampling**: Qwen's recommended settings for coding in thinking mode. - **A short system prompt for code**: exact names and signatures, complete code without "TODO", edge cases, no invented APIs, a final check before answering, secure defaults, answers in the user's language. -- **`flycoder0.3pro` has its own recipe**: a six-step workflow (contract, plan, code, tests, check, report), rules for coding agents, a worked example that shows the expected answer, and multi-token prediction. Details: [Documentation.md, section 6.5](Documentation.md#65-what-makes-flycoder03pro-different). +- **`flycoder0.3pro` adds rules for coding agents and multi-token prediction** on Qwen3.8 27B. Details: [Documentation.md, section 6.5](Documentation.md#65-what-makes-flycoder03pro-different). ## Install under a short name @@ -38,7 +38,15 @@ The installer picks the variant from your memory and names it `flycoder`. Option ## Measured quality and speed -The 0.3 reference run (`docs/bench/cpu-nothink-0.3.json`) is being recorded. The 0.2 run stays in [docs/bench/cpu-nothink.json](docs/bench/cpu-nothink.json). +20 coding problems with hidden tests (`bench/`), on a 4-core Linux server without a GPU, thinking off, one attempt per problem: + +| Profile | Base | Passed | Generation | +|---|---|---|---| +| `flycoder0.3` | Qwen3.5 9B | 9/20 | 3.9 tok/s | +| `flycoder0.3fast` | Qwen3.5 4B | 5/20 | 6.1 tok/s | +| 0.2 beta (did not run on every 16 GB Mac) | Gemma 4 12B | 14/20 | 3.4 tok/s | + +Three of the 9B's failures are answers cut off at 2,048 tokens: with thinking off it reasons in code comments; keep thinking on (the default) for code you ship. On a Mac, speeds are much higher. Details and the system-prompt comparison: [Documentation.md, section 7](Documentation.md#7-benchmark). ## FlyBrain: less RAM for simple requests (optional) diff --git a/tests/modelfile.test.mjs b/tests/modelfile.test.mjs index 4690172..07fd664 100644 --- a/tests/modelfile.test.mjs +++ b/tests/modelfile.test.mjs @@ -2,7 +2,6 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import { execFileSync } from 'node:child_process'; -import { PROBLEMS } from '../bench/problems.mjs'; const read = file => fs.readFileSync(new URL(`../${file}`, import.meta.url), 'utf8'); // Minimal Modelfile reader: FROM, REQUIRES, PARAMETER, and the triple-quoted SYSTEM and MESSAGE blocks. @@ -55,38 +54,20 @@ test('the 0.3, fast and lite experts share one system prompt, apart from the bas for (const v of SHARED) assert.deepEqual(parse(read(v.file)).messages, [], `${v.file} has no example conversation`); }); -test('flycoder0.3pro has its own agent workflow on top of the shared rules', () => { - const pro = parse(read(PRO.file)).system; - assert.match(pro, /^You are FlyCoder 0\.3 pro,/); - for (const step of ['1. Contract', '2. Plan', '3. Code', '4. Tests', '5. Check', '6. Report', 'As an agent with tools']) assert.ok(pro.includes(step), `pro prompt has ${step}`); - for (const rule of ['No placeholders, no "TODO"', 'never invent functions', 'parameterized SQL', 'Never claim a result you did not see', "write them in the user's language"]) - assert.ok(pro.includes(rule), `pro prompt keeps the rule: ${rule}`); - assert.ok(pro.length < 3000, 'keep the pro prompt short'); -}); - -test("the pro example conversation shows code that passes its own tests", () => { - const messages = parse(read(PRO.file)).messages; - assert.deepEqual(messages.map(m => m.role), ['user', 'assistant']); - // An example that solves a bench problem would hand the model the answer and inflate its score. - for (const p of PROBLEMS) assert.ok(!messages[0].content.includes(p.entry), `the example must not be the bench problem ${p.id}`); - assert.ok(!messages[1].content.includes('"""'), 'triple double quotes would end the MESSAGE block'); - const [code, tests] = [...messages[1].content.matchAll(/```python\n([\s\S]*?)```/g)].map(m => m[1]); - assert.match(tests, /pytest/); - // Run the example with plain asserts (pytest is not needed): the valid and invalid cases of its tests. - const [valid, invalid] = [...tests.matchAll(/"text", (\[[^\]]*\])/g)].map(m => JSON.parse(m[1].replace(/,\s*\]$/, ']'))); - assert.ok(valid.length >= 5 && invalid.length >= 5); - const check = `${code}\nfor text in ${JSON.stringify(valid)}:\n assert is_valid_ipv4(text), text\n` + - `for text in ${JSON.stringify(invalid)}:\n assert not is_valid_ipv4(text), text\nprint("ok")\n`; - assert.equal(execFileSync('python3', ['-c', check], { encoding: 'utf8' }).trim(), 'ok'); -}); - -test('Modelfile.router is a tiny deterministic classifier that answers simple or hard', () => { - const m = parse(read(ROUTER.file)); - assert.deepEqual(m.from, [ROUTER.gguf]); - assert.deepEqual(m.unknown, []); - assert.deepEqual(m.params, ROUTER.params); - assert.match(m.system, /"simple"/); assert.match(m.system, /"hard"/); - assert.match(m.system, /When unsure, answer "hard"/); +test('flycoder0.3pro keeps every shared rule and adds the agent rules', () => { + // Measured on the 9B base: the shared prompt solved 9/20 bench problems, a six-step workflow with a worked example 6/20. + const shared = parse(read('Modelfile')).system; + const m = parse(read(PRO.file)); + assert.match(m.system, /^You are FlyCoder 0\.3 pro,/); + assert.ok(m.system.includes(PRO.base), 'pro prompt names the real base model'); + const rules = shared.split('\n').filter(l => l.startsWith('- ') && !l.startsWith('- Never claim')); + assert.ok(rules.length >= 8); + for (const rule of rules) assert.ok(m.system.includes(rule), `pro prompt keeps: ${rule}`); + assert.match(m.system, /Never claim that you ran code or tests unless a tool actually ran them/); + assert.match(m.system, /As an agent with tools: read the relevant files before editing them/); + assert.match(m.system, /Never run a destructive command/); + assert.deepEqual(m.messages, [], 'no example conversation: it cost quality in the A/B bench'); + assert.ok(m.system.length < 3000, 'keep the pro prompt short'); }); test('installer and publisher agree with the Modelfiles and the package version', () => {