Skip to content

Commit 600c6ce

Browse files
authored
Merge pull request #73 from OpenAF/codex/revamp-mcpdynamic-for-llm-fallback
Add connection-level fallback to MCP dynamic selection
2 parents 8c37e62 + a923115 commit 600c6ce

File tree

7 files changed

+1227
-70
lines changed

7 files changed

+1227
-70
lines changed

.package.yaml

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@ repository:
4848
url: https://openaf.io/opacks/mini-a.opack
4949
description: Mini-A is a minimalist autonomous agent that uses LLMs, shell commands and/or MCP stdio or http(s) servers to achieve user-defined goals. It is designed to be simple, flexible, and easy to use.
5050
name: mini-a
51-
main: ''
52-
mainJob: mini-a-main.yaml
51+
main: mini-a-con.js
52+
mainJob: ''
5353
license: Apache 2.0 license
5454
version: '20251108'
5555
dependencies:
@@ -93,7 +93,8 @@ files:
9393
- mcps/mcp-weather.yaml
9494
- mcps/mcp-web.yaml
9595
- mini-a-con.js
96-
- mini-a-main.yaml
96+
- mini-a-main.yaml.old
97+
- mini-a-modelman.js
9798
- mini-a-modes.yaml
9899
- mini-a-utils.js
99100
- mini-a-web.sh
@@ -120,7 +121,7 @@ filesHash:
120121
LICENSE: 7df059597099bb7dcf25d2a9aedfaf4465f72d8d
121122
README.md: c06f60ae19a9872b1eef1a2dc5da6867ffe4f7ca
122123
TESTS_MODELS.md: 8c09f3ca8112e4b8bd007fc096fcac8946d25cf7
123-
USAGE.md: eb83a7aa4e3bd902a391638e74d88ae775c43597
124+
USAGE.md: c1d80828785d5f2dca31656365c898f6dd935c8c
124125
examples/README.md: 0d77434a52d4b21fdd033d980086fa530964e9d8
125126
examples/changelog-gen.yaml: 4f329832fb682821b7357c5f0a2d4f5f811d29f7
126127
examples/document.yaml: 19871e428a840eb2b726a4167664a0f27926a66d
@@ -148,13 +149,14 @@ filesHash:
148149
mcps/mcp-time.yaml: fb1a07bd28cf5284bb4178694411d4a8a736ba33
149150
mcps/mcp-weather.yaml: dd339672ebcd6094e8f2ef87fc32a6d95b53bc68
150151
mcps/mcp-web.yaml: a3c9cd4f02b7874f8faa2a097d445e2f2ca451b9
151-
mini-a-con.js: bd05f76db77c7c754dbf6d8fc1e99ab3a389a5d8
152-
mini-a-main.yaml: 4a358bb2b569a8393ac9b8ddbda1599cd9621a02
152+
mini-a-con.js: 66086fadafaeabe8d390353e70f9d606dd434eaa
153+
mini-a-main.yaml.old: a4ece7b69a410fe2adc1938819901255d22c5cbb
154+
mini-a-modelman.js: 3039a8fa9b947634c61d5136a497208bc883973a
153155
mini-a-modes.yaml: 3e9cf81240092c5db5eda1b64ada21a299fafe91
154156
mini-a-utils.js: 8663ea34f3b64d0a5367e4ab6556766332967f11
155157
mini-a-web.sh: 31c3f5a5b5cf0b6795f2079ed5821091c48b4184
156158
mini-a-web.yaml: cb838c26c0273b40f3190458126f716cfc9848a0
157-
mini-a.js: 1808747e45419c334ee31d83f8840998b6df6c8b
159+
mini-a.js: f452df30792a9e599d37299f422cfc23fac8e7cc
158160
mini-a.sh: 0b8991af4014610f24260d0db202a2cd9f6e9f33
159161
mini-a.yaml: ecf2e8c567562be695169ee29428ecb22f575d26
160162
public/index.md: 475c7278c9bc504b8612a1e7f4ac72f1defc959d

README.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,10 @@ mini-a goal="help me plan a vacation in Lisbon" chatbotmode=true
6464
- **Multi-Model Support** - Works with OpenAI, Google Gemini, GitHub Models, AWS Bedrock, Ollama, and more
6565
- **Dual-Model Cost Optimization** - Use a low-cost model for routine steps with smart escalation (see [USAGE.md](USAGE.md#dual-model-setup-cost-optimization))
6666
- **MCP Integration** - Seamless integration with Model Context Protocol servers (STDIO & HTTP)
67+
- **Dynamic Tool Selection** - Intelligent filtering of MCP tools using stemming, synonyms, n-grams, and fuzzy matching (`mcpdynamic=true`)
68+
- **Tool Caching** - Smart caching for deterministic and read-only tools to avoid redundant operations
69+
- **Circuit Breakers** - Automatic connection health management with cooldown periods
70+
- **Lazy Initialization** - Deferred MCP connection establishment for faster startup (`mcplazy=true`)
6771
- **Built-in MCP Servers** - Database, file system, network, time/timezone, email, S3, RSS, Yahoo Finance, SSH, and more
6872
- **MCP Self-Hosting** - Expose Mini-A itself as a MCP server via `mcps/mcp-mini-a.yaml` (remote callers can run goals with limited formatting/planning overrides while privileged flags stay server-side)
6973
- **Optional Shell Access** - Execute shell commands with safety controls and sandboxing

USAGE.md

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -353,7 +353,7 @@ The `start()` method accepts various configuration options:
353353
#### MCP (Model Context Protocol) Integration
354354
- **`mcp`** (string): MCP configuration in JSON format (single object or array for multiple connections)
355355
- **`usetools`** (boolean, default: false): Register MCP tools directly on the model instead of expanding the system prompt with tool schemas
356-
- **`mcpdynamic`** (boolean, default: false): When `usetools=true`, analyze the goal and only register the MCP tools that appear relevant, falling back to all tools if no clear match is found
356+
- **`mcpdynamic`** (boolean, default: false): When `usetools=true`, analyze the goal and only register the MCP tools that appear relevant, consulting the available LLMs to pick a promising connection when heuristics fail and only falling back to all tools if no confident choice is produced
357357
- **`mcplazy`** (boolean, default: false): Defer MCP connection initialization until a tool is first executed; useful when configuring many optional integrations
358358
- **`toolcachettl`** (number, optional): Override the default cache duration (milliseconds) for deterministic tool results when no per-tool metadata is provided
359359

@@ -371,11 +371,12 @@ Tools advertise determinism via MCP metadata (e.g., `annotations.readOnlyHint`,
371371

372372
Set `usetools=true mcpdynamic=true` when you want Mini-A to narrow the registered MCP tools to only those that look useful for the current goal. The agent evaluates the candidate list in stages:
373373

374-
1. **Keyword heuristics**: quick matching on tool names, descriptions, and goal keywords.
374+
1. **Keyword heuristics**: advanced matching on tool names, descriptions, and goal keywords using stemming (word root extraction), synonym matching (semantic equivalents), n-gram extraction (multi-word phrases), and fuzzy matching (typo tolerance via Levenshtein distance).
375375
2. **Low-cost model inference**: if `OAF_LC_MODEL` is configured, the cheaper model proposes a shortlist.
376376
3. **Primary model inference**: the main model performs the same selection when the low-cost tier does not return results.
377+
4. **Connection chooser fallback**: when no tools are selected, Mini-A asks the low-cost model (then the primary model if needed) to pick the single most helpful MCP connection and its tools before falling back further.
377378

378-
If every stage returns an empty list (or errors), Mini-A logs the issue and falls back to registering the full tool catalog so nothing is accidentally hidden. Selection happens per MCP connection, and you will see `mcp` log entries showing which tools were registered. Leave `mcpdynamic` false when you prefer the traditional register everything behaviour or when your model lacks tool-calling support.
379+
Only when every stage returns an empty list (or errors) does Mini-A log the issue and register the full tool catalog so nothing is accidentally hidden. Selection happens per MCP connection, and you will see `mcp` log entries showing which tools were registered. The new `tool_selection` metrics track which selection method was used (keyword, llm_lc, llm_main, connection_chooser, or fallback_all). Leave `mcpdynamic` false when you prefer the traditional "register everything" behaviour or when your model lacks tool-calling support.
379380

380381
#### Knowledge and Context
381382
- **`knowledge`** (string): Additional context or knowledge for the agent (can be text or file path)
@@ -573,16 +574,23 @@ When the model responds with multiple independent tool calls in the same step, M
573574
Pair `usetools=true` with `mcpdynamic=true` to let Mini-A narrow the registered tool set via intelligent filtering. This feature is especially useful when working with large MCP catalogs where registering all tools would overwhelm the context window.
574575

575576
**Selection strategy (multi-stage):**
576-
1. **Keyword heuristics** – Quick matching on tool names, descriptions, and goal keywords
577+
1. **Keyword heuristics** – Advanced matching on tool names, descriptions, and goal keywords using:
578+
- **Stemming** – Reduces words to root forms (search/searching/searched → search)
579+
- **Synonym matching** – Recognizes semantic equivalents (find=search, file=document, etc.)
580+
- **N-gram extraction** – Captures multi-word phrases like "file system" or "database query"
581+
- **Fuzzy matching** – Tolerates typos using Levenshtein distance (≤2 character changes)
577582
2. **Low-cost LLM inference** – If `OAF_LC_MODEL` is configured, the cheaper model proposes a shortlist
578583
3. **Primary model inference** – The main model performs selection when the low-cost tier doesn't return results
579-
4. **Fallback to full catalog** – If all stages return empty, Mini-A registers everything to ensure no tools are hidden
584+
4. **Connection chooser fallback** – When no tools match, asks LLM to choose the most relevant MCP connection
585+
5. **Fallback to full catalog** – If all stages return empty, Mini-A registers everything to ensure no tools are hidden
580586

581587
**Benefits:**
582-
- Reduced context window usage
583-
- Faster tool registration
588+
- Reduced context window usage through intelligent filtering
589+
- Faster tool registration with multi-stage selection
584590
- Lower token costs for large tool catalogs
591+
- Semantic understanding beyond exact keyword matches
585592
- Graceful degradation when filtering fails
593+
- Detailed metrics tracking selection method effectiveness
586594

587595
### Tool Caching & Optimization
588596

@@ -1311,6 +1319,9 @@ Mini-A records extensive counters that help track behaviour and costs:
13111319
| `performance` | `steps_taken`, `total_session_time_ms`, `avg_step_time_ms`, `max_context_tokens`, `llm_estimated_tokens`, `llm_actual_tokens`, `llm_normal_tokens`, `llm_lc_tokens` | Execution pacing and token consumption for cost analysis. |
13121320
| `behavior_patterns` | `escalations`, `retries`, `consecutive_errors`, `consecutive_thoughts`, `json_parse_failures`, `action_loops_detected`, `thinking_loops_detected`, `similar_thoughts_detected` | Signals that highlight unhealthy loops or parser problems. |
13131321
| `summarization` | `summaries_made`, `summaries_skipped`, `summaries_forced`, `context_summarizations`, `summaries_tokens_reduced`, `summaries_original_tokens`, `summaries_final_tokens` | Auto-summarization activity and token savings. |
1322+
| `tool_selection` | `dynamic_used`, `keyword`, `llm_lc`, `llm_main`, `connection_chooser_lc`, `connection_chooser_main`, `fallback_all` | Dynamic tool selection metrics tracking how tools are selected when `mcpdynamic=true`. Shows usage of keyword matching, LLM-based selection (low-cost and main models), connection-level chooser fallbacks, and full catalog fallback. Includes stemming, synonym matching, n-grams, and fuzzy matching capabilities. |
1323+
| `tool_cache` | `hits`, `misses`, `total_requests`, `hit_rate` | Tool result caching metrics for deterministic and read-only MCP tools. Tracks cache effectiveness and provides hit rate percentage. |
1324+
| `mcp_resilience` | `circuit_breaker_trips`, `circuit_breaker_resets`, `lazy_init_success`, `lazy_init_failed` | MCP resilience and optimization metrics. Circuit breaker trips/resets track connection health management. Lazy initialization metrics show deferred MCP connection establishment when `mcplazy=true`. |
13141325

13151326
These counters mirror what is exported via `ow.metrics.add('mini-a', ...)`, so the same structure appears in Prometheus/Grafana when scraped through OpenAF.
13161327

mini-a-con.js

Lines changed: 201 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,150 @@
44

55
try {
66
plugin("Console")
7-
var args = isDef(global._args) ? global._args : processExpr(" ")
7+
var args = processExpr(" ")
8+
9+
// Init
10+
if (!(isString(args.libs) && args.libs.trim().length > 0)) {
11+
var envLibs = args.OAF_MINI_A_LIBS || getEnv("OAF_MINI_A_LIBS")
12+
if (isString(envLibs) && envLibs.trim().length > 0) {
13+
args.libs = envLibs.trim()
14+
//log("Using libs from OAF_MINI_A_LIBS environment variable.")
15+
}
16+
//global._args = args
17+
}
18+
19+
(function(args) {
20+
if (args.__modeApplied === true) return
21+
if (!isString(args.mode)) return
22+
var modeName = args.mode.trim()
23+
if (modeName.length === 0) return
24+
25+
var modesPath = getOPackPath("mini-a") + "/mini-a-modes.yaml"
26+
var presets = {}
27+
try {
28+
var loaded = io.readFileYAML(modesPath)
29+
if (isMap(loaded) && isMap(loaded.modes)) {
30+
presets = loaded.modes
31+
} else if (isMap(loaded)) {
32+
presets = loaded
33+
} else {
34+
presets = {}
35+
}
36+
} catch(e) {
37+
var errMsg = (isDef(e) && isString(e.message)) ? e.message : e
38+
logWarn(`Failed to load mode presets for '${modeName}': ${errMsg}`)
39+
args.__modeApplied = true
40+
return
41+
}
42+
43+
// Load custom modes from user's home directory
44+
function resolveCanonicalPath(basePath, fileName) {
45+
return io.fileInfo((basePath || ".") + "/" + fileName).canonicalPath
46+
}
47+
var modesHome = isDef(__gHDir) ? __gHDir() : java.lang.System.getProperty("user.home")
48+
var customModesPath = resolveCanonicalPath(modesHome, ".openaf-mini-a_modes.yaml")
49+
if (io.fileExists(customModesPath)) {
50+
try {
51+
var customLoaded = io.readFileYAML(customModesPath)
52+
var customPresets = {}
53+
if (isMap(customLoaded) && isMap(customLoaded.modes)) {
54+
customPresets = customLoaded.modes
55+
} else if (isMap(customLoaded)) {
56+
customPresets = customLoaded
57+
}
58+
// Merge custom modes with default modes (custom overrides defaults)
59+
if (isMap(customPresets) && Object.keys(customPresets).length > 0) {
60+
presets = merge(presets, customPresets)
61+
}
62+
} catch(e) {
63+
var errMsg = (isDef(e) && isString(e.message)) ? e.message : e
64+
logWarn(`Failed to load custom mode presets from '${customModesPath}': ${errMsg}`)
65+
}
66+
}
67+
68+
if (!isMap(presets) || Object.keys(presets).length === 0) {
69+
logWarn(`Mode '${modeName}' requested but no presets are defined.`)
70+
args.__modeApplied = true
71+
return
72+
}
73+
74+
var keys = Object.keys(presets)
75+
var resolvedKey
76+
for (var i = 0; i < keys.length; i++) {
77+
var key = keys[i]
78+
if (key === modeName || key.toLowerCase() === modeName.toLowerCase()) {
79+
resolvedKey = key
80+
break
81+
}
82+
}
83+
84+
if (isUnDef(resolvedKey)) {
85+
logWarn(`Mode '${modeName}' not found. Available modes: ${keys.join(", ")}`)
86+
args.__modeApplied = true
87+
return
88+
}
89+
90+
var preset = presets[resolvedKey]
91+
if (!isMap(preset)) {
92+
logWarn(`Mode '${resolvedKey}' preset is invalid.`)
93+
args.__modeApplied = true
94+
return
95+
}
96+
97+
var applied = []
98+
var paramsSource = preset.params
99+
var applyParam = function(key, value) {
100+
if (isString(key) && key.length > 0) {
101+
args[key] = value
102+
applied.push(key)
103+
}
104+
}
105+
106+
if (isArray(paramsSource)) {
107+
paramsSource.forEach(function(entry) {
108+
if (!isMap(entry)) return
109+
Object.keys(entry).forEach(function(paramKey) {
110+
applyParam(paramKey, entry[paramKey])
111+
})
112+
})
113+
} else if (isMap(paramsSource)) {
114+
Object.keys(paramsSource).forEach(function(paramKey) {
115+
applyParam(paramKey, paramsSource[paramKey])
116+
})
117+
} else if (isDef(paramsSource)) {
118+
logWarn(`Mode '${resolvedKey}' has unsupported params definition.`)
119+
}
120+
121+
var infoMsg = `Mode '${resolvedKey}' enabled`
122+
if (isString(preset.description) && preset.description.length > 0) {
123+
infoMsg += `: ${preset.description}`
124+
}
125+
log(infoMsg)
126+
127+
if (applied.length > 0) {
128+
log(`Mode '${resolvedKey}' applied defaults for: ${applied.join(", ")}`)
129+
} else {
130+
log(`Mode '${resolvedKey}' did not change any arguments (overrides already provided).`)
131+
}
132+
133+
args.mode = resolvedKey
134+
args.__modeApplied = true
135+
})(args)
136+
137+
// Choose
138+
if (toBoolean(args.modelman) === true) {
139+
// Start model management mode
140+
load("mini-a-modelman.js")
141+
exit(0)
142+
} else if (toBoolean(args.web) === true || toBoolean(args.onport) === true) {
143+
// Start web mode
144+
oJobRunFile(getOPackPath("mini-a") + "/mini-a-web.yaml", args, genUUID(), __, false)
145+
exit(0)
146+
} else if (isDef(args.goal)) {
147+
// Start cli mode
148+
oJobRunFile(getOPackPath("mini-a") + "/mini-a.yaml", args, genUUID(), __, false)
149+
exit(0)
150+
}
8151

9152
// Helper functions
10153
// ----------------
@@ -66,7 +209,7 @@ try {
66209
var consoleReader = __
67210
var commandHistory = __
68211
var lastConversationStats = __
69-
var slashCommands = ["help", "set", "toggle", "unset", "show", "reset", "last", "clear", "context", "compact", "summarize", "history", "exit", "quit"]
212+
var slashCommands = ["help", "set", "toggle", "unset", "show", "reset", "last", "clear", "context", "compact", "summarize", "history", "model", "exit", "quit"]
70213
var resumeConversation = parseBoolean(findArgumentValue(args, "resume")) === true
71214
var conversationArgValue = findArgumentValue(args, "conversation")
72215
var initialConversationPath = isString(conversationArgValue) && conversationArgValue.trim().length > 0
@@ -1126,6 +1269,7 @@ try {
11261269
" " + colorifyText("/compact", "BOLD") + colorifyText(" [n] Summarize old context, keep last n messages", hintColor),
11271270
" " + colorifyText("/summarize", "BOLD") + colorifyText(" [n] Compact and display an LLM-generated conversation summary", hintColor),
11281271
" " + colorifyText("/history", "BOLD") + colorifyText(" [n] Show the last n conversation turns", hintColor),
1272+
" " + colorifyText("/model", "BOLD") + colorifyText(" [target] Choose a different model (target: model or modellc)", hintColor),
11291273
" " + colorifyText("/exit", "BOLD") + colorifyText(" Leave the console", hintColor)
11301274
]
11311275
print( ow.format.withSideLine( lines.join("\n"), __, promptColor, hintColor, ow.format.withSideLineThemes().openCurvedRect) )
@@ -1250,6 +1394,61 @@ try {
12501394
}
12511395
continue
12521396
}
1397+
if (command === "model" || command.indexOf("model ") === 0) {
1398+
var target = "model" // default to model
1399+
if (command.indexOf("model ") === 0) {
1400+
var targetArg = command.substring(6).trim().toLowerCase()
1401+
if (targetArg === "modellc" || targetArg === "lc") {
1402+
target = "modellc"
1403+
} else if (targetArg === "model") {
1404+
target = "model"
1405+
} else {
1406+
print(colorifyText("Invalid target. Use 'model' or 'modellc'.", errorColor))
1407+
continue
1408+
}
1409+
}
1410+
try {
1411+
// Store original args and set temporary args for model manager
1412+
var originalGlobalArgs = clone(args)
1413+
global._args = merge(args, { __noprint: true })
1414+
1415+
// Set up result capture mechanism
1416+
global.__mini_a_con_capture_model = true
1417+
global.__mini_a_con_model_result = __
1418+
1419+
// Load the model manager (which will execute mainOAFModel and store result)
1420+
var modelManPath = getOPackPath("mini-a") + "/mini-a-modelman.js"
1421+
load(modelManPath)
1422+
1423+
// Get the captured result
1424+
var selectedModel = global.__mini_a_con_model_result
1425+
1426+
// Clean up
1427+
delete global.__mini_a_con_capture_model
1428+
delete global.__mini_a_con_model_result
1429+
args = originalGlobalArgs
1430+
1431+
if (isMap(selectedModel)) {
1432+
sessionOptions[target] = af.toSLON(selectedModel)
1433+
1434+
/*if (target == "model") args.model = af.toSLON(selectedModel)
1435+
else if (target == "lowcost") args.modellc = af.toSLON(selectedModel)*/
1436+
1437+
print(colorifyText("Model definition set for " + target + ".", successColor))
1438+
//print(colorifyText("Value: " + modelSLON, hintColor))
1439+
} else {
1440+
print(colorifyText("No model selected.", hintColor))
1441+
}
1442+
} catch (modelError) {
1443+
printErr(ansiColor("ITALIC," + errorColor, "!!") + colorifyText(" Failed to load model: " + modelError, errorColor))
1444+
$err(modelError)
1445+
// Clean up on error
1446+
delete global.__mini_a_con_capture_model
1447+
delete global.__mini_a_con_model_result
1448+
if (isDef(originalGlobalArgs)) args = originalGlobalArgs
1449+
}
1450+
continue
1451+
}
12531452
if (command.indexOf("toggle ") === 0) {
12541453
toggleOption(command.substring(7).trim())
12551454
continue

0 commit comments

Comments
 (0)