mirror of
https://github.com/sipeed/picoclaw.git
synced 2026-06-12 18:08:54 +00:00
feat(model): llm rate limiting (#2198)
* feat(model): rate limiting * fix(agent): preserve per-model identity in rate limiting and fallback * fix test
This commit is contained in:
+21
-5
@@ -119,9 +119,18 @@ func NewAgentLoop(
|
||||
) *AgentLoop {
|
||||
registry := NewAgentRegistry(cfg, provider)
|
||||
|
||||
// Set up shared fallback chain
|
||||
// Set up shared fallback chain with rate limiting.
|
||||
cooldown := providers.NewCooldownTracker()
|
||||
fallbackChain := providers.NewFallbackChain(cooldown)
|
||||
rl := providers.NewRateLimiterRegistry()
|
||||
// Register rate limiters for all agents' candidates so that RPM limits
|
||||
// configured in ModelConfig are enforced before each LLM call.
|
||||
for _, agentID := range registry.ListAgentIDs() {
|
||||
if agent, ok := registry.GetAgent(agentID); ok {
|
||||
rl.RegisterCandidates(agent.Candidates)
|
||||
rl.RegisterCandidates(agent.LightCandidates)
|
||||
}
|
||||
}
|
||||
fallbackChain := providers.NewFallbackChain(cooldown, rl)
|
||||
|
||||
// Create state manager using default agent's workspace for channel recording
|
||||
defaultAgent := registry.GetDefaultAgent()
|
||||
@@ -1032,8 +1041,15 @@ func (al *AgentLoop) ReloadProviderAndConfig(
|
||||
al.cfg = cfg
|
||||
al.registry = registry
|
||||
|
||||
// Also update fallback chain with new config
|
||||
al.fallback = providers.NewFallbackChain(providers.NewCooldownTracker())
|
||||
// Also update fallback chain with new config; rebuild rate limiter registry.
|
||||
newRL := providers.NewRateLimiterRegistry()
|
||||
for _, agentID := range registry.ListAgentIDs() {
|
||||
if agent, ok := registry.GetAgent(agentID); ok {
|
||||
newRL.RegisterCandidates(agent.Candidates)
|
||||
newRL.RegisterCandidates(agent.LightCandidates)
|
||||
}
|
||||
}
|
||||
al.fallback = providers.NewFallbackChain(providers.NewCooldownTracker(), newRL)
|
||||
|
||||
al.mu.Unlock()
|
||||
|
||||
@@ -3229,7 +3245,7 @@ func (al *AgentLoop) buildCommandsRuntime(agent *AgentInstance, opts *processOpt
|
||||
return "", fmt.Errorf("failed to initialize model %q: %w", value, err)
|
||||
}
|
||||
|
||||
nextCandidates := resolveModelCandidates(cfg, cfg.Agents.Defaults.Provider, modelCfg.Model, agent.Fallbacks)
|
||||
nextCandidates := resolveModelCandidates(cfg, cfg.Agents.Defaults.Provider, value, agent.Fallbacks)
|
||||
if len(nextCandidates) == 0 {
|
||||
return "", fmt.Errorf("model %q did not resolve to any provider candidates", value)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user