mirror of
https://github.com/sipeed/picoclaw.git
synced 2026-05-25 16:00:35 +00:00
15a70ac45c
* feat(seahorse): implement short-term memory engine of seahorse Add pkg/seahorse/ module implementing a SQLite-backed DAG-based summary hierarchy for context management, ported from lossless-claw's LCM design: - types.go + short_constants.go: core types (Message, Summary, Conversation, ContextItem) and configuration constants (fanout, token targets, thresholds) - migration.go: idempotent DB schema with FTS5 trigram tokenizer for CJK - store.go: full SQLite CRUD (conversations, messages, summaries DAG, context_items with ordinal gap numbering, FTS5 search) - short_engine.go: Engine lifecycle (NewEngine, Ingest, Assemble, Compact), session pattern filtering (ignore/stateless glob→regex compilation), per-session mutex via sync.Map - short_assembler.go: budget-aware context assembly with fresh tail protection (32 messages), oldest-first eviction, summary XML formatting, RebuildContextItems - short_compaction.go: leaf compaction (messages→summary) and condensed compaction (summaries→higher-level summary), 3-level LLM escalation, CompactUntilUnder for emergency overflow - short_retrieval.go: lookupByID, FTS5/LIKE search, recursive expand with token cap - context_seahorse.go: agent.ContextManager adapter, registered as "seahorse", provider↔seahorse message type conversion (ToolCalls, tool_result) * fix(seahorse): correct 3 adapter bugs in context management - TokenCount: use full message (Content+ToolCalls+Media) instead of Content-only - Empty Content: rebuild Content from tool_result Parts when stored empty - Duplicate summaries: summaries only in Summary field, not in History messages - Grep: fix SearchResult.Snippet→Content for summaries - Schema: fix FTS5 SQL uses VIRTUAL TABLE not TEMP TABLE - TestFTS5SQLConstants: verify FTS5 SQL syntax correctness - Test: fix flaky TestCompactLeaf * fix(agent): ingest steering messages into seahorse SQLite Steering messages were only persisted to session JSONL but not ingested into seahorse SQLite, causing them to be missing from context assembly. Added `ts.ingestMessage(turnCtx, al, pm)` call in the steering message injection block alongside the existing JSONL persistence. Test: TestSeahorseSteeringMessageIngested verifies steering messages appear in seahorse SQLite DB after being processed. * fix(seahorse): address 3 blocking bugs from code review - Fix resequenceContextItemsTx scan error handling (store.go:850) Changed `return err` to `return scanErr` to properly propagate scan errors instead of returning nil (which silently corrupts data) - Fix sql.NullString for INTEGER column (store.go:847) Changed `mid` from sql.NullString to sql.NullInt64 since message_id is INTEGER in schema. Removed unnecessary strconv.ParseInt call. - Fix compactCondensed fallback deleting non-candidate items Added ReplaceContextItemsWithSummary method for per-item deletion when candidates are not contiguous in ordinal space. Optimized to use range deletion when candidates are consecutive. * fix(seahorse): pass Budget to Compact for correct condensed threshold Issue #4 from PR review: When Budget was not passed to seahorse.Compact, it defaulted to `tokensBefore * 0.75`, making `tokensBefore > budget` always true and causing condensed compaction to trigger unnecessarily. Changes: - context_seahorse.go: Forward Budget from CompactRequest to CompactInput - loop.go: Pass Budget (ContextWindow) in all 3 Compact calls - Add test verifying condensed is skipped when tokens < threshold - Fix lint issues in store.go and store_test.go * fix(seahorse): add mutex for assembler lazy initialization Issue #5 from PR review: The check-then-create pattern for e.assembler was a data race when multiple goroutines called Assemble() concurrently: if e.assembler == nil { e.assembler = &Assembler{...} } Changes: - Add assemblerMu sync.Mutex to Engine struct - Add initAssemblerOnce() using double-checked locking (same pattern as initCompactionOnce) - Add TestAssemblerLazyInitRace to verify thread-safety * fix(seahorse): handle non-consecutive depths in selectShallowestCondensationCandidate Issue #8 from PR review: the loop iterated depth 0, 1, 2... assuming consecutive keys, but break when key was missing caused deeper depths to never be checked. Fix: collect all existing depth keys, sort, then iterate in order. * fix(seahorse): wrap DeleteMessagesAfterID and appendContextItems in transactions - DeleteMessagesAfterID: wrap all DELETE operations in a transaction for atomicity, remove redundant manual FTS delete (handled by trigger) - appendContextItems: use transaction to fix read-then-write race condition - Add GetMaxOrdinalTx and resolveItemTokenCountTx for transaction-scoped queries - Remove unused resolveItemTokenCount function Fixes PR review issues 6 and 7. * fix(seahorse): derive readable content from Parts and cap CompactUntilUnder iterations - Derive readable content from MessageParts in AddMessageWithParts so FTS5 indexing and summary formatting can access tool call information - formatMessagesForSummary and truncateSummary now fall back to Parts when Content is empty, fixing blank summaries for Part-based messages - Add MaxCompactIterations (20) to prevent CompactUntilUnder infinite loops; exceeded iterations are logged as warnings
213 lines
6.1 KiB
Go
213 lines
6.1 KiB
Go
package seahorse
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// ParseLastDuration parses a "last" duration string like "6h", "7d", "2w", "1m".
|
|
// Returns the duration and nil error, or zero and error if invalid.
|
|
func ParseLastDuration(s string) (time.Duration, error) {
|
|
if s == "" {
|
|
return 0, fmt.Errorf("empty duration")
|
|
}
|
|
|
|
re := regexp.MustCompile(`^(\d+)([hdwm])$`)
|
|
matches := re.FindStringSubmatch(s)
|
|
if matches == nil {
|
|
return 0, fmt.Errorf("invalid duration format: %q (use format like 6h, 7d, 2w, 1m)", s)
|
|
}
|
|
|
|
value, _ := strconv.Atoi(matches[1])
|
|
unit := matches[2]
|
|
|
|
switch unit {
|
|
case "h":
|
|
return time.Duration(value) * time.Hour, nil
|
|
case "d":
|
|
return time.Duration(value) * 24 * time.Hour, nil
|
|
case "w":
|
|
return time.Duration(value) * 7 * 24 * time.Hour, nil
|
|
case "m":
|
|
return time.Duration(value) * 30 * 24 * time.Hour, nil
|
|
default:
|
|
return 0, fmt.Errorf("unknown unit: %q", unit)
|
|
}
|
|
}
|
|
|
|
// GrepInput controls search across summaries and messages.
|
|
type GrepInput struct {
|
|
Pattern string `json:"pattern"`
|
|
Scope string `json:"scope,omitempty"` // "both" (default), "summary", or "message"
|
|
Role string `json:"role,omitempty"` // "user", "assistant", or "" (all)
|
|
AllConversations bool `json:"allConversations,omitempty"`
|
|
Since *time.Time `json:"since,omitempty"`
|
|
Before *time.Time `json:"before,omitempty"`
|
|
Last string `json:"last,omitempty"` // shortcut: "6h", "7d", "2w", "1m"
|
|
Limit int `json:"limit,omitempty"`
|
|
}
|
|
|
|
// GrepResult contains search results.
|
|
type GrepResult struct {
|
|
Success bool `json:"success"`
|
|
Summaries []GrepSummaryResult `json:"summaries"`
|
|
Messages []GrepMessageResult `json:"messages"`
|
|
TotalSummaries int `json:"totalSummaries"`
|
|
TotalMessages int `json:"totalMessages"`
|
|
Hint string `json:"hint,omitempty"`
|
|
}
|
|
|
|
// GrepSummaryResult is a summary match from grep.
|
|
type GrepSummaryResult struct {
|
|
ID string `json:"id"`
|
|
Content string `json:"content"`
|
|
Depth int `json:"depth"`
|
|
Kind SummaryKind `json:"kind"`
|
|
ConversationID int64 `json:"conversationId"`
|
|
// Rank is the bm25 relevance score (negative value, closer to 0 = better match).
|
|
// Examples: -0.5 = excellent match, -2.0 = good match, -10.0 = partial match.
|
|
Rank float64 `json:"rank,omitempty"`
|
|
}
|
|
|
|
// GrepMessageResult is a message match from grep.
|
|
type GrepMessageResult struct {
|
|
ID int64 `json:"id,string"`
|
|
Snippet string `json:"snippet"`
|
|
Role string `json:"role"`
|
|
ConversationID int64 `json:"conversationId"`
|
|
Rank float64 `json:"rank,omitempty"` // Relevance score (lower = better match)
|
|
}
|
|
|
|
// ExpandMessagesResult contains expanded messages.
|
|
type ExpandMessagesResult struct {
|
|
Messages []Message `json:"messages"`
|
|
TokenCount int `json:"tokenCount"`
|
|
}
|
|
|
|
// Grep searches summaries and messages for matching content.
|
|
func (r *RetrievalEngine) Grep(ctx context.Context, input GrepInput) (*GrepResult, error) {
|
|
if input.Pattern == "" {
|
|
return nil, fmt.Errorf("grep: pattern is required")
|
|
}
|
|
|
|
limit := input.Limit
|
|
if limit == 0 {
|
|
limit = 20
|
|
}
|
|
|
|
// Handle Last parameter: convert to Since
|
|
since := input.Since
|
|
if input.Last != "" {
|
|
dur, err := ParseLastDuration(input.Last)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("grep: invalid last: %w", err)
|
|
}
|
|
t := time.Now().UTC().Add(-dur)
|
|
since = &t
|
|
}
|
|
|
|
// Auto-detect mode: use LIKE if pattern contains %, otherwise full-text
|
|
mode := ""
|
|
if strings.Contains(input.Pattern, "%") {
|
|
mode = "like"
|
|
}
|
|
|
|
searchInput := SearchInput{
|
|
Pattern: input.Pattern,
|
|
Mode: mode,
|
|
Role: input.Role,
|
|
AllConversations: input.AllConversations,
|
|
Since: since,
|
|
Before: input.Before,
|
|
Limit: limit,
|
|
}
|
|
|
|
result := &GrepResult{
|
|
Success: true,
|
|
Summaries: make([]GrepSummaryResult, 0),
|
|
Messages: make([]GrepMessageResult, 0),
|
|
TotalSummaries: 0,
|
|
TotalMessages: 0,
|
|
}
|
|
|
|
// Determine scope
|
|
scope := input.Scope
|
|
if scope == "" {
|
|
scope = "both"
|
|
}
|
|
|
|
// Search summaries if requested
|
|
if scope == "both" || scope == "summary" {
|
|
sumResults, err := r.store.SearchSummaries(ctx, searchInput)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("search summaries: %w", err)
|
|
}
|
|
for _, sr := range sumResults {
|
|
if sr.SummaryID != "" {
|
|
result.Summaries = append(result.Summaries, GrepSummaryResult{
|
|
ID: sr.SummaryID,
|
|
Content: sr.Content,
|
|
Depth: sr.Depth,
|
|
Kind: sr.Kind,
|
|
ConversationID: sr.ConversationID,
|
|
Rank: sr.Rank,
|
|
})
|
|
}
|
|
}
|
|
if len(sumResults) > 0 {
|
|
result.TotalSummaries = sumResults[0].TotalCount
|
|
}
|
|
}
|
|
|
|
// Search messages if requested
|
|
if scope == "both" || scope == "message" {
|
|
msgResults, err := r.store.SearchMessages(ctx, searchInput)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("search messages: %w", err)
|
|
}
|
|
for _, sr := range msgResults {
|
|
if sr.MessageID > 0 {
|
|
result.Messages = append(result.Messages, GrepMessageResult{
|
|
ID: sr.MessageID,
|
|
Snippet: sr.Snippet,
|
|
Role: sr.Role,
|
|
ConversationID: sr.ConversationID,
|
|
Rank: sr.Rank,
|
|
})
|
|
}
|
|
}
|
|
if len(msgResults) > 0 {
|
|
result.TotalMessages = msgResults[0].TotalCount
|
|
}
|
|
}
|
|
|
|
// Add hint if no results
|
|
if len(result.Summaries) == 0 && len(result.Messages) == 0 {
|
|
result.Hint = "No matches. Try: %keyword% for fuzzy search, or all_conversations: true"
|
|
}
|
|
|
|
return result, nil
|
|
}
|
|
|
|
// ExpandMessages retrieves full message content by IDs.
|
|
func (r *RetrievalEngine) ExpandMessages(ctx context.Context, messageIDs []int64) (*ExpandMessagesResult, error) {
|
|
result := &ExpandMessagesResult{
|
|
Messages: make([]Message, 0, len(messageIDs)),
|
|
}
|
|
|
|
for _, msgID := range messageIDs {
|
|
msg, err := r.store.GetMessageByID(ctx, msgID)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
result.Messages = append(result.Messages, *msg)
|
|
result.TokenCount += msg.TokenCount
|
|
}
|
|
|
|
return result, nil
|
|
}
|