diff options
Diffstat (limited to 'internal/store/store.go')
| -rw-r--r-- | internal/store/store.go | 130 |
1 files changed, 127 insertions, 3 deletions
diff --git a/internal/store/store.go b/internal/store/store.go index 88c9991..a3c6b03 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1,11 +1,14 @@ // Package store persists proxy history to SQLite in WAL mode. Request // and response bytes are stored as-received where possible (see the // Exact fields) rather than re-serialized from a parsed representation. +// An FTS5 index (history_fts) mirrors method/host/path and the raw +// request/response text for full-text search - see Search. package store import ( "database/sql" "fmt" + "strings" "time" _ "modernc.org/sqlite" @@ -28,6 +31,11 @@ CREATE TABLE IF NOT EXISTS history ( error TEXT NOT NULL DEFAULT '', source TEXT NOT NULL DEFAULT 'proxy' ); + +CREATE VIRTUAL TABLE IF NOT EXISTS history_fts USING fts5( + method, host, path, request_text, response_text, + tokenize = 'unicode61 remove_diacritics 2' +); ` // Store is a handle to the history database. Safe for concurrent use. @@ -63,6 +71,16 @@ func Open(path string) (*Store, error) { // Added after the initial schema; ignore the "duplicate column" error // on databases that already have it. db.Exec("ALTER TABLE history ADD COLUMN source TEXT NOT NULL DEFAULT 'proxy'") + // Backfill history_fts for rows inserted before it existed. A no-op + // once caught up, since every Insert keeps both tables in sync. + if _, err := db.Exec(` + INSERT INTO history_fts (rowid, method, host, path, request_text, response_text) + SELECT id, method, host, path, CAST(request_raw AS TEXT), COALESCE(CAST(response_raw AS TEXT), '') + FROM history WHERE id NOT IN (SELECT rowid FROM history_fts) + `); err != nil { + db.Close() + return nil, fmt.Errorf("backfill search index: %w", err) + } return &Store{db: db}, nil } @@ -106,7 +124,7 @@ type Summary struct { Source string } -// Insert stores e and returns its assigned ID. +// Insert stores e (and indexes it for search) and returns its assigned ID. func (s *Store) Insert(e *Entry) (int64, error) { var statusCode any if e.StatusCode != 0 { @@ -116,7 +134,14 @@ func (s *Store) Insert(e *Entry) (int64, error) { if source == "" { source = "proxy" } - res, err := s.db.Exec( + + tx, err := s.db.Begin() + if err != nil { + return 0, fmt.Errorf("begin insert: %w", err) + } + defer tx.Rollback() + + res, err := tx.Exec( `INSERT INTO history (started_at, duration_ms, method, scheme, host, path, status_code, request_raw, response_raw, request_exact, response_exact, error, source) @@ -127,7 +152,23 @@ func (s *Store) Insert(e *Entry) (int64, error) { if err != nil { return 0, fmt.Errorf("insert history entry: %w", err) } - return res.LastInsertId() + id, err := res.LastInsertId() + if err != nil { + return 0, fmt.Errorf("insert history entry: %w", err) + } + + if _, err := tx.Exec( + `INSERT INTO history_fts (rowid, method, host, path, request_text, response_text) + VALUES (?, ?, ?, ?, ?, ?)`, + id, e.Method, e.Host, e.Path, string(e.RequestRaw), string(e.ResponseRaw), + ); err != nil { + return 0, fmt.Errorf("index history entry: %w", err) + } + + if err := tx.Commit(); err != nil { + return 0, fmt.Errorf("commit insert: %w", err) + } + return id, nil } // List returns up to limit history summaries older than beforeID (or the @@ -165,6 +206,57 @@ func (s *Store) List(limit int, beforeID int64) ([]Summary, error) { return out, rows.Err() } +// Search returns up to limit history summaries older than beforeID (or +// the most recent if beforeID is 0) matching query, newest first among +// equally-ranked matches, best-relevance first otherwise. query is +// mostly plain text - "example.com", "x-forwarded-for", "192.168.1.1" +// all just work - plus a little FTS5 syntax for power users: AND/OR/NOT, +// and column filters like host:example.com. See prepareFTSQuery for why +// plain terms need help: FTS5's query grammar treats characters like +// '.', '-', '/', '@' as syntax, so an unquoted domain name is a parse +// error, not a search term, unless quoted first. +func (s *Store) Search(query string, limit int, beforeID int64) ([]Summary, error) { + if limit <= 0 || limit > 1000 { + limit = 200 + } + if beforeID <= 0 { + beforeID = 1<<63 - 1 + } + query = prepareFTSQuery(query) + // history_fts must appear unaliased for MATCH/bm25 to resolve against + // it - this SQLite build doesn't support querying FTS5 through a + // table alias (confirmed directly against the sqlite3 CLI: aliasing + // it raises "no such column"). + rows, err := s.db.Query( + `SELECT h.id, h.started_at, h.duration_ms, h.method, h.scheme, h.host, h.path, + COALESCE(h.status_code, 0), length(h.request_raw), COALESCE(length(h.response_raw), 0), + h.error, h.source + FROM history_fts + JOIN history h ON h.id = history_fts.rowid + WHERE history_fts MATCH ? AND h.id < ? + ORDER BY bm25(history_fts), h.id DESC LIMIT ?`, + query, beforeID, limit, + ) + if err != nil { + return nil, fmt.Errorf("search history: %w", err) + } + defer rows.Close() + + var out []Summary + for rows.Next() { + var sum Summary + var startedAt, durationMs int64 + if err := rows.Scan(&sum.ID, &startedAt, &durationMs, &sum.Method, &sum.Scheme, &sum.Host, &sum.Path, + &sum.StatusCode, &sum.ReqSize, &sum.RespSize, &sum.Error, &sum.Source); err != nil { + return nil, fmt.Errorf("scan search row: %w", err) + } + sum.StartedAt = time.UnixMilli(startedAt) + sum.Duration = time.Duration(durationMs) * time.Millisecond + out = append(out, sum) + } + return out, rows.Err() +} + // Get returns the full entry (including raw bytes) for id. func (s *Store) Get(id int64) (*Entry, error) { row := s.db.QueryRow( @@ -188,6 +280,38 @@ func (s *Store) Get(id int64) (*Entry, error) { return &e, nil } +// prepareFTSQuery makes free text safe to hand to FTS5's MATCH, whose +// query grammar reserves a wide range of punctuation ('.', '-', '/', +// '@', '(', ')', and more - confirmed empirically, not just from docs) +// as syntax. A bareword containing any of it is a parse error, not a +// literal search term, which would make searching for the most common +// things in HTTP traffic (domains, paths, hyphenated headers, IPs) fail +// by default. So: quote every plain token as an FTS5 phrase, which is +// syntactically valid regardless of its contents, while still +// recognizing AND/OR/NOT and column:value filters for anyone using them +// deliberately. +// +// Known limitation: this splits on whitespace, so a query the user +// already wrote as a multi-word "quoted phrase" gets re-split and each +// word individually re-quoted (still a valid query, just no longer an +// exact-adjacency phrase match). Fine for the common case of typing +// plain, unquoted search terms, which is what this exists for. +func prepareFTSQuery(q string) string { + fields := strings.Fields(q) + for i, f := range fields { + switch strings.ToUpper(f) { + case "AND", "OR", "NOT": + continue + } + if col, val, ok := strings.Cut(f, ":"); ok && col != "" && val != "" { + fields[i] = col + `:"` + strings.ReplaceAll(val, `"`, `""`) + `"` + continue + } + fields[i] = `"` + strings.ReplaceAll(f, `"`, `""`) + `"` + } + return strings.Join(fields, " ") +} + func boolToInt(b bool) int { if b { return 1 |