srdusr
aboutsummaryrefslogtreecommitdiffstats
path: root/internal/store/store.go
diff options
context:
space:
mode:
Diffstat (limited to 'internal/store/store.go')
-rw-r--r--internal/store/store.go130
1 files changed, 127 insertions, 3 deletions
diff --git a/internal/store/store.go b/internal/store/store.go
index 88c9991..a3c6b03 100644
--- a/internal/store/store.go
+++ b/internal/store/store.go
@@ -1,11 +1,14 @@
// Package store persists proxy history to SQLite in WAL mode. Request
// and response bytes are stored as-received where possible (see the
// Exact fields) rather than re-serialized from a parsed representation.
+// An FTS5 index (history_fts) mirrors method/host/path and the raw
+// request/response text for full-text search - see Search.
package store
import (
"database/sql"
"fmt"
+ "strings"
"time"
_ "modernc.org/sqlite"
@@ -28,6 +31,11 @@ CREATE TABLE IF NOT EXISTS history (
error TEXT NOT NULL DEFAULT '',
source TEXT NOT NULL DEFAULT 'proxy'
);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS history_fts USING fts5(
+ method, host, path, request_text, response_text,
+ tokenize = 'unicode61 remove_diacritics 2'
+);
`
// Store is a handle to the history database. Safe for concurrent use.
@@ -63,6 +71,16 @@ func Open(path string) (*Store, error) {
// Added after the initial schema; ignore the "duplicate column" error
// on databases that already have it.
db.Exec("ALTER TABLE history ADD COLUMN source TEXT NOT NULL DEFAULT 'proxy'")
+ // Backfill history_fts for rows inserted before it existed. A no-op
+ // once caught up, since every Insert keeps both tables in sync.
+ if _, err := db.Exec(`
+ INSERT INTO history_fts (rowid, method, host, path, request_text, response_text)
+ SELECT id, method, host, path, CAST(request_raw AS TEXT), COALESCE(CAST(response_raw AS TEXT), '')
+ FROM history WHERE id NOT IN (SELECT rowid FROM history_fts)
+ `); err != nil {
+ db.Close()
+ return nil, fmt.Errorf("backfill search index: %w", err)
+ }
return &Store{db: db}, nil
}
@@ -106,7 +124,7 @@ type Summary struct {
Source string
}
-// Insert stores e and returns its assigned ID.
+// Insert stores e (and indexes it for search) and returns its assigned ID.
func (s *Store) Insert(e *Entry) (int64, error) {
var statusCode any
if e.StatusCode != 0 {
@@ -116,7 +134,14 @@ func (s *Store) Insert(e *Entry) (int64, error) {
if source == "" {
source = "proxy"
}
- res, err := s.db.Exec(
+
+ tx, err := s.db.Begin()
+ if err != nil {
+ return 0, fmt.Errorf("begin insert: %w", err)
+ }
+ defer tx.Rollback()
+
+ res, err := tx.Exec(
`INSERT INTO history
(started_at, duration_ms, method, scheme, host, path, status_code,
request_raw, response_raw, request_exact, response_exact, error, source)
@@ -127,7 +152,23 @@ func (s *Store) Insert(e *Entry) (int64, error) {
if err != nil {
return 0, fmt.Errorf("insert history entry: %w", err)
}
- return res.LastInsertId()
+ id, err := res.LastInsertId()
+ if err != nil {
+ return 0, fmt.Errorf("insert history entry: %w", err)
+ }
+
+ if _, err := tx.Exec(
+ `INSERT INTO history_fts (rowid, method, host, path, request_text, response_text)
+ VALUES (?, ?, ?, ?, ?, ?)`,
+ id, e.Method, e.Host, e.Path, string(e.RequestRaw), string(e.ResponseRaw),
+ ); err != nil {
+ return 0, fmt.Errorf("index history entry: %w", err)
+ }
+
+ if err := tx.Commit(); err != nil {
+ return 0, fmt.Errorf("commit insert: %w", err)
+ }
+ return id, nil
}
// List returns up to limit history summaries older than beforeID (or the
@@ -165,6 +206,57 @@ func (s *Store) List(limit int, beforeID int64) ([]Summary, error) {
return out, rows.Err()
}
+// Search returns up to limit history summaries older than beforeID (or
+// the most recent if beforeID is 0) matching query, newest first among
+// equally-ranked matches, best-relevance first otherwise. query is
+// mostly plain text - "example.com", "x-forwarded-for", "192.168.1.1"
+// all just work - plus a little FTS5 syntax for power users: AND/OR/NOT,
+// and column filters like host:example.com. See prepareFTSQuery for why
+// plain terms need help: FTS5's query grammar treats characters like
+// '.', '-', '/', '@' as syntax, so an unquoted domain name is a parse
+// error, not a search term, unless quoted first.
+func (s *Store) Search(query string, limit int, beforeID int64) ([]Summary, error) {
+ if limit <= 0 || limit > 1000 {
+ limit = 200
+ }
+ if beforeID <= 0 {
+ beforeID = 1<<63 - 1
+ }
+ query = prepareFTSQuery(query)
+ // history_fts must appear unaliased for MATCH/bm25 to resolve against
+ // it - this SQLite build doesn't support querying FTS5 through a
+ // table alias (confirmed directly against the sqlite3 CLI: aliasing
+ // it raises "no such column").
+ rows, err := s.db.Query(
+ `SELECT h.id, h.started_at, h.duration_ms, h.method, h.scheme, h.host, h.path,
+ COALESCE(h.status_code, 0), length(h.request_raw), COALESCE(length(h.response_raw), 0),
+ h.error, h.source
+ FROM history_fts
+ JOIN history h ON h.id = history_fts.rowid
+ WHERE history_fts MATCH ? AND h.id < ?
+ ORDER BY bm25(history_fts), h.id DESC LIMIT ?`,
+ query, beforeID, limit,
+ )
+ if err != nil {
+ return nil, fmt.Errorf("search history: %w", err)
+ }
+ defer rows.Close()
+
+ var out []Summary
+ for rows.Next() {
+ var sum Summary
+ var startedAt, durationMs int64
+ if err := rows.Scan(&sum.ID, &startedAt, &durationMs, &sum.Method, &sum.Scheme, &sum.Host, &sum.Path,
+ &sum.StatusCode, &sum.ReqSize, &sum.RespSize, &sum.Error, &sum.Source); err != nil {
+ return nil, fmt.Errorf("scan search row: %w", err)
+ }
+ sum.StartedAt = time.UnixMilli(startedAt)
+ sum.Duration = time.Duration(durationMs) * time.Millisecond
+ out = append(out, sum)
+ }
+ return out, rows.Err()
+}
+
// Get returns the full entry (including raw bytes) for id.
func (s *Store) Get(id int64) (*Entry, error) {
row := s.db.QueryRow(
@@ -188,6 +280,38 @@ func (s *Store) Get(id int64) (*Entry, error) {
return &e, nil
}
+// prepareFTSQuery makes free text safe to hand to FTS5's MATCH, whose
+// query grammar reserves a wide range of punctuation ('.', '-', '/',
+// '@', '(', ')', and more - confirmed empirically, not just from docs)
+// as syntax. A bareword containing any of it is a parse error, not a
+// literal search term, which would make searching for the most common
+// things in HTTP traffic (domains, paths, hyphenated headers, IPs) fail
+// by default. So: quote every plain token as an FTS5 phrase, which is
+// syntactically valid regardless of its contents, while still
+// recognizing AND/OR/NOT and column:value filters for anyone using them
+// deliberately.
+//
+// Known limitation: this splits on whitespace, so a query the user
+// already wrote as a multi-word "quoted phrase" gets re-split and each
+// word individually re-quoted (still a valid query, just no longer an
+// exact-adjacency phrase match). Fine for the common case of typing
+// plain, unquoted search terms, which is what this exists for.
+func prepareFTSQuery(q string) string {
+ fields := strings.Fields(q)
+ for i, f := range fields {
+ switch strings.ToUpper(f) {
+ case "AND", "OR", "NOT":
+ continue
+ }
+ if col, val, ok := strings.Cut(f, ":"); ok && col != "" && val != "" {
+ fields[i] = col + `:"` + strings.ReplaceAll(val, `"`, `""`) + `"`
+ continue
+ }
+ fields[i] = `"` + strings.ReplaceAll(f, `"`, `""`) + `"`
+ }
+ return strings.Join(fields, " ")
+}
+
func boolToInt(b bool) int {
if b {
return 1