diff options
Diffstat (limited to 'internal/store')
| -rw-r--r-- | internal/store/store.go | 114 | ||||
| -rw-r--r-- | internal/store/store_test.go | 95 |
2 files changed, 194 insertions, 15 deletions
diff --git a/internal/store/store.go b/internal/store/store.go index eb017db..d04887b 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -8,6 +8,8 @@ package store import ( "database/sql" "fmt" + "regexp" + "strconv" "strings" "time" @@ -238,6 +240,14 @@ func (s *Store) List(limit int, beforeID int64) ([]Summary, error) { // plain terms need help: FTS5's query grammar treats characters like // '.', '-', '/', '@' as syntax, so an unquoted domain name is a parse // error, not a search term, unless quoted first. +// +// Two more filters are recognized ahead of the FTS5 layer, since neither +// fits it - status_code isn't a text column FTS5 can index, and doesn't +// benefit from full-text matching (numeric comparison, not word search), +// and source is a plain low-cardinality column better matched exactly: +// status:404, status:>=400, status:!=200, status:4xx (also 2xx/3xx/5xx), +// and source:proxy / source:repeater / source:intruder. These combine +// with AND against any remaining free-text/FTS5 portion of the query. func (s *Store) Search(query string, limit int, beforeID int64) ([]Summary, error) { if limit <= 0 || limit > 1000 { limit = 200 @@ -245,21 +255,47 @@ func (s *Store) Search(query string, limit int, beforeID int64) ([]Summary, erro if beforeID <= 0 { beforeID = 1<<63 - 1 } - query = prepareFTSQuery(query) - // history_fts must appear unaliased for MATCH/bm25 to resolve against - // it - this SQLite build doesn't support querying FTS5 through a - // table alias (confirmed directly against the sqlite3 CLI: aliasing - // it raises "no such column"). - rows, err := s.db.Query( - `SELECT h.id, h.started_at, h.duration_ms, h.method, h.scheme, h.host, h.path, - COALESCE(h.status_code, 0), length(h.request_raw), COALESCE(length(h.response_raw), 0), - h.error, h.source - FROM history_fts - JOIN history h ON h.id = history_fts.rowid - WHERE history_fts MATCH ? AND h.id < ? - ORDER BY bm25(history_fts), h.id DESC LIMIT ?`, - query, beforeID, limit, - ) + + remaining, pred := extractStructured(query) + where := []string{"h.id < ?"} + args := []any{beforeID} + if pred.statusSQL != "" { + where = append(where, pred.statusSQL) + args = append(args, pred.statusArgs...) + } + if pred.source != "" { + where = append(where, "h.source = ?") + args = append(args, pred.source) + } + + var q string + if remaining == "" { + // No free-text component left - query history directly, no + // FTS5 join or ranking needed. + q = `SELECT h.id, h.started_at, h.duration_ms, h.method, h.scheme, h.host, h.path, + COALESCE(h.status_code, 0), length(h.request_raw), COALESCE(length(h.response_raw), 0), + h.error, h.source + FROM history h + WHERE ` + strings.Join(where, " AND ") + ` + ORDER BY h.id DESC LIMIT ?` + } else { + // history_fts must appear unaliased for MATCH/bm25 to resolve + // against it - this SQLite build doesn't support querying FTS5 + // through a table alias (confirmed directly against the sqlite3 + // CLI: aliasing it raises "no such column"). + where = append([]string{"history_fts MATCH ?"}, where...) + args = append([]any{prepareFTSQuery(remaining)}, args...) + q = `SELECT h.id, h.started_at, h.duration_ms, h.method, h.scheme, h.host, h.path, + COALESCE(h.status_code, 0), length(h.request_raw), COALESCE(length(h.response_raw), 0), + h.error, h.source + FROM history_fts + JOIN history h ON h.id = history_fts.rowid + WHERE ` + strings.Join(where, " AND ") + ` + ORDER BY bm25(history_fts), h.id DESC LIMIT ?` + } + args = append(args, limit) + + rows, err := s.db.Query(q, args...) if err != nil { return nil, fmt.Errorf("search history: %w", err) } @@ -280,6 +316,54 @@ func (s *Store) Search(query string, limit int, beforeID int64) ([]Summary, erro return out, rows.Err() } +// structuredPredicate holds filters extracted from a search query that +// get applied as real SQL predicates instead of going through FTS5. +type structuredPredicate struct { + statusSQL string + statusArgs []any + source string +} + +var ( + statusExactRe = regexp.MustCompile(`^(>=|<=|!=|>|<|=)?(\d{3})$`) + statusRangeRe = regexp.MustCompile(`^([2-5])xx$`) +) + +// extractStructured pulls status:/source: tokens out of query, returning +// what's left (for the FTS5 layer, if anything) and the predicates found. +func extractStructured(query string) (remaining string, pred structuredPredicate) { + fields := strings.Fields(query) + kept := fields[:0:0] + for _, f := range fields { + lower := strings.ToLower(f) + switch { + case strings.HasPrefix(lower, "status:"): + val := strings.ToLower(strings.TrimPrefix(f, "status:")) + if m := statusRangeRe.FindStringSubmatch(val); m != nil { + lo, _ := strconv.Atoi(m[1] + "00") + pred.statusSQL = "status_code >= ? AND status_code < ?" + pred.statusArgs = []any{lo, lo + 100} + continue + } + if m := statusExactRe.FindStringSubmatch(val); m != nil { + op := m[1] + if op == "" { + op = "=" + } + n, _ := strconv.Atoi(m[2]) + pred.statusSQL = "status_code " + op + " ?" + pred.statusArgs = []any{n} + continue + } + case strings.HasPrefix(lower, "source:"): + pred.source = strings.ToLower(strings.TrimPrefix(f, "source:")) + continue + } + kept = append(kept, f) + } + return strings.Join(kept, " "), pred +} + // Get returns the full entry (including raw bytes) for id. func (s *Store) Get(id int64) (*Entry, error) { row := s.db.QueryRow( diff --git a/internal/store/store_test.go b/internal/store/store_test.go new file mode 100644 index 0000000..7999ac3 --- /dev/null +++ b/internal/store/store_test.go @@ -0,0 +1,95 @@ +package store + +import "testing" + +func TestExtractStructured(t *testing.T) { + tests := []struct { + name string + query string + remaining string + statusSQL string + statusArgs []any + source string + }{ + { + name: "plain text only", + query: "admin login", + remaining: "admin login", + }, + { + name: "exact status", + query: "status:404", + remaining: "", + statusSQL: "status_code = ?", + statusArgs: []any{404}, + }, + { + name: "status with operator", + query: "status:>=400", + remaining: "", + statusSQL: "status_code >= ?", + statusArgs: []any{400}, + }, + { + name: "status not-equal", + query: "status:!=200", + remaining: "", + statusSQL: "status_code != ?", + statusArgs: []any{200}, + }, + { + name: "status range shorthand", + query: "status:4xx", + remaining: "", + statusSQL: "status_code >= ? AND status_code < ?", + statusArgs: []any{400, 500}, + }, + { + name: "source filter", + query: "source:repeater", + remaining: "", + source: "repeater", + }, + { + name: "combined with free text", + query: "admin status:>=400 source:proxy", + remaining: "admin", + statusSQL: "status_code >= ?", + statusArgs: []any{400}, + source: "proxy", + }, + { + name: "not a real status token falls through as text", + query: "status:banana", + remaining: "status:banana", + }, + { + name: "malformed 4-digit status falls through as text", + query: "status:4004", + remaining: "status:4004", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + remaining, pred := extractStructured(tt.query) + if remaining != tt.remaining { + t.Errorf("remaining = %q, want %q", remaining, tt.remaining) + } + if pred.statusSQL != tt.statusSQL { + t.Errorf("statusSQL = %q, want %q", pred.statusSQL, tt.statusSQL) + } + if len(pred.statusArgs) != len(tt.statusArgs) { + t.Fatalf("statusArgs = %v, want %v", pred.statusArgs, tt.statusArgs) + } + for i := range pred.statusArgs { + if pred.statusArgs[i] != tt.statusArgs[i] { + t.Errorf("statusArgs[%d] = %v, want %v", i, pred.statusArgs[i], tt.statusArgs[i]) + } + } + if pred.source != tt.source { + t.Errorf("source = %q, want %q", pred.source, tt.source) + } + }) + } +} |