From 8d15c2e0b326933f8fc912e3b13f37e78a9bc0b6 Mon Sep 17 00:00:00 2001 From: srdusr <99972264+srdusr@users.noreply.github.com> Date: Wed, 14 Feb 2024 00:37:00 +0200 Subject: History view: SQLite storage, daemon/TUI split over Unix socket Implements build-order step 3. Adds: - internal/store: SQLite (WAL, single-writer) history table, raw request/response blobs plus metadata for the list view. - internal/proxy: request/response capture wired into forward(). HTTP/1.1 legs are captured byte-exact via a teeConn that records wire bytes as they're read, taken right after the message is fully drained (so no manual re-reading/replaying is needed - RoundTrip's own streaming does the draining). HTTP/2 legs (no meaningful "raw bytes" of their own - multiplexed, HPACK-compressed framing) are reconstructed instead, and marked as such in storage. - internal/ipc: JSON-over-Unix-socket protocol between mitmuxd (owns the proxy and the DB) and any client - list/get for queries, subscribe for a live push stream of newly captured entries. Keeps the proxy engine independent of the UI, per the architecture sketch. - cmd/mitmux: Bubble Tea TUI - a live-updating history table and a request/response detail view with raw bytes. Two real bugs surfaced during testing and got fixed before commit: 1. http.Transport's HTTP/2 auto-dispatch does a literal *tls.Conn type assertion on the dialed connection; wrapping it in a capturing teeConn broke that silently, and HTTP/2 framing got parsed as HTTP/1.1 text. Fixed by dropping http.Transport for the upstream leg entirely in favor of an explicit per-protocol round trip (see PLAN.md stack note). 2. singleConnListener wrapped the client teeConn *inside* a closeSignalConn, so ConnContext's type assertion for it silently failed and HTTP/1.1 client-side capture never activated. Fixed the wrap order; verified via direct SQLite inspection that request_exact flips back to 1 and the stored bytes are genuinely wire-exact (preserved chunked-encoding framing, original header casing/order). Verified live: plain HTTP, HTTPS H1.1, HTTPS H2, and a POST with a body, checked against the raw stored bytes directly in SQLite; IPC list/get/ subscribe against a throwaway client; and the TUI driven end-to-end in a tmux session (list, detail view, tab between request/response, live update on a new request while sitting on the list). --- internal/store/store.go | 186 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 internal/store/store.go (limited to 'internal/store') diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 0000000..c0219e4 --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,186 @@ +// Package store persists proxy history to SQLite in WAL mode. Request +// and response bytes are stored as-received where possible (see the +// Exact fields) rather than re-serialized from a parsed representation. +package store + +import ( + "database/sql" + "fmt" + "time" + + _ "modernc.org/sqlite" +) + +const schema = ` +CREATE TABLE IF NOT EXISTS history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + started_at INTEGER NOT NULL, + duration_ms INTEGER NOT NULL, + method TEXT NOT NULL, + scheme TEXT NOT NULL, + host TEXT NOT NULL, + path TEXT NOT NULL, + status_code INTEGER, + request_raw BLOB NOT NULL, + response_raw BLOB, + request_exact INTEGER NOT NULL, + response_exact INTEGER NOT NULL, + error TEXT NOT NULL DEFAULT '' +); +` + +// Store is a handle to the history database. Safe for concurrent use. +type Store struct { + db *sql.DB +} + +// Open opens (creating if needed) the SQLite database at path in WAL mode. +func Open(path string) (*Store, error) { + db, err := sql.Open("sqlite", path) + if err != nil { + return nil, fmt.Errorf("open db: %w", err) + } + // modernc.org/sqlite has no real connection pooling benefit here and + // SQLite only supports one writer at a time; serializing access + // through a single connection avoids SQLITE_BUSY entirely. + db.SetMaxOpenConns(1) + + for _, pragma := range []string{ + "PRAGMA journal_mode = WAL", + "PRAGMA synchronous = NORMAL", + "PRAGMA foreign_keys = ON", + } { + if _, err := db.Exec(pragma); err != nil { + db.Close() + return nil, fmt.Errorf("%s: %w", pragma, err) + } + } + if _, err := db.Exec(schema); err != nil { + db.Close() + return nil, fmt.Errorf("create schema: %w", err) + } + return &Store{db: db}, nil +} + +// Close closes the underlying database. +func (s *Store) Close() error { + return s.db.Close() +} + +// Entry is one captured request/response pair. +type Entry struct { + ID int64 + StartedAt time.Time + Duration time.Duration + Method string + Scheme string + Host string + Path string + StatusCode int // 0 if no response was received + RequestRaw []byte + ResponseRaw []byte // nil if no response was received + RequestExact bool // true if RequestRaw is wire-exact, false if reconstructed (e.g. HTTP/2) + ResponseExact bool + Error string // network/transport error, if the request never got a response +} + +// Summary is the lightweight metadata used for the history list view - +// no request/response bodies. +type Summary struct { + ID int64 + StartedAt time.Time + Duration time.Duration + Method string + Scheme string + Host string + Path string + StatusCode int + ReqSize int + RespSize int + Error string +} + +// Insert stores e and returns its assigned ID. +func (s *Store) Insert(e *Entry) (int64, error) { + var statusCode any + if e.StatusCode != 0 { + statusCode = e.StatusCode + } + res, err := s.db.Exec( + `INSERT INTO history + (started_at, duration_ms, method, scheme, host, path, status_code, + request_raw, response_raw, request_exact, response_exact, error) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + e.StartedAt.UnixMilli(), e.Duration.Milliseconds(), e.Method, e.Scheme, e.Host, e.Path, + statusCode, e.RequestRaw, e.ResponseRaw, boolToInt(e.RequestExact), boolToInt(e.ResponseExact), e.Error, + ) + if err != nil { + return 0, fmt.Errorf("insert history entry: %w", err) + } + return res.LastInsertId() +} + +// List returns up to limit history summaries older than beforeID (or the +// most recent if beforeID is 0), newest first. +func (s *Store) List(limit int, beforeID int64) ([]Summary, error) { + if limit <= 0 || limit > 1000 { + limit = 200 + } + if beforeID <= 0 { + beforeID = 1<<63 - 1 + } + rows, err := s.db.Query( + `SELECT id, started_at, duration_ms, method, scheme, host, path, + COALESCE(status_code, 0), length(request_raw), COALESCE(length(response_raw), 0), error + FROM history WHERE id < ? ORDER BY id DESC LIMIT ?`, + beforeID, limit, + ) + if err != nil { + return nil, fmt.Errorf("list history: %w", err) + } + defer rows.Close() + + var out []Summary + for rows.Next() { + var sum Summary + var startedAt, durationMs int64 + if err := rows.Scan(&sum.ID, &startedAt, &durationMs, &sum.Method, &sum.Scheme, &sum.Host, &sum.Path, + &sum.StatusCode, &sum.ReqSize, &sum.RespSize, &sum.Error); err != nil { + return nil, fmt.Errorf("scan history row: %w", err) + } + sum.StartedAt = time.UnixMilli(startedAt) + sum.Duration = time.Duration(durationMs) * time.Millisecond + out = append(out, sum) + } + return out, rows.Err() +} + +// Get returns the full entry (including raw bytes) for id. +func (s *Store) Get(id int64) (*Entry, error) { + row := s.db.QueryRow( + `SELECT id, started_at, duration_ms, method, scheme, host, path, + COALESCE(status_code, 0), request_raw, response_raw, + request_exact, response_exact, error + FROM history WHERE id = ?`, + id, + ) + var e Entry + var startedAt, durationMs int64 + var reqExact, respExact int + if err := row.Scan(&e.ID, &startedAt, &durationMs, &e.Method, &e.Scheme, &e.Host, &e.Path, + &e.StatusCode, &e.RequestRaw, &e.ResponseRaw, &reqExact, &respExact, &e.Error); err != nil { + return nil, fmt.Errorf("get history entry %d: %w", id, err) + } + e.StartedAt = time.UnixMilli(startedAt) + e.Duration = time.Duration(durationMs) * time.Millisecond + e.RequestExact = reqExact != 0 + e.ResponseExact = respExact != 0 + return &e, nil +} + +func boolToInt(b bool) int { + if b { + return 1 + } + return 0 +} -- cgit v1.2.3