1 files changed, 148 insertions, 0 deletions
diff --git a/scripts/extract_texts.js b/scripts/extract_texts.js
new file mode 100644
index 0000000..8e8bdd3
--- /dev/null
+++ b/scripts/extract_texts.js
@@ -0,0 +1,148 @@
+#!/usr/bin/env node
+/*
+  Extract paragraphs from mirrored sites under similar/ to build a large texts.json.
+  - Scans HTML files in similar/play.typeracer.com, similar/monkeytype.com, etc.
+  - Extracts visible text from common content tags, splits into paragraphs, filters by length.
+  - Deduplicates and shuffles, attaches category from source directory, and attribution as the source path.
+  - Writes to repo-root texts.json for both CLI and Web to use.
+*/
+const fs = require('fs');
+const path = require('path');
+
+const ROOT = path.resolve(__dirname, '..');
+const SIMILAR_DIR = path.join(ROOT, 'similar');
+const OUTPUT = path.join(ROOT, 'texts.json');
+
+const CONTENT_TAGS = ['p', 'article', 'main', 'section'];
+
+function findHtmlFiles(dir) {
+    const results = [];
+    let entries;
+    try {
+        entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+        return results;
+    }
+    for (const entry of entries) {
+        const full = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            results.push(...findHtmlFiles(full));
+        } else if (entry.isFile() && entry.name.toLowerCase().endsWith('.html')) {
+            results.push(full);
+        }
+    }
+    return results;
+}
+
+function stripBoilerplateTags(html) {
+    return html.replace(/<(script|style|nav|footer|header|noscript)[^>]*>[\s\S]*?<\/\1>/gi, ' ');
+}
+
+function decodeEntities(text) {
+    return text
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'");
+}
+
+function extractTagText(html, tag) {
+    const regex = new RegExp(`<${tag}[^>]*>([\\s\\S]*?)<\\/${tag}>`, 'gi');
+    const out = [];
+    let match;
+    while ((match = regex.exec(html)) !== null) {
+        const text = decodeEntities(match[1].replace(/<[^>]+>/g, ' '));
+        out.push(text);
+    }
+    return out;
+}
+
+function isLikelyVisibleText(text) {
+    const t = text.replace(/\s+/g, ' ').trim();
+    if (!t) return false;
+    if (t.length < 60) return false; // avoid too-short snippets
+    // avoid nav/footer boilerplate
+    if (/©|copyright|cookie|privacy|terms|policy|subscribe|sign in|login|menu|footer|header/i.test(t)) return false;
+    return true;
+}
+
+function splitIntoParagraphs(text) {
+    const blocks = text
+        .split(/\n\s*\n|\r\n\r\n/)
+        .map(s => s.replace(/\s+/g, ' ').trim())
+        .filter(Boolean);
+    const paras = [];
+    for (const b of blocks) {
+        if (b.length <= 400) {
+            paras.push(b);
+        } else {
+            let start = 0;
+            while (start < b.length) {
+                const end = Math.min(start + 350, b.length);
+                const slice = b.slice(start, end);
+                const lastPeriod = slice.lastIndexOf('. ');
+                const lastComma = slice.lastIndexOf(', ');
+                const cut = lastPeriod > 150 ? lastPeriod + 1 : (lastComma > 150 ? lastComma + 1 : slice.length);
+                paras.push(slice.slice(0, cut).trim());
+                start += cut;
+            }
+        }
+    }
+    return paras;
+}
+
+try {
+    const htmlFiles = findHtmlFiles(SIMILAR_DIR);
+    const items = [];
+    const seen = new Set();
+
+    for (const file of htmlFiles) {
+        const rel = path.relative(SIMILAR_DIR, file);
+        const parts = rel.split(path.sep);
+        const category = parts[0]?.replace(/\W+/g, '').toLowerCase() || 'general';
+        const attribution = `similar/${rel}`;
+
+        const html = stripBoilerplateTags(fs.readFileSync(file, 'utf8'));
+        const textBits = [];
+        for (const tag of CONTENT_TAGS) {
+            for (const text of extractTagText(html, tag)) {
+                if (isLikelyVisibleText(text)) textBits.push(text);
+            }
+        }
+
+        const combined = textBits.join('\n\n');
+        if (!combined.trim()) continue;
+
+        const paras = splitIntoParagraphs(combined)
+            .map(s => s.replace(/\s+/g, ' ').trim())
+            .filter(s => s.length >= 80 && s.length <= 400);
+
+        for (const content of paras) {
+            const key = content.toLowerCase();
+            if (seen.has(key)) continue;
+            seen.add(key);
+            items.push({ category, content, attribution });
+        }
+    }
+
+    // Shuffle
+    for (let i = items.length - 1; i > 0; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        [items[i], items[j]] = [items[j], items[i]];
+    }
+
+    // If not enough, keep existing texts.json and merge
+    let existing = [];
+    if (fs.existsSync(OUTPUT)) {
+        try { existing = JSON.parse(fs.readFileSync(OUTPUT, 'utf8')); } catch {}
+    }
+    const merged = [...items, ...existing].slice(0, 5000); // cap to 5k entries
+
+    fs.writeFileSync(OUTPUT, JSON.stringify(merged, null, 2));
+    console.log(`Wrote ${merged.length} texts to ${OUTPUT}`);
+} catch (err) {
+    console.error('extract_texts failed:', err);
+    process.exit(1);
+}