Feed Hunter portal: dark theme dashboard on localhost:8888

- Web portal with 5 views: dashboard, feed, investigations, sims, status - Enhanced triage with 40+ claim patterns - Position monitor script - Pipeline report generator - Systemd service for portal
2026-02-08 00:06:24 -06:00
parent 8638500190
commit 5ce3e812a1
9 changed files with 2137 additions and 993 deletions
--- a/projects/feed-hunter/run-and-report.sh
+++ b/projects/feed-hunter/run-and-report.sh
@ -0,0 +1,333 @@
+#!/bin/bash
+# Feed Hunter — Full pipeline with HTML reporting
+# Usage: ./run-and-report.sh [scroll_pages] [output_dir]
+#
+# Runs: scrape → triage → investigate → simulate → generate HTML report
+
+set -e
+
+PAGES=${1:-8}
+OUTPUT_DIR=${2:-"reports/$(date +%Y%m%d-%H%M%S)"}
+BASE="/home/wdjones/.openclaw/workspace"
+SKILL="$BASE/skills/deep-scraper/scripts"
+PROJECT="$BASE/projects/feed-hunter"
+DATA="$BASE/data/x-feed"
+
+echo "=== Feed Hunter Pipeline + Report ==="
+echo "$(date '+%Y-%m-%d %H:%M:%S %Z')"
+echo "Output: $OUTPUT_DIR"
+
+# Create output directory
+mkdir -p "$OUTPUT_DIR"
+
+# Ensure Chrome is running with debug port
+if ! curl -s http://127.0.0.1:9222/json >/dev/null 2>&1; then
+    echo "Starting Chrome..."
+    bash "$SKILL/launch-chrome-debug.sh"
+fi
+
+# Stage 1: Scrape
+echo ""
+echo "--- Stage 1: Scrape ($PAGES pages) ---"
+python3 -u "$SKILL/scrape-x-feed.py" --port 9222 --scroll-pages "$PAGES" 2>&1 | tee "$OUTPUT_DIR/scrape.log"
+
+# Find latest scrape
+LATEST=$(ls -dt "$DATA"/20* | head -1)
+echo "Latest scrape: $LATEST"
+
+# Stage 2: Triage
+echo ""
+echo "--- Stage 2: Triage ---"
+python3 "$SKILL/triage-posts.py" "$LATEST/posts.json" 2>&1 | tee "$OUTPUT_DIR/triage.log"
+
+# Stage 3: Generate investigation tasks
+TRIAGE="$LATEST/triage.json"
+INVESTIGATION_COUNT=0
+if [ -f "$TRIAGE" ]; then
+    INVESTIGATION_COUNT=$(python3 -c "import json; d=json.load(open('$TRIAGE')); print(len(d.get('investigation_queue',[])))")
+    if [ "$INVESTIGATION_COUNT" -gt 0 ]; then
+        echo ""
+        echo "--- Stage 3: Investigation Tasks ---"
+        python3 "$PROJECT/investigate.py" "$TRIAGE" --output "$LATEST/investigations" 2>&1 | tee "$OUTPUT_DIR/investigate.log"
+    else
+        echo ""
+        echo ">>> No posts worth investigating this run."
+    fi
+else
+    echo ">>> No triage output found."
+fi
+
+# Stage 4: Update simulations if new investigations found
+SIMULATION_UPDATES=0
+if [ -f "$PROJECT/data/investigations" ] && [ "$INVESTIGATION_COUNT" -gt 0 ]; then
+    echo ""
+    echo "--- Stage 4: Simulation Updates ---"
+    python3 "$PROJECT/simulator.py" --check-investigations 2>&1 | tee "$OUTPUT_DIR/simulation.log" || true
+    SIMULATION_UPDATES=1
+fi
+
+# Stage 5: Generate HTML Report
+echo ""
+echo "--- Stage 5: Generate Report ---"
+
+# Get summary stats
+POSTS_COUNT=$(python3 -c "import json; d=json.load(open('$LATEST/posts.json')); print(len(d.get('posts',[])))")
+HIGH_VALUE=0
+WORTH_INVESTIGATING=0
+DISMISSED=0
+
+if [ -f "$TRIAGE" ]; then
+    HIGH_VALUE=$(python3 -c "import json; d=json.load(open('$TRIAGE')); print(len(d.get('high_value',[])))")
+    WORTH_INVESTIGATING=$(python3 -c "import json; d=json.load(open('$TRIAGE')); print(len(d.get('worth_investigating',[])))")
+    DISMISSED=$(python3 -c "import json; d=json.load(open('$TRIAGE')); print(len(d.get('dismissed',[])))")
+fi
+
+# Get investigation results
+VERIFIED_CLAIMS=0
+ACTIONABLE_STRATEGIES=0
+if [ -d "$LATEST/investigations" ]; then
+    VERIFIED_CLAIMS=$(find "$LATEST/investigations" -name "*.json" -exec grep -l '"verdict".*"VERIFIED"' {} \; | wc -l)
+    ACTIONABLE_STRATEGIES=$(find "$LATEST/investigations" -name "*.json" -exec grep -l '"actionable".*true' {} \; | wc -l)
+fi
+
+# Get simulation status
+ACTIVE_POSITIONS=0
+TOTAL_PNL=0
+if [ -f "$PROJECT/data/simulations/active.json" ]; then
+    ACTIVE_POSITIONS=$(python3 -c "import json; d=json.load(open('$PROJECT/data/simulations/active.json')); print(len(d.get('positions',[])))" 2>/dev/null || echo "0")
+    TOTAL_PNL=$(python3 -c "import json; d=json.load(open('$PROJECT/data/simulations/active.json')); print(sum(p.get('unrealized_pnl',0) for p in d.get('positions',[])))" 2>/dev/null || echo "0")
+fi
+
+cat > "$OUTPUT_DIR/report.html" << EOF
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Feed Hunter Report - $(date '+%Y-%m-%d %H:%M')</title>
+    <style>
+        body {
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+            background: #0d1117;
+            color: #f0f6fc;
+            line-height: 1.6;
+            margin: 0;
+            padding: 2rem;
+        }
+        .container {
+            max-width: 1200px;
+            margin: 0 auto;
+        }
+        .header {
+            text-align: center;
+            border-bottom: 2px solid #30363d;
+            padding-bottom: 2rem;
+            margin-bottom: 2rem;
+        }
+        .header h1 {
+            color: #58a6ff;
+            font-size: 2.5rem;
+            margin-bottom: 0.5rem;
+        }
+        .header .subtitle {
+            color: #8b949e;
+            font-size: 1.1rem;
+        }
+        .metrics {
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
+            gap: 1.5rem;
+            margin-bottom: 3rem;
+        }
+        .metric-card {
+            background: #161b22;
+            border: 1px solid #30363d;
+            border-radius: 8px;
+            padding: 1.5rem;
+            text-align: center;
+        }
+        .metric-value {
+            font-size: 2.5rem;
+            font-weight: bold;
+            margin-bottom: 0.5rem;
+        }
+        .metric-label {
+            color: #8b949e;
+            font-size: 0.9rem;
+            text-transform: uppercase;
+            letter-spacing: 0.5px;
+        }
+        .positive { color: #3fb950; }
+        .negative { color: #f85149; }
+        .neutral { color: #58a6ff; }
+        .section {
+            background: #161b22;
+            border: 1px solid #30363d;
+            border-radius: 8px;
+            padding: 2rem;
+            margin-bottom: 2rem;
+        }
+        .section h2 {
+            color: #f0f6fc;
+            border-bottom: 1px solid #30363d;
+            padding-bottom: 1rem;
+            margin-bottom: 1.5rem;
+        }
+        .status-good {
+            color: #3fb950;
+            font-weight: bold;
+        }
+        .status-warn {
+            color: #ffab40;
+            font-weight: bold;
+        }
+        .status-error {
+            color: #f85149;
+            font-weight: bold;
+        }
+        .log-snippet {
+            background: #0d1117;
+            border: 1px solid #30363d;
+            border-radius: 4px;
+            padding: 1rem;
+            font-family: 'Monaco', 'Menlo', monospace;
+            font-size: 0.85rem;
+            max-height: 200px;
+            overflow-y: auto;
+            white-space: pre-wrap;
+            margin: 1rem 0;
+        }
+        .timestamp {
+            color: #8b949e;
+            font-size: 0.9rem;
+            text-align: right;
+            margin-top: 2rem;
+            border-top: 1px solid #30363d;
+            padding-top: 1rem;
+        }
+        .link-button {
+            display: inline-block;
+            background: #58a6ff;
+            color: #fff;
+            padding: 0.75rem 1.5rem;
+            border-radius: 6px;
+            text-decoration: none;
+            font-weight: bold;
+            margin: 0.5rem 0.5rem 0.5rem 0;
+            transition: background 0.2s;
+        }
+        .link-button:hover {
+            background: #4493e1;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <div class="header">
+            <h1>🖤 Feed Hunter Report</h1>
+            <div class="subtitle">Pipeline execution completed at $(date '+%Y-%m-%d %H:%M:%S %Z')</div>
+        </div>
+
+        <div class="metrics">
+            <div class="metric-card">
+                <div class="metric-value neutral">$POSTS_COUNT</div>
+                <div class="metric-label">Posts Scraped</div>
+            </div>
+            <div class="metric-card">
+                <div class="metric-value $([ $HIGH_VALUE -gt 0 ] && echo 'positive' || echo 'neutral')">$HIGH_VALUE</div>
+                <div class="metric-label">High Value</div>
+            </div>
+            <div class="metric-card">
+                <div class="metric-value $([ $INVESTIGATION_COUNT -gt 0 ] && echo 'positive' || echo 'neutral')">$INVESTIGATION_COUNT</div>
+                <div class="metric-label">Investigated</div>
+            </div>
+            <div class="metric-card">
+                <div class="metric-value $([ $VERIFIED_CLAIMS -gt 0 ] && echo 'positive' || echo 'neutral')">$VERIFIED_CLAIMS</div>
+                <div class="metric-label">Verified Claims</div>
+            </div>
+            <div class="metric-card">
+                <div class="metric-value $([ $ACTIVE_POSITIONS -gt 0 ] && echo 'positive' || echo 'neutral')">$ACTIVE_POSITIONS</div>
+                <div class="metric-label">Active Positions</div>
+            </div>
+            <div class="metric-card">
+                <div class="metric-value $(python3 -c "print('positive' if $TOTAL_PNL >= 0 else 'negative')")">$${TOTAL_PNL}</div>
+                <div class="metric-label">Total P&L</div>
+            </div>
+        </div>
+
+        <div class="section">
+            <h2>📊 Pipeline Summary</h2>
+            <p><strong>Scraping:</strong> Collected $POSTS_COUNT posts from $PAGES pages</p>
+            <p><strong>Triaging:</strong> $HIGH_VALUE high-value, $WORTH_INVESTIGATING worth investigating, $DISMISSED dismissed</p>
+            <p><strong>Investigation:</strong> $INVESTIGATION_COUNT posts analyzed, $VERIFIED_CLAIMS claims verified</p>
+            <p><strong>Simulations:</strong> $ACTIVE_POSITIONS active positions, \$$TOTAL_PNL unrealized P&L</p>
+        </div>
+
+        <div class="section">
+            <h2>🔍 Investigation Highlights</h2>
+            $(if [ $VERIFIED_CLAIMS -gt 0 ]; then
+                echo "<div class='status-good'>✓ Found $VERIFIED_CLAIMS verified claims worth tracking</div>"
+            else
+                echo "<div class='status-warn'>⚠ No verified claims found in this run</div>"
+            fi)
+            
+            $(if [ $ACTIONABLE_STRATEGIES -gt 0 ]; then
+                echo "<div class='status-good'>✓ Identified $ACTIONABLE_STRATEGIES actionable strategies</div>"
+            else
+                echo "<div class='status-warn'>⚠ No actionable strategies identified</div>"
+            fi)
+        </div>
+
+        <div class="section">
+            <h2>📈 Simulation Status</h2>
+            $(if [ $ACTIVE_POSITIONS -gt 0 ]; then
+                echo "<div class='status-good'>✓ $ACTIVE_POSITIONS positions actively tracked</div>"
+                echo "<div>Current P&L: <span class='$(python3 -c "print('positive' if $TOTAL_PNL >= 0 else 'negative')")'>\$$TOTAL_PNL</span></div>"
+            else
+                echo "<div class='status-warn'>⚠ No active positions</div>"
+            fi)
+        </div>
+
+        <div class="section">
+            <h2>📱 Quick Actions</h2>
+            <a href="http://localhost:8888/" class="link-button">Open Portal Dashboard</a>
+            <a href="http://localhost:8888/feed" class="link-button">View Latest Feed</a>
+            <a href="http://localhost:8888/investigations" class="link-button">Investigation Reports</a>
+            <a href="http://localhost:8888/simulations" class="link-button">Simulation Tracker</a>
+        </div>
+
+        <div class="section">
+            <h2>📋 Execution Logs</h2>
+            
+            <h3>Scraping Output</h3>
+            <div class="log-snippet">$(tail -n 10 "$OUTPUT_DIR/scrape.log" 2>/dev/null || echo "No scrape log found")</div>
+            
+            <h3>Triage Output</h3>
+            <div class="log-snippet">$(tail -n 10 "$OUTPUT_DIR/triage.log" 2>/dev/null || echo "No triage log found")</div>
+            
+            $(if [ $INVESTIGATION_COUNT -gt 0 ]; then
+                echo "<h3>Investigation Output</h3>"
+                echo "<div class='log-snippet'>$(tail -n 10 "$OUTPUT_DIR/investigate.log" 2>/dev/null || echo "No investigation log found")</div>"
+            fi)
+        </div>
+
+        <div class="timestamp">
+            Report generated: $(date '+%Y-%m-%d %H:%M:%S %Z')<br>
+            Data directory: $LATEST<br>
+            Report saved: $OUTPUT_DIR/report.html
+        </div>
+    </div>
+</body>
+</html>
+EOF
+
+echo ""
+echo "=== Pipeline + Report Complete ==="
+echo "📊 Dashboard: http://localhost:8888/"
+echo "📄 Report: $OUTPUT_DIR/report.html"
+echo "📁 Data: $LATEST"
+echo ""
+echo "Summary:"
+echo "  Posts: $POSTS_COUNT | High Value: $HIGH_VALUE | Investigated: $INVESTIGATION_COUNT"
+echo "  Verified: $VERIFIED_CLAIMS | Actionable: $ACTIONABLE_STRATEGIES"
+echo "  Active Positions: $ACTIVE_POSITIONS | P&L: \$$TOTAL_PNL"