Case
5ce3e812a1
- Web portal with 5 views: dashboard, feed, investigations, sims, status - Enhanced triage with 40+ claim patterns - Position monitor script - Pipeline report generator - Systemd service for portal
333 lines
12 KiB
Bash
Executable File
333 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
# Feed Hunter — Full pipeline with HTML reporting
|
|
# Usage: ./run-and-report.sh [scroll_pages] [output_dir]
|
|
#
|
|
# Runs: scrape → triage → investigate → simulate → generate HTML report
|
|
|
|
set -e
|
|
|
|
PAGES=${1:-8}
|
|
OUTPUT_DIR=${2:-"reports/$(date +%Y%m%d-%H%M%S)"}
|
|
BASE="/home/wdjones/.openclaw/workspace"
|
|
SKILL="$BASE/skills/deep-scraper/scripts"
|
|
PROJECT="$BASE/projects/feed-hunter"
|
|
DATA="$BASE/data/x-feed"
|
|
|
|
echo "=== Feed Hunter Pipeline + Report ==="
|
|
echo "$(date '+%Y-%m-%d %H:%M:%S %Z')"
|
|
echo "Output: $OUTPUT_DIR"
|
|
|
|
# Create output directory
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
# Ensure Chrome is running with debug port
|
|
if ! curl -s http://127.0.0.1:9222/json >/dev/null 2>&1; then
|
|
echo "Starting Chrome..."
|
|
bash "$SKILL/launch-chrome-debug.sh"
|
|
fi
|
|
|
|
# Stage 1: Scrape
|
|
echo ""
|
|
echo "--- Stage 1: Scrape ($PAGES pages) ---"
|
|
python3 -u "$SKILL/scrape-x-feed.py" --port 9222 --scroll-pages "$PAGES" 2>&1 | tee "$OUTPUT_DIR/scrape.log"
|
|
|
|
# Find latest scrape
|
|
LATEST=$(ls -dt "$DATA"/20* | head -1)
|
|
echo "Latest scrape: $LATEST"
|
|
|
|
# Stage 2: Triage
|
|
echo ""
|
|
echo "--- Stage 2: Triage ---"
|
|
python3 "$SKILL/triage-posts.py" "$LATEST/posts.json" 2>&1 | tee "$OUTPUT_DIR/triage.log"
|
|
|
|
# Stage 3: Generate investigation tasks
|
|
TRIAGE="$LATEST/triage.json"
|
|
INVESTIGATION_COUNT=0
|
|
if [ -f "$TRIAGE" ]; then
|
|
INVESTIGATION_COUNT=$(python3 -c "import json; d=json.load(open('$TRIAGE')); print(len(d.get('investigation_queue',[])))")
|
|
if [ "$INVESTIGATION_COUNT" -gt 0 ]; then
|
|
echo ""
|
|
echo "--- Stage 3: Investigation Tasks ---"
|
|
python3 "$PROJECT/investigate.py" "$TRIAGE" --output "$LATEST/investigations" 2>&1 | tee "$OUTPUT_DIR/investigate.log"
|
|
else
|
|
echo ""
|
|
echo ">>> No posts worth investigating this run."
|
|
fi
|
|
else
|
|
echo ">>> No triage output found."
|
|
fi
|
|
|
|
# Stage 4: Update simulations if new investigations found
|
|
SIMULATION_UPDATES=0
|
|
if [ -f "$PROJECT/data/investigations" ] && [ "$INVESTIGATION_COUNT" -gt 0 ]; then
|
|
echo ""
|
|
echo "--- Stage 4: Simulation Updates ---"
|
|
python3 "$PROJECT/simulator.py" --check-investigations 2>&1 | tee "$OUTPUT_DIR/simulation.log" || true
|
|
SIMULATION_UPDATES=1
|
|
fi
|
|
|
|
# Stage 5: Generate HTML Report
|
|
echo ""
|
|
echo "--- Stage 5: Generate Report ---"
|
|
|
|
# Get summary stats
|
|
POSTS_COUNT=$(python3 -c "import json; d=json.load(open('$LATEST/posts.json')); print(len(d.get('posts',[])))")
|
|
HIGH_VALUE=0
|
|
WORTH_INVESTIGATING=0
|
|
DISMISSED=0
|
|
|
|
if [ -f "$TRIAGE" ]; then
|
|
HIGH_VALUE=$(python3 -c "import json; d=json.load(open('$TRIAGE')); print(len(d.get('high_value',[])))")
|
|
WORTH_INVESTIGATING=$(python3 -c "import json; d=json.load(open('$TRIAGE')); print(len(d.get('worth_investigating',[])))")
|
|
DISMISSED=$(python3 -c "import json; d=json.load(open('$TRIAGE')); print(len(d.get('dismissed',[])))")
|
|
fi
|
|
|
|
# Get investigation results
|
|
VERIFIED_CLAIMS=0
|
|
ACTIONABLE_STRATEGIES=0
|
|
if [ -d "$LATEST/investigations" ]; then
|
|
VERIFIED_CLAIMS=$(find "$LATEST/investigations" -name "*.json" -exec grep -l '"verdict".*"VERIFIED"' {} \; | wc -l)
|
|
ACTIONABLE_STRATEGIES=$(find "$LATEST/investigations" -name "*.json" -exec grep -l '"actionable".*true' {} \; | wc -l)
|
|
fi
|
|
|
|
# Get simulation status
|
|
ACTIVE_POSITIONS=0
|
|
TOTAL_PNL=0
|
|
if [ -f "$PROJECT/data/simulations/active.json" ]; then
|
|
ACTIVE_POSITIONS=$(python3 -c "import json; d=json.load(open('$PROJECT/data/simulations/active.json')); print(len(d.get('positions',[])))" 2>/dev/null || echo "0")
|
|
TOTAL_PNL=$(python3 -c "import json; d=json.load(open('$PROJECT/data/simulations/active.json')); print(sum(p.get('unrealized_pnl',0) for p in d.get('positions',[])))" 2>/dev/null || echo "0")
|
|
fi
|
|
|
|
cat > "$OUTPUT_DIR/report.html" << EOF
|
|
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Feed Hunter Report - $(date '+%Y-%m-%d %H:%M')</title>
|
|
<style>
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
|
background: #0d1117;
|
|
color: #f0f6fc;
|
|
line-height: 1.6;
|
|
margin: 0;
|
|
padding: 2rem;
|
|
}
|
|
.container {
|
|
max-width: 1200px;
|
|
margin: 0 auto;
|
|
}
|
|
.header {
|
|
text-align: center;
|
|
border-bottom: 2px solid #30363d;
|
|
padding-bottom: 2rem;
|
|
margin-bottom: 2rem;
|
|
}
|
|
.header h1 {
|
|
color: #58a6ff;
|
|
font-size: 2.5rem;
|
|
margin-bottom: 0.5rem;
|
|
}
|
|
.header .subtitle {
|
|
color: #8b949e;
|
|
font-size: 1.1rem;
|
|
}
|
|
.metrics {
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
|
gap: 1.5rem;
|
|
margin-bottom: 3rem;
|
|
}
|
|
.metric-card {
|
|
background: #161b22;
|
|
border: 1px solid #30363d;
|
|
border-radius: 8px;
|
|
padding: 1.5rem;
|
|
text-align: center;
|
|
}
|
|
.metric-value {
|
|
font-size: 2.5rem;
|
|
font-weight: bold;
|
|
margin-bottom: 0.5rem;
|
|
}
|
|
.metric-label {
|
|
color: #8b949e;
|
|
font-size: 0.9rem;
|
|
text-transform: uppercase;
|
|
letter-spacing: 0.5px;
|
|
}
|
|
.positive { color: #3fb950; }
|
|
.negative { color: #f85149; }
|
|
.neutral { color: #58a6ff; }
|
|
.section {
|
|
background: #161b22;
|
|
border: 1px solid #30363d;
|
|
border-radius: 8px;
|
|
padding: 2rem;
|
|
margin-bottom: 2rem;
|
|
}
|
|
.section h2 {
|
|
color: #f0f6fc;
|
|
border-bottom: 1px solid #30363d;
|
|
padding-bottom: 1rem;
|
|
margin-bottom: 1.5rem;
|
|
}
|
|
.status-good {
|
|
color: #3fb950;
|
|
font-weight: bold;
|
|
}
|
|
.status-warn {
|
|
color: #ffab40;
|
|
font-weight: bold;
|
|
}
|
|
.status-error {
|
|
color: #f85149;
|
|
font-weight: bold;
|
|
}
|
|
.log-snippet {
|
|
background: #0d1117;
|
|
border: 1px solid #30363d;
|
|
border-radius: 4px;
|
|
padding: 1rem;
|
|
font-family: 'Monaco', 'Menlo', monospace;
|
|
font-size: 0.85rem;
|
|
max-height: 200px;
|
|
overflow-y: auto;
|
|
white-space: pre-wrap;
|
|
margin: 1rem 0;
|
|
}
|
|
.timestamp {
|
|
color: #8b949e;
|
|
font-size: 0.9rem;
|
|
text-align: right;
|
|
margin-top: 2rem;
|
|
border-top: 1px solid #30363d;
|
|
padding-top: 1rem;
|
|
}
|
|
.link-button {
|
|
display: inline-block;
|
|
background: #58a6ff;
|
|
color: #fff;
|
|
padding: 0.75rem 1.5rem;
|
|
border-radius: 6px;
|
|
text-decoration: none;
|
|
font-weight: bold;
|
|
margin: 0.5rem 0.5rem 0.5rem 0;
|
|
transition: background 0.2s;
|
|
}
|
|
.link-button:hover {
|
|
background: #4493e1;
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<div class="header">
|
|
<h1>🖤 Feed Hunter Report</h1>
|
|
<div class="subtitle">Pipeline execution completed at $(date '+%Y-%m-%d %H:%M:%S %Z')</div>
|
|
</div>
|
|
|
|
<div class="metrics">
|
|
<div class="metric-card">
|
|
<div class="metric-value neutral">$POSTS_COUNT</div>
|
|
<div class="metric-label">Posts Scraped</div>
|
|
</div>
|
|
<div class="metric-card">
|
|
<div class="metric-value $([ $HIGH_VALUE -gt 0 ] && echo 'positive' || echo 'neutral')">$HIGH_VALUE</div>
|
|
<div class="metric-label">High Value</div>
|
|
</div>
|
|
<div class="metric-card">
|
|
<div class="metric-value $([ $INVESTIGATION_COUNT -gt 0 ] && echo 'positive' || echo 'neutral')">$INVESTIGATION_COUNT</div>
|
|
<div class="metric-label">Investigated</div>
|
|
</div>
|
|
<div class="metric-card">
|
|
<div class="metric-value $([ $VERIFIED_CLAIMS -gt 0 ] && echo 'positive' || echo 'neutral')">$VERIFIED_CLAIMS</div>
|
|
<div class="metric-label">Verified Claims</div>
|
|
</div>
|
|
<div class="metric-card">
|
|
<div class="metric-value $([ $ACTIVE_POSITIONS -gt 0 ] && echo 'positive' || echo 'neutral')">$ACTIVE_POSITIONS</div>
|
|
<div class="metric-label">Active Positions</div>
|
|
</div>
|
|
<div class="metric-card">
|
|
<div class="metric-value $(python3 -c "print('positive' if $TOTAL_PNL >= 0 else 'negative')")">$${TOTAL_PNL}</div>
|
|
<div class="metric-label">Total P&L</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h2>📊 Pipeline Summary</h2>
|
|
<p><strong>Scraping:</strong> Collected $POSTS_COUNT posts from $PAGES pages</p>
|
|
<p><strong>Triaging:</strong> $HIGH_VALUE high-value, $WORTH_INVESTIGATING worth investigating, $DISMISSED dismissed</p>
|
|
<p><strong>Investigation:</strong> $INVESTIGATION_COUNT posts analyzed, $VERIFIED_CLAIMS claims verified</p>
|
|
<p><strong>Simulations:</strong> $ACTIVE_POSITIONS active positions, \$$TOTAL_PNL unrealized P&L</p>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h2>🔍 Investigation Highlights</h2>
|
|
$(if [ $VERIFIED_CLAIMS -gt 0 ]; then
|
|
echo "<div class='status-good'>✓ Found $VERIFIED_CLAIMS verified claims worth tracking</div>"
|
|
else
|
|
echo "<div class='status-warn'>⚠ No verified claims found in this run</div>"
|
|
fi)
|
|
|
|
$(if [ $ACTIONABLE_STRATEGIES -gt 0 ]; then
|
|
echo "<div class='status-good'>✓ Identified $ACTIONABLE_STRATEGIES actionable strategies</div>"
|
|
else
|
|
echo "<div class='status-warn'>⚠ No actionable strategies identified</div>"
|
|
fi)
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h2>📈 Simulation Status</h2>
|
|
$(if [ $ACTIVE_POSITIONS -gt 0 ]; then
|
|
echo "<div class='status-good'>✓ $ACTIVE_POSITIONS positions actively tracked</div>"
|
|
echo "<div>Current P&L: <span class='$(python3 -c "print('positive' if $TOTAL_PNL >= 0 else 'negative')")'>\$$TOTAL_PNL</span></div>"
|
|
else
|
|
echo "<div class='status-warn'>⚠ No active positions</div>"
|
|
fi)
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h2>📱 Quick Actions</h2>
|
|
<a href="http://localhost:8888/" class="link-button">Open Portal Dashboard</a>
|
|
<a href="http://localhost:8888/feed" class="link-button">View Latest Feed</a>
|
|
<a href="http://localhost:8888/investigations" class="link-button">Investigation Reports</a>
|
|
<a href="http://localhost:8888/simulations" class="link-button">Simulation Tracker</a>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h2>📋 Execution Logs</h2>
|
|
|
|
<h3>Scraping Output</h3>
|
|
<div class="log-snippet">$(tail -n 10 "$OUTPUT_DIR/scrape.log" 2>/dev/null || echo "No scrape log found")</div>
|
|
|
|
<h3>Triage Output</h3>
|
|
<div class="log-snippet">$(tail -n 10 "$OUTPUT_DIR/triage.log" 2>/dev/null || echo "No triage log found")</div>
|
|
|
|
$(if [ $INVESTIGATION_COUNT -gt 0 ]; then
|
|
echo "<h3>Investigation Output</h3>"
|
|
echo "<div class='log-snippet'>$(tail -n 10 "$OUTPUT_DIR/investigate.log" 2>/dev/null || echo "No investigation log found")</div>"
|
|
fi)
|
|
</div>
|
|
|
|
<div class="timestamp">
|
|
Report generated: $(date '+%Y-%m-%d %H:%M:%S %Z')<br>
|
|
Data directory: $LATEST<br>
|
|
Report saved: $OUTPUT_DIR/report.html
|
|
</div>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
EOF
|
|
|
|
echo ""
|
|
echo "=== Pipeline + Report Complete ==="
|
|
echo "📊 Dashboard: http://localhost:8888/"
|
|
echo "📄 Report: $OUTPUT_DIR/report.html"
|
|
echo "📁 Data: $LATEST"
|
|
echo ""
|
|
echo "Summary:"
|
|
echo " Posts: $POSTS_COUNT | High Value: $HIGH_VALUE | Investigated: $INVESTIGATION_COUNT"
|
|
echo " Verified: $VERIFIED_CLAIMS | Actionable: $ACTIONABLE_STRATEGIES"
|
|
echo " Active Positions: $ACTIVE_POSITIONS | P&L: \$$TOTAL_PNL" |