Feed Hunter: deep scraper skill, pipeline, simulator, first investigation
- Built deep-scraper skill (CDP-based X feed extraction) - Three-stage pipeline: scrape → triage → investigate - Paper trading simulator with position tracking - First live investigation: verified kch123 Polymarket profile ($9.3M P&L) - Opened first paper position: Seahawks Super Bowl @ 68c - Telegram alerts with inline action buttons - Portal build in progress (night shift)
This commit is contained in:
59
projects/feed-hunter/run-pipeline.sh
Executable file
59
projects/feed-hunter/run-pipeline.sh
Executable file
@ -0,0 +1,59 @@
|
||||
#!/bin/bash
|
||||
# Feed Hunter — Full pipeline run
|
||||
# Usage: ./run-pipeline.sh [scroll_pages]
|
||||
#
|
||||
# Runs: scrape → triage → generate investigation tasks
|
||||
# Agent handles investigation + alerts separately
|
||||
|
||||
set -e
|
||||
|
||||
PAGES=${1:-8}
|
||||
BASE="/home/wdjones/.openclaw/workspace"
|
||||
SKILL="$BASE/skills/deep-scraper/scripts"
|
||||
PROJECT="$BASE/projects/feed-hunter"
|
||||
DATA="$BASE/data/x-feed"
|
||||
|
||||
echo "=== Feed Hunter Pipeline ==="
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S %Z')"
|
||||
|
||||
# Ensure Chrome is running with debug port
|
||||
if ! curl -s http://127.0.0.1:9222/json >/dev/null 2>&1; then
|
||||
echo "Starting Chrome..."
|
||||
bash "$SKILL/launch-chrome-debug.sh"
|
||||
fi
|
||||
|
||||
# Stage 1: Scrape
|
||||
echo ""
|
||||
echo "--- Stage 1: Scrape ($PAGES pages) ---"
|
||||
python3 -u "$SKILL/scrape-x-feed.py" --port 9222 --scroll-pages "$PAGES"
|
||||
|
||||
# Find latest scrape
|
||||
LATEST=$(ls -dt "$DATA"/20* | head -1)
|
||||
echo "Latest scrape: $LATEST"
|
||||
|
||||
# Stage 2: Triage
|
||||
echo ""
|
||||
echo "--- Stage 2: Triage ---"
|
||||
python3 "$SKILL/triage-posts.py" "$LATEST/posts.json"
|
||||
|
||||
# Stage 3: Generate investigation tasks
|
||||
TRIAGE="$LATEST/triage.json"
|
||||
if [ -f "$TRIAGE" ]; then
|
||||
QUEUE=$(python3 -c "import json; d=json.load(open('$TRIAGE')); print(len(d.get('investigation_queue',[])))")
|
||||
if [ "$QUEUE" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "--- Stage 3: Investigation Tasks ---"
|
||||
python3 "$PROJECT/investigate.py" "$TRIAGE" --output "$LATEST/investigations"
|
||||
echo ""
|
||||
echo ">>> $QUEUE posts queued for investigation"
|
||||
echo ">>> Agent should read: $LATEST/investigations/"
|
||||
else
|
||||
echo ""
|
||||
echo ">>> No posts worth investigating this run."
|
||||
fi
|
||||
else
|
||||
echo ">>> No triage output found."
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Pipeline complete: $LATEST ==="
|
||||
Reference in New Issue
Block a user