Night shift: tweet analyzer, data connectors, feed monitor, market watch portal

This commit is contained in:
2026-02-12 00:16:41 -06:00
parent f623cba45c
commit 07f1448d57
20 changed files with 1825 additions and 388 deletions

View File

@ -0,0 +1,51 @@
# Data Source Connectors
Standalone Python scripts for fetching crypto/market data. Each has CLI with `--pretty` (JSON formatting) and `--summary` (human-readable output).
## defillama.py ✅ (no auth needed)
DefiLlama API — DeFi protocol data, token prices, yield farming opportunities.
```bash
./defillama.py protocols --limit 10 --summary # Top protocols by TVL
./defillama.py tvl aave --pretty # TVL for specific protocol
./defillama.py prices coingecko:bitcoin coingecko:ethereum --summary
./defillama.py yields --limit 20 --stablecoins --summary # Top stablecoin yields
```
**Endpoints used:** api.llama.fi/protocols, api.llama.fi/tvl/{name}, coins.llama.fi/prices, yields.llama.fi/pools
## coinglass.py 🔑 (API key recommended)
Coinglass — funding rates, open interest, long/short ratios.
```bash
export COINGLASS_API_KEY=your_key # Get at coinglass.com/pricing
./coinglass.py funding --summary
./coinglass.py oi --summary
./coinglass.py long-short --summary
```
**Note:** Free internal API endpoints often return empty data. API key required for reliable access.
## arkham.py 🔑 (API key required)
Arkham Intelligence — whale wallet tracking, token transfers, entity search.
```bash
export ARKHAM_API_KEY=your_key # Sign up at platform.arkhamintelligence.com
./arkham.py notable --summary # List known whale addresses
./arkham.py address vitalik --summary # Address intelligence (supports name shortcuts)
./arkham.py transfers 0x1234... --limit 10 --pretty
./arkham.py search "binance" --pretty
```
**Built-in shortcuts:** vitalik, justin-sun, binance-hot, coinbase-prime, aave-treasury, uniswap-deployer
## Programmatic Usage
```python
from tools.data_sources.defillama import get_protocols, get_prices, get_yield_pools
from tools.data_sources.coinglass import get_funding_rates
from tools.data_sources.arkham import get_address_info, NOTABLE_ADDRESSES
```

4
tools/data_sources/__init__.py Executable file
View File

@ -0,0 +1,4 @@
"""Crypto & market data source connectors."""
from pathlib import Path
DATA_SOURCES_DIR = Path(__file__).parent

167
tools/data_sources/arkham.py Executable file
View File

@ -0,0 +1,167 @@
#!/usr/bin/env python3
"""Arkham Intelligence connector — whale tracking, token flows, address intelligence.
Requires API key for most endpoints. Set ARKHAM_API_KEY env var.
Sign up at https://platform.arkhamintelligence.com
"""
import argparse
import json
import os
import sys
from typing import Any
import requests
BASE = "https://api.arkhamintelligence.com"
TIMEOUT = 30
NOTABLE_ADDRESSES = {
"vitalik": "0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045",
"justin-sun": "0x3DdfA8eC3052539b6C9549F12cEA2C295cfF5296",
"binance-hot": "0x28C6c06298d514Db089934071355E5743bf21d60",
"coinbase-prime": "0xA9D1e08C7793af67e9d92fe308d5697FB81d3E43",
"aave-treasury": "0x464C71f6c2F760DdA6093dCB91C24c39e5d6e18c",
"uniswap-deployer": "0x41653c7d61609D856f29355E404F310Ec4142Cfb",
}
def _get(path: str, params: dict | None = None) -> Any:
key = os.environ.get("ARKHAM_API_KEY")
headers = {"User-Agent": "Mozilla/5.0"}
if key:
headers["API-Key"] = key
r = requests.get(f"{BASE}/{path}", params=params, headers=headers, timeout=TIMEOUT)
if r.status_code in (401, 403) or "api key" in r.text.lower():
raise EnvironmentError(
"Arkham API key required. Set ARKHAM_API_KEY env var.\n"
"Sign up at https://platform.arkhamintelligence.com"
)
r.raise_for_status()
return r.json()
def resolve_address(name_or_addr: str) -> str:
return NOTABLE_ADDRESSES.get(name_or_addr.lower(), name_or_addr)
# ── Data fetchers ───────────────────────────────────────────────────────────
def get_address_info(address: str) -> dict:
return _get(f"intelligence/address/{resolve_address(address)}")
def get_transfers(address: str, limit: int = 20) -> dict:
return _get("token/transfers", {"address": resolve_address(address), "limit": limit})
def search_entity(query: str) -> dict:
return _get("intelligence/search", {"query": query})
# ── Summary helpers ─────────────────────────────────────────────────────────
def summary_address(data: dict) -> str:
lines = ["═══ Address Intelligence ═══", ""]
if isinstance(data, dict):
entity = data.get("entity", {}) or {}
if entity:
lines.append(f" Entity: {entity.get('name', 'Unknown')}")
lines.append(f" Type: {entity.get('type', 'Unknown')}")
lines.append(f" Address: {data.get('address', '?')}")
labels = data.get("labels", [])
if labels:
lines.append(f" Labels: {', '.join(str(l) for l in labels)}")
else:
lines.append(f" {data}")
return "\n".join(lines)
def summary_transfers(data) -> str:
lines = ["═══ Recent Transfers ═══", ""]
transfers = data if isinstance(data, list) else (data.get("transfers", data.get("data", [])) if isinstance(data, dict) else [])
if not transfers:
lines.append(" No transfers found.")
return "\n".join(lines)
for t in transfers[:15]:
token = t.get("token", {}).get("symbol", "?") if isinstance(t.get("token"), dict) else "?"
amount = t.get("amount", t.get("value", "?"))
fr = t.get("from", {})
to = t.get("to", {})
fl = (fr.get("label") or fr.get("address", "?")[:12]) if isinstance(fr, dict) else str(fr)[:12]
tl = (to.get("label") or to.get("address", "?")[:12]) if isinstance(to, dict) else str(to)[:12]
lines.append(f" {token:<8} {str(amount):>15} {fl}{tl}")
return "\n".join(lines)
def summary_notable() -> str:
lines = ["═══ Notable/Whale Addresses ═══", ""]
for name, addr in NOTABLE_ADDRESSES.items():
lines.append(f" {name:<20} {addr}")
lines.append("")
lines.append(" Use these as shortcuts: arkham.py address vitalik")
return "\n".join(lines)
# ── CLI ─────────────────────────────────────────────────────────────────────
def main():
common = argparse.ArgumentParser(add_help=False)
common.add_argument("--pretty", action="store_true", help="Pretty-print JSON output")
common.add_argument("--summary", action="store_true", help="Human-readable summary")
parser = argparse.ArgumentParser(description="Arkham Intelligence connector", parents=[common])
sub = parser.add_subparsers(dest="command", required=True)
p_addr = sub.add_parser("address", help="Address intelligence", parents=[common])
p_addr.add_argument("address", help="Ethereum address or notable name")
p_tx = sub.add_parser("transfers", help="Recent token transfers", parents=[common])
p_tx.add_argument("address")
p_tx.add_argument("--limit", type=int, default=20)
p_search = sub.add_parser("search", help="Search entities", parents=[common])
p_search.add_argument("query")
sub.add_parser("notable", help="List notable/whale addresses", parents=[common])
args = parser.parse_args()
try:
if args.command == "notable":
if args.summary:
print(summary_notable())
else:
json.dump(NOTABLE_ADDRESSES, sys.stdout, indent=2 if args.pretty else None)
print()
return
if args.command == "address":
data = get_address_info(args.address)
if args.summary:
print(summary_address(data)); return
result = data
elif args.command == "transfers":
data = get_transfers(args.address, args.limit)
if args.summary:
print(summary_transfers(data)); return
result = data
elif args.command == "search":
result = search_entity(args.query)
else:
parser.print_help(); return
json.dump(result, sys.stdout, indent=2 if args.pretty else None)
print()
except EnvironmentError as e:
print(str(e), file=sys.stderr); sys.exit(1)
except requests.HTTPError as e:
detail = e.response.text[:200] if e.response is not None else ""
print(json.dumps({"error": str(e), "detail": detail}), file=sys.stderr); sys.exit(1)
except Exception as e:
print(json.dumps({"error": f"{type(e).__name__}: {e}"}), file=sys.stderr); sys.exit(1)
if __name__ == "__main__":
main()

181
tools/data_sources/coinglass.py Executable file
View File

@ -0,0 +1,181 @@
#!/usr/bin/env python3
"""Coinglass data connector — funding rates, open interest, long/short ratios.
Uses the free fapi.coinglass.com internal API where available.
Some endpoints may return empty data without authentication.
Set COINGLASS_API_KEY env var for authenticated access to open-api.coinglass.com.
"""
import argparse
import json
import os
import sys
from typing import Any
import requests
FREE_BASE = "https://fapi.coinglass.com/api"
AUTH_BASE = "https://open-api.coinglass.com/public/v2"
TIMEOUT = 30
def _free_get(path: str, params: dict | None = None) -> Any:
headers = {
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
"Referer": "https://www.coinglass.com/",
}
r = requests.get(f"{FREE_BASE}/{path}", params=params, headers=headers, timeout=TIMEOUT)
r.raise_for_status()
data = r.json()
if data.get("code") == "0" or data.get("success"):
return data.get("data", [])
raise ValueError(f"API error: {data.get('msg', 'unknown')}")
def _auth_get(path: str, params: dict | None = None) -> Any:
key = os.environ.get("COINGLASS_API_KEY")
if not key:
raise EnvironmentError("COINGLASS_API_KEY not set. Get one at https://www.coinglass.com/pricing")
headers = {"coinglassSecret": key}
r = requests.get(f"{AUTH_BASE}/{path}", params=params, headers=headers, timeout=TIMEOUT)
r.raise_for_status()
data = r.json()
if data.get("success") or data.get("code") == "0":
return data.get("data", [])
raise ValueError(f"API error: {data.get('msg', 'unknown')}")
# ── Data fetchers ───────────────────────────────────────────────────────────
def get_funding_rates() -> list[dict]:
"""Funding rates across exchanges."""
try:
data = _free_get("fundingRate/v2/home")
if data:
return data
except Exception:
pass
return _auth_get("funding")
def get_open_interest() -> list[dict]:
"""Aggregated open interest data."""
try:
data = _free_get("openInterest/v3/home")
if data:
return data
except Exception:
pass
return _auth_get("open_interest")
def get_long_short_ratio() -> list[dict]:
"""Global long/short account ratios."""
try:
data = _free_get("futures/longShort/v2/home")
if data:
return data
except Exception:
pass
return _auth_get("long_short")
# ── Summary helpers ─────────────────────────────────────────────────────────
def _no_data_msg(name: str) -> str:
return (f"No {name} data available (free API may be restricted).\n"
"Set COINGLASS_API_KEY for full access: https://www.coinglass.com/pricing")
def summary_funding(data: list[dict]) -> str:
if not data:
return _no_data_msg("funding rate")
lines = ["═══ Funding Rates ═══", ""]
for item in data[:20]:
symbol = item.get("symbol", item.get("coin", "?"))
rate = None
if "uMarginList" in item:
for m in item["uMarginList"]:
rate = m.get("rate")
if rate is not None:
break
else:
rate = item.get("rate")
if rate is not None:
lines.append(f" {symbol:<10} {float(rate)*100:>8.4f}%")
else:
lines.append(f" {symbol:<10} (rate unavailable)")
return "\n".join(lines)
def summary_oi(data: list[dict]) -> str:
if not data:
return _no_data_msg("open interest")
lines = ["═══ Open Interest ═══", ""]
for item in data[:20]:
symbol = item.get("symbol", item.get("coin", "?"))
oi = item.get("openInterest", item.get("oi", 0))
lines.append(f" {symbol:<10} OI: ${float(oi):>15,.0f}")
return "\n".join(lines)
def summary_ls(data: list[dict]) -> str:
if not data:
return _no_data_msg("long/short")
lines = ["═══ Long/Short Ratios ═══", ""]
for item in data[:20]:
symbol = item.get("symbol", item.get("coin", "?"))
long_rate = item.get("longRate", item.get("longRatio", "?"))
short_rate = item.get("shortRate", item.get("shortRatio", "?"))
lines.append(f" {symbol:<10} Long: {long_rate} Short: {short_rate}")
return "\n".join(lines)
# ── CLI ─────────────────────────────────────────────────────────────────────
def main():
common = argparse.ArgumentParser(add_help=False)
common.add_argument("--pretty", action="store_true", help="Pretty-print JSON output")
common.add_argument("--summary", action="store_true", help="Human-readable summary")
parser = argparse.ArgumentParser(description="Coinglass data connector", parents=[common])
sub = parser.add_subparsers(dest="command", required=True)
sub.add_parser("funding", help="Funding rates across exchanges", parents=[common])
sub.add_parser("oi", help="Open interest overview", parents=[common])
sub.add_parser("long-short", help="Long/short ratios", parents=[common])
args = parser.parse_args()
try:
if args.command == "funding":
data = get_funding_rates()
if args.summary:
print(summary_funding(data)); return
result = data
elif args.command == "oi":
data = get_open_interest()
if args.summary:
print(summary_oi(data)); return
result = data
elif args.command == "long-short":
data = get_long_short_ratio()
if args.summary:
print(summary_ls(data)); return
result = data
else:
parser.print_help(); return
json.dump(result, sys.stdout, indent=2 if args.pretty else None)
print()
except EnvironmentError as e:
print(str(e), file=sys.stderr); sys.exit(1)
except requests.HTTPError as e:
print(json.dumps({"error": str(e)}), file=sys.stderr); sys.exit(1)
except Exception as e:
print(json.dumps({"error": f"{type(e).__name__}: {e}"}), file=sys.stderr); sys.exit(1)
if __name__ == "__main__":
main()

176
tools/data_sources/defillama.py Executable file
View File

@ -0,0 +1,176 @@
#!/usr/bin/env python3
"""DefiLlama API connector — TVL, token prices, yield/APY data.
No authentication required. All endpoints are free.
API base: https://api.llama.fi | Prices: https://coins.llama.fi | Yields: https://yields.llama.fi
"""
import argparse
import json
import sys
from typing import Any
import requests
BASE = "https://api.llama.fi"
COINS_BASE = "https://coins.llama.fi"
YIELDS_BASE = "https://yields.llama.fi"
TIMEOUT = 30
def _get(url: str, params: dict | None = None) -> Any:
r = requests.get(url, params=params, timeout=TIMEOUT)
r.raise_for_status()
return r.json()
# ── Protocol / TVL ──────────────────────────────────────────────────────────
def get_protocols(limit: int = 20) -> list[dict]:
"""Top protocols by TVL."""
data = _get(f"{BASE}/protocols")
# Sort by tvl descending, filter out CEXes
protos = [p for p in data if p.get("category") != "CEX" and p.get("tvl")]
protos.sort(key=lambda p: p.get("tvl", 0), reverse=True)
return protos[:limit]
def get_tvl(protocol: str) -> dict:
"""Get current TVL for a specific protocol (slug name)."""
val = _get(f"{BASE}/tvl/{protocol}")
return {"protocol": protocol, "tvl": val}
def get_protocol_detail(protocol: str) -> dict:
"""Full protocol details including chain breakdowns."""
return _get(f"{BASE}/protocol/{protocol}")
# ── Token Prices ────────────────────────────────────────────────────────────
def get_prices(coins: list[str]) -> dict:
"""Get current prices. Coins format: 'coingecko:ethereum', 'ethereum:0x...', etc."""
joined = ",".join(coins)
data = _get(f"{COINS_BASE}/prices/current/{joined}")
return data.get("coins", {})
# ── Yields / APY ────────────────────────────────────────────────────────────
def get_yield_pools(limit: int = 30, min_tvl: float = 1_000_000, stablecoin_only: bool = False) -> list[dict]:
"""Top yield pools sorted by APY."""
data = _get(f"{YIELDS_BASE}/pools")
pools = data.get("data", [])
# Filter
pools = [p for p in pools if (p.get("tvlUsd") or 0) >= min_tvl and (p.get("apy") or 0) > 0]
if stablecoin_only:
pools = [p for p in pools if p.get("stablecoin")]
pools.sort(key=lambda p: p.get("apy", 0), reverse=True)
return pools[:limit]
# ── Summary helpers ─────────────────────────────────────────────────────────
def _fmt_usd(v: float) -> str:
if v >= 1e9:
return f"${v/1e9:.2f}B"
if v >= 1e6:
return f"${v/1e6:.1f}M"
return f"${v:,.0f}"
def summary_protocols(protos: list[dict]) -> str:
lines = ["═══ Top Protocols by TVL ═══", ""]
for i, p in enumerate(protos, 1):
lines.append(f" {i:>2}. {p['name']:<25} TVL: {_fmt_usd(p.get('tvl', 0)):>12} chain: {p.get('chain', '?')}")
return "\n".join(lines)
def summary_prices(prices: dict) -> str:
lines = ["═══ Token Prices ═══", ""]
for coin, info in prices.items():
lines.append(f" {info.get('symbol', coin):<10} ${info['price']:>12,.2f} (confidence: {info.get('confidence', '?')})")
return "\n".join(lines)
def summary_yields(pools: list[dict]) -> str:
lines = ["═══ Top Yield Pools ═══", ""]
for i, p in enumerate(pools, 1):
lines.append(
f" {i:>2}. {p.get('symbol','?'):<25} APY: {p.get('apy',0):>8.2f}% "
f"TVL: {_fmt_usd(p.get('tvlUsd',0)):>10} {p.get('chain','?')}/{p.get('project','?')}"
)
return "\n".join(lines)
# ── CLI ─────────────────────────────────────────────────────────────────────
def main():
common = argparse.ArgumentParser(add_help=False)
common.add_argument("--pretty", action="store_true", help="Pretty-print JSON output")
common.add_argument("--summary", action="store_true", help="Human-readable summary")
parser = argparse.ArgumentParser(description="DefiLlama data connector", parents=[common])
sub = parser.add_subparsers(dest="command", required=True)
# protocols
p_proto = sub.add_parser("protocols", help="Top protocols by TVL", parents=[common])
p_proto.add_argument("--limit", type=int, default=20)
# tvl
p_tvl = sub.add_parser("tvl", help="TVL for a specific protocol", parents=[common])
p_tvl.add_argument("protocol", help="Protocol slug (e.g. aave, lido)")
# prices
p_price = sub.add_parser("prices", help="Token prices", parents=[common])
p_price.add_argument("coins", nargs="+", help="Coin IDs: coingecko:ethereum, ethereum:0x...")
# yields
p_yield = sub.add_parser("yields", help="Top yield pools", parents=[common])
p_yield.add_argument("--limit", type=int, default=30)
p_yield.add_argument("--min-tvl", type=float, default=1_000_000)
p_yield.add_argument("--stablecoins", action="store_true")
args = parser.parse_args()
try:
if args.command == "protocols":
data = get_protocols(args.limit)
if args.summary:
print(summary_protocols(data))
return
result = [{"name": p["name"], "tvl": p.get("tvl"), "chain": p.get("chain"), "category": p.get("category"), "symbol": p.get("symbol")} for p in data]
elif args.command == "tvl":
result = get_tvl(args.protocol)
if args.summary:
print(f"{args.protocol}: {_fmt_usd(result['tvl'])}")
return
elif args.command == "prices":
result = get_prices(args.coins)
if args.summary:
print(summary_prices(result))
return
elif args.command == "yields":
data = get_yield_pools(args.limit, args.min_tvl, args.stablecoins)
if args.summary:
print(summary_yields(data))
return
result = [{"symbol": p.get("symbol"), "apy": p.get("apy"), "tvlUsd": p.get("tvlUsd"), "chain": p.get("chain"), "project": p.get("project"), "pool": p.get("pool")} for p in data]
else:
parser.print_help()
return
indent = 2 if args.pretty else None
json.dump(result, sys.stdout, indent=indent)
print()
except requests.HTTPError as e:
print(json.dumps({"error": str(e)}), file=sys.stderr)
sys.exit(1)
except Exception as e:
print(json.dumps({"error": f"Unexpected: {e}"}), file=sys.stderr)
sys.exit(1)
if __name__ == "__main__":
main()