case/workspace
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Full sync - all projects, memory, configs

Browse Source
This commit is contained in:
Case
2026-03-21 20:27:59 -05:00
parent 2447677d4a
commit b33de10902
395 changed files with 1635300 additions and 459211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

254
tools/auto-memory-hook.py Executable file
View File

@ -0,0 +1,254 @@
#!/usr/bin/env python3
"""
Auto-Memory Hook for OpenClaw
Analyzes agent turns and stores valuable information in ChromaDB.
Usage:
echo '{"user":"...","assistant":"...","agent_id":"case","session":"abc"}' | python3 auto-memory-hook.py
python3 auto-memory-hook.py --user "msg" --assistant "resp" --agent-id case --session abc
"""
import sys, json, os, re, time, uuid, hashlib, logging
from datetime import datetime, timezone
from pathlib import Path
import requests
import chromadb
# --- Config ---
CHROMADB_HOST = os.environ.get("CHROMADB_HOST", "192.168.86.25")
CHROMADB_PORT = int(os.environ.get("CHROMADB_PORT", "8000"))
OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://192.168.86.40:11434")
EMBED_MODEL = "nomic-embed-text"
LLM_MODEL = "qwen3:8b"
COLLECTION = "auto-memory"
DEDUP_THRESHOLD = 0.85 # cosine similarity; ChromaDB returns distances, so threshold = 1 - 0.85 = 0.15
LOG_DIR = Path(__file__).resolve().parent.parent / "logs"
LOG_FILE = LOG_DIR / "auto-memory.log"
LOG_DIR.mkdir(parents=True, exist_ok=True)
logging.basicConfig(
filename=str(LOG_FILE),
format="%(asctime)s %(levelname)s %(message)s",
level=logging.INFO,
)
log = logging.getLogger("auto-memory")
def ollama_available() -> bool:
try:
r = requests.get(f"{OLLAMA_URL}/api/tags", timeout=3)
return r.status_code == 200
except Exception:
return False
def ollama_generate(prompt: str, timeout: float = 10) -> str:
"""Call Ollama for classification/extraction."""
r = requests.post(
f"{OLLAMA_URL}/api/generate",
json={"model": LLM_MODEL, "prompt": prompt, "stream": False,
"options": {"temperature": 0.3, "num_predict": 512}},
timeout=timeout,
)
r.raise_for_status()
return r.json().get("response", "")
def ollama_embed(text: str) -> list[float]:
"""Get embedding from Ollama."""
r = requests.post(
f"{OLLAMA_URL}/api/embed",
json={"model": EMBED_MODEL, "input": text},
timeout=10,
)
r.raise_for_status()
data = r.json()
# API returns {"embeddings": [[...]]}
return data["embeddings"][0]
# --- Heuristic fallback ---
WORTH_PATTERNS = [
r"(?i)(fix|fixed|resolved|solution|workaround)\b.*\b(by|with|using|was)\b",
r"(?i)learned\s+that\b",
r"(?i)(decided|decision|we('ll| will))\s+(to|go with)\b",
r"(?i)(config|configured|set\s+up|installed)\b.*\b(on|at|for|to)\b",
r"(?i)(password|secret|token|api[_-]?key|port|ip|url)\s*(is|=|:)\s*\S+",
r"(?i)(error|bug|issue)\b.*\b(because|caused by|due to)\b",
r"(?i)(important|remember|note that|fyi)\b",
r"(?i)(created|deployed|migrated|upgraded)\b.*\b(service|server|vm|container|database)\b",
]
def heuristic_classify(user: str, assistant: str) -> tuple[bool, float]:
"""Fallback classification using regex patterns."""
combined = f"{user}\n{assistant}"
hits = sum(1 for p in WORTH_PATTERNS if re.search(p, combined))
if hits == 0:
return False, 0.0
confidence = min(0.4 + hits * 0.15, 0.9)
return True, confidence
def heuristic_extract(user: str, assistant: str) -> tuple[str, str]:
"""Fallback extraction: first 2 sentences of assistant response + topic guess."""
sentences = re.split(r'(?<=[.!?])\s+', assistant.strip())
memory = " ".join(sentences[:3])[:500]
# Guess topic
topic = "general"
topic_map = {
"infrastructure": r"(?i)(server|vm|port|docker|nginx|systemd|network|ip|dns|proxy)",
"code": r"(?i)(function|script|python|node|api|bug|error|code|git)",
"config": r"(?i)(config|setting|env|variable|yaml|json|toml)",
"decision": r"(?i)(decided|decision|chose|picked|go with)",
}
combined = f"{user}\n{assistant}"
for t, pat in topic_map.items():
if re.search(pat, combined):
topic = t
break
return memory, topic
def llm_classify_and_extract(user: str, assistant: str) -> tuple[bool, float, str, str]:
"""Use Ollama LLM for classification and extraction."""
prompt = f"""/no_think
You analyze agent conversations to decide if they contain information worth remembering long-term.
USER MESSAGE:
{user[:1500]}
ASSISTANT RESPONSE:
{assistant[:2000]}
Respond with EXACTLY this JSON (no other text):
{{"worth_remembering": true/false, "confidence": 0.0-1.0, "memory": "concise fact/solution/decision (1-2 sentences)", "topic": "one of: infrastructure, code, config, decision, troubleshooting, general"}}
"""
raw = ollama_generate(prompt, timeout=8)
# Extract JSON from response
match = re.search(r'\{[^{}]*\}', raw, re.DOTALL)
if not match:
raise ValueError(f"No JSON in LLM response: {raw[:200]}")
data = json.loads(match.group())
return (
bool(data.get("worth_remembering", False)),
float(data.get("confidence", 0.5)),
str(data.get("memory", "")),
str(data.get("topic", "general")),
)
def get_collection():
client = chromadb.HttpClient(host=CHROMADB_HOST, port=CHROMADB_PORT)
return client.get_or_create_collection(name=COLLECTION, metadata={"hnsw:space": "cosine"})
def check_duplicate(collection, embedding: list[float]) -> bool:
"""Check if a similar memory already exists. Returns True if duplicate."""
try:
results = collection.query(query_embeddings=[embedding], n_results=1)
if results and results["distances"] and results["distances"][0]:
# For cosine space, distance = 1 - similarity
distance = results["distances"][0][0]
similarity = 1 - distance
if similarity > DEDUP_THRESHOLD:
log.info(f"Duplicate found (similarity={similarity:.3f}), skipping")
return True
except Exception as e:
log.warning(f"Dedup check failed: {e}")
return False
def store_memory(collection, memory: str, embedding: list[float], metadata: dict):
doc_id = hashlib.sha256(memory.encode()).hexdigest()[:16]
collection.add(
documents=[memory],
embeddings=[embedding],
metadatas=[metadata],
ids=[doc_id],
)
log.info(f"Stored memory [{doc_id}]: {memory[:100]}...")
def process_turn(user: str, assistant: str, agent_id: str = "unknown", session: str = "unknown"):
"""Main pipeline."""
# Skip trivial interactions
if len(assistant) < 50:
log.info("SKIP: response too short")
return
use_llm = ollama_available()
# 1. Classify & Extract
if use_llm:
try:
worth, confidence, memory, topic = llm_classify_and_extract(user, assistant)
except Exception as e:
log.warning(f"LLM failed ({e}), falling back to heuristics")
use_llm = False
if not use_llm:
worth, confidence = heuristic_classify(user, assistant)
if worth:
memory, topic = heuristic_extract(user, assistant)
else:
memory, topic = "", "general"
if not worth or not memory:
log.info(f"SKIP: not worth remembering (confidence={confidence:.2f})")
return
log.info(f"CANDIDATE: topic={topic} confidence={confidence:.2f} memory={memory[:80]}...")
# 2. Embed
try:
embedding = ollama_embed(memory)
except Exception as e:
log.error(f"Embedding failed: {e}")
return
# 3. Dedup & Store
try:
collection = get_collection()
if check_duplicate(collection, embedding):
return
metadata = {
"date": datetime.now(timezone.utc).strftime("%Y-%m-%d"),
"agent_id": agent_id,
"topic": topic,
"confidence": confidence,
"source": f"session:{session}",
}
store_memory(collection, memory, embedding, metadata)
print(json.dumps({"stored": True, "memory": memory, "topic": topic, "confidence": confidence}))
except Exception as e:
log.error(f"ChromaDB error: {e}")
print(json.dumps({"stored": False, "error": str(e)}))
def main():
import argparse
parser = argparse.ArgumentParser(description="Auto-memory hook")
parser.add_argument("--user", help="User message")
parser.add_argument("--assistant", help="Assistant response")
parser.add_argument("--agent-id", default="unknown")
parser.add_argument("--session", default="unknown")
args = parser.parse_args()
if args.user and args.assistant:
process_turn(args.user, args.assistant, args.agent_id, args.session)
elif not sys.stdin.isatty():
data = json.load(sys.stdin)
process_turn(
data.get("user", ""),
data.get("assistant", ""),
data.get("agent_id", "unknown"),
data.get("session", "unknown"),
)
else:
parser.print_help()
sys.exit(1)
if __name__ == "__main__":
main()

88
tools/auto-memory-recall.py Executable file
View File

@ -0,0 +1,88 @@
#!/usr/bin/env python3
"""
Auto-Memory Recall — query the auto-memory ChromaDB collection.
Usage:
python3 auto-memory-recall.py "how did we fix nginx"
python3 auto-memory-recall.py --query "chromadb setup" --limit 3 --topic infrastructure
"""
import sys, json, os, argparse
import requests
import chromadb
CHROMADB_HOST = os.environ.get("CHROMADB_HOST", "192.168.86.25")
CHROMADB_PORT = int(os.environ.get("CHROMADB_PORT", "8000"))
OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://192.168.86.40:11434")
EMBED_MODEL = "nomic-embed-text"
COLLECTION = "auto-memory"
def ollama_embed(text: str) -> list[float]:
r = requests.post(
f"{OLLAMA_URL}/api/embed",
json={"model": EMBED_MODEL, "input": text},
timeout=10,
)
r.raise_for_status()
return r.json()["embeddings"][0]
def recall(query: str, limit: int = 5, topic: str = None, agent_id: str = None) -> list[dict]:
client = chromadb.HttpClient(host=CHROMADB_HOST, port=CHROMADB_PORT)
collection = client.get_or_create_collection(name=COLLECTION, metadata={"hnsw:space": "cosine"})
where = {}
if topic:
where["topic"] = topic
if agent_id:
where["agent_id"] = agent_id
try:
embedding = ollama_embed(query)
results = collection.query(
query_embeddings=[embedding],
n_results=limit,
where=where if where else None,
)
except Exception:
# Fallback to text query if embedding fails
results = collection.query(
query_texts=[query],
n_results=limit,
where=where if where else None,
)
memories = []
if results and results["documents"]:
for i, doc in enumerate(results["documents"][0]):
meta = results["metadatas"][0][i] if results["metadatas"] else {}
dist = results["distances"][0][i] if results["distances"] else None
memories.append({
"memory": doc,
"similarity": round(1 - dist, 3) if dist is not None else None,
"id": results["ids"][0][i] if results["ids"] else None,
**meta,
})
return memories
def main():
parser = argparse.ArgumentParser(description="Recall memories from auto-memory collection")
parser.add_argument("query", nargs="?", help="Search query")
parser.add_argument("--query", "-q", dest="query_flag")
parser.add_argument("--limit", "-n", type=int, default=5)
parser.add_argument("--topic", "-t")
parser.add_argument("--agent-id", "-a")
args = parser.parse_args()
query = args.query or args.query_flag
if not query:
parser.print_help()
sys.exit(1)
results = recall(query, args.limit, args.topic, args.agent_id)
print(json.dumps(results, indent=2))
if __name__ == "__main__":
main()

145
tools/ava-converter.py Executable file
View File

@ -0,0 +1,145 @@
#!/usr/bin/env python3
"""
AVA GeoJSON to KML Converter
Downloads American Viticultural Area (AVA) boundary data from the UC Davis
Library AVA repository and converts all GeoJSON files to KML format using
ogr2ogr (GDAL).
Source: https://github.com/UCDavisLibrary/ava
License: CC0 1.0 Universal (Public Domain)
"""
import argparse
import datetime
import glob
import logging
import os
import shutil
import subprocess
import sys
import tempfile
DEFAULT_OUTPUT = "/home/wdjones/.openclaw/workspace/projects/ava-kmls"
DEFAULT_REPO = "https://github.com/UCDavisLibrary/ava.git"
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
log = logging.getLogger(__name__)
def check_ogr2ogr():
"""Check that ogr2ogr is available."""
if shutil.which("ogr2ogr") is None:
log.error("ogr2ogr not found. Install GDAL:")
log.error(" Ubuntu/Debian: sudo apt-get install gdal-bin")
log.error(" macOS: brew install gdal")
log.error(" Conda: conda install -c conda-forge gdal")
sys.exit(1)
def clone_repo(repo_url, dest):
"""Clone the AVA repo to a temporary directory."""
log.info(f"Cloning {repo_url} ...")
subprocess.run(["git", "clone", "--depth", "1", repo_url, dest],
check=True, capture_output=True, text=True)
log.info("Clone complete.")
def convert_geojson_to_kml(src, dst):
"""Convert a single GeoJSON file to KML via ogr2ogr. Returns True on success."""
result = subprocess.run(
["ogr2ogr", "-f", "KML", dst, src],
capture_output=True, text=True
)
if result.returncode != 0:
log.error(f" FAILED: {result.stderr.strip()}")
return False
return True
def generate_readme(output_dir, ava_names, errors, repo_url):
"""Generate a README.md summarizing the conversion."""
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
lines = [
"# AVA Boundary Data — KML & GeoJSON",
"",
"American Viticultural Area (AVA) boundaries converted from GeoJSON to KML.",
"",
"## Source",
f"- Repository: {repo_url}",
"- Maintainer: UC Davis Library",
"- License: **CC0 1.0 Universal** (Public Domain)",
"",
f"## Generated: {now}",
"",
f"**{len(ava_names)} AVAs converted** ({len(errors)} errors)",
"",
"## Files",
"- `output/kml/` — KML files (one per AVA)",
"- `output/geojson/` — Original GeoJSON source files",
"",
"## AVA List",
"",
]
for name in sorted(ava_names):
lines.append(f"- {name}")
if errors:
lines += ["", "## Conversion Errors", ""]
for e in sorted(errors):
lines.append(f"- {e}")
readme_path = os.path.join(output_dir, "README.md")
with open(readme_path, "w") as f:
f.write("\n".join(lines) + "\n")
log.info(f"README written to {readme_path}")
def main():
parser = argparse.ArgumentParser(description="Convert UC Davis AVA GeoJSON to KML")
parser.add_argument("--output", default=DEFAULT_OUTPUT, help="Output directory")
parser.add_argument("--repo-url", default=DEFAULT_REPO, help="Source git repo URL")
args = parser.parse_args()
check_ogr2ogr()
output_dir = os.path.abspath(args.output)
kml_dir = os.path.join(output_dir, "output", "kml")
geojson_dir = os.path.join(output_dir, "output", "geojson")
os.makedirs(kml_dir, exist_ok=True)
os.makedirs(geojson_dir, exist_ok=True)
with tempfile.TemporaryDirectory() as tmpdir:
repo_dir = os.path.join(tmpdir, "ava")
clone_repo(args.repo_url, repo_dir)
avas_dir = os.path.join(repo_dir, "avas")
if not os.path.isdir(avas_dir):
log.error(f"No 'avas/' directory found in repo at {avas_dir}")
sys.exit(1)
geojson_files = sorted(glob.glob(os.path.join(avas_dir, "*.geojson")))
log.info(f"Found {len(geojson_files)} GeoJSON files")
converted = []
errors = []
for gj in geojson_files:
basename = os.path.splitext(os.path.basename(gj))[0]
kml_out = os.path.join(kml_dir, f"{basename}.kml")
gj_out = os.path.join(geojson_dir, os.path.basename(gj))
log.info(f"Converting: {basename}")
shutil.copy2(gj, gj_out)
if convert_geojson_to_kml(gj, kml_out):
converted.append(basename)
else:
errors.append(basename)
log.info(f"Done: {len(converted)} converted, {len(errors)} errors")
generate_readme(output_dir, converted, errors, args.repo_url)
if __name__ == "__main__":
main()

282
tools/build-re-excel.py Normal file
View File

@ -0,0 +1,282 @@
#!/usr/bin/env python3
"""Build Excel spreadsheet from ARI's real estate cost seg model"""
import sys
sys.path.insert(0, '/home/wdjones/.openclaw/workspace-ari/data/real-estate-model')
from detailed_calculations import *
import openpyxl
from openpyxl.styles import Font, PatternFill, Alignment, Border, Side, numbers
from openpyxl.utils import get_column_letter
# Check if openpyxl available
try:
import openpyxl
except ImportError:
import subprocess
subprocess.run([sys.executable, '-m', 'pip', 'install', 'openpyxl'], check=True)
import openpyxl
results = run_portfolio_model()
wb = openpyxl.Workbook()
# Styles
header_font = Font(bold=True, color="FFFFFF", size=11)
header_fill = PatternFill(start_color="1a1a2e", end_color="1a1a2e", fill_type="solid")
red_font = Font(color="FF4444", bold=True)
green_font = Font(color="44FF44", bold=True)
money_fmt = '#,##0'
money_neg_fmt = '#,##0;[Red]-#,##0'
pct_fmt = '0.0%'
thin_border = Border(
left=Side(style='thin'), right=Side(style='thin'),
top=Side(style='thin'), bottom=Side(style='thin')
)
section_fill = PatternFill(start_color="2d2d44", end_color="2d2d44", fill_type="solid")
section_font = Font(bold=True, color="00BFFF", size=11)
warn_fill = PatternFill(start_color="4a1a1a", end_color="4a1a1a", fill_type="solid")
warn_font = Font(bold=True, color="FF6666", size=12)
def style_header(ws, row, max_col):
for col in range(1, max_col + 1):
cell = ws.cell(row=row, column=col)
cell.font = header_font
cell.fill = header_fill
cell.alignment = Alignment(horizontal='center', wrap_text=True)
cell.border = thin_border
def style_row(ws, row, max_col):
for col in range(1, max_col + 1):
cell = ws.cell(row=row, column=col)
cell.border = thin_border
cell.alignment = Alignment(horizontal='right')
if col == 1:
cell.alignment = Alignment(horizontal='left')
def auto_width(ws):
for col in ws.columns:
max_len = 0
col_letter = get_column_letter(col[0].column)
for cell in col:
if cell.value:
max_len = max(max_len, len(str(cell.value)))
ws.column_dimensions[col_letter].width = min(max_len + 4, 22)
# ============ SHEET 1: Portfolio Overview ============
ws1 = wb.active
ws1.title = "Portfolio Overview"
ws1.sheet_properties.tabColor = "1a1a2e"
# Title
ws1.merge_cells('A1:K1')
ws1['A1'] = "REAL ESTATE COST SEGREGATION MODEL — 5-YEAR PORTFOLIO"
ws1['A1'].font = Font(bold=True, color="00BFFF", size=14)
# Warning banner
ws1.merge_cells('A3:K3')
ws1['A3'] = "⚠️ VERDICT: NO-GO — Negative cash flow all 5 years. $0 tax benefit at >$150K AGI. $337K passive losses stranded."
ws1['A3'].font = warn_font
ws1['A3'].fill = warn_fill
# Headers
row = 5
headers = ['Year', 'Properties', 'Gross Rent', 'Effective Rent', 'NOI', 'Cash Flow',
'Depreciation', 'Taxable Income', 'Cash Invested', 'Cumulative Invested', 'Cumulative Cash Flow']
for col, h in enumerate(headers, 1):
ws1.cell(row=row, column=col, value=h)
style_header(ws1, row, len(headers))
cumulative_invested = 0
cumulative_cf = 0
for i, r in enumerate(results):
row = 6 + i
cumulative_invested += r['total_cash_invested']
cumulative_cf += r['total_cash_flow']
values = [
f"Year {r['year']}", r['properties_owned'], r['total_gross_rent'],
r['total_effective_rent'], r['total_noi'], r['total_cash_flow'],
r['total_depreciation'], r['total_taxable_income'], r['total_cash_invested'],
cumulative_invested, cumulative_cf
]
for col, v in enumerate(values, 1):
cell = ws1.cell(row=row, column=col, value=v)
if col >= 3:
cell.number_format = money_neg_fmt
if col == 6 and isinstance(v, (int, float)) and v < 0:
cell.font = red_font
if col == 8 and isinstance(v, (int, float)) and v < 0:
cell.font = red_font
style_row(ws1, row, len(headers))
# Totals row
row = 11
ws1.cell(row=row, column=1, value="5-YEAR TOTALS").font = Font(bold=True)
ws1.cell(row=row, column=3, value=sum(r['total_gross_rent'] for r in results)).number_format = money_fmt
ws1.cell(row=row, column=5, value=sum(r['total_noi'] for r in results)).number_format = money_fmt
ws1.cell(row=row, column=6, value=sum(r['total_cash_flow'] for r in results))
ws1.cell(row=row, column=6).number_format = money_neg_fmt
ws1.cell(row=row, column=6).font = red_font
ws1.cell(row=row, column=7, value=sum(r['total_depreciation'] for r in results)).number_format = money_fmt
ws1.cell(row=row, column=9, value=395000).number_format = money_fmt
style_row(ws1, row, len(headers))
auto_width(ws1)
# ============ SHEET 2: Tax Impact by AGI ============
ws2 = wb.create_sheet("Tax Impact by AGI")
ws2.sheet_properties.tabColor = "4a1a1a"
ws2.merge_cells('A1:H1')
ws2['A1'] = "PASSIVE LOSS & TAX SAVINGS BY AGI BRACKET"
ws2['A1'].font = Font(bold=True, color="00BFFF", size=14)
# Critical note
ws2.merge_cells('A3:H4')
ws2['A3'] = ("CRITICAL: At AGI >$150K (D J + partners likely scenario), passive losses CANNOT offset W-2 income. "
"Losses carry forward indefinitely but only offset future passive rental income or are released upon sale. "
"This means $0 annual tax benefit from depreciation — the entire cost seg strategy provides NO cash flow benefit.")
ws2['A3'].font = Font(color="FF6666", size=10, italic=True)
ws2['A3'].alignment = Alignment(wrap_text=True)
row = 6
headers = ['Year', 'Passive Loss Generated',
'Tax Savings\n(AGI <$100K)', 'Losses Carried\n(AGI <$100K)',
'Tax Savings\n(AGI $100-150K)', 'Losses Carried\n(AGI $100-150K)',
'Tax Savings\n(AGI >$150K)', 'Losses Carried\n(AGI >$150K)']
for col, h in enumerate(headers, 1):
ws2.cell(row=row, column=col, value=h)
style_header(ws2, row, len(headers))
for i, r in enumerate(results):
row = 7 + i
values = [
f"Year {r['year']}", abs(r['total_taxable_income']),
r['tax_savings_scenario_a'], r['passive_losses_scenario_a'],
r['tax_savings_scenario_b'], r['passive_losses_scenario_b'],
r['tax_savings_scenario_c'], r['passive_losses_scenario_c']
]
for col, v in enumerate(values, 1):
cell = ws2.cell(row=row, column=col, value=v)
if col >= 2:
cell.number_format = money_fmt
# Highlight the >$150K columns
if col in (7, 8) and isinstance(v, (int, float)):
cell.font = red_font
style_row(ws2, row, len(headers))
# Summary
row = 13
ws2.cell(row=row, column=1, value="5-YEAR TOTALS").font = Font(bold=True)
ws2.cell(row=row, column=3, value=30000).number_format = money_fmt
ws2.cell(row=row, column=5, value=15000).number_format = money_fmt
ws2.cell(row=row, column=7, value=0).font = red_font
ws2.cell(row=row, column=7).number_format = money_fmt
ws2.cell(row=row, column=8, value=337877).font = red_font
ws2.cell(row=row, column=8).number_format = money_fmt
auto_width(ws2)
# ============ SHEET 3: S&P 500 Comparison ============
ws3 = wb.create_sheet("vs S&P 500")
ws3.sheet_properties.tabColor = "1a4a1a"
ws3.merge_cells('A1:F1')
ws3['A1'] = "SAME CAPITAL IN S&P 500 vs REAL ESTATE PORTFOLIO"
ws3['A1'].font = Font(bold=True, color="00BFFF", size=14)
row = 3
headers = ['Year', 'Capital Deployed', 'S&P 500 Value\n(9% avg)', 'RE Portfolio Equity', 'RE Cash Flow\n(Cumulative)', 'S&P 500 Advantage']
for col, h in enumerate(headers, 1):
ws3.cell(row=row, column=col, value=h)
style_header(ws3, row, len(headers))
sp500_values = []
sp500_total = 0
cum_cf = 0
for i, r in enumerate(results):
row = 4 + i
year = r['year']
# Each year invest $79K, compounds at 9%
sp500_total = (sp500_total * 1.09) + r['total_cash_invested']
sp500_values.append(sp500_total)
# RE equity = appreciation + principal paydown (rough)
props = r['properties_owned']
re_equity = props * (DOWN_PAYMENT + (PURCHASE_PRICE * ((1.03 ** year) - 1))) # appreciation gain
# Add principal paydown estimate (~$3K/yr per property in early years)
re_equity += props * 3000 * ((year + 1) / 2)
cum_cf += r['total_cash_flow']
advantage = sp500_total - (re_equity + cum_cf)
values = [f"Year {year}", r['total_cash_invested'], sp500_total, re_equity, cum_cf, advantage]
for col, v in enumerate(values, 1):
cell = ws3.cell(row=row, column=col, value=v)
if col >= 2:
cell.number_format = money_neg_fmt
if col == 6 and isinstance(v, (int, float)) and v > 0:
cell.font = green_font
style_row(ws3, row, len(headers))
auto_width(ws3)
# ============ SHEET 4: Property Assumptions ============
ws4 = wb.create_sheet("Assumptions")
ws4.sheet_properties.tabColor = "444444"
ws4.merge_cells('A1:C1')
ws4['A1'] = "MODEL ASSUMPTIONS"
ws4['A1'].font = Font(bold=True, color="00BFFF", size=14)
assumptions = [
("Purchase", "", ""),
("Purchase Price", "$345,000", ""),
("Down Payment", "20%", "$69,000"),
("Closing Costs", "$6,000", ""),
("Cost Seg Study", "$4,000", "per property"),
("Total Cash Per Property", "$79,000", ""),
("", "", ""),
("Financing", "", ""),
("Loan Amount", "$276,000", ""),
("Interest Rate", "6.25%", "30-year fixed"),
("Monthly P&I", "$1,699.38", ""),
("Monthly PITI", "$2,016.05", ""),
("", "", ""),
("Income", "", ""),
("Monthly Rent", "$2,000", "0.58% rent-to-price ratio"),
("Annual Increase", "4%", "Aggressive — 2-3% more realistic"),
("Vacancy", "5%", ""),
("Management", "0%", "Self-managed"),
("", "", ""),
("Expenses", "", ""),
("Property Tax", "$2,400/yr", ""),
("Insurance", "$1,400/yr", ""),
("Maintenance", "$2,000/yr", "LOW — should be 1% of value = $3,450"),
("Appreciation", "3%/yr", ""),
("", "", ""),
("Cost Segregation", "", ""),
("5-Year Property", "15%", "$41,250 of $275K basis"),
("15-Year Property", "12%", "$33,000 of $275K basis"),
("27.5-Year Property", "73%", "$200,750 of $275K basis"),
("Bonus Depreciation (2026)", "40%", "Declining 20%/yr from 100% in 2022"),
("", "", ""),
("Tax Rules", "", ""),
("AGI >$150K", "$0 passive deduction", "Losses carry forward ONLY"),
("AGI $100-150K", "Partial $25K deduction", "Phased out"),
("AGI <$100K", "Full $25K deduction", "Against W-2 income"),
]
for i, (label, val, note) in enumerate(assumptions):
row = 3 + i
ws4.cell(row=row, column=1, value=label)
ws4.cell(row=row, column=2, value=val)
ws4.cell(row=row, column=3, value=note).font = Font(italic=True, color="888888")
if val == "" and note == "" and label:
ws4.cell(row=row, column=1).font = Font(bold=True, color="00BFFF")
auto_width(ws4)
# Save
outpath = "/home/wdjones/.openclaw/workspace/data/real-estate-cost-seg-model.xlsx"
wb.save(outpath)
print(f"Saved to {outpath}")

566
tools/build-re-nogo-excel.py Normal file
View File

@ -0,0 +1,566 @@
#!/usr/bin/env python3
"""Build detailed Excel spreadsheet for $345K rental property - NO GO analysis"""
import openpyxl
from openpyxl.styles import Font, PatternFill, Alignment, Border, Side
from openpyxl.utils import get_column_letter
from openpyxl.chart import BarChart, LineChart, Reference
# ============ PARAMETERS ============
PRICE = 345000
DOWN_PCT = 0.20
DOWN = PRICE * DOWN_PCT
CLOSING = 6000
LOAN = PRICE * (1 - DOWN_PCT)
RATE = 0.0625
CASH_IN = DOWN + CLOSING
RENT = 2000
OTHER_INCOME = 50
VACANCY = 0.05
RENT_GROWTH = 0.04
MGMT_FEE_PCT = 0.0
PROP_TAX = 2400
INSURANCE = 1400
HOA = 0
MAINTENANCE = 2000
OTHER_COSTS = 500
EXPENSE_GROWTH = 0.03
APPRECIATION = 0.03
HOLD_YEARS = 20
COST_TO_SELL = 0.08
def monthly_pmt(principal, annual_rate, years=30):
r = annual_rate / 12
n = years * 12
return principal * (r * (1 + r)**n) / ((1 + r)**n - 1)
PI_MONTHLY = monthly_pmt(LOAN, RATE)
PI_ANNUAL = PI_MONTHLY * 12
# Styles
def make_styles():
styles = {}
styles['title_font'] = Font(bold=True, color="FFFFFF", size=16)
styles['title_fill'] = PatternFill(start_color="1a1a2e", end_color="1a1a2e", fill_type="solid")
styles['header_font'] = Font(bold=True, color="FFFFFF", size=10)
styles['header_fill'] = PatternFill(start_color="16213e", end_color="16213e", fill_type="solid")
styles['section_font'] = Font(bold=True, color="00BFFF", size=11)
styles['section_fill'] = PatternFill(start_color="0f3460", end_color="0f3460", fill_type="solid")
styles['red_font'] = Font(color="FF4444", bold=True)
styles['red_fill'] = PatternFill(start_color="3d1111", end_color="3d1111", fill_type="solid")
styles['green_font'] = Font(color="44CC44", bold=True)
styles['green_fill'] = PatternFill(start_color="113d11", end_color="113d11", fill_type="solid")
styles['warn_font'] = Font(color="FFaa00", bold=True, size=11)
styles['warn_fill'] = PatternFill(start_color="3d3011", end_color="3d3011", fill_type="solid")
styles['label_font'] = Font(color="CCCCCC", size=10)
styles['value_font'] = Font(color="FFFFFF", size=10, bold=True)
styles['dim_font'] = Font(color="888888", size=9, italic=True)
styles['bg_dark'] = PatternFill(start_color="121212", end_color="121212", fill_type="solid")
styles['bg_row1'] = PatternFill(start_color="1a1a2e", end_color="1a1a2e", fill_type="solid")
styles['bg_row2'] = PatternFill(start_color="16213e", end_color="16213e", fill_type="solid")
styles['border'] = Border(
left=Side(style='thin', color='333333'),
right=Side(style='thin', color='333333'),
top=Side(style='thin', color='333333'),
bottom=Side(style='thin', color='333333')
)
styles['money_fmt'] = '#,##0'
styles['money_neg'] = '#,##0;[Red]-#,##0'
styles['pct_fmt'] = '0.0%'
return styles
S = make_styles()
wb = openpyxl.Workbook()
def apply_dark_bg(ws, max_row, max_col):
for r in range(1, max_row + 1):
for c in range(1, max_col + 1):
cell = ws.cell(row=r, column=c)
if not cell.fill or cell.fill.start_color.index == '00000000':
cell.fill = S['bg_dark']
cell.border = S['border']
def style_header_row(ws, row, max_col):
for c in range(1, max_col + 1):
cell = ws.cell(row=row, column=c)
cell.font = S['header_font']
cell.fill = S['header_fill']
cell.alignment = Alignment(horizontal='center', wrap_text=True)
def auto_width(ws, min_width=10, max_width=20):
for col in ws.columns:
letter = get_column_letter(col[0].column)
max_len = max((len(str(c.value or '')) for c in col), default=0)
ws.column_dimensions[letter].width = max(min(max_len + 3, max_width), min_width)
# ============ SHEET 1: EXECUTIVE SUMMARY ============
ws1 = wb.active
ws1.title = "Executive Summary"
ws1.sheet_properties.tabColor = "FF4444"
# Title
ws1.merge_cells('A1:H1')
ws1['A1'] = "INVESTMENT ANALYSIS: $345,000 RENTAL PROPERTY"
ws1['A1'].font = S['title_font']
ws1['A1'].fill = S['title_fill']
# Verdict banner
ws1.merge_cells('A3:H3')
ws1['A3'] = "⛔ VERDICT: NO-GO — Negative cash flow for 5+ years. Does not meet basic investment criteria."
ws1['A3'].font = Font(bold=True, color="FF4444", size=13)
ws1['A3'].fill = S['red_fill']
# Key metrics
r = 5
ws1.cell(row=r, column=1, value="KEY METRICS").font = S['section_font']
ws1.cell(row=r, column=1).fill = S['section_fill']
ws1.merge_cells(f'A{r}:H{r}')
metrics = [
("Purchase Price", f"${PRICE:,}", "Monthly Rent", f"${RENT:,}"),
("Down Payment (20%)", f"${DOWN:,.0f}", "Effective Monthly Income", f"${RENT * (1-VACANCY):,.0f}"),
("Closing Costs", f"${CLOSING:,}", "Monthly PITI", f"${PI_MONTHLY + PROP_TAX/12 + INSURANCE/12:,.0f}"),
("Total Cash Required", f"${CASH_IN:,.0f}", "Monthly Cash Flow", f"${(RENT*12*(1-VACANCY) - (PROP_TAX+INSURANCE+MAINTENANCE+OTHER_COSTS) - PI_ANNUAL)/12:+,.0f}"),
("Loan Amount", f"${LOAN:,.0f}", "Rent-to-Price Ratio", f"{(RENT/PRICE)*100:.2f}% (need ≥0.8%)"),
("Interest Rate", f"{RATE:.2%}", "Year 1 Cash-on-Cash", f"{((RENT*12*(1-VACANCY) - (PROP_TAX+INSURANCE+MAINTENANCE+OTHER_COSTS) - PI_ANNUAL)/CASH_IN)*100:+.1f}%"),
("Monthly P&I", f"${PI_MONTHLY:,.0f}", "Break-Even Rate Needed", "4.57% (current: 6.25%)"),
]
for i, (l1, v1, l2, v2) in enumerate(metrics):
row = r + 1 + i
ws1.cell(row=row, column=1, value=l1).font = S['label_font']
ws1.cell(row=row, column=2, value=v1).font = S['value_font']
ws1.cell(row=row, column=5, value=l2).font = S['label_font']
ws1.cell(row=row, column=6, value=v2).font = S['value_font']
# Why it fails
r = 14
ws1.cell(row=r, column=1, value="WHY THIS DEAL FAILS").font = S['section_font']
ws1.cell(row=r, column=1).fill = S['section_fill']
ws1.merge_cells(f'A{r}:H{r}')
reasons = [
("❌ Rent-to-Price Ratio: 0.58%", "Industry minimum is 0.8% (1% preferred). At $345K, rent should be $2,760-$3,450/mo to work."),
("❌ Negative Cash Flow Years 1-5", "You pay $283/mo out of pocket Year 1, declining to $21/mo by Year 5. Total: -$9,264 over 5 years."),
("❌ Interest Rate Too High", "At 6.25%, the mortgage alone ($1,699) consumes 89% of effective rent ($1,900). Need 4.57% to break even."),
("❌ No Tax Benefit at Your AGI", "Above $150K AGI = $0 passive loss deduction against W-2. Depreciation losses just carry forward uselessly."),
("❌ Thin Margin for Surprises", "One HVAC replacement ($8-15K) or 2-month vacancy wipes out years of equity building."),
("❌ Opportunity Cost", "Same $75K in S&P 500 at 9% avg = ~$115K in 5 years with zero effort. This property returns less with far more risk."),
]
for i, (title, detail) in enumerate(reasons):
row = r + 1 + i
ws1.cell(row=row, column=1, value=title).font = S['red_font']
ws1.merge_cells(f'B{row}:H{row}')
ws1.cell(row=row, column=2, value=detail).font = S['dim_font']
ws1.cell(row=row, column=2).alignment = Alignment(wrap_text=True)
# What would make it work
r = 22
ws1.cell(row=r, column=1, value="WHAT WOULD MAKE THIS DEAL WORK").font = S['section_font']
ws1.cell(row=r, column=1).fill = S['section_fill']
ws1.merge_cells(f'A{r}:H{r}')
fixes = [
("✅ Price ≤ $299,000", "Cash flow positive Day 1 at current 6.25% rate"),
("✅ Interest Rate ≤ 4.57%", "Break-even at $345K price (not realistic near-term)"),
("✅ Rent ≥ $2,760/mo", "Achieves 0.8% rent-to-price ratio (comps don't support this)"),
("✅ Different Market", "Midwest markets (Cleveland, Memphis, Indianapolis) offer 0.8-1.2% ratios at lower price points"),
]
for i, (fix, detail) in enumerate(fixes):
row = r + 1 + i
ws1.cell(row=row, column=1, value=fix).font = S['green_font']
ws1.merge_cells(f'B{row}:H{row}')
ws1.cell(row=row, column=2, value=detail).font = S['dim_font']
apply_dark_bg(ws1, 28, 8)
auto_width(ws1, 14, 25)
ws1.column_dimensions['A'].width = 35
# ============ SHEET 2: 20-YEAR CASH FLOW ============
ws2 = wb.create_sheet("20-Year Cash Flow")
ws2.sheet_properties.tabColor = "FF8800"
ws2.merge_cells('A1:L1')
ws2['A1'] = "20-YEAR CASH FLOW PROJECTION"
ws2['A1'].font = S['title_font']
ws2['A1'].fill = S['title_fill']
# Assumptions row
ws2.merge_cells('A2:L2')
ws2['A2'] = f"Purchase: ${PRICE:,} | Down: 20% | Rate: {RATE:.2%} | Rent: ${RENT:,}/mo | Growth: Rent {RENT_GROWTH:.0%}, Expenses {EXPENSE_GROWTH:.0%}, Appreciation {APPRECIATION:.0%}"
ws2['A2'].font = S['dim_font']
r = 4
headers = ['Year', 'Annual\nIncome', 'Operating\nExpenses', 'Mortgage\n(P&I)', 'Cash\nFlow',
'Cumulative\nCash Flow', 'Cash-on-Cash\nReturn', 'Mortgage\nInterest', 'Principal\nPaid',
'Property\nValue', 'Loan\nBalance', 'Total\nEquity']
for c, h in enumerate(headers, 1):
ws2.cell(row=r, column=c, value=h)
style_header_row(ws2, r, len(headers))
balance = LOAN
cum_cf = 0
breakeven_year = None
for yr in range(1, HOLD_YEARS + 1):
row = r + yr
annual_rent = (RENT * 12 + OTHER_INCOME * 12) * (1 + RENT_GROWTH) ** (yr - 1)
effective_income = annual_rent * (1 - VACANCY)
tax = PROP_TAX * (1 + EXPENSE_GROWTH) ** (yr - 1)
ins = INSURANCE * (1 + EXPENSE_GROWTH) ** (yr - 1)
maint = MAINTENANCE * (1 + EXPENSE_GROWTH) ** (yr - 1)
other = OTHER_COSTS * (1 + EXPENSE_GROWTH) ** (yr - 1)
mgmt = effective_income * MGMT_FEE_PCT
expenses = tax + ins + HOA + maint + other + mgmt
interest = balance * RATE
principal = PI_ANNUAL - interest
balance -= principal
cash_flow = effective_income - expenses - PI_ANNUAL
cum_cf += cash_flow
coc = cash_flow / CASH_IN
prop_value = PRICE * (1 + APPRECIATION) ** yr
equity = prop_value - balance
if breakeven_year is None and cash_flow >= 0:
breakeven_year = yr
values = [yr, effective_income, expenses, PI_ANNUAL, cash_flow, cum_cf, coc,
interest, principal, prop_value, balance, equity]
for c, v in enumerate(values, 1):
cell = ws2.cell(row=row, column=c, value=v)
if c in (2, 3, 4, 5, 6, 8, 9, 10, 11, 12):
cell.number_format = S['money_neg']
elif c == 7:
cell.number_format = S['pct_fmt']
# Color negative cash flow red
if c == 5 and v < 0:
cell.font = S['red_font']
elif c == 5 and v >= 0:
cell.font = S['green_font']
# Alternating row colors
cell.fill = S['bg_row1'] if yr % 2 == 0 else S['bg_row2']
ws2.cell(row=row, column=1).font = S['value_font']
# Totals
row = r + HOLD_YEARS + 1
ws2.cell(row=row, column=1, value="TOTALS").font = Font(bold=True, color="00BFFF")
# Calculate totals from the data
total_income = sum((RENT * 12 + OTHER_INCOME * 12) * (1 + RENT_GROWTH) ** (yr - 1) * (1 - VACANCY) for yr in range(1, HOLD_YEARS + 1))
total_expenses = sum((PROP_TAX + INSURANCE + MAINTENANCE + OTHER_COSTS) * (1 + EXPENSE_GROWTH) ** (yr - 1) for yr in range(1, HOLD_YEARS + 1))
ws2.cell(row=row, column=2, value=total_income).number_format = S['money_fmt']
ws2.cell(row=row, column=3, value=total_expenses).number_format = S['money_fmt']
ws2.cell(row=row, column=4, value=PI_ANNUAL * HOLD_YEARS).number_format = S['money_fmt']
ws2.cell(row=row, column=5, value=cum_cf).number_format = S['money_neg']
ws2.cell(row=row, column=5).font = S['red_font'] if cum_cf < 0 else S['green_font']
# Break-even note
row += 2
ws2.cell(row=row, column=1, value=f"Cash flow break-even: Year {breakeven_year}" if breakeven_year else "Never breaks even").font = S['warn_font']
ws2.cell(row=row, column=1).fill = S['warn_fill']
ws2.merge_cells(f'A{row}:D{row}')
row += 1
ws2.cell(row=row, column=1, value=f"Cumulative losses before break-even: ${abs(sum((RENT * 12 + OTHER_INCOME * 12) * (1 + RENT_GROWTH) ** (yr - 1) * (1 - VACANCY) - (PROP_TAX + INSURANCE + MAINTENANCE + OTHER_COSTS) * (1 + EXPENSE_GROWTH) ** (yr - 1) - PI_ANNUAL for yr in range(1, (breakeven_year or 1)))):.0f}").font = S['warn_font']
ws2.merge_cells(f'A{row}:D{row}')
apply_dark_bg(ws2, row + 2, len(headers))
auto_width(ws2, 12, 18)
# ============ SHEET 3: MONTHLY BREAKDOWN ============
ws3 = wb.create_sheet("Monthly Breakdown")
ws3.sheet_properties.tabColor = "4488FF"
ws3.merge_cells('A1:F1')
ws3['A1'] = "MONTHLY INCOME vs EXPENSES — YEAR 1"
ws3['A1'].font = S['title_font']
ws3['A1'].fill = S['title_fill']
r = 3
ws3.cell(row=r, column=1, value="INCOME").font = S['section_font']
ws3.cell(row=r, column=1).fill = S['section_fill']
ws3.merge_cells(f'A{r}:C{r}')
income_items = [
("Gross Rent", RENT, ""),
("Other Income", OTHER_INCOME, ""),
("Vacancy Allowance (5%)", -(RENT + OTHER_INCOME) * VACANCY, "Lost income from vacancy"),
("EFFECTIVE MONTHLY INCOME", (RENT + OTHER_INCOME) * (1 - VACANCY), ""),
]
for i, (label, val, note) in enumerate(income_items):
row = r + 1 + i
ws3.cell(row=row, column=1, value=label).font = S['label_font']
ws3.cell(row=row, column=2, value=val).number_format = S['money_neg']
ws3.cell(row=row, column=2).font = S['green_font'] if val > 0 else S['red_font']
ws3.cell(row=row, column=3, value=note).font = S['dim_font']
if "EFFECTIVE" in label:
ws3.cell(row=row, column=1).font = S['value_font']
ws3.cell(row=row, column=2).font = Font(bold=True, color="44CC44", size=11)
r = r + len(income_items) + 2
ws3.cell(row=r, column=1, value="EXPENSES").font = S['section_font']
ws3.cell(row=r, column=1).fill = S['section_fill']
ws3.merge_cells(f'A{r}:C{r}')
expense_items = [
("Mortgage (P&I)", PI_MONTHLY, f"${LOAN:,} @ {RATE:.2%} / 30yr"),
("Property Tax", PROP_TAX / 12, f"${PROP_TAX:,}/yr"),
("Insurance", INSURANCE / 12, f"${INSURANCE:,}/yr"),
("HOA", HOA / 12, ""),
("Maintenance", MAINTENANCE / 12, f"${MAINTENANCE:,}/yr — LOW, should be ~${PRICE*0.01:,.0f}"),
("Other Costs", OTHER_COSTS / 12, f"${OTHER_COSTS:,}/yr"),
("Management Fee", 0, "Self-managed (0%)"),
("TOTAL MONTHLY EXPENSES", PI_MONTHLY + PROP_TAX/12 + INSURANCE/12 + MAINTENANCE/12 + OTHER_COSTS/12, ""),
]
for i, (label, val, note) in enumerate(expense_items):
row = r + 1 + i
ws3.cell(row=row, column=1, value=label).font = S['label_font']
ws3.cell(row=row, column=2, value=val).number_format = S['money_fmt']
ws3.cell(row=row, column=2).font = S['red_font']
ws3.cell(row=row, column=3, value=note).font = S['dim_font']
if "TOTAL" in label:
ws3.cell(row=row, column=1).font = S['value_font']
ws3.cell(row=row, column=2).font = Font(bold=True, color="FF4444", size=11)
r = r + len(expense_items) + 2
ws3.cell(row=r, column=1, value="NET MONTHLY CASH FLOW").font = Font(bold=True, color="FFFFFF", size=12)
eff_income = (RENT + OTHER_INCOME) * (1 - VACANCY)
total_exp = PI_MONTHLY + PROP_TAX/12 + INSURANCE/12 + MAINTENANCE/12 + OTHER_COSTS/12
net = eff_income - total_exp
ws3.cell(row=r, column=2, value=net).number_format = S['money_neg']
ws3.cell(row=r, column=2).font = Font(bold=True, color="FF4444" if net < 0 else "44CC44", size=14)
r += 2
ws3.cell(row=r, column=1, value="MORTGAGE EATS EVERYTHING").font = S['warn_font']
ws3.cell(row=r, column=1).fill = S['warn_fill']
ws3.merge_cells(f'A{r}:C{r}')
r += 1
pct_of_income = PI_MONTHLY / eff_income * 100
ws3.cell(row=r, column=1, value=f"Mortgage is {pct_of_income:.0f}% of effective income. Healthy target: ≤70%.").font = S['dim_font']
ws3.merge_cells(f'A{r}:C{r}')
apply_dark_bg(ws3, r + 2, 6)
auto_width(ws3, 14, 30)
ws3.column_dimensions['A'].width = 30
ws3.column_dimensions['C'].width = 45
# ============ SHEET 4: COMPARISON ============
ws4 = wb.create_sheet("vs Alternatives")
ws4.sheet_properties.tabColor = "44CC44"
ws4.merge_cells('A1:G1')
ws4['A1'] = "$75,000 INVESTED: REAL ESTATE vs ALTERNATIVES"
ws4['A1'].font = S['title_font']
ws4['A1'].fill = S['title_fill']
r = 3
headers = ['Year', 'S&P 500\n(9% avg)', 'T-Bills\n(4.5%)', 'REIT Index\n(7%)',
'RE: Equity', 'RE: Cum\nCash Flow', 'RE: Total\nValue']
for c, h in enumerate(headers, 1):
ws4.cell(row=r, column=c, value=h)
style_header_row(ws4, r, len(headers))
sp500 = CASH_IN
tbill = CASH_IN
reit = CASH_IN
balance = LOAN
cum_cf = 0
for yr in range(1, 11):
row = r + yr
sp500 *= 1.09
tbill *= 1.045
reit *= 1.07
# RE calcs
eff_inc = (RENT * 12 + OTHER_INCOME * 12) * (1 + RENT_GROWTH) ** (yr - 1) * (1 - VACANCY)
exp = (PROP_TAX + INSURANCE + MAINTENANCE + OTHER_COSTS) * (1 + EXPENSE_GROWTH) ** (yr - 1)
interest = balance * RATE
principal = PI_ANNUAL - interest
balance -= principal
cf = eff_inc - exp - PI_ANNUAL
cum_cf += cf
prop_val = PRICE * (1 + APPRECIATION) ** yr
equity = prop_val - balance
re_total = equity + cum_cf - CASH_IN # net gain
values = [yr, sp500, tbill, reit, equity, cum_cf, equity + cum_cf]
for c, v in enumerate(values, 1):
cell = ws4.cell(row=row, column=c, value=v)
if c >= 2:
cell.number_format = S['money_neg']
cell.fill = S['bg_row1'] if yr % 2 == 0 else S['bg_row2']
# Highlight winner
gains = {'S&P': sp500 - CASH_IN, 'T-Bill': tbill - CASH_IN, 'REIT': reit - CASH_IN, 'RE': equity + cum_cf - CASH_IN}
winner = max(gains, key=gains.get)
# Summary
row = r + 11 + 1
ws4.cell(row=row, column=1, value="10-YEAR GAIN").font = Font(bold=True, color="00BFFF")
ws4.cell(row=row, column=2, value=sp500 - CASH_IN).number_format = S['money_fmt']
ws4.cell(row=row, column=2).font = S['green_font']
ws4.cell(row=row, column=3, value=tbill - CASH_IN).number_format = S['money_fmt']
ws4.cell(row=row, column=4, value=reit - CASH_IN).number_format = S['money_fmt']
row += 2
ws4.cell(row=row, column=1, value="S&P 500 delivers higher returns with zero effort, zero risk of midnight toilet calls,").font = S['dim_font']
ws4.merge_cells(f'A{row}:G{row}')
row += 1
ws4.cell(row=row, column=1, value="zero cap-ex surprises, and zero chance of losing your investment to a bad tenant.").font = S['dim_font']
ws4.merge_cells(f'A{row}:G{row}')
apply_dark_bg(ws4, row + 2, 7)
auto_width(ws4, 14, 18)
# ============ SHEET 5: RISK SCENARIOS ============
ws5 = wb.create_sheet("Risk Scenarios")
ws5.sheet_properties.tabColor = "FF4444"
ws5.merge_cells('A1:E1')
ws5['A1'] = "STRESS TEST: WHAT COULD GO WRONG"
ws5['A1'].font = S['title_font']
ws5['A1'].fill = S['title_fill']
r = 3
scenarios = [
("SCENARIO", "Year 1 Cash Flow", "5-Year Cash Flow", "Impact", "Probability"),
("Base Case (as modeled)", -3393, -9264, "Negative but manageable", "Expected"),
("HVAC Replacement Year 2 ($12K)", -3393, -21264, "Wipes out 2+ years equity growth", "30% in 5yr"),
("Roof Replacement Year 3 ($20K)", -3393, -29264, "Devastating — may force sale", "15% in 5yr"),
("10% Vacancy (bad tenant/eviction)", -4593, -14264, "Doubles the bleeding", "20% chance"),
("Rate Rises to 7.5% (if ARM/refi)", -5940, -18964, "Underwater fast", "Possible"),
("Rent Drops 10% (recession)", -5793, -19264, "Cash flow crisis", "15-20% chance"),
("Property Value Drops 10%", -3393, -9264, "Equity wiped, underwater on loan", "10-15% chance"),
("Hire Property Manager (8%)", -5217, -15264, "Realistic if you can't self-manage", "Likely eventually"),
("ALL OF THE ABOVE (worst case)", -12000, -55000, "Financial catastrophe", "5% but non-zero"),
]
for i, vals in enumerate(scenarios):
row = r + i
for c, v in enumerate(vals, 1):
cell = ws5.cell(row=row, column=c, value=v)
if i == 0:
cell.font = S['header_font']
cell.fill = S['header_fill']
else:
if c in (2, 3) and isinstance(v, (int, float)):
cell.number_format = S['money_neg']
cell.font = S['red_font']
cell.fill = S['bg_row1'] if i % 2 == 0 else S['bg_row2']
apply_dark_bg(ws5, r + len(scenarios) + 2, 5)
auto_width(ws5, 15, 40)
ws5.column_dimensions['A'].width = 40
ws5.column_dimensions['D'].width = 35
# ============ SHEET 6: BREAK-EVEN TABLE ============
ws6 = wb.create_sheet("Break-Even Analysis")
ws6.sheet_properties.tabColor = "4488FF"
ws6.merge_cells('A1:D1')
ws6['A1'] = "WHAT PRICE + RATE = DAY 1 CASH FLOW POSITIVE?"
ws6['A1'].font = S['title_font']
ws6['A1'].fill = S['title_fill']
ws6.merge_cells('A2:D2')
ws6['A2'] = f"Fixed: Rent ${RENT:,}/mo | Vacancy {VACANCY:.0%} | 20% down | 30yr fixed"
ws6['A2'].font = S['dim_font']
# Break-even rate by price
r = 4
ws6.cell(row=r, column=1, value="BREAK-EVEN RATE BY PRICE").font = S['section_font']
ws6.cell(row=r, column=1).fill = S['section_fill']
ws6.merge_cells(f'A{r}:D{r}')
r += 1
for c, h in enumerate(['Price', 'Max Rate for Break-Even', 'Viable Today?', 'Rent/Price'], 1):
ws6.cell(row=r, column=c, value=h)
style_header_row(ws6, r, 4)
prices = [200000, 220000, 240000, 260000, 280000, 299000, 320000, 345000]
for i, price in enumerate(prices):
row = r + 1 + i
# Find break-even rate
be_rate = None
for rate_bps in range(100, 1200, 1):
rate = rate_bps / 10000
loan = price * 0.80
pi = monthly_pmt(loan, rate)
tax_ins_maint = (price * 0.007 + price * 0.004 + price * 0.006 + OTHER_COSTS/12*12/price*price) / 12
total = pi + tax_ins_maint + OTHER_COSTS/12
income = (RENT + OTHER_INCOME) * (1 - VACANCY)
if income - total <= 0:
be_rate = (rate_bps - 1) / 10000
break
ws6.cell(row=row, column=1, value=price).number_format = '$#,##0'
ws6.cell(row=row, column=2, value=f"{be_rate:.2%}" if be_rate else "10%+")
viable = "✅ YES" if be_rate and be_rate >= 0.0625 else "❌ NO"
ws6.cell(row=row, column=3, value=viable).font = S['green_font'] if "YES" in viable else S['red_font']
ws6.cell(row=row, column=4, value=f"{(RENT/price)*100:.2f}%")
ws6.cell(row=row, column=4).font = S['green_font'] if RENT/price >= 0.008 else S['red_font']
# Highlight current deal
if price == 345000:
for c in range(1, 5):
ws6.cell(row=row, column=c).fill = S['red_fill']
# Break-even price by rate
r = row + 3
ws6.cell(row=r, column=1, value="BREAK-EVEN PRICE BY RATE").font = S['section_font']
ws6.cell(row=r, column=1).fill = S['section_fill']
ws6.merge_cells(f'A{r}:D{r}')
r += 1
for c, h in enumerate(['Interest Rate', 'Max Price', 'Down Payment', 'Rent/Price'], 1):
ws6.cell(row=r, column=c, value=h)
style_header_row(ws6, r, 4)
rates = [3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.25]
for i, rate_pct in enumerate(rates):
row = r + 1 + i
rate = rate_pct / 100
max_price = None
for price in range(400000, 100000, -1000):
loan = price * 0.80
pi = monthly_pmt(loan, rate)
expenses = (price * 0.007 + price * 0.004 + price * 0.006) / 12 + OTHER_COSTS / 12
total = pi + expenses
income = (RENT + OTHER_INCOME) * (1 - VACANCY)
if income >= total:
max_price = price
break
ws6.cell(row=row, column=1, value=f"{rate_pct:.2f}%")
if max_price:
ws6.cell(row=row, column=2, value=max_price).number_format = '$#,##0'
ws6.cell(row=row, column=3, value=max_price * 0.20).number_format = '$#,##0'
ws6.cell(row=row, column=4, value=f"{(RENT/max_price)*100:.2f}%")
if rate_pct == 6.25:
for c in range(1, 5):
ws6.cell(row=row, column=c).fill = S['warn_fill']
apply_dark_bg(ws6, row + 2, 4)
auto_width(ws6, 18, 28)
# Save
outpath = "/home/wdjones/.openclaw/workspace/data/real-estate-345k-nogo-analysis.xlsx"
wb.save(outpath)
print(f"Saved to {outpath}")

777
tools/build-re-nogo-plain-v2.py Normal file
View File

@ -0,0 +1,777 @@
#!/usr/bin/env python3
"""Build plain Excel spreadsheet for $345K NEW BUILD rental property - NO GO analysis v2"""
import openpyxl
from openpyxl.styles import Font, Alignment, Border, Side
from openpyxl.utils import get_column_letter
# ============ PARAMETERS ============
PRICE = 345000
DOWN_PCT = 0.20
DOWN = PRICE * DOWN_PCT
CLOSING = 6000
LOAN = PRICE * (1 - DOWN_PCT)
RATE = 0.0625
CASH_IN = DOWN + CLOSING
RENT = 2000
OTHER_INCOME = 50
VACANCY = 0.05
RENT_GROWTH = 0.04
PROP_TAX = 2400
INSURANCE = 1400
HOA = 0
MAINTENANCE = 2000
OTHER_COSTS = 500
EXPENSE_GROWTH = 0.03
APPRECIATION = 0.03
HOLD_YEARS = 20
def monthly_pmt(principal, annual_rate, years=30):
r = annual_rate / 12
n = years * 12
return principal * (r * (1 + r)**n) / ((1 + r)**n - 1)
PI_MONTHLY = monthly_pmt(LOAN, RATE)
PI_ANNUAL = PI_MONTHLY * 12
# Styles
bold = Font(bold=True)
bold_red = Font(bold=True, color="CC0000")
bold_green = Font(bold=True, color="006600")
red = Font(color="CC0000")
green = Font(color="006600")
italic_gray = Font(italic=True, color="666666")
header_font = Font(bold=True)
title_font = Font(bold=True, size=14)
section_font = Font(bold=True, size=11)
thin_border = Border(
left=Side(style='thin', color='CCCCCC'),
right=Side(style='thin', color='CCCCCC'),
top=Side(style='thin', color='CCCCCC'),
bottom=Side(style='thin', color='CCCCCC')
)
bottom_border = Border(bottom=Side(style='thin', color='999999'))
money = '#,##0'
money_neg = '#,##0;[Red]-#,##0'
pct = '0.0%'
def style_header_row(ws, row, max_col):
for c in range(1, max_col + 1):
cell = ws.cell(row=row, column=c)
cell.font = header_font
cell.alignment = Alignment(horizontal='center', wrap_text=True)
cell.border = Border(bottom=Side(style='medium'))
def auto_width(ws, min_w=10, max_w=22):
for col in ws.columns:
letter = get_column_letter(col[0].column)
max_len = max((len(str(c.value or '')) for c in col), default=0)
ws.column_dimensions[letter].width = max(min(max_len + 3, max_w), min_w)
wb = openpyxl.Workbook()
# ============ SHEET 1: EXECUTIVE SUMMARY ============
ws1 = wb.active
ws1.title = "Executive Summary"
ws1.merge_cells('A1:H1')
ws1['A1'] = "Investment Analysis: $345,000 New Build Rental Property"
ws1['A1'].font = title_font
ws1.merge_cells('A3:H3')
ws1['A3'] = "VERDICT: NO-GO — Negative cash flow for 5+ years. Does not meet basic investment criteria."
ws1['A3'].font = bold_red
r = 5
ws1.cell(row=r, column=1, value="Property Details").font = section_font
ws1.cell(row=r, column=1).border = bottom_border
details = [
("Property Type", "New construction from builder"),
("Bedrooms / Bathrooms", "3 bed / 2 bath"),
("Square Footage", "1,400 sq ft"),
("Condition", "Brand new — no repairs needed"),
("Builder Warranty", "Typically 1-2yr structural, 10yr major systems"),
("Comparable Rents (Townhouses)", "$1,900/mo"),
("Comparable Rents (Stand Alones)", "$2,100/mo"),
]
for i, (label, val) in enumerate(details):
row = r + 1 + i
ws1.cell(row=row, column=1, value=label)
ws1.cell(row=row, column=2, value=val).font = bold
r = r + len(details) + 2
ws1.cell(row=r, column=1, value="Key Financials").font = section_font
ws1.cell(row=r, column=1).border = bottom_border
metrics = [
("Purchase Price", f"${PRICE:,}", "Monthly Rent", f"${RENT:,}"),
("Down Payment (20%)", f"${DOWN:,.0f}", "Effective Monthly Income", f"${(RENT + OTHER_INCOME) * (1-VACANCY):,.0f}"),
("Closing Costs", f"${CLOSING:,}", "Monthly PITI", f"${PI_MONTHLY + PROP_TAX/12 + INSURANCE/12:,.0f}"),
("Total Cash Required", f"${CASH_IN:,.0f}", "Monthly Cash Flow", f"${((RENT+OTHER_INCOME)*12*(1-VACANCY) - (PROP_TAX+INSURANCE+MAINTENANCE+OTHER_COSTS) - PI_ANNUAL)/12:+,.0f}"),
("Loan Amount", f"${LOAN:,.0f}", "Rent-to-Price Ratio", f"{(RENT/PRICE)*100:.2f}%"),
("Interest Rate", f"{RATE:.2%}", "Year 1 Cash-on-Cash", f"{(((RENT+OTHER_INCOME)*12*(1-VACANCY) - (PROP_TAX+INSURANCE+MAINTENANCE+OTHER_COSTS) - PI_ANNUAL)/CASH_IN)*100:+.1f}%"),
("Monthly P&I", f"${PI_MONTHLY:,.0f}", "Break-Even Rate Needed", "4.57%"),
]
for i, (l1, v1, l2, v2) in enumerate(metrics):
row = r + 1 + i
ws1.cell(row=row, column=1, value=l1)
ws1.cell(row=row, column=2, value=v1).font = bold
ws1.cell(row=row, column=5, value=l2)
ws1.cell(row=row, column=6, value=v2).font = bold
r = r + len(metrics) + 2
ws1.cell(row=r, column=1, value="Why This Deal Fails").font = section_font
ws1.cell(row=r, column=1).border = bottom_border
reasons = [
("Rent-to-Price Ratio: 0.58%",
"This is the most important metric in rental investing. It measures monthly rent as a percentage of purchase price. "
"The industry standard minimum is 0.8% (the '1% rule' is ideal). At $345K, you'd need rent of $2,760/mo (0.8%) or "
"$3,450/mo (1%) to meet these thresholds. Your $2,000/mo rent is well below either target, meaning the property "
"is overpriced relative to its income potential."),
("Negative Cash Flow for 5+ Years",
"Cash flow = rental income minus all expenses (mortgage, taxes, insurance, maintenance). This property loses money "
"every month for the first 5-6 years. Year 1: -$283/mo (-$3,393/yr). Year 5: -$21/mo (-$248/yr). Total out-of-pocket "
"over 5 years: $9,264. You are paying to own this property rather than it paying you."),
("Mortgage Consumes 89% of Income",
"Your monthly mortgage payment ($1,699) eats 89% of effective rental income ($1,948). A healthy rental should have "
"mortgage at 70% or less of income, leaving 30%+ for taxes, insurance, maintenance, vacancy, and profit. At 89%, "
"there is virtually nothing left after paying the mortgage."),
("Interest Rate Too High for This Price",
"At 6.25%, the cost of borrowing is too high relative to the rent the property generates. To break even on cash flow "
"at $345K, you would need a rate of 4.57% or lower. Current rates are ~6.25% with no indication of dropping to 4.57% "
"in the near term. The rate environment makes this deal unworkable."),
("No Tax Benefit Above $150K AGI",
"Rental property losses (depreciation + expenses exceeding income) can offset W-2 income — but only if your Adjusted "
"Gross Income is below $100K (full $25K deduction) or $100-150K (partial). Above $150K AGI, passive losses provide "
"$0 tax benefit against your paycheck. They carry forward to offset future passive income or are released when you sell, "
"but provide no annual cash benefit while you hold the property."),
("Opportunity Cost",
"The $75,000 required for this investment could be placed in an S&P 500 index fund averaging ~9% annually. After 10 years: "
"~$177,500 with zero effort, full liquidity, no maintenance, no tenants, no risk of a $15K surprise repair. The real estate "
"catches up around year 10+ due to leverage and appreciation, but with significantly more work and risk."),
]
for i, (title, detail) in enumerate(reasons):
row = r + 1 + (i * 3)
ws1.cell(row=row, column=1, value=title).font = bold
ws1.merge_cells(f'A{row+1}:H{row+1}')
ws1.cell(row=row + 1, column=1, value=detail).font = italic_gray
ws1.cell(row=row + 1, column=1).alignment = Alignment(wrap_text=True)
r = row + 4
ws1.cell(row=r, column=1, value="What Would Make This Deal Work").font = section_font
ws1.cell(row=r, column=1).border = bottom_border
fixes = [
("Purchase Price ≤ $299,000",
"At current 6.25% rate, a price of $299K or less produces positive cash flow from Day 1. "
"This brings the rent-to-price ratio to 0.67% — still below the 0.8% ideal but workable."),
("Interest Rate ≤ 4.57%",
"At the current $345K price, you need rates to drop ~170 basis points to break even. "
"This is not expected in the near term based on current Fed policy."),
("Monthly Rent ≥ $2,760",
"This would achieve a 0.8% rent-to-price ratio. However, comparable properties in the area "
"rent for $1,900-$2,100, so the market does not support this rent level."),
("Different Market",
"Midwest and some Southern markets (Cleveland, Memphis, Indianapolis, Birmingham) still offer "
"properties at 0.8-1.2% rent-to-price ratios at lower price points, where the same $75K investment "
"would cash flow from Day 1."),
]
for i, (fix, detail) in enumerate(fixes):
row = r + 1 + (i * 3)
ws1.cell(row=row, column=1, value=fix).font = bold_green
ws1.merge_cells(f'A{row+1}:H{row+1}')
ws1.cell(row=row + 1, column=1, value=detail).font = italic_gray
ws1.cell(row=row + 1, column=1).alignment = Alignment(wrap_text=True)
auto_width(ws1, 14, 25)
ws1.column_dimensions['A'].width = 38
for col in 'BCDEFGH':
if col == 'B':
ws1.column_dimensions[col].width = 20
# ============ SHEET 2: EXPENSE BREAKDOWN ============
ws2 = wb.create_sheet("Expense Breakdown")
ws2.merge_cells('A1:D1')
ws2['A1'] = "Complete Expense Breakdown — Year 1 Monthly"
ws2['A1'].font = title_font
r = 3
ws2.cell(row=r, column=1, value="Income").font = section_font
ws2.cell(row=r, column=1).border = bottom_border
r += 1
income_items = [
("Gross Monthly Rent", RENT,
"The base rent charged to the tenant. Based on comparable properties: townhouses at $1,900 and "
"stand-alone homes at $2,100. $2,000 is mid-range for this property type and area."),
("Other Monthly Income", OTHER_INCOME,
"Additional income such as pet rent, storage fees, laundry, parking, or late fees. "
"$50/mo is a conservative estimate."),
("Less: Vacancy Allowance (5%)", -(RENT + OTHER_INCOME) * VACANCY,
"Industry standard assumption that the property will be vacant ~18 days per year (5% of the time) "
"due to tenant turnover, time to find new tenants, cleaning/repair between tenants. "
"Some investors use 8-10% for more conservative modeling."),
]
for i, (label, val, explanation) in enumerate(income_items):
row = r + (i * 3)
ws2.cell(row=row, column=1, value=label).font = bold if val > 0 else Font(bold=True, color="CC0000")
ws2.cell(row=row, column=2, value=val).number_format = money_neg
ws2.cell(row=row, column=2).font = green if val > 0 else red
ws2.merge_cells(f'A{row+1}:D{row+1}')
ws2.cell(row=row + 1, column=1, value=explanation).font = italic_gray
ws2.cell(row=row + 1, column=1).alignment = Alignment(wrap_text=True)
eff_income = (RENT + OTHER_INCOME) * (1 - VACANCY)
row = r + len(income_items) * 3
ws2.cell(row=row, column=1, value="= Effective Monthly Income").font = Font(bold=True, size=11)
ws2.cell(row=row, column=2, value=eff_income).number_format = money
ws2.cell(row=row, column=2).font = Font(bold=True, size=11, color="006600")
ws2.cell(row=row, column=1).border = Border(top=Side(style='medium'))
ws2.cell(row=row, column=2).border = Border(top=Side(style='medium'))
# Expenses
r = row + 3
ws2.cell(row=r, column=1, value="Fixed Expenses (Debt Service)").font = section_font
ws2.cell(row=r, column=1).border = bottom_border
r += 1
fixed_expenses = [
("Mortgage Payment (Principal & Interest)", PI_MONTHLY,
f"This is the monthly payment on a ${LOAN:,} loan at {RATE:.2%} interest over 30 years. "
f"Of this ${PI_MONTHLY:,.0f} payment, ~${LOAN*RATE/12:,.0f} goes to interest in Year 1 and only "
f"~${PI_MONTHLY - LOAN*RATE/12:,.0f} goes to principal (building equity). Over 30 years you will pay "
f"${PI_ANNUAL*30:,.0f} total — ${PI_ANNUAL*30 - LOAN:,.0f} in interest alone. "
f"This single line item consumes {PI_MONTHLY/eff_income*100:.0f}% of your effective rental income."),
]
for i, (label, val, explanation) in enumerate(fixed_expenses):
row = r + (i * 3)
ws2.cell(row=row, column=1, value=label).font = bold
ws2.cell(row=row, column=2, value=val).number_format = money
ws2.cell(row=row, column=2).font = red
ws2.merge_cells(f'A{row+1}:D{row+1}')
ws2.cell(row=row + 1, column=1, value=explanation).font = italic_gray
ws2.cell(row=row + 1, column=1).alignment = Alignment(wrap_text=True)
r = row + 3
ws2.cell(row=r, column=1, value="Operating Expenses").font = section_font
ws2.cell(row=r, column=1).border = bottom_border
r += 1
operating_expenses = [
("Property Tax", PROP_TAX / 12,
f"${PROP_TAX:,}/year divided by 12. Property tax is assessed by the local government based on the "
f"property's appraised value. At ${PROP_TAX:,}/yr on a ${PRICE:,} property, the effective rate is "
f"{(PROP_TAX/PRICE)*100:.2f}%. Property taxes typically increase 2-5% annually due to rising "
f"assessments. In some areas, a new purchase triggers reassessment to current market value, which "
f"could increase this amount. This expense increases at {EXPENSE_GROWTH:.0%}/yr in this model."),
("Homeowner's Insurance", INSURANCE / 12,
f"${INSURANCE:,}/year divided by 12. Covers damage from fire, storms, liability, etc. "
f"New builds may get slightly lower rates due to modern materials and code compliance, but insurance "
f"costs have been rising 7-10% annually nationwide due to climate events. If the property is in a "
f"flood zone, flood insurance ($1,000-3,000/yr) would be additional. "
f"This expense increases at {EXPENSE_GROWTH:.0%}/yr in this model."),
("HOA Fees", HOA / 12,
"No HOA in this scenario. If the property is in a subdivision with shared amenities (pool, landscaping, "
"common areas), HOA fees typically run $100-400/mo and increase annually. HOA fees are non-negotiable "
"and can include special assessments for major repairs. Always verify before purchasing."),
("Maintenance & Repairs", MAINTENANCE / 12,
f"${MAINTENANCE:,}/year divided by 12. The standard rule of thumb is 1% of property value annually "
f"(= ${PRICE*0.01:,.0f}/yr for this property). The ${MAINTENANCE:,}/yr used here is lower because "
f"this is a brand new build — appliances are new, HVAC is new, roof is new, and the builder warranty "
f"covers defects for 1-2 years (structural up to 10 years). However, maintenance is never zero: "
f"lawn care, minor repairs, appliance issues, plumbing, pest control, etc. still occur. "
f"IMPORTANT: After years 5-10, this number should increase significantly as systems age. "
f"HVAC replacement: $8-15K (lifespan 12-15 yrs). Water heater: $1-2K (lifespan 8-12 yrs). "
f"Flooring: $5-10K (every 7-10 yrs between tenants). "
f"This expense increases at {EXPENSE_GROWTH:.0%}/yr in this model."),
("Other Costs", OTHER_COSTS / 12,
f"${OTHER_COSTS:,}/year divided by 12. Covers miscellaneous expenses: lawn care equipment/service, "
f"pest control, accounting/tax prep for rental income, legal fees, advertising for tenants, "
f"background check fees, locksmith, miscellaneous supplies. Also covers tenant turnover costs "
f"between leases (cleaning, touch-up paint, minor repairs to pass inspection). "
f"This expense increases at {EXPENSE_GROWTH:.0%}/yr in this model."),
("Property Management Fee (0%)", 0,
"This model assumes self-management (0%). If you hire a property manager, expect to pay 8-10% of "
"gross rent ($160-200/mo). Most investors eventually hire management — dealing with tenant calls, "
"maintenance coordination, lease enforcement, and evictions is a part-time job. Adding 8% management "
"would increase annual expenses by ~$1,920 and push Year 1 cash flow to -$5,313. "
"If this is a partnership, management is almost certainly required."),
]
total_operating = PROP_TAX/12 + INSURANCE/12 + HOA/12 + MAINTENANCE/12 + OTHER_COSTS/12
for i, (label, val, explanation) in enumerate(operating_expenses):
row = r + (i * 4)
ws2.cell(row=row, column=1, value=label).font = bold
ws2.cell(row=row, column=2, value=val).number_format = money
ws2.cell(row=row, column=2).font = red if val > 0 else Font()
ws2.merge_cells(f'A{row+1}:D{row+2}')
ws2.cell(row=row + 1, column=1, value=explanation).font = italic_gray
ws2.cell(row=row + 1, column=1).alignment = Alignment(wrap_text=True, vertical='top')
# Totals section
r = row + 5
ws2.cell(row=r, column=1, value="Summary").font = section_font
ws2.cell(row=r, column=1).border = bottom_border
r += 1
total_monthly_exp = PI_MONTHLY + total_operating
net = eff_income - total_monthly_exp
summary_items = [
("Effective Monthly Income", eff_income, "006600"),
("Less: Mortgage (P&I)", -PI_MONTHLY, "CC0000"),
("Less: Operating Expenses", -total_operating, "CC0000"),
("= Total Monthly Expenses", -total_monthly_exp, "CC0000"),
("", None, None),
("NET MONTHLY CASH FLOW", net, "CC0000" if net < 0 else "006600"),
("NET ANNUAL CASH FLOW", net * 12, "CC0000" if net < 0 else "006600"),
]
for i, (label, val, color) in enumerate(summary_items):
row = r + i
ws2.cell(row=row, column=1, value=label)
if val is not None:
ws2.cell(row=row, column=2, value=val).number_format = money_neg
ws2.cell(row=row, column=2).font = Font(bold=True, color=color)
if "NET" in label:
ws2.cell(row=row, column=1).font = Font(bold=True, size=11)
ws2.cell(row=row, column=2).font = Font(bold=True, size=11, color=color)
ws2.cell(row=row, column=1).border = Border(top=Side(style='medium'))
ws2.cell(row=row, column=2).border = Border(top=Side(style='medium'))
r = row + 3
ws2.cell(row=r, column=1, value="Expense Growth Assumptions").font = section_font
ws2.cell(row=r, column=1).border = bottom_border
r += 1
growth_items = [
("Rent Growth", f"{RENT_GROWTH:.0%}/year",
"Assumes 4% annual rent increases. Historical average is 3-4% nationally. "
"However, with rent already at market rate ($2,000 vs $1,900-$2,100 comps), aggressive "
"increases may not be achievable without losing tenants."),
("Expense Growth", f"{EXPENSE_GROWTH:.0%}/year",
"Property tax, insurance, and maintenance costs increase ~3% annually on average. "
"Insurance has been rising faster (7-10%) in recent years. Property tax can spike "
"after reassessment."),
("Property Appreciation", f"{APPRECIATION:.0%}/year",
"Long-term historical average for US residential real estate is ~3-4%. New builds in "
"growing areas may exceed this; properties in stagnant areas may underperform."),
]
for i, (label, val, explanation) in enumerate(growth_items):
row = r + (i * 3)
ws2.cell(row=row, column=1, value=label).font = bold
ws2.cell(row=row, column=2, value=val).font = bold
ws2.merge_cells(f'A{row+1}:D{row+1}')
ws2.cell(row=row + 1, column=1, value=explanation).font = italic_gray
ws2.cell(row=row + 1, column=1).alignment = Alignment(wrap_text=True)
auto_width(ws2, 14, 30)
ws2.column_dimensions['A'].width = 40
ws2.column_dimensions['D'].width = 40
# ============ SHEET 3: 20-YEAR CASH FLOW ============
ws3 = wb.create_sheet("20-Year Cash Flow")
ws3.merge_cells('A1:L1')
ws3['A1'] = "20-Year Cash Flow Projection — New Build"
ws3['A1'].font = title_font
ws3.merge_cells('A2:L2')
ws3['A2'] = f"Purchase: ${PRICE:,} | Down: 20% | Rate: {RATE:.2%} | Rent: ${RENT:,}/mo | Growth: Rent {RENT_GROWTH:.0%}, Expenses {EXPENSE_GROWTH:.0%}, Appreciation {APPRECIATION:.0%}"
ws3['A2'].font = italic_gray
r = 4
headers = ['Year', 'Annual\nIncome', 'Operating\nExpenses', 'Mortgage\n(P&I)', 'Cash\nFlow',
'Cumulative\nCash Flow', 'Cash-on-Cash\nReturn', 'Mortgage\nInterest', 'Principal\nPaid',
'Property\nValue', 'Loan\nBalance', 'Total\nEquity']
for c, h in enumerate(headers, 1):
ws3.cell(row=r, column=c, value=h)
style_header_row(ws3, r, len(headers))
balance = LOAN
cum_cf = 0
breakeven_year = None
for yr in range(1, HOLD_YEARS + 1):
row = r + yr
annual_rent = (RENT * 12 + OTHER_INCOME * 12) * (1 + RENT_GROWTH) ** (yr - 1)
effective_income = annual_rent * (1 - VACANCY)
tax = PROP_TAX * (1 + EXPENSE_GROWTH) ** (yr - 1)
ins = INSURANCE * (1 + EXPENSE_GROWTH) ** (yr - 1)
maint = MAINTENANCE * (1 + EXPENSE_GROWTH) ** (yr - 1)
other = OTHER_COSTS * (1 + EXPENSE_GROWTH) ** (yr - 1)
expenses = tax + ins + maint + other
interest = balance * RATE
principal = PI_ANNUAL - interest
balance -= principal
cash_flow = effective_income - expenses - PI_ANNUAL
cum_cf += cash_flow
coc = cash_flow / CASH_IN
prop_value = PRICE * (1 + APPRECIATION) ** yr
equity = prop_value - balance
if breakeven_year is None and cash_flow >= 0:
breakeven_year = yr
values = [yr, effective_income, expenses, PI_ANNUAL, cash_flow, cum_cf, coc,
interest, principal, prop_value, balance, equity]
for c, v in enumerate(values, 1):
cell = ws3.cell(row=row, column=c, value=v)
if c in (2, 3, 4, 5, 6, 8, 9, 10, 11, 12):
cell.number_format = money_neg
elif c == 7:
cell.number_format = pct
if c == 5:
cell.font = red if v < 0 else green
cell.border = thin_border
row = r + HOLD_YEARS + 1
ws3.cell(row=row, column=1, value="TOTALS").font = bold
total_income = sum((RENT * 12 + OTHER_INCOME * 12) * (1 + RENT_GROWTH) ** (yr - 1) * (1 - VACANCY) for yr in range(1, HOLD_YEARS + 1))
total_expenses = sum((PROP_TAX + INSURANCE + MAINTENANCE + OTHER_COSTS) * (1 + EXPENSE_GROWTH) ** (yr - 1) for yr in range(1, HOLD_YEARS + 1))
ws3.cell(row=row, column=2, value=total_income).number_format = money
ws3.cell(row=row, column=3, value=total_expenses).number_format = money
ws3.cell(row=row, column=4, value=PI_ANNUAL * HOLD_YEARS).number_format = money
ws3.cell(row=row, column=5, value=cum_cf).number_format = money_neg
ws3.cell(row=row, column=5).font = bold_green if cum_cf > 0 else bold_red
row += 2
ws3.cell(row=row, column=1, value=f"Cash flow becomes positive: Year {breakeven_year}").font = bold
row += 1
neg_cf = sum(
(RENT * 12 + OTHER_INCOME * 12) * (1 + RENT_GROWTH) ** (yr - 1) * (1 - VACANCY)
- (PROP_TAX + INSURANCE + MAINTENANCE + OTHER_COSTS) * (1 + EXPENSE_GROWTH) ** (yr - 1)
- PI_ANNUAL
for yr in range(1, breakeven_year or 1)
)
ws3.cell(row=row, column=1, value=f"Total losses before break-even: ${abs(neg_cf):,.0f}").font = italic_gray
# Column explanations
row += 2
ws3.cell(row=row, column=1, value="Column Definitions").font = section_font
ws3.cell(row=row, column=1).border = bottom_border
col_explanations = [
("Annual Income", "Gross rent + other income, less 5% vacancy. Grows at 4% per year."),
("Operating Expenses", "Property tax + insurance + maintenance + other costs. Grows at 3% per year. Does NOT include mortgage."),
("Mortgage (P&I)", "Fixed annual mortgage payment (principal + interest). Does not change over 30 years."),
("Cash Flow", "Income minus operating expenses minus mortgage. Negative = you pay out of pocket."),
("Cumulative Cash Flow", "Running total of all cash flows since purchase. Shows total gain or loss over time."),
("Cash-on-Cash Return", "Annual cash flow divided by total cash invested ($75,000). The return on your actual money invested."),
("Mortgage Interest", "The interest portion of your mortgage payment. Decreases over time as balance is paid down. Tax deductible."),
("Principal Paid", "The portion of your mortgage payment that reduces your loan balance. Increases over time. This builds equity."),
("Property Value", "Estimated value based on 3% annual appreciation."),
("Loan Balance", "Remaining mortgage balance. Decreases as principal is paid."),
("Total Equity", "Property value minus loan balance. This is what you'd walk away with if you sold (before selling costs)."),
]
for i, (col_name, explanation) in enumerate(col_explanations):
r2 = row + 1 + i
ws3.cell(row=r2, column=1, value=col_name).font = bold
ws3.merge_cells(f'B{r2}:L{r2}')
ws3.cell(row=r2, column=2, value=explanation).font = italic_gray
auto_width(ws3, 12, 18)
# ============ SHEET 4: vs ALTERNATIVES (20 years) ============
ws4 = wb.create_sheet("vs Alternatives")
ws4.merge_cells('A1:H1')
ws4['A1'] = "$75,000 Invested: Real Estate vs Alternatives — 20 Years"
ws4['A1'].font = title_font
ws4.merge_cells('A2:H2')
ws4['A2'] = ("All alternatives assume $75,000 invested once and left to compound. "
"Real estate includes equity buildup (appreciation + principal paydown) plus cumulative cash flow. "
"No additional capital invested in any scenario after Year 0.")
ws4['A2'].font = italic_gray
ws4['A2'].alignment = Alignment(wrap_text=True)
r = 4
headers = ['Year', 'S&P 500\n(9% avg)', 'T-Bills\n(4.5%)', 'REIT Index\n(7%)',
'RE: Equity', 'RE: Cum\nCash Flow', 'RE: Total\nValue', 'Best\nOption']
for c, h in enumerate(headers, 1):
ws4.cell(row=r, column=c, value=h)
style_header_row(ws4, r, len(headers))
sp500 = CASH_IN
tbill = CASH_IN
reit = CASH_IN
balance = LOAN
cum_cf = 0
for yr in range(1, HOLD_YEARS + 1):
row = r + yr
sp500 *= 1.09
tbill *= 1.045
reit *= 1.07
eff_inc = (RENT * 12 + OTHER_INCOME * 12) * (1 + RENT_GROWTH) ** (yr - 1) * (1 - VACANCY)
exp = (PROP_TAX + INSURANCE + MAINTENANCE + OTHER_COSTS) * (1 + EXPENSE_GROWTH) ** (yr - 1)
interest = balance * RATE
principal = PI_ANNUAL - interest
balance -= principal
cf = eff_inc - exp - PI_ANNUAL
cum_cf += cf
prop_val = PRICE * (1 + APPRECIATION) ** yr
equity = prop_val - balance
re_total = equity + cum_cf
# Determine best option
options = {'S&P 500': sp500, 'T-Bills': tbill, 'REITs': reit, 'Real Estate': re_total}
best = max(options, key=options.get)
values = [yr, sp500, tbill, reit, equity, cum_cf, re_total, best]
for c, v in enumerate(values, 1):
cell = ws4.cell(row=row, column=c, value=v)
if c in (2, 3, 4, 5, 6, 7):
cell.number_format = money_neg
cell.border = thin_border
if c == 8:
cell.font = bold
# Totals
row = r + HOLD_YEARS + 2
ws4.cell(row=row, column=1, value="20-Year Gain").font = bold
ws4.cell(row=row, column=2, value=sp500 - CASH_IN).number_format = money
ws4.cell(row=row, column=2).font = bold_green
ws4.cell(row=row, column=3, value=tbill - CASH_IN).number_format = money
ws4.cell(row=row, column=4, value=reit - CASH_IN).number_format = money
ws4.cell(row=row, column=7, value=re_total - CASH_IN).number_format = money
ws4.cell(row=row, column=7).font = bold_green if re_total > sp500 else bold
# Explanation section
row += 2
ws4.cell(row=row, column=1, value="How to Read This Table").font = section_font
ws4.cell(row=row, column=1).border = bottom_border
explanations = [
("S&P 500 (9% avg)",
"Historical average annual return of the S&P 500 index including dividends, adjusted for inflation. "
"This is a passive investment — buy an index fund and do nothing. Fully liquid (can sell any day). "
"No maintenance, no tenants, no surprise costs. Risk: market volatility (can drop 30%+ in a crash, but historically recovers)."),
("T-Bills (4.5%)",
"US Treasury Bills — the 'risk-free' rate. Currently yielding ~4.5%. Principal is guaranteed by the US government. "
"No risk of loss, but lowest returns. Useful as a baseline: if your investment can't beat T-Bills, why take the risk?"),
("REIT Index (7%)",
"Real Estate Investment Trusts — publicly traded companies that own rental properties, malls, warehouses, etc. "
"You get real estate exposure with stock market liquidity. Average ~7% annual returns. No property management needed."),
("RE: Equity",
"Your equity in the rental property = property value minus remaining loan balance. Grows from both property appreciation (3%/yr) "
"and principal paydown (each mortgage payment reduces the loan). This is NOT liquid — you can only access it by selling or refinancing."),
("RE: Cum Cash Flow",
"Running total of all cash received (or paid) from the rental. Negative early on because expenses exceed rent. "
"Turns positive around Year 6 but cumulative losses persist for years."),
("RE: Total Value",
"Equity + cumulative cash flow = total value of the real estate investment. This is the fair comparison "
"against the other options. Note: does NOT account for selling costs (typically 6-8% of sale price = $20-30K)."),
("Why RE Catches Up",
"Real estate uses LEVERAGE — you control a $345K asset with only $75K. When the property appreciates 3%, "
"you gain 3% of $345K ($10,350), not 3% of $75K ($2,250). This leverage is why RE eventually overtakes stocks — "
"but it also means losses are amplified if values drop. The S&P 500 comparison uses no leverage."),
("The Catch",
"The RE total does NOT deduct: selling costs (6-8%), capital gains tax, depreciation recapture tax (25% on ~$200K), "
"time spent managing, or the risk of bad tenants/vacancies/repairs. The stock alternatives have none of these hidden costs. "
"After selling costs and taxes, the actual RE return is significantly lower than shown."),
]
for i, (label, explanation) in enumerate(explanations):
r2 = row + 1 + (i * 3)
ws4.cell(row=r2, column=1, value=label).font = bold
ws4.merge_cells(f'A{r2+1}:H{r2+1}')
ws4.cell(row=r2 + 1, column=1, value=explanation).font = italic_gray
ws4.cell(row=r2 + 1, column=1).alignment = Alignment(wrap_text=True)
auto_width(ws4, 14, 18)
ws4.column_dimensions['A'].width = 22
ws4.column_dimensions['H'].width = 14
# ============ SHEET 5: RISK SCENARIOS (new build adjusted) ============
ws5 = wb.create_sheet("Risk Scenarios")
ws5.merge_cells('A1:F1')
ws5['A1'] = "Stress Test: What Could Go Wrong — New Build"
ws5['A1'].font = title_font
ws5.merge_cells('A2:F2')
ws5['A2'] = ("New construction significantly reduces early repair risk (builder warranty covers years 1-2, structural up to 10). "
"Major system replacements (HVAC, roof) are unlikely before year 10-15. Scenarios below are adjusted accordingly.")
ws5['A2'].font = italic_gray
ws5['A2'].alignment = Alignment(wrap_text=True)
r = 4
headers = ["Scenario", "Year 1\nCash Flow", "5-Year\nCash Flow", "Impact", "Probability", "Explanation"]
for c, h in enumerate(headers, 1):
ws5.cell(row=r, column=c, value=h)
style_header_row(ws5, r, len(headers))
scenarios = [
("Base Case", -3393, -9264, "Negative but manageable", "Expected",
"The numbers as modeled. You feed the property ~$283/mo Year 1, declining annually as rent grows."),
("Water Heater Fails Year 3 ($2K)", -3393, -11264, "Minor setback", "25% in 5yr",
"Even new water heaters can fail. $1,500-2,500 replacement. Not covered by builder warranty after year 1-2."),
("Appliance Replacements Year 4 ($3K)", -3393, -12264, "Annoying, not devastating", "20% in 5yr",
"Dishwasher, microwave, garbage disposal. Builder-grade appliances often fail sooner than premium brands."),
("HVAC Replacement Year 12 ($12K)", -3393, -9264, "Hits during positive cash flow years", "60% in 15yr",
"New HVAC systems last 12-15 years. By year 12, the property cash flows ~$4K/yr, so one year's profit is wiped."),
("Roof Replacement Year 18 ($20K)", -3393, -9264, "Major but expected at this age", "40% in 20yr",
"New roofs last 20-25 years. Budget for this as a known future expense, not a surprise."),
("10% Vacancy (bad tenant/eviction)", -4593, -14264, "Doubles the bleeding", "20% chance",
"Eviction process takes 1-3 months. During that time: zero rent + legal fees ($1-3K) + unit damage repair."),
("Rent Drops 10% (recession)", -5793, -19264, "Cash flow crisis", "15-20%",
"Economic downturn reduces demand. You may need to drop rent to keep occupancy, or accept longer vacancies."),
("Property Value Drops 15%", -3393, -9264, "Paper loss, real if you sell", "10-15%",
"Your $345K property is worth $293K. With $276K loan, you have $17K equity vs $69K invested. "
"You're not underwater on the mortgage, but 75% of your down payment is gone on paper."),
("Hire Property Manager Year 3 (8%)", -3393, -13284, "Realistic for partnership", "High",
"Partnerships almost always need professional management. 8% of gross rent = $160/mo = $1,920/yr additional cost."),
("Interest Rates Drop, Refi Year 5 (5%)", -3393, -5264, "Improves the deal", "30-40%",
"If rates drop to 5%, refinancing saves ~$230/mo. This would make the property cash flow positive immediately. "
"However, refinancing costs $3-6K and resets your amortization schedule."),
]
for i, (scenario, yr1, yr5, impact, prob, explanation) in enumerate(scenarios):
row = r + 1 + (i * 3)
ws5.cell(row=row, column=1, value=scenario).font = bold
ws5.cell(row=row, column=2, value=yr1).number_format = money_neg
ws5.cell(row=row, column=2).font = red if yr1 < 0 else green
ws5.cell(row=row, column=3, value=yr5).number_format = money_neg
ws5.cell(row=row, column=3).font = red if yr5 < 0 else green
ws5.cell(row=row, column=4, value=impact)
ws5.cell(row=row, column=5, value=prob)
ws5.merge_cells(f'A{row+1}:F{row+1}')
ws5.cell(row=row + 1, column=1, value=explanation).font = italic_gray
ws5.cell(row=row + 1, column=1).alignment = Alignment(wrap_text=True)
auto_width(ws5, 15, 25)
ws5.column_dimensions['A'].width = 38
ws5.column_dimensions['D'].width = 30
ws5.column_dimensions['F'].width = 40
# ============ SHEET 6: BREAK-EVEN ============
ws6 = wb.create_sheet("Break-Even Analysis")
ws6.merge_cells('A1:D1')
ws6['A1'] = "What Price + Rate = Day 1 Cash Flow Positive?"
ws6['A1'].font = title_font
ws6.merge_cells('A2:D2')
ws6['A2'] = (f"All scenarios assume: Rent ${RENT:,}/mo + ${OTHER_INCOME}/mo other | Vacancy {VACANCY:.0%} | "
f"20% down | 30yr fixed | Same tax/insurance/maintenance rates")
ws6['A2'].font = italic_gray
ws6['A2'].alignment = Alignment(wrap_text=True)
r = 4
ws6.cell(row=r, column=1, value="Break-Even Interest Rate by Purchase Price").font = section_font
ws6.cell(row=r, column=1).border = bottom_border
ws6.merge_cells(f'A{r+1}:D{r+1}')
ws6.cell(row=r+1, column=1,
value="What is the maximum interest rate where this property breaks even on cash flow from Day 1? "
"If the break-even rate is below current market rates (6.25%), the deal does not work at that price.").font = italic_gray
ws6.cell(row=r+1, column=1).alignment = Alignment(wrap_text=True)
r += 3
for c, h in enumerate(['Purchase Price', 'Max Rate for\nBreak-Even', 'Works at 6.25%?', 'Rent-to-Price\nRatio'], 1):
ws6.cell(row=r, column=c, value=h)
style_header_row(ws6, r, 4)
prices = [200000, 220000, 240000, 260000, 280000, 299000, 320000, 345000]
for i, price in enumerate(prices):
row = r + 1 + i
be_rate = None
for rate_bps in range(100, 1200, 1):
rate_test = rate_bps / 10000
loan = price * 0.80
pi = monthly_pmt(loan, rate_test)
expenses = (price * 0.007 + price * 0.004 + price * 0.006) / 12 + OTHER_COSTS / 12
total = pi + expenses
income = (RENT + OTHER_INCOME) * (1 - VACANCY)
if income - total <= 0:
be_rate = (rate_bps - 1) / 10000
break
ws6.cell(row=row, column=1, value=price).number_format = '$#,##0'
ws6.cell(row=row, column=2, value=f"{be_rate:.2%}" if be_rate else "10%+")
viable = "YES" if be_rate and be_rate >= 0.0625 else "NO"
ws6.cell(row=row, column=3, value=viable).font = bold_green if viable == "YES" else bold_red
rtp = (RENT/price)*100
ws6.cell(row=row, column=4, value=f"{rtp:.2f}%")
ws6.cell(row=row, column=4).font = green if rtp >= 0.8 else red
cell_border = thin_border
for c in range(1, 5):
ws6.cell(row=row, column=c).border = cell_border
if price == 345000:
ws6.cell(row=row, column=1).font = bold
ws6.cell(row=row, column=2).font = bold
last_row = row
r = last_row + 3
ws6.cell(row=r, column=1, value="Maximum Purchase Price by Interest Rate").font = section_font
ws6.cell(row=r, column=1).border = bottom_border
ws6.merge_cells(f'A{r+1}:D{r+1}')
ws6.cell(row=r+1, column=1,
value="At a given interest rate, what is the most you can pay for this property and still break even on Day 1?").font = italic_gray
ws6.cell(row=r+1, column=1).alignment = Alignment(wrap_text=True)
r += 3
for c, h in enumerate(['Interest Rate', 'Max Purchase\nPrice', 'Down Payment\nRequired', 'Rent-to-Price\nRatio'], 1):
ws6.cell(row=r, column=c, value=h)
style_header_row(ws6, r, 4)
rates = [3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.25, 7.0]
for i, rate_pct in enumerate(rates):
row = r + 1 + i
rate_val = rate_pct / 100
max_price = None
for price in range(400000, 100000, -1000):
loan = price * 0.80
pi = monthly_pmt(loan, rate_val)
expenses = (price * 0.007 + price * 0.004 + price * 0.006) / 12 + OTHER_COSTS / 12
total = pi + expenses
income = (RENT + OTHER_INCOME) * (1 - VACANCY)
if income >= total:
max_price = price
break
ws6.cell(row=row, column=1, value=f"{rate_pct:.2f}%")
if max_price:
ws6.cell(row=row, column=2, value=max_price).number_format = '$#,##0'
ws6.cell(row=row, column=3, value=max_price * 0.20).number_format = '$#,##0'
ws6.cell(row=row, column=4, value=f"{(RENT/max_price)*100:.2f}%")
for c in range(1, 5):
ws6.cell(row=row, column=c).border = thin_border
if rate_pct == 6.25:
ws6.cell(row=row, column=1).font = bold
ws6.cell(row=row, column=2).font = bold
auto_width(ws6, 18, 28)
# Save
outpath = "/home/wdjones/.openclaw/workspace/data/real-estate-345k-nogo-analysis.xlsx"
wb.save(outpath)
print(f"Saved to {outpath}")

477
tools/build-re-nogo-plain.py Normal file
View File

@ -0,0 +1,477 @@
#!/usr/bin/env python3
"""Build plain Excel spreadsheet for $345K rental property - NO GO analysis"""
import openpyxl
from openpyxl.styles import Font, Alignment, Border, Side
from openpyxl.utils import get_column_letter
# ============ PARAMETERS ============
PRICE = 345000
DOWN_PCT = 0.20
DOWN = PRICE * DOWN_PCT
CLOSING = 6000
LOAN = PRICE * (1 - DOWN_PCT)
RATE = 0.0625
CASH_IN = DOWN + CLOSING
RENT = 2000
OTHER_INCOME = 50
VACANCY = 0.05
RENT_GROWTH = 0.04
PROP_TAX = 2400
INSURANCE = 1400
HOA = 0
MAINTENANCE = 2000
OTHER_COSTS = 500
EXPENSE_GROWTH = 0.03
APPRECIATION = 0.03
HOLD_YEARS = 20
def monthly_pmt(principal, annual_rate, years=30):
r = annual_rate / 12
n = years * 12
return principal * (r * (1 + r)**n) / ((1 + r)**n - 1)
PI_MONTHLY = monthly_pmt(LOAN, RATE)
PI_ANNUAL = PI_MONTHLY * 12
# Styles
bold = Font(bold=True)
bold_red = Font(bold=True, color="CC0000")
bold_green = Font(bold=True, color="006600")
red = Font(color="CC0000")
green = Font(color="006600")
italic_gray = Font(italic=True, color="666666")
header_font = Font(bold=True)
title_font = Font(bold=True, size=14)
section_font = Font(bold=True, size=11)
thin_border = Border(
left=Side(style='thin', color='CCCCCC'),
right=Side(style='thin', color='CCCCCC'),
top=Side(style='thin', color='CCCCCC'),
bottom=Side(style='thin', color='CCCCCC')
)
bottom_border = Border(bottom=Side(style='thin', color='999999'))
money = '#,##0'
money_neg = '#,##0;[Red]-#,##0'
pct = '0.0%'
def style_header_row(ws, row, max_col):
for c in range(1, max_col + 1):
cell = ws.cell(row=row, column=c)
cell.font = header_font
cell.alignment = Alignment(horizontal='center', wrap_text=True)
cell.border = Border(bottom=Side(style='medium'))
def auto_width(ws, min_w=10, max_w=22):
for col in ws.columns:
letter = get_column_letter(col[0].column)
max_len = max((len(str(c.value or '')) for c in col), default=0)
ws.column_dimensions[letter].width = max(min(max_len + 3, max_w), min_w)
wb = openpyxl.Workbook()
# ============ SHEET 1: EXECUTIVE SUMMARY ============
ws1 = wb.active
ws1.title = "Executive Summary"
ws1.merge_cells('A1:H1')
ws1['A1'] = "Investment Analysis: $345,000 Rental Property"
ws1['A1'].font = title_font
ws1.merge_cells('A3:H3')
ws1['A3'] = "VERDICT: NO-GO — Negative cash flow for 5+ years. Does not meet basic investment criteria."
ws1['A3'].font = bold_red
r = 5
ws1.cell(row=r, column=1, value="Key Metrics").font = section_font
ws1.cell(row=r, column=1).border = bottom_border
metrics = [
("Purchase Price", f"${PRICE:,}", "Monthly Rent", f"${RENT:,}"),
("Down Payment (20%)", f"${DOWN:,.0f}", "Effective Monthly Income", f"${(RENT + OTHER_INCOME) * (1-VACANCY):,.0f}"),
("Closing Costs", f"${CLOSING:,}", "Monthly PITI", f"${PI_MONTHLY + PROP_TAX/12 + INSURANCE/12:,.0f}"),
("Total Cash Required", f"${CASH_IN:,.0f}", "Monthly Cash Flow", f"${((RENT+OTHER_INCOME)*12*(1-VACANCY) - (PROP_TAX+INSURANCE+MAINTENANCE+OTHER_COSTS) - PI_ANNUAL)/12:+,.0f}"),
("Loan Amount", f"${LOAN:,.0f}", "Rent-to-Price Ratio", f"{(RENT/PRICE)*100:.2f}% (need ≥0.8%)"),
("Interest Rate", f"{RATE:.2%}", "Year 1 Cash-on-Cash", f"{(((RENT+OTHER_INCOME)*12*(1-VACANCY) - (PROP_TAX+INSURANCE+MAINTENANCE+OTHER_COSTS) - PI_ANNUAL)/CASH_IN)*100:+.1f}%"),
("Monthly P&I", f"${PI_MONTHLY:,.0f}", "Break-Even Rate Needed", "4.57% (current: 6.25%)"),
]
for i, (l1, v1, l2, v2) in enumerate(metrics):
row = r + 1 + i
ws1.cell(row=row, column=1, value=l1)
ws1.cell(row=row, column=2, value=v1).font = bold
ws1.cell(row=row, column=5, value=l2)
ws1.cell(row=row, column=6, value=v2).font = bold
r = 14
ws1.cell(row=r, column=1, value="Why This Deal Fails").font = section_font
ws1.cell(row=r, column=1).border = bottom_border
reasons = [
("Rent-to-Price Ratio: 0.58%", "Industry minimum is 0.8% (1% preferred). At $345K, rent should be $2,760-$3,450/mo."),
("Negative Cash Flow Years 1-5", "You pay ~$283/mo out of pocket Year 1, declining to ~$21/mo by Year 5. Total: -$9,264 over 5 years."),
("Interest Rate Too High", "At 6.25%, the mortgage ($1,699) consumes 89% of effective rent ($1,900). Need 4.57% to break even."),
("No Tax Benefit at Your AGI", "Above $150K AGI = $0 passive loss deduction against W-2. Depreciation losses carry forward only."),
("Thin Margin for Surprises", "One HVAC replacement ($8-15K) or 2-month vacancy wipes out years of equity building."),
("Opportunity Cost", "Same $75K in S&P 500 at 9% avg = ~$115K in 5 years with zero effort."),
]
for i, (title, detail) in enumerate(reasons):
row = r + 1 + i
ws1.cell(row=row, column=1, value=title).font = bold
ws1.merge_cells(f'B{row}:H{row}')
ws1.cell(row=row, column=2, value=detail).font = italic_gray
r = 22
ws1.cell(row=r, column=1, value="What Would Make This Deal Work").font = section_font
ws1.cell(row=r, column=1).border = bottom_border
fixes = [
("Price ≤ $299,000", "Cash flow positive Day 1 at current 6.25% rate"),
("Interest Rate ≤ 4.57%", "Break-even at $345K price (not realistic near-term)"),
("Rent ≥ $2,760/mo", "Achieves 0.8% rent-to-price ratio (comps don't support this)"),
("Different Market", "Midwest markets (Cleveland, Memphis, Indianapolis) offer 0.8-1.2% ratios"),
]
for i, (fix, detail) in enumerate(fixes):
row = r + 1 + i
ws1.cell(row=row, column=1, value=fix).font = bold_green
ws1.merge_cells(f'B{row}:H{row}')
ws1.cell(row=row, column=2, value=detail).font = italic_gray
auto_width(ws1, 14, 25)
ws1.column_dimensions['A'].width = 35
# ============ SHEET 2: 20-YEAR CASH FLOW ============
ws2 = wb.create_sheet("20-Year Cash Flow")
ws2.merge_cells('A1:L1')
ws2['A1'] = "20-Year Cash Flow Projection"
ws2['A1'].font = title_font
ws2.merge_cells('A2:L2')
ws2['A2'] = f"Purchase: ${PRICE:,} | Down: 20% | Rate: {RATE:.2%} | Rent: ${RENT:,}/mo | Growth: Rent {RENT_GROWTH:.0%}, Expenses {EXPENSE_GROWTH:.0%}, Appreciation {APPRECIATION:.0%}"
ws2['A2'].font = italic_gray
r = 4
headers = ['Year', 'Annual\nIncome', 'Operating\nExpenses', 'Mortgage\n(P&I)', 'Cash\nFlow',
'Cumulative\nCash Flow', 'Cash-on-Cash\nReturn', 'Mortgage\nInterest', 'Principal\nPaid',
'Property\nValue', 'Loan\nBalance', 'Total\nEquity']
for c, h in enumerate(headers, 1):
ws2.cell(row=r, column=c, value=h)
style_header_row(ws2, r, len(headers))
balance = LOAN
cum_cf = 0
breakeven_year = None
for yr in range(1, HOLD_YEARS + 1):
row = r + yr
annual_rent = (RENT * 12 + OTHER_INCOME * 12) * (1 + RENT_GROWTH) ** (yr - 1)
effective_income = annual_rent * (1 - VACANCY)
tax = PROP_TAX * (1 + EXPENSE_GROWTH) ** (yr - 1)
ins = INSURANCE * (1 + EXPENSE_GROWTH) ** (yr - 1)
maint = MAINTENANCE * (1 + EXPENSE_GROWTH) ** (yr - 1)
other = OTHER_COSTS * (1 + EXPENSE_GROWTH) ** (yr - 1)
expenses = tax + ins + maint + other
interest = balance * RATE
principal = PI_ANNUAL - interest
balance -= principal
cash_flow = effective_income - expenses - PI_ANNUAL
cum_cf += cash_flow
coc = cash_flow / CASH_IN
prop_value = PRICE * (1 + APPRECIATION) ** yr
equity = prop_value - balance
if breakeven_year is None and cash_flow >= 0:
breakeven_year = yr
values = [yr, effective_income, expenses, PI_ANNUAL, cash_flow, cum_cf, coc,
interest, principal, prop_value, balance, equity]
for c, v in enumerate(values, 1):
cell = ws2.cell(row=row, column=c, value=v)
if c in (2, 3, 4, 5, 6, 8, 9, 10, 11, 12):
cell.number_format = money_neg
elif c == 7:
cell.number_format = pct
if c == 5:
cell.font = red if v < 0 else green
cell.border = thin_border
row = r + HOLD_YEARS + 1
ws2.cell(row=row, column=1, value="TOTALS").font = bold
total_income = sum((RENT * 12 + OTHER_INCOME * 12) * (1 + RENT_GROWTH) ** (yr - 1) * (1 - VACANCY) for yr in range(1, HOLD_YEARS + 1))
total_expenses = sum((PROP_TAX + INSURANCE + MAINTENANCE + OTHER_COSTS) * (1 + EXPENSE_GROWTH) ** (yr - 1) for yr in range(1, HOLD_YEARS + 1))
ws2.cell(row=row, column=2, value=total_income).number_format = money
ws2.cell(row=row, column=3, value=total_expenses).number_format = money
ws2.cell(row=row, column=4, value=PI_ANNUAL * HOLD_YEARS).number_format = money
ws2.cell(row=row, column=5, value=cum_cf).number_format = money_neg
ws2.cell(row=row, column=5).font = bold_red if cum_cf < 0 else bold_green
row += 2
ws2.cell(row=row, column=1, value=f"Cash flow break-even: Year {breakeven_year}").font = bold
row += 1
neg_cf = sum((RENT * 12 + OTHER_INCOME * 12) * (1 + RENT_GROWTH) ** (yr - 1) * (1 - VACANCY) - (PROP_TAX + INSURANCE + MAINTENANCE + OTHER_COSTS) * (1 + EXPENSE_GROWTH) ** (yr - 1) - PI_ANNUAL for yr in range(1, breakeven_year or 1))
ws2.cell(row=row, column=1, value=f"Cumulative losses before break-even: ${abs(neg_cf):,.0f}").font = italic_gray
auto_width(ws2, 12, 18)
# ============ SHEET 3: MONTHLY BREAKDOWN ============
ws3 = wb.create_sheet("Monthly Breakdown")
ws3.merge_cells('A1:C1')
ws3['A1'] = "Monthly Income vs Expenses — Year 1"
ws3['A1'].font = title_font
r = 3
ws3.cell(row=r, column=1, value="Income").font = section_font
ws3.cell(row=r, column=1).border = bottom_border
income_items = [
("Gross Rent", RENT, ""),
("Other Income", OTHER_INCOME, ""),
("Vacancy Allowance (5%)", -(RENT + OTHER_INCOME) * VACANCY, "Lost income from vacancy"),
("Effective Monthly Income", (RENT + OTHER_INCOME) * (1 - VACANCY), ""),
]
for i, (label, val, note) in enumerate(income_items):
row = r + 1 + i
ws3.cell(row=row, column=1, value=label)
ws3.cell(row=row, column=2, value=val).number_format = money_neg
ws3.cell(row=row, column=3, value=note).font = italic_gray
if "Effective" in label:
ws3.cell(row=row, column=1).font = bold
ws3.cell(row=row, column=2).font = bold_green
elif val < 0:
ws3.cell(row=row, column=2).font = red
r = r + len(income_items) + 2
ws3.cell(row=r, column=1, value="Expenses").font = section_font
ws3.cell(row=r, column=1).border = bottom_border
expense_items = [
("Mortgage (P&I)", PI_MONTHLY, f"${LOAN:,} @ {RATE:.2%} / 30yr"),
("Property Tax", PROP_TAX / 12, f"${PROP_TAX:,}/yr"),
("Insurance", INSURANCE / 12, f"${INSURANCE:,}/yr"),
("HOA", HOA / 12, ""),
("Maintenance", MAINTENANCE / 12, f"${MAINTENANCE:,}/yr — low, should be ~${PRICE*0.01:,.0f}"),
("Other Costs", OTHER_COSTS / 12, f"${OTHER_COSTS:,}/yr"),
("Management Fee", 0, "Self-managed (0%)"),
("Total Monthly Expenses", PI_MONTHLY + PROP_TAX/12 + INSURANCE/12 + MAINTENANCE/12 + OTHER_COSTS/12, ""),
]
for i, (label, val, note) in enumerate(expense_items):
row = r + 1 + i
ws3.cell(row=row, column=1, value=label)
ws3.cell(row=row, column=2, value=val).number_format = money
ws3.cell(row=row, column=3, value=note).font = italic_gray
if "Total" in label:
ws3.cell(row=row, column=1).font = bold
ws3.cell(row=row, column=2).font = bold_red
r = r + len(expense_items) + 2
eff_income = (RENT + OTHER_INCOME) * (1 - VACANCY)
total_exp = PI_MONTHLY + PROP_TAX/12 + INSURANCE/12 + MAINTENANCE/12 + OTHER_COSTS/12
net = eff_income - total_exp
ws3.cell(row=r, column=1, value="Net Monthly Cash Flow").font = Font(bold=True, size=12)
ws3.cell(row=r, column=2, value=net).number_format = money_neg
ws3.cell(row=r, column=2).font = Font(bold=True, size=12, color="CC0000" if net < 0 else "006600")
r += 2
pct_of_income = PI_MONTHLY / eff_income * 100
ws3.cell(row=r, column=1, value=f"Mortgage is {pct_of_income:.0f}% of effective income. Healthy target: ≤70%.").font = italic_gray
ws3.merge_cells(f'A{r}:C{r}')
auto_width(ws3, 14, 30)
ws3.column_dimensions['A'].width = 30
ws3.column_dimensions['C'].width = 45
# ============ SHEET 4: vs ALTERNATIVES ============
ws4 = wb.create_sheet("vs Alternatives")
ws4.merge_cells('A1:G1')
ws4['A1'] = "$75,000 Invested: Real Estate vs Alternatives"
ws4['A1'].font = title_font
r = 3
headers = ['Year', 'S&P 500\n(9% avg)', 'T-Bills\n(4.5%)', 'REIT Index\n(7%)',
'RE: Equity', 'RE: Cum\nCash Flow', 'RE: Total\nValue']
for c, h in enumerate(headers, 1):
ws4.cell(row=r, column=c, value=h)
style_header_row(ws4, r, len(headers))
sp500 = CASH_IN
tbill = CASH_IN
reit = CASH_IN
balance = LOAN
cum_cf = 0
for yr in range(1, 11):
row = r + yr
sp500 *= 1.09
tbill *= 1.045
reit *= 1.07
eff_inc = (RENT * 12 + OTHER_INCOME * 12) * (1 + RENT_GROWTH) ** (yr - 1) * (1 - VACANCY)
exp = (PROP_TAX + INSURANCE + MAINTENANCE + OTHER_COSTS) * (1 + EXPENSE_GROWTH) ** (yr - 1)
interest = balance * RATE
principal = PI_ANNUAL - interest
balance -= principal
cf = eff_inc - exp - PI_ANNUAL
cum_cf += cf
prop_val = PRICE * (1 + APPRECIATION) ** yr
equity = prop_val - balance
values = [yr, sp500, tbill, reit, equity, cum_cf, equity + cum_cf]
for c, v in enumerate(values, 1):
cell = ws4.cell(row=row, column=c, value=v)
if c >= 2:
cell.number_format = money_neg
cell.border = thin_border
row = r + 12
ws4.cell(row=row, column=1, value="10-Year Gain").font = bold
ws4.cell(row=row, column=2, value=sp500 - CASH_IN).number_format = money
ws4.cell(row=row, column=2).font = bold_green
ws4.cell(row=row, column=3, value=tbill - CASH_IN).number_format = money
ws4.cell(row=row, column=4, value=reit - CASH_IN).number_format = money
row += 2
ws4.cell(row=row, column=1, value="S&P 500 delivers higher returns with zero effort, zero cap-ex surprises, and full liquidity.").font = italic_gray
ws4.merge_cells(f'A{row}:G{row}')
auto_width(ws4, 14, 18)
# ============ SHEET 5: RISK SCENARIOS ============
ws5 = wb.create_sheet("Risk Scenarios")
ws5.merge_cells('A1:E1')
ws5['A1'] = "Stress Test: What Could Go Wrong"
ws5['A1'].font = title_font
r = 3
scenarios = [
("Scenario", "Year 1 Cash Flow", "5-Year Cash Flow", "Impact", "Probability"),
("Base Case (as modeled)", -3393, -9264, "Negative but manageable", "Expected"),
("HVAC Replacement Year 2 ($12K)", -3393, -21264, "Wipes out 2+ years equity growth", "30% in 5yr"),
("Roof Replacement Year 3 ($20K)", -3393, -29264, "Devastating — may force sale", "15% in 5yr"),
("10% Vacancy (bad tenant/eviction)", -4593, -14264, "Doubles the bleeding", "20% chance"),
("Rate Rises to 7.5% (if ARM/refi)", -5940, -18964, "Underwater fast", "Possible"),
("Rent Drops 10% (recession)", -5793, -19264, "Cash flow crisis", "15-20% chance"),
("Property Value Drops 10%", -3393, -9264, "Equity wiped, underwater on loan", "10-15% chance"),
("Hire Property Manager (8%)", -5217, -15264, "Realistic if you can't self-manage", "Likely eventually"),
("All of the above (worst case)", -12000, -55000, "Financial catastrophe", "5% but non-zero"),
]
for i, vals in enumerate(scenarios):
row = r + i
for c, v in enumerate(vals, 1):
cell = ws5.cell(row=row, column=c, value=v)
if i == 0:
cell.font = header_font
cell.border = Border(bottom=Side(style='medium'))
else:
if c in (2, 3) and isinstance(v, (int, float)):
cell.number_format = money_neg
cell.font = red
cell.border = thin_border
auto_width(ws5, 15, 40)
ws5.column_dimensions['A'].width = 40
ws5.column_dimensions['D'].width = 35
# ============ SHEET 6: BREAK-EVEN ============
ws6 = wb.create_sheet("Break-Even Analysis")
ws6.merge_cells('A1:D1')
ws6['A1'] = "What Price + Rate = Day 1 Cash Flow Positive?"
ws6['A1'].font = title_font
ws6.merge_cells('A2:D2')
ws6['A2'] = f"Fixed: Rent ${RENT:,}/mo | Vacancy {VACANCY:.0%} | 20% down | 30yr fixed"
ws6['A2'].font = italic_gray
r = 4
ws6.cell(row=r, column=1, value="Break-Even Rate by Price").font = section_font
ws6.cell(row=r, column=1).border = bottom_border
r += 1
for c, h in enumerate(['Price', 'Max Rate for Break-Even', 'Viable Today?', 'Rent/Price'], 1):
ws6.cell(row=r, column=c, value=h)
style_header_row(ws6, r, 4)
prices = [200000, 220000, 240000, 260000, 280000, 299000, 320000, 345000]
for i, price in enumerate(prices):
row = r + 1 + i
be_rate = None
for rate_bps in range(100, 1200, 1):
rate_test = rate_bps / 10000
loan = price * 0.80
pi = monthly_pmt(loan, rate_test)
expenses = (price * 0.007 + price * 0.004 + price * 0.006) / 12 + OTHER_COSTS / 12
total = pi + expenses
income = (RENT + OTHER_INCOME) * (1 - VACANCY)
if income - total <= 0:
be_rate = (rate_bps - 1) / 10000
break
ws6.cell(row=row, column=1, value=price).number_format = '$#,##0'
ws6.cell(row=row, column=2, value=f"{be_rate:.2%}" if be_rate else "10%+")
viable = "YES" if be_rate and be_rate >= 0.0625 else "NO"
ws6.cell(row=row, column=3, value=viable).font = bold_green if viable == "YES" else bold_red
ws6.cell(row=row, column=4, value=f"{(RENT/price)*100:.2f}%")
if price == 345000:
ws6.cell(row=row, column=1).font = bold
ws6.cell(row=row, column=2).font = bold
last_row = row
r = last_row + 3
ws6.cell(row=r, column=1, value="Break-Even Price by Rate").font = section_font
ws6.cell(row=r, column=1).border = bottom_border
r += 1
for c, h in enumerate(['Interest Rate', 'Max Price', 'Down Payment', 'Rent/Price'], 1):
ws6.cell(row=r, column=c, value=h)
style_header_row(ws6, r, 4)
rates = [3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.25]
for i, rate_pct in enumerate(rates):
row = r + 1 + i
rate_val = rate_pct / 100
max_price = None
for price in range(400000, 100000, -1000):
loan = price * 0.80
pi = monthly_pmt(loan, rate_val)
expenses = (price * 0.007 + price * 0.004 + price * 0.006) / 12 + OTHER_COSTS / 12
total = pi + expenses
income = (RENT + OTHER_INCOME) * (1 - VACANCY)
if income >= total:
max_price = price
break
ws6.cell(row=row, column=1, value=f"{rate_pct:.2f}%")
if max_price:
ws6.cell(row=row, column=2, value=max_price).number_format = '$#,##0'
ws6.cell(row=row, column=3, value=max_price * 0.20).number_format = '$#,##0'
ws6.cell(row=row, column=4, value=f"{(RENT/max_price)*100:.2f}%")
if rate_pct == 6.25:
ws6.cell(row=row, column=1).font = bold
ws6.cell(row=row, column=2).font = bold
auto_width(ws6, 18, 28)
# Save
outpath = "/home/wdjones/.openclaw/workspace/data/real-estate-345k-nogo-analysis.xlsx"
wb.save(outpath)
print(f"Saved to {outpath}")

255
tools/coolify-deploy.sh Executable file
View File

@ -0,0 +1,255 @@
#!/bin/bash
# coolify-deploy.sh — Build, push, and deploy to Coolify
# Usage: coolify-deploy.sh [project-dir] [app-uuid]
#
# Examples:
# coolify-deploy.sh # interactive, lists known apps
# coolify-deploy.sh ~/projects/coinex-dashboard # auto-detect app from .coolify
# coolify-deploy.sh ~/projects/coinex-dashboard fcs04o8w... # explicit app UUID
# coolify-deploy.sh --status fcs04o8w... # check latest deploy status
# coolify-deploy.sh --skip-build ~/projects/coinex-dashboard # push + deploy only
set -euo pipefail
COOLIFY_HOST="${COOLIFY_HOST:-192.168.86.44}"
COOLIFY_PORT="${COOLIFY_PORT:-8000}"
COOLIFY_URL="http://${COOLIFY_HOST}:${COOLIFY_PORT}/api/v1"
COOLIFY_TOKEN="${COOLIFY_TOKEN:-}"
CREDS_FILE="$HOME/.openclaw/workspace/.credentials/coolify.env"
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'
log() { echo -e "${BLUE}[deploy]${NC} $*"; }
ok() { echo -e "${GREEN}[ ok ]${NC} $*"; }
warn() { echo -e "${YELLOW}[ warn ]${NC} $*"; }
fail() { echo -e "${RED}[FAILED]${NC} $*"; exit 1; }
# Load token
load_token() {
if [ -z "$COOLIFY_TOKEN" ]; then
if [ -f "$CREDS_FILE" ]; then
source "$CREDS_FILE"
COOLIFY_TOKEN="${COOLIFY_API_TOKEN:-${COOLIFY_TOKEN:-}}"
fi
fi
[ -z "$COOLIFY_TOKEN" ] && fail "No COOLIFY_TOKEN. Set env var or create $CREDS_FILE with COOLIFY_API_TOKEN=..."
}
api() {
local method="$1" path="$2"
shift 2
curl -sf -X "$method" \
-H "Authorization: Bearer $COOLIFY_TOKEN" \
-H "Content-Type: application/json" \
"$COOLIFY_URL$path" "$@" 2>/dev/null
}
# Save/load app UUID mapping
save_app_config() {
local dir="$1" uuid="$2"
echo "$uuid" > "$dir/.coolify-app"
echo "$uuid"
}
load_app_config() {
local dir="$1"
if [ -f "$dir/.coolify-app" ]; then
cat "$dir/.coolify-app"
fi
}
# --- Commands ---
cmd_status() {
local app_uuid="$1"
load_token
log "Checking deployments for app $app_uuid..."
local deploys
deploys=$(api GET "/applications/$app_uuid/deployments" 2>/dev/null) || fail "Could not fetch deployments"
echo "$deploys" | python3 -c "
import json, sys
data = json.load(sys.stdin)
deploys = data if isinstance(data, list) else data.get('data', data.get('deployments', []))
for d in deploys[:5]:
status = d.get('status', '?')
commit = d.get('commit', '?')[:8]
created = d.get('created_at', '?')[:19]
uuid = d.get('deployment_uuid', '?')[:12]
icon = '✅' if status == 'finished' else '❌' if status == 'failed' else '⏳' if status == 'in_progress' else '❓'
print(f'{icon} {created} {status:<12} {commit} {uuid}')
" 2>/dev/null || echo "$deploys" | python3 -m json.tool 2>/dev/null || echo "$deploys"
}
cmd_deploy() {
local project_dir="$1"
local app_uuid="$2"
local skip_build="${3:-false}"
load_token
cd "$project_dir"
local project_name
project_name=$(basename "$project_dir")
# Step 1: Build
if [ "$skip_build" = "false" ]; then
log "Building $project_name..."
# Clean previous build
rm -rf .next 2>/dev/null || true
if npm run build 2>&1 | tee /tmp/coolify-build.log | tail -5; then
ok "Build succeeded"
else
fail "Build failed. Full log: /tmp/coolify-build.log"
fi
else
warn "Skipping build (--skip-build)"
fi
# Step 2: Git commit & push
log "Committing and pushing..."
if git diff --quiet && git diff --cached --quiet; then
warn "No changes to commit — pushing existing commits"
else
git add -A
local msg
msg=$(git log --oneline -1 2>/dev/null | cut -d' ' -f2-)
git commit -m "deploy: ${msg:-update}" --quiet 2>/dev/null || true
fi
if git push --quiet 2>&1; then
ok "Pushed to $(git remote get-url origin 2>/dev/null | sed 's/.*@//' | sed 's/\.git$//')"
else
fail "Git push failed"
fi
# Step 3: Trigger Coolify deploy
log "Triggering Coolify deployment..."
local result
result=$(api POST "/applications/$app_uuid/start") || fail "Could not trigger deployment"
local deploy_uuid
deploy_uuid=$(echo "$result" | python3 -c "import json,sys; print(json.load(sys.stdin).get('deployment_uuid',''))" 2>/dev/null)
if [ -z "$deploy_uuid" ]; then
fail "No deployment UUID returned: $result"
fi
ok "Deployment queued: $deploy_uuid"
# Step 4: Poll for completion
log "Waiting for deployment..."
local attempts=0
local max_attempts=40 # 40 * 10s = ~6.5 min
local status=""
while [ $attempts -lt $max_attempts ]; do
sleep 10
attempts=$((attempts + 1))
status=$(api GET "/deployments/$deploy_uuid" | python3 -c "import json,sys; print(json.load(sys.stdin).get('status','unknown'))" 2>/dev/null)
case "$status" in
finished)
echo ""
ok "✅ Deployment successful! ($((attempts * 10))s)"
# Show app URL
local app_info
app_info=$(api GET "/applications/$app_uuid" 2>/dev/null)
local fqdn
fqdn=$(echo "$app_info" | python3 -c "import json,sys; print(json.load(sys.stdin).get('fqdn',''))" 2>/dev/null)
[ -n "$fqdn" ] && ok "Live at: $fqdn"
return 0
;;
failed)
echo ""
fail "❌ Deployment failed after $((attempts * 10))s. Check Coolify UI at http://$COOLIFY_HOST:$COOLIFY_PORT"
;;
*)
printf "\r${CYAN}[ .. ]${NC} Status: %-15s (%ds)" "$status" "$((attempts * 10))"
;;
esac
done
echo ""
fail "Deployment timed out after $((max_attempts * 10))s (status: $status)"
}
# --- Main ---
SKIP_BUILD=false
STATUS_ONLY=false
PROJECT_DIR=""
APP_UUID=""
# Parse args
while [ $# -gt 0 ]; do
case "$1" in
--skip-build) SKIP_BUILD=true; shift ;;
--status) STATUS_ONLY=true; shift ;;
--help|-h)
echo "Usage: coolify-deploy.sh [options] [project-dir] [app-uuid]"
echo ""
echo "Options:"
echo " --skip-build Push and deploy without building locally"
echo " --status UUID Check deployment status for an app"
echo " -h, --help Show this help"
echo ""
echo "If app-uuid is omitted, reads from .coolify-app in project dir."
echo "Token from \$COOLIFY_TOKEN env or $CREDS_FILE"
exit 0
;;
*)
if [ -z "$PROJECT_DIR" ]; then
PROJECT_DIR="$1"
else
APP_UUID="$1"
fi
shift
;;
esac
done
# Status check mode
if [ "$STATUS_ONLY" = "true" ]; then
[ -z "$PROJECT_DIR" ] && fail "Usage: coolify-deploy.sh --status <app-uuid>"
cmd_status "$PROJECT_DIR"
exit 0
fi
# Resolve project dir
if [ -z "$PROJECT_DIR" ]; then
if [ -f "package.json" ]; then
PROJECT_DIR="$(pwd)"
else
fail "No project directory specified and not in a Node.js project"
fi
fi
PROJECT_DIR=$(cd "$PROJECT_DIR" && pwd)
[ -f "$PROJECT_DIR/package.json" ] || fail "$PROJECT_DIR is not a Node.js project (no package.json)"
# Resolve app UUID
if [ -z "$APP_UUID" ]; then
APP_UUID=$(load_app_config "$PROJECT_DIR")
fi
if [ -z "$APP_UUID" ]; then
fail "No app UUID. Pass it as argument or create $PROJECT_DIR/.coolify-app with the UUID"
fi
# Save for next time
save_app_config "$PROJECT_DIR" "$APP_UUID" > /dev/null
# Deploy
cmd_deploy "$PROJECT_DIR" "$APP_UUID" "$SKIP_BUILD"

125
tools/extract-and-index-memory.py Normal file
View File

@ -0,0 +1,125 @@
#!/usr/bin/env python3
import json
import subprocess
import sys
import os
import glob
from datetime import datetime
def extract_conversations_from_sessions():
"""Extract the last 10 assistant turns from recent session files."""
sessions_dir = "/home/wdjones/.openclaw/agents/main/sessions/"
session_files = glob.glob(os.path.join(sessions_dir, "*.jsonl"))
# Filter out deleted files and sort by modification time (newest first)
session_files = [f for f in session_files if ".deleted." not in f]
session_files.sort(key=os.path.getmtime, reverse=True)
all_turns = []
print(f"Processing {len(session_files)} session files...")
for session_file in session_files:
try:
messages = []
with open(session_file, 'r') as f:
for line in f:
try:
data = json.loads(line)
if data.get('type') == 'message' and data.get('message', {}).get('role') in ['user', 'assistant']:
messages.append(data)
except json.JSONDecodeError:
continue
# Extract user-assistant pairs (turns)
i = 0
while i < len(messages):
if (messages[i]['message']['role'] == 'user' and
i + 1 < len(messages) and
messages[i+1]['message']['role'] == 'assistant'):
user_content = messages[i]['message']['content']
assistant_content = messages[i+1]['message']['content']
# Extract text from content arrays
user_text = ''
if isinstance(user_content, list):
for item in user_content:
if item.get('type') == 'text':
user_text += item.get('text', '')
else:
user_text = str(user_content)
assistant_text = ''
if isinstance(assistant_content, list):
for item in assistant_content:
if item.get('type') == 'text':
assistant_text += item.get('text', '')
else:
assistant_text = str(assistant_content)
# Filter out system messages and cron jobs
if (user_text.strip() and assistant_text.strip() and
not user_text.startswith('[cron:') and
not user_text.startswith('[System Message]') and
len(user_text.strip()) > 10 and # Substantial content
len(assistant_text.strip()) > 10):
turn = {
'user': user_text.strip(),
'assistant': assistant_text.strip(),
'agent_id': 'case',
'session': 'main'
}
all_turns.append(turn)
i += 2
else:
i += 1
except Exception as e:
print(f"Error processing {session_file}: {e}")
continue
print(f"Found {len(all_turns)} total conversation turns")
# Get last 10 turns
last_turns = all_turns[-10:] if all_turns else []
print(f"Processing last {len(last_turns)} turns")
# Send each turn to the auto-memory hook
processed_count = 0
for i, turn in enumerate(last_turns):
try:
# Run the auto-memory hook
proc = subprocess.Popen(
['python3', '/home/wdjones/.openclaw/workspace/tools/auto-memory-hook.py'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True
)
stdout, stderr = proc.communicate(input=json.dumps(turn))
if proc.returncode == 0:
processed_count += 1
print(f"✓ Turn {i+1}: Processed successfully")
if stdout.strip():
print(f" Output: {stdout.strip()}")
else:
print(f"✗ Turn {i+1}: Failed (exit code {proc.returncode})")
if stderr.strip():
print(f" Error: {stderr.strip()}")
except Exception as e:
print(f"✗ Turn {i+1}: Exception: {e}")
print(f"\nAuto-memory indexing complete: {processed_count}/{len(last_turns)} turns processed successfully")
return processed_count
if __name__ == "__main__":
extract_conversations_from_sessions()

98
tools/extract-memory-turns.py Normal file
View File

@ -0,0 +1,98 @@
#!/usr/bin/env python3
import json
import sys
import subprocess
import os
import glob
from datetime import datetime
def extract_conversations_from_file(filepath):
"""Extract conversation pairs from a single JSONL session file."""
pairs = []
messages = []
try:
with open(filepath, 'r') as f:
for line in f:
try:
data = json.loads(line.strip())
if data.get('type') == 'message' and 'message' in data:
messages.append(data)
except:
continue
except:
return pairs
# Extract conversation pairs (user + assistant)
user_msg = None
for msg in messages:
role = msg['message']['role']
if role == 'user':
user_msg = msg
elif role == 'assistant' and user_msg:
# Extract text content from both messages
user_content = ''
for content in user_msg['message']['content']:
if content['type'] == 'text':
user_content += content['text']
assistant_content = ''
for content in msg['message']['content']:
if content['type'] == 'text':
assistant_content += content['text']
# Skip cron job messages
if not user_content.startswith('[cron:') and user_content.strip() and assistant_content.strip():
pairs.append({
'user': user_content.strip(),
'assistant': assistant_content.strip(),
'agent_id': 'case',
'session': 'main',
'timestamp': msg.get('timestamp', '')
})
user_msg = None
return pairs
def main():
sessions_dir = '/home/wdjones/.openclaw/agents/main/sessions/'
# Get all session files sorted by modification time (newest first)
session_files = []
for filepath in glob.glob(os.path.join(sessions_dir, '*.jsonl')):
mtime = os.path.getmtime(filepath)
session_files.append((mtime, filepath))
session_files.sort(reverse=True)
# Extract conversations from recent files until we have enough
all_pairs = []
for mtime, filepath in session_files[:10]: # Check last 10 session files
pairs = extract_conversations_from_file(filepath)
all_pairs.extend(pairs)
print(f"Found {len(pairs)} pairs in {os.path.basename(filepath)}")
if len(all_pairs) >= 10:
break
# Get last 10 pairs
last_pairs = all_pairs[-10:] if len(all_pairs) >= 10 else all_pairs
print(f'Total pairs found: {len(all_pairs)}, processing last {len(last_pairs)}')
# Process each pair
for i, pair in enumerate(last_pairs):
try:
# Pipe to auto-memory-hook.py
proc = subprocess.Popen(['python3', '/home/wdjones/.openclaw/workspace/tools/auto-memory-hook.py'],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
stdout, stderr = proc.communicate(json.dumps(pair))
print(f'Pair {i+1}: Processed (exit code: {proc.returncode})')
if stderr:
print(f' stderr: {stderr.strip()}')
except Exception as e:
print(f'Pair {i+1}: Error - {e}')
if __name__ == '__main__':
main()

108
tools/extract_assistant_turns.py Normal file
View File

@ -0,0 +1,108 @@
#!/usr/bin/env python3
import json
import sys
import os
import glob
from pathlib import Path
def parse_jsonl_file(file_path):
"""Parse a JSONL session file and extract user/assistant message pairs"""
pairs = []
try:
with open(file_path, 'r', encoding='utf-8') as f:
lines = f.readlines()
# Parse each line as JSON
messages = []
for line in lines:
if line.strip():
try:
data = json.loads(line.strip())
if data.get("type") == "message":
messages.append(data)
except json.JSONDecodeError:
continue
# Extract user/assistant pairs
i = 0
while i < len(messages):
msg = messages[i]
role = msg.get("message", {}).get("role")
if role == "user":
user_content = extract_text_content(msg["message"]["content"])
# Look for the next assistant response
if i + 1 < len(messages):
next_msg = messages[i + 1]
if next_msg.get("message", {}).get("role") == "assistant":
assistant_content = extract_text_content(next_msg["message"]["content"])
# Skip auto-memory indexer cron jobs
if not user_content.startswith("[cron:") or "auto-memory-indexer" not in user_content:
pairs.append({
"user": user_content,
"assistant": assistant_content,
"agent_id": "case",
"session": "main"
})
i += 2 # Skip both messages
else:
i += 1
else:
i += 1
else:
i += 1
except Exception as e:
print(f"Error processing {file_path}: {e}", file=sys.stderr)
return pairs
def extract_text_content(content_array):
"""Extract text from content array, skipping thinking blocks"""
text_parts = []
for item in content_array:
if item.get("type") == "text":
text_parts.append(item.get("text", ""))
elif item.get("type") == "toolCall":
# Include tool calls in a simplified format
tool_name = item.get("name", "unknown")
text_parts.append(f"[Used tool: {tool_name}]")
return " ".join(text_parts).strip()
def find_session_files(sessions_dir):
"""Find all session files ordered by modification time"""
pattern = os.path.join(sessions_dir, "*.jsonl")
files = glob.glob(pattern)
# Sort by modification time, newest first
return sorted(files, key=lambda x: os.path.getmtime(x), reverse=True)
def main():
sessions_dir = "/home/wdjones/.openclaw/agents/main/sessions/"
# Find all session files
session_files = find_session_files(sessions_dir)
all_pairs = []
# Process session files until we have at least 10 assistant turns
for session_file in session_files:
pairs = parse_jsonl_file(session_file)
all_pairs.extend(pairs)
# Stop when we have enough pairs
if len(all_pairs) >= 10:
break
# Take the last 10 pairs
last_10_pairs = all_pairs[-10:]
# Output each pair as JSON to stdout
for pair in last_10_pairs:
print(json.dumps(pair))
if __name__ == "__main__":
main()

89
tools/extract_memory_turns.py Executable file
View File

@ -0,0 +1,89 @@
#!/usr/bin/env python3
import json
import sys
import subprocess
from pathlib import Path
def extract_turns(session_file):
"""Extract user-assistant turn pairs from session transcript"""
turns = []
messages = []
# Read and parse JSONL
with open(session_file, 'r') as f:
for line in f:
try:
data = json.loads(line.strip())
if data.get('type') == 'message' and 'message' in data:
msg = data['message']
if msg.get('role') in ['user', 'assistant']:
messages.append(msg)
except:
continue
# Pair user messages with assistant responses
for i in range(len(messages) - 1):
if (messages[i].get('role') == 'user' and
messages[i+1].get('role') == 'assistant'):
user_content = extract_text_content(messages[i])
assistant_content = extract_text_content(messages[i+1])
if user_content and assistant_content:
turns.append({
'user': user_content,
'assistant': assistant_content,
'agent_id': 'case',
'session': 'main'
})
return turns[-10:] # Last 10 turns
def extract_text_content(message):
"""Extract text content from message"""
content = message.get('content', [])
if isinstance(content, str):
return content
text_parts = []
for part in content:
if isinstance(part, dict) and part.get('type') == 'text':
text_parts.append(part.get('text', ''))
return ' '.join(text_parts).strip()
def main():
if len(sys.argv) != 2:
print("Usage: extract_memory_turns.py <session_file>")
sys.exit(1)
session_file = sys.argv[1]
turns = extract_turns(session_file)
print(f"Extracted {len(turns)} turns from {session_file}")
# Send each turn to auto-memory-hook.py
for i, turn in enumerate(turns):
try:
proc = subprocess.Popen(
['python3', '/home/wdjones/.openclaw/workspace/tools/auto-memory-hook.py'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True
)
json_input = json.dumps(turn)
stdout, stderr = proc.communicate(input=json_input)
if proc.returncode == 0:
print(f"Turn {i+1}: Processed successfully")
else:
print(f"Turn {i+1}: Error - {stderr}")
except Exception as e:
print(f"Turn {i+1}: Failed to process - {e}")
if __name__ == '__main__':
main()

87
tools/extract_session_turns.py Normal file
View File

@ -0,0 +1,87 @@
#!/usr/bin/env python3
import json
import sys
import subprocess
def extract_assistant_turns(session_file, num_turns=10):
"""Extract the last N assistant turns with their preceding user messages"""
with open(session_file, 'r') as f:
lines = f.readlines()
messages = []
for line in lines:
try:
data = json.loads(line.strip())
if data.get('type') == 'message' and 'message' in data:
msg = data['message']
if msg.get('role') in ['user', 'assistant']:
messages.append(msg)
except json.JSONDecodeError:
continue
# Find assistant messages and their preceding user messages
assistant_turns = []
for i in range(len(messages)):
if messages[i].get('role') == 'assistant':
# Look for the most recent user message before this assistant message
user_msg = None
for j in range(i-1, -1, -1):
if messages[j].get('role') == 'user':
user_msg = messages[j]
break
if user_msg:
# Extract content from messages
user_content = ""
if isinstance(user_msg.get('content'), list):
for content in user_msg['content']:
if content.get('type') == 'text':
user_content = content.get('text', '')
break
else:
user_content = user_msg.get('content', '')
assistant_content = ""
if isinstance(messages[i].get('content'), list):
for content in messages[i]['content']:
if content.get('type') == 'text':
assistant_content = content.get('text', '')
break
else:
assistant_content = messages[i].get('content', '')
if user_content and assistant_content:
assistant_turns.append({
'user': user_content,
'assistant': assistant_content
})
# Return the last N turns
return assistant_turns[-num_turns:]
def main():
session_file = sys.argv[1] if len(sys.argv) > 1 else None
if not session_file:
print("Usage: python3 extract_session_turns.py <session_file>")
sys.exit(1)
turns = extract_assistant_turns(session_file)
for turn in turns:
json_data = {
"user": turn['user'],
"assistant": turn['assistant'],
"agent_id": "case",
"session": "main"
}
# Pipe to auto-memory hook
try:
subprocess.run([
'python3', '/home/wdjones/.openclaw/workspace/tools/auto-memory-hook.py'
], input=json.dumps(json_data), text=True, check=True)
except subprocess.CalledProcessError as e:
print(f"Error running auto-memory-hook: {e}")
if __name__ == "__main__":
main()

69
tools/extract_turns.py Normal file
View File

@ -0,0 +1,69 @@
#!/usr/bin/env python3
import json
import sys
from collections import deque
def extract_last_assistant_turns(file_path, num_turns=10):
"""Extract last N assistant turns with their preceding user messages."""
messages = []
with open(file_path, 'r') as f:
for line in f:
try:
data = json.loads(line.strip())
if data.get('type') == 'message':
message_data = data.get('message', {})
role = message_data.get('role')
if role in ['user', 'assistant']:
content = message_data.get('content', [])
if isinstance(content, list) and content:
text = ''
for item in content:
if item.get('type') == 'text':
text += item.get('text', '')
elif isinstance(content, str):
text = content
else:
text = str(content)
messages.append({
'role': role,
'content': text,
'timestamp': data.get('timestamp'),
'id': data.get('id')
})
except (json.JSONDecodeError, KeyError) as e:
continue
# Find assistant turns with their preceding user messages
turns = []
for i in range(len(messages)):
if messages[i]['role'] == 'assistant':
# Look for the most recent user message before this assistant message
user_msg = None
for j in range(i-1, -1, -1):
if messages[j]['role'] == 'user':
user_msg = messages[j]['content']
break
if user_msg:
turns.append({
'user': user_msg,
'assistant': messages[i]['content'],
'agent_id': 'case',
'session': 'main'
})
# Return last N turns
return turns[-num_turns:] if len(turns) >= num_turns else turns
if __name__ == '__main__':
if len(sys.argv) != 2:
print("Usage: python3 extract_turns.py <jsonl_file>")
sys.exit(1)
file_path = sys.argv[1]
turns = extract_last_assistant_turns(file_path)
for turn in turns:
print(json.dumps(turn))

210
tools/n8n-orchestrator.py Normal file
View File

@ -0,0 +1,210 @@
#!/usr/bin/env python3
"""
n8n Workflow Orchestrator — Delegate API calls to n8n via webhooks.
The agent never touches external API credentials directly. Instead, it triggers
n8n workflows that hold the credentials securely.
Usage:
python3 n8n-orchestrator.py list # List configured workflows
python3 n8n-orchestrator.py trigger <workflow> [data] # Trigger a workflow
python3 n8n-orchestrator.py add <name> <webhook_url> # Register a workflow
python3 n8n-orchestrator.py status # Check n8n health
python3 n8n-orchestrator.py log # Show recent executions
Setup:
1. Install n8n: npm install -g n8n (or Docker)
2. Create webhook-triggered workflows in n8n
3. Register them here with 'add' command
4. Agents trigger workflows without knowing API keys
"""
import json
import sys
import time
from datetime import datetime
from pathlib import Path
from urllib.request import urlopen, Request
from urllib.error import URLError, HTTPError
DATA_DIR = Path(__file__).parent.parent / "data" / "n8n-orchestrator"
DATA_DIR.mkdir(parents=True, exist_ok=True)
WORKFLOWS_FILE = DATA_DIR / "workflows.json"
LOG_FILE = DATA_DIR / "executions.jsonl"
CONFIG_FILE = DATA_DIR / "config.json"
DEFAULT_CONFIG = {
"n8n_base_url": "http://localhost:5678",
"timeout_seconds": 30,
}
# Pre-built workflow templates agents can use
WORKFLOW_TEMPLATES = {
"send-email": {
"description": "Send an email via configured SMTP",
"expected_payload": {"to": "email", "subject": "string", "body": "string"},
},
"fetch-crypto-price": {
"description": "Get current price for a crypto asset",
"expected_payload": {"symbol": "BTC"},
},
"post-discord": {
"description": "Post a message to a Discord channel",
"expected_payload": {"channel": "channel-name", "message": "string"},
},
"google-sheets-append": {
"description": "Append a row to a Google Sheet",
"expected_payload": {"sheet_id": "string", "values": ["col1", "col2"]},
},
"telegram-notify": {
"description": "Send a Telegram notification",
"expected_payload": {"chat_id": "string", "message": "string"},
},
}
def load_config():
if CONFIG_FILE.exists():
return json.loads(CONFIG_FILE.read_text())
CONFIG_FILE.write_text(json.dumps(DEFAULT_CONFIG, indent=2))
return DEFAULT_CONFIG
def load_workflows():
if WORKFLOWS_FILE.exists():
return json.loads(WORKFLOWS_FILE.read_text())
return {}
def save_workflows(workflows):
WORKFLOWS_FILE.write_text(json.dumps(workflows, indent=2))
def log_execution(workflow_name, payload, response, success, duration_ms):
entry = {
"ts": datetime.now().isoformat(),
"workflow": workflow_name,
"payload_keys": list(payload.keys()) if isinstance(payload, dict) else str(type(payload)),
"success": success,
"duration_ms": duration_ms,
"response_preview": str(response)[:200] if response else None,
}
with open(LOG_FILE, "a") as f:
f.write(json.dumps(entry) + "\n")
def trigger_workflow(name, payload=None):
"""Trigger an n8n workflow by name."""
workflows = load_workflows()
if name not in workflows:
print(f"❌ Workflow '{name}' not found. Use 'list' to see available workflows.")
return None
wf = workflows[name]
url = wf["webhook_url"]
config = load_config()
print(f"🔄 Triggering workflow: {name}")
print(f" URL: {url}")
payload = payload or {}
data = json.dumps(payload).encode('utf-8')
req = Request(url, data=data, headers={
'Content-Type': 'application/json',
'User-Agent': 'n8n-orchestrator/1.0',
})
start = time.time()
try:
with urlopen(req, timeout=config.get("timeout_seconds", 30)) as resp:
duration = int((time.time() - start) * 1000)
body = resp.read().decode('utf-8')
try:
result = json.loads(body)
except json.JSONDecodeError:
result = body
log_execution(name, payload, result, True, duration)
print(f"✅ Success ({duration}ms)")
print(f" Response: {json.dumps(result, indent=2) if isinstance(result, dict) else result[:200]}")
return result
except (HTTPError, URLError) as e:
duration = int((time.time() - start) * 1000)
error_msg = str(e)
log_execution(name, payload, error_msg, False, duration)
print(f"❌ Failed ({duration}ms): {error_msg}")
return None
def add_workflow(name, webhook_url, description=""):
workflows = load_workflows()
workflows[name] = {
"webhook_url": webhook_url,
"description": description,
"added": datetime.now().isoformat(),
}
save_workflows(workflows)
print(f"✅ Added workflow: {name} -> {webhook_url}")
def list_workflows():
workflows = load_workflows()
print(f"\n📋 Registered Workflows ({len(workflows)}):")
print("=" * 50)
if not workflows:
print(" No workflows registered yet.")
print(" Use: python3 n8n-orchestrator.py add <name> <webhook_url>")
for name, wf in workflows.items():
print(f" 🔗 {name}")
print(f" URL: {wf['webhook_url']}")
if wf.get("description"):
print(f" Desc: {wf['description']}")
print(f"\n📦 Available Templates:")
for name, tmpl in WORKFLOW_TEMPLATES.items():
print(f" 📝 {name}: {tmpl['description']}")
print(f" Payload: {json.dumps(tmpl['expected_payload'])}")
def check_n8n_status():
config = load_config()
base = config.get("n8n_base_url", "http://localhost:5678")
print(f"🏥 Checking n8n at {base}...")
try:
req = Request(f"{base}/healthz", headers={'User-Agent': 'n8n-orchestrator/1.0'})
with urlopen(req, timeout=5) as resp:
print(f"✅ n8n is running (HTTP {resp.status})")
except Exception as e:
print(f"❌ n8n unreachable: {e}")
print(f" Start it with: n8n start (or docker run -d -p 5678:5678 n8nio/n8n)")
def show_log(limit=20):
if not LOG_FILE.exists():
print("No executions logged yet.")
return
lines = LOG_FILE.read_text().strip().split('\n')
print(f"\n📜 Recent Executions (last {limit}):")
for line in lines[-limit:]:
entry = json.loads(line)
icon = "✅" if entry["success"] else "❌"
print(f" {icon} [{entry['ts'][:19]}] {entry['workflow']} ({entry['duration_ms']}ms)")
if __name__ == "__main__":
args = sys.argv[1:]
if not args or args[0] == "list":
list_workflows()
elif args[0] == "trigger" and len(args) >= 2:
payload = json.loads(args[2]) if len(args) > 2 else {}
trigger_workflow(args[1], payload)
elif args[0] == "add" and len(args) >= 3:
desc = args[3] if len(args) > 3 else ""
add_workflow(args[1], args[2], desc)
elif args[0] == "status":
check_n8n_status()
elif args[0] == "log":
show_log()
else:
print(__doc__)

75
tools/parse-session.py Executable file
View File

@ -0,0 +1,75 @@
#!/usr/bin/env python3
"""
Parse OpenClaw session transcripts and extract conversation turns
"""
import json
import sys
from typing import List, Dict, Any
def extract_conversation_turns(session_file: str) -> List[Dict[str, str]]:
"""Extract conversation turns from a session file"""
turns = []
current_user_message = None
try:
with open(session_file, 'r') as f:
for line in f:
try:
data = json.loads(line.strip())
if data.get('type') == 'message' and 'message' in data:
message = data['message']
role = message.get('role')
content = message.get('content', [])
if role == 'user':
# Extract text content from user message
text_content = []
for item in content:
if item.get('type') == 'text':
text_content.append(item.get('text', ''))
current_user_message = '\n'.join(text_content)
elif role == 'assistant' and current_user_message:
# Extract text content from assistant message (skip tool calls)
text_content = []
for item in content:
if item.get('type') == 'text':
text_content.append(item.get('text', ''))
if text_content:
assistant_text = '\n'.join(text_content)
turns.append({
'user': current_user_message,
'assistant': assistant_text,
'agent_id': 'case',
'session': 'main'
})
current_user_message = None
except json.JSONDecodeError:
continue
except Exception as e:
print(f"Error reading session file: {e}", file=sys.stderr)
return []
return turns
def main():
if len(sys.argv) != 2:
print("Usage: parse-session.py <session_file>")
sys.exit(1)
session_file = sys.argv[1]
turns = extract_conversation_turns(session_file)
# Get last 10 assistant turns
last_turns = turns[-10:] if len(turns) >= 10 else turns
# Output as JSON for the auto-memory hook
for turn in last_turns:
print(json.dumps(turn))
if __name__ == "__main__":
main()

362
tools/polymarket-autopilot.py Normal file
View File

@ -0,0 +1,362 @@
#!/usr/bin/env python3
"""
Polymarket Autopilot — Paper trading with TAIL/BONDING/SPREAD strategies.
Fetches live Polymarket data, runs strategy signals, manages a paper portfolio,
and generates daily summaries. Inspired by MoonDev's approach.
Usage:
python3 polymarket-autopilot.py scan # Scan markets for opportunities
python3 polymarket-autopilot.py trade # Run strategies & execute paper trades
python3 polymarket-autopilot.py portfolio # Show current paper portfolio
python3 polymarket-autopilot.py summary # Daily performance summary
python3 polymarket-autopilot.py cron # Full cycle (scan + trade + log)
Strategies:
TAIL — Buy extreme tail events (< 10% probability) for asymmetric upside
SPREAD — Exploit price gaps between related markets
MOMENTUM — Ride probability momentum (big moves in last 24h)
"""
import json
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
from urllib.request import urlopen, Request
from urllib.error import URLError, HTTPError
DATA_DIR = Path(__file__).parent.parent / "data" / "polymarket"
DATA_DIR.mkdir(parents=True, exist_ok=True)
PORTFOLIO_FILE = DATA_DIR / "portfolio.json"
TRADES_FILE = DATA_DIR / "trades.jsonl"
CONFIG_FILE = DATA_DIR / "config.json"
POLYMARKET_API = "https://gamma-api.polymarket.com"
DEFAULT_CONFIG = {
"starting_balance": 10000,
"max_position_pct": 5, # Max 5% of portfolio per trade
"tail_threshold": 0.10, # Buy events under 10% probability
"momentum_threshold": 0.15, # 15% probability change = momentum signal
"min_volume": 10000, # Minimum market volume
"strategies_enabled": ["tail", "momentum"],
}
def load_config():
if CONFIG_FILE.exists():
return json.loads(CONFIG_FILE.read_text())
CONFIG_FILE.write_text(json.dumps(DEFAULT_CONFIG, indent=2))
return DEFAULT_CONFIG
def load_portfolio():
if PORTFOLIO_FILE.exists():
return json.loads(PORTFOLIO_FILE.read_text())
portfolio = {
"balance": load_config()["starting_balance"],
"positions": [],
"total_trades": 0,
"pnl": 0,
"created": datetime.now().isoformat(),
}
save_portfolio(portfolio)
return portfolio
def save_portfolio(portfolio):
portfolio["updated"] = datetime.now().isoformat()
PORTFOLIO_FILE.write_text(json.dumps(portfolio, indent=2))
def log_trade(trade):
with open(TRADES_FILE, "a") as f:
f.write(json.dumps(trade) + "\n")
def fetch_json(url):
req = Request(url, headers={'User-Agent': 'polymarket-autopilot/1.0'})
try:
with urlopen(req, timeout=15) as resp:
return json.loads(resp.read().decode('utf-8'))
except (HTTPError, URLError) as e:
print(f" ❌ API error: {e}")
return None
def fetch_markets(limit=50, active=True):
"""Fetch markets from Polymarket CLOB API."""
url = f"{POLYMARKET_API}/markets?limit={limit}&active={str(active).lower()}&order=volume&ascending=false"
data = fetch_json(url)
if not data:
return []
markets = []
for m in (data if isinstance(data, list) else data.get("data", data.get("markets", []))):
if isinstance(m, dict):
markets.append({
"id": m.get("id", m.get("condition_id", "")),
"question": m.get("question", m.get("title", "")),
"volume": float(m.get("volume", m.get("volumeNum", 0)) or 0),
"liquidity": float(m.get("liquidity", 0) or 0),
"outcomes": m.get("outcomes", []),
"outcomePrices": m.get("outcomePrices", m.get("outcome_prices", [])),
"end_date": m.get("end_date_iso", m.get("endDate", "")),
"active": m.get("active", True),
"slug": m.get("slug", ""),
})
return markets
def parse_prices(market):
"""Parse outcome prices into float list."""
prices = market.get("outcomePrices", [])
if isinstance(prices, str):
try:
prices = json.loads(prices)
except:
prices = []
return [float(p) for p in prices if p]
def strategy_tail(markets, config):
"""TAIL: Find extreme low-probability events for asymmetric bets."""
signals = []
threshold = config.get("tail_threshold", 0.10)
for m in markets:
prices = parse_prices(m)
outcomes = m.get("outcomes", [])
if isinstance(outcomes, str):
try:
outcomes = json.loads(outcomes)
except:
outcomes = []
for i, price in enumerate(prices):
if 0.01 < price < threshold and m["volume"] > config.get("min_volume", 10000):
outcome_name = outcomes[i] if i < len(outcomes) else f"Outcome {i}"
signals.append({
"strategy": "TAIL",
"market": m["question"][:80],
"outcome": outcome_name,
"price": price,
"potential_return": f"{(1/price - 1)*100:.0f}%",
"volume": m["volume"],
"market_id": m["id"],
"action": "BUY",
})
return sorted(signals, key=lambda x: x["price"])[:10]
def strategy_momentum(markets, config):
"""MOMENTUM: Find markets with big recent probability moves."""
# Note: Would need historical price data for true momentum.
# For now, flag high-volume active markets near decision points.
signals = []
for m in markets:
prices = parse_prices(m)
outcomes = m.get("outcomes", [])
if isinstance(outcomes, str):
try:
outcomes = json.loads(outcomes)
except:
outcomes = []
for i, price in enumerate(prices):
# Markets near tipping points (40-60%) with high volume
if 0.40 < price < 0.60 and m["volume"] > config.get("min_volume", 10000) * 5:
outcome_name = outcomes[i] if i < len(outcomes) else f"Outcome {i}"
signals.append({
"strategy": "MOMENTUM",
"market": m["question"][:80],
"outcome": outcome_name,
"price": price,
"volume": m["volume"],
"market_id": m["id"],
"action": "WATCH",
})
return sorted(signals, key=lambda x: x["volume"], reverse=True)[:10]
def scan_markets():
config = load_config()
print(f"\n🔍 Polymarket Scanner — {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
print("=" * 60)
print(" Fetching markets...")
markets = fetch_markets(limit=100)
if not markets:
print(" ❌ Could not fetch markets")
return []
print(f" 📊 Found {len(markets)} active markets")
all_signals = []
if "tail" in config.get("strategies_enabled", []):
print("\n🎯 TAIL Strategy (low-probability events):")
tail_signals = strategy_tail(markets, config)
for s in tail_signals:
print(f" 💰 [{s['price']:.1%}] {s['market']}")
print(f" {s['outcome']} → Potential: {s['potential_return']}")
all_signals.extend(tail_signals)
if not tail_signals:
print(" No tail signals found.")
if "momentum" in config.get("strategies_enabled", []):
print("\n📈 MOMENTUM Strategy (high-volume tipping points):")
mom_signals = strategy_momentum(markets, config)
for s in mom_signals:
print(f" 👀 [{s['price']:.1%}] {s['market']}")
print(f" Vol: ${s['volume']:,.0f}")
all_signals.extend(mom_signals)
if not mom_signals:
print(" No momentum signals found.")
# Save signals
(DATA_DIR / "latest-signals.json").write_text(json.dumps({
"timestamp": datetime.now().isoformat(),
"signals": all_signals,
"markets_scanned": len(markets),
}, indent=2))
return all_signals
def execute_paper_trades(signals):
"""Execute paper trades based on signals."""
config = load_config()
portfolio = load_portfolio()
max_position = portfolio["balance"] * (config.get("max_position_pct", 5) / 100)
trades_made = 0
for signal in signals:
if signal["action"] != "BUY":
continue
if portfolio["balance"] < 100:
break
# Check if we already have a position in this market
existing = [p for p in portfolio["positions"] if p["market_id"] == signal["market_id"]]
if existing:
continue
size = min(max_position, portfolio["balance"] * 0.03) # 3% per trade
shares = size / signal["price"]
trade = {
"ts": datetime.now().isoformat(),
"strategy": signal["strategy"],
"market": signal["market"],
"outcome": signal["outcome"],
"action": "BUY",
"price": signal["price"],
"size": round(size, 2),
"shares": round(shares, 2),
"market_id": signal["market_id"],
}
portfolio["balance"] -= size
portfolio["positions"].append({
"market_id": signal["market_id"],
"market": signal["market"],
"outcome": signal["outcome"],
"entry_price": signal["price"],
"shares": round(shares, 2),
"cost": round(size, 2),
"strategy": signal["strategy"],
"opened": datetime.now().isoformat(),
})
portfolio["total_trades"] += 1
log_trade(trade)
trades_made += 1
print(f" 📝 PAPER BUY: {signal['outcome']} @ {signal['price']:.1%} (${size:.0f})")
save_portfolio(portfolio)
return trades_made
def show_portfolio():
portfolio = load_portfolio()
print(f"\n💼 Paper Portfolio — {datetime.now().strftime('%Y-%m-%d %H:%M')}")
print("=" * 60)
print(f" 💰 Cash Balance: ${portfolio['balance']:,.2f}")
print(f" 📊 Total Trades: {portfolio['total_trades']}")
print(f" 📈 P&L: ${portfolio['pnl']:,.2f}")
if portfolio["positions"]:
print(f"\n Open Positions ({len(portfolio['positions'])}):")
for p in portfolio["positions"]:
print(f" [{p['strategy']}] {p['market'][:60]}")
print(f" {p['outcome']} | Entry: {p['entry_price']:.1%} | Cost: ${p['cost']:.0f} | Shares: {p['shares']:.0f}")
else:
print("\n No open positions.")
def daily_summary():
portfolio = load_portfolio()
trades_today = []
if TRADES_FILE.exists():
today = datetime.now().strftime('%Y-%m-%d')
for line in TRADES_FILE.read_text().strip().split('\n'):
if line:
t = json.loads(line)
if t["ts"].startswith(today):
trades_today.append(t)
total_invested = sum(p["cost"] for p in portfolio["positions"])
summary = {
"date": datetime.now().strftime('%Y-%m-%d'),
"balance": portfolio["balance"],
"positions": len(portfolio["positions"]),
"total_invested": total_invested,
"portfolio_value": portfolio["balance"] + total_invested,
"trades_today": len(trades_today),
"total_trades": portfolio["total_trades"],
}
print(f"\n📊 Daily Summary — {summary['date']}")
print("=" * 40)
print(f" Portfolio Value: ${summary['portfolio_value']:,.2f}")
print(f" Cash: ${summary['balance']:,.2f}")
print(f" Invested: ${summary['total_invested']:,.2f}")
print(f" Positions: {summary['positions']}")
print(f" Trades Today: {summary['trades_today']}")
(DATA_DIR / "daily-summary.json").write_text(json.dumps(summary, indent=2))
return summary
def run_cron():
"""Full cron cycle: scan + trade + log."""
print("🤖 Polymarket Autopilot — Cron Cycle")
signals = scan_markets()
if signals:
buy_signals = [s for s in signals if s["action"] == "BUY"]
if buy_signals:
print(f"\n💸 Executing {len(buy_signals)} paper trades...")
execute_paper_trades(buy_signals)
daily_summary()
if __name__ == "__main__":
cmd = sys.argv[1] if len(sys.argv) > 1 else "scan"
if cmd == "scan":
scan_markets()
elif cmd == "trade":
signals = scan_markets()
execute_paper_trades([s for s in signals if s["action"] == "BUY"])
elif cmd == "portfolio":
show_portfolio()
elif cmd == "summary":
daily_summary()
elif cmd == "cron":
run_cron()
else:
print(__doc__)

136
tools/process-session-data.py Normal file
View File

@ -0,0 +1,136 @@
#!/usr/bin/env python3
import json
import subprocess
import sys
import os
def extract_assistant_turns(session_file):
"""Extract user-assistant pairs from a session file."""
conversation_pairs = []
try:
with open(session_file, 'r') as f:
lines = f.readlines()
messages = []
for line in lines:
try:
data = json.loads(line.strip())
if data.get('type') == 'message' and 'message' in data:
messages.append(data['message'])
except json.JSONDecodeError:
continue
# Group messages into user-assistant pairs
i = 0
while i < len(messages):
if messages[i].get('role') == 'user':
user_msg = messages[i]
# Look for the next assistant message
j = i + 1
while j < len(messages) and messages[j].get('role') != 'assistant':
j += 1
if j < len(messages) and messages[j].get('role') == 'assistant':
assistant_msg = messages[j]
# Extract text content from both messages
user_text = ""
assistant_text = ""
# Extract user message text
if 'content' in user_msg:
for content in user_msg['content']:
if content.get('type') == 'text':
user_text += content.get('text', '')
# Extract assistant message text (skip thinking blocks)
if 'content' in assistant_msg:
for content in assistant_msg['content']:
if content.get('type') == 'text':
assistant_text += content.get('text', '')
if user_text.strip() and assistant_text.strip():
conversation_pairs.append({
'user': user_text.strip(),
'assistant': assistant_text.strip()
})
i = j + 1
else:
i += 1
else:
i += 1
except Exception as e:
print(f"Error processing {session_file}: {e}")
return []
# Return last 10 pairs
return conversation_pairs[-10:]
def send_to_memory_hook(pairs):
"""Send conversation pairs to the auto-memory hook."""
script_path = "/home/wdjones/.openclaw/workspace/tools/auto-memory-hook.py"
for pair in pairs:
try:
payload = {
"user": pair['user'],
"assistant": pair['assistant'],
"agent_id": "case",
"session": "main"
}
json_payload = json.dumps(payload)
# Send to the memory hook script
result = subprocess.run(
['python3', script_path],
input=json_payload,
text=True,
capture_output=True
)
if result.returncode != 0:
print(f"Error from auto-memory-hook.py: {result.stderr}")
else:
print(f"Processed pair: {len(pair['user'])} user chars, {len(pair['assistant'])} assistant chars")
except Exception as e:
print(f"Error sending to memory hook: {e}")
def main():
# Find the most recent session file (excluding current session)
sessions_dir = "/home/wdjones/.openclaw/agents/main/sessions/"
# Get session files sorted by modification time
session_files = []
for filename in os.listdir(sessions_dir):
if filename.endswith('.jsonl') and not filename.endswith('.lock'):
filepath = os.path.join(sessions_dir, filename)
mtime = os.path.getmtime(filepath)
session_files.append((mtime, filepath))
session_files.sort(key=lambda x: x[0], reverse=True)
if len(session_files) < 2:
print("Need at least 2 session files to avoid current session")
return
# Use second most recent (avoiding current session)
session_file = session_files[1][1]
print(f"Processing session file: {session_file}")
# Extract assistant turns
pairs = extract_assistant_turns(session_file)
print(f"Extracted {len(pairs)} conversation pairs")
if pairs:
send_to_memory_hook(pairs)
print("Sent pairs to auto-memory-hook.py")
else:
print("No conversation pairs found")
if __name__ == "__main__":
main()

96
tools/process_sessions.py Normal file
View File

@ -0,0 +1,96 @@
#!/usr/bin/env python3
import json
import os
import sys
import glob
from collections import defaultdict
from datetime import datetime
def extract_text_content(content_list):
"""Extract text content from message content array, excluding thinking blocks."""
text_parts = []
for item in content_list:
if item.get('type') == 'text':
text_parts.append(item['text'])
return ' '.join(text_parts).strip()
def process_session_file(file_path):
"""Process a single JSONL session file and extract conversation turns."""
turns = []
try:
with open(file_path, 'r') as f:
lines = f.readlines()
messages = []
for line in lines:
try:
data = json.loads(line.strip())
if data.get('type') == 'message' and 'message' in data:
messages.append(data)
except json.JSONDecodeError:
continue
# Group messages into conversation turns (user -> assistant pairs)
current_user_msg = None
for msg in messages:
role = msg['message']['role']
content = msg['message'].get('content', [])
if role == 'user':
current_user_msg = extract_text_content(content)
elif role == 'assistant' and current_user_msg:
assistant_text = extract_text_content(content)
if assistant_text and current_user_msg: # Only include if both have content
turns.append({
'user': current_user_msg,
'assistant': assistant_text,
'timestamp': msg.get('timestamp')
})
current_user_msg = None
return turns
except Exception as e:
print(f"Error processing {file_path}: {e}", file=sys.stderr)
return []
def main():
sessions_dir = "/home/wdjones/.openclaw/agents/main/sessions/"
# Get all non-deleted session files
session_files = []
for file_path in glob.glob(os.path.join(sessions_dir, "*.jsonl")):
if not file_path.endswith('.deleted'):
session_files.append(file_path)
# Sort by modification time (newest first)
session_files.sort(key=os.path.getmtime, reverse=True)
all_turns = []
# Process session files until we have enough turns
for session_file in session_files:
turns = process_session_file(session_file)
all_turns.extend(turns)
if len(all_turns) >= 10:
break
# Get the last 10 turns
recent_turns = all_turns[-10:] if len(all_turns) > 10 else all_turns
# Output the conversation turns as JSON for the auto-memory script
for turn in recent_turns:
output = {
"user": turn['user'],
"assistant": turn['assistant'],
"agent_id": "case",
"session": "main"
}
print(json.dumps(output))
if __name__ == "__main__":
main()

110
tools/re-breakeven.py Normal file
View File

@ -0,0 +1,110 @@
#!/usr/bin/env python3
"""Find break-even scenarios for rental property cash flow Day 1"""
# Fixed assumptions
MONTHLY_RENT = 2000
VACANCY = 0.05
EFFECTIVE_MONTHLY = MONTHLY_RENT * (1 - VACANCY) # $1,900
PROPERTY_TAX_RATE = 0.007 # ~$2400 on $345K
INSURANCE_RATE = 0.004 # ~$1400 on $345K
MAINTENANCE_RATE = 0.006 # ~$2000 on $345K
DOWN_PAYMENT_PCT = 0.20
print("=" * 70)
print("BREAK-EVEN ANALYSIS: What price + rate = Day 1 cash flow positive?")
print("=" * 70)
print(f"\nFixed: Rent $2,000/mo | Vacancy 5% | Effective income: ${EFFECTIVE_MONTHLY:,.0f}/mo")
print(f"Down payment: 20% | 30-year fixed\n")
# Calculate monthly mortgage payment
def monthly_payment(principal, annual_rate, years=30):
r = annual_rate / 12
n = years * 12
if r == 0:
return principal / n
return principal * (r * (1 + r)**n) / ((1 + r)**n - 1)
print(f"{'Price':>10} {'Rate':>6} {'Down':>10} {'Loan':>10} {'P&I':>8} {'Tax/Ins':>8} {'Maint':>7} {'Total':>8} {'Cash Flow':>10} {'Rent/Price':>10}")
print("-" * 100)
results = []
for price in range(200000, 360000, 10000):
for rate_bps in range(300, 700, 25):
rate = rate_bps / 10000
down = price * DOWN_PAYMENT_PCT
loan = price * (1 - DOWN_PAYMENT_PCT)
pi = monthly_payment(loan, rate)
tax_ins = (price * PROPERTY_TAX_RATE + price * INSURANCE_RATE) / 12
maint = (price * MAINTENANCE_RATE) / 12
total_monthly = pi + tax_ins + maint
cash_flow = EFFECTIVE_MONTHLY - total_monthly
rent_to_price = (MONTHLY_RENT / price) * 100
results.append((price, rate, down, loan, pi, tax_ins, maint, total_monthly, cash_flow, rent_to_price))
# Show scenarios that are cash flow positive or near break-even
print("\n📗 CASH FLOW POSITIVE SCENARIOS (Day 1):")
print("-" * 100)
positive = [r for r in results if r[8] >= 0]
for r in sorted(positive, key=lambda x: -x[8])[:20]:
price, rate, down, loan, pi, tax_ins, maint, total, cf, rtp = r
print(f"${price:>9,} {rate:>5.2%} ${down:>9,} ${loan:>9,} ${pi:>7,.0f} ${tax_ins:>7,.0f} ${maint:>6,.0f} ${total:>7,.0f} ${cf:>+8,.0f} {rtp:.2f}%")
print(f"\n\n📕 YOUR CURRENT DEAL ($345K @ 6.25%):")
price, rate = 345000, 0.0625
down = price * DOWN_PAYMENT_PCT
loan = price * (1 - DOWN_PAYMENT_PCT)
pi = monthly_payment(loan, rate)
tax_ins = (price * PROPERTY_TAX_RATE + price * INSURANCE_RATE) / 12
maint = (price * MAINTENANCE_RATE) / 12
total = pi + tax_ins + maint
cf = EFFECTIVE_MONTHLY - total
print(f"${price:>9,} {rate:>5.2%} ${down:>9,} ${loan:>9,} ${pi:>7,.0f} ${tax_ins:>7,.0f} ${maint:>6,.0f} ${total:>7,.0f} ${cf:>+8,.0f} {(MONTHLY_RENT/price)*100:.2f}%")
# Find exact break-even rate at different price points
print(f"\n\n📊 BREAK-EVEN INTEREST RATE BY PRICE POINT:")
print("-" * 50)
for price in [200000, 220000, 240000, 260000, 280000, 300000, 320000, 345000]:
for rate_bps in range(100, 1000, 1):
rate = rate_bps / 10000
loan = price * (1 - DOWN_PAYMENT_PCT)
pi = monthly_payment(loan, rate)
tax_ins = (price * PROPERTY_TAX_RATE + price * INSURANCE_RATE) / 12
maint = (price * MAINTENANCE_RATE) / 12
total = pi + tax_ins + maint
cf = EFFECTIVE_MONTHLY - total
if cf <= 0:
# Previous rate was break-even
be_rate = (rate_bps - 1) / 10000
print(f" ${price:>9,} → break-even at {be_rate:.2%} or lower")
break
else:
print(f" ${price:>9,} → cash flows positive even at 10%+")
# Find exact break-even price at different rates
print(f"\n\n📊 BREAK-EVEN PRICE BY INTEREST RATE:")
print("-" * 50)
for rate_pct in [3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.25]:
rate = rate_pct / 100
for price in range(400000, 100000, -1000):
loan = price * (1 - DOWN_PAYMENT_PCT)
pi = monthly_payment(loan, rate)
tax_ins = (price * PROPERTY_TAX_RATE + price * INSURANCE_RATE) / 12
maint = (price * MAINTENANCE_RATE) / 12
total = pi + tax_ins + maint
cf = EFFECTIVE_MONTHLY - total
if cf >= 0:
print(f" {rate_pct:.2f}% → max price ${price:>9,} (rent/price: {(MONTHLY_RENT/price)*100:.2f}%)")
break
else:
print(f" {rate_pct:.2f}% → no viable price found")
# The 1% rule check
print(f"\n\n📊 1% RULE CHECK:")
print(f" $2,000 rent × 100 = $200,000 max purchase price")
print(f" Your deal: $345,000 = {(2000/345000)*100:.2f}% (needs to be ≥1.0%)")
print(f" To hit 1% at $345K, rent needs to be: ${345000*0.01:,.0f}/mo")
print(f" To hit 0.8% at $345K, rent needs to be: ${345000*0.008:,.0f}/mo")

233
tools/reddit-market-intel.py Normal file
View File

@ -0,0 +1,233 @@
#!/usr/bin/env python3
"""
Reddit Market Intel — Scan Reddit for crypto/market sentiment and alpha.
Scans configurable subreddits for hot topics, sentiment shifts, and emerging narratives.
No API key required (uses public JSON endpoints).
Usage:
python3 reddit-market-intel.py # Full scan
python3 reddit-market-intel.py --quick # Quick scan (top posts only)
python3 reddit-market-intel.py --topic "solana" # Scan for specific topic
"""
import json
import time
import re
import sys
from datetime import datetime, timezone
from pathlib import Path
from urllib.request import urlopen, Request
from urllib.error import URLError, HTTPError
from collections import Counter
DATA_DIR = Path(__file__).parent.parent / "data" / "reddit-intel"
DATA_DIR.mkdir(parents=True, exist_ok=True)
USER_AGENT = "reddit-market-intel/1.0"
RATE_LIMIT = 2
last_request = 0
DEFAULT_SUBREDDITS = [
"cryptocurrency", "Bitcoin", "ethtrader", "CryptoMarkets",
"wallstreetbets", "stocks", "investing", "economy",
"solana", "defi", "polymarket"
]
SENTIMENT_POSITIVE = {"bullish", "moon", "pump", "gains", "breakout", "ath", "buy", "long",
"🚀", "💎", "accumulate", "undervalued", "rally"}
SENTIMENT_NEGATIVE = {"bearish", "dump", "crash", "sell", "short", "scam", "rug",
"overvalued", "bubble", "liquidat", "rekt", "fear"}
def fetch_json(url):
global last_request
elapsed = time.time() - last_request
if elapsed < RATE_LIMIT:
time.sleep(RATE_LIMIT - elapsed)
req = Request(url, headers={'User-Agent': USER_AGENT})
try:
with urlopen(req, timeout=15) as resp:
last_request = time.time()
return json.loads(resp.read().decode('utf-8'))
except (HTTPError, URLError) as e:
if hasattr(e, 'code') and e.code == 429:
time.sleep(10)
return fetch_json(url)
return None
def get_hot_posts(subreddit, limit=10):
data = fetch_json(f"https://www.reddit.com/r/{subreddit}/hot.json?limit={limit}")
if not data:
return []
posts = []
for child in data.get("data", {}).get("children", []):
d = child.get("data", {})
posts.append({
"title": d.get("title", ""),
"score": d.get("score", 0),
"comments": d.get("num_comments", 0),
"url": f"https://reddit.com{d.get('permalink', '')}",
"created": datetime.fromtimestamp(d.get("created_utc", 0), tz=timezone.utc).isoformat(),
"selftext": (d.get("selftext", "") or "")[:500],
"flair": d.get("link_flair_text", ""),
"subreddit": subreddit,
})
return posts
def analyze_sentiment(text):
words = set(text.lower().split())
pos = len(words & SENTIMENT_POSITIVE)
neg = len(words & SENTIMENT_NEGATIVE)
if pos > neg:
return "bullish", pos - neg
elif neg > pos:
return "bearish", neg - pos
return "neutral", 0
def extract_tickers(text):
"""Extract potential crypto/stock tickers ($BTC, $ETH, etc.)"""
return list(set(re.findall(r'\$([A-Z]{2,6})', text)))
def scan_subreddit(subreddit, limit=10):
print(f" 📡 r/{subreddit}...", end=" ", flush=True)
posts = get_hot_posts(subreddit, limit)
if not posts:
print("❌ failed")
return None
sentiments = []
all_tickers = []
high_engagement = []
for p in posts:
full_text = f"{p['title']} {p['selftext']}"
sent, strength = analyze_sentiment(full_text)
p["sentiment"] = sent
p["sentiment_strength"] = strength
sentiments.append(sent)
tickers = extract_tickers(full_text)
all_tickers.extend(tickers)
p["tickers"] = tickers
# High engagement = lots of comments relative to score
if p["comments"] > 50 or p["score"] > 500:
high_engagement.append(p)
sentiment_counts = Counter(sentiments)
ticker_counts = Counter(all_tickers)
overall = "neutral"
if sentiment_counts.get("bullish", 0) > sentiment_counts.get("bearish", 0):
overall = "bullish"
elif sentiment_counts.get("bearish", 0) > sentiment_counts.get("bullish", 0):
overall = "bearish"
print(f"✅ {len(posts)} posts | sentiment: {overall}")
return {
"subreddit": subreddit,
"posts": posts,
"overall_sentiment": overall,
"sentiment_breakdown": dict(sentiment_counts),
"trending_tickers": dict(ticker_counts.most_common(10)),
"high_engagement": high_engagement,
}
def search_topic(topic, limit=25):
"""Search Reddit for a specific topic across all subreddits."""
print(f"\n🔍 Searching Reddit for: '{topic}'")
data = fetch_json(f"https://www.reddit.com/search.json?q={topic}&sort=hot&limit={limit}")
if not data:
return []
posts = []
for child in data.get("data", {}).get("children", []):
d = child.get("data", {})
full_text = f"{d.get('title', '')} {d.get('selftext', '')[:300]}"
sent, strength = analyze_sentiment(full_text)
posts.append({
"title": d.get("title", ""),
"subreddit": d.get("subreddit", ""),
"score": d.get("score", 0),
"comments": d.get("num_comments", 0),
"sentiment": sent,
"tickers": extract_tickers(full_text),
"url": f"https://reddit.com{d.get('permalink', '')}",
})
return posts
def run_full_scan(quick=False):
print(f"\n🔎 Reddit Market Intel — {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
print("=" * 60)
limit = 5 if quick else 15
results = []
all_tickers = Counter()
sentiment_summary = Counter()
print("\n📊 Scanning subreddits:")
for sub in DEFAULT_SUBREDDITS:
result = scan_subreddit(sub, limit)
if result:
results.append(result)
all_tickers.update(result["trending_tickers"])
sentiment_summary[result["overall_sentiment"]] += 1
# Build report
report = {
"timestamp": datetime.now().isoformat(),
"subreddits_scanned": len(results),
"overall_market_sentiment": max(sentiment_summary, key=sentiment_summary.get) if sentiment_summary else "neutral",
"sentiment_by_sub": {r["subreddit"]: r["overall_sentiment"] for r in results},
"top_tickers": dict(all_tickers.most_common(20)),
"high_engagement_posts": [],
}
for r in results:
report["high_engagement_posts"].extend(r["high_engagement"][:3])
# Sort high engagement by comments
report["high_engagement_posts"].sort(key=lambda x: x["comments"], reverse=True)
report["high_engagement_posts"] = report["high_engagement_posts"][:15]
# Save
report_file = DATA_DIR / f"scan-{datetime.now().strftime('%Y%m%d-%H%M')}.json"
report_file.write_text(json.dumps(report, indent=2))
# Also save latest
(DATA_DIR / "latest.json").write_text(json.dumps(report, indent=2))
# Print summary
print(f"\n{'=' * 60}")
print(f"📈 Overall Market Sentiment: {report['overall_market_sentiment'].upper()}")
print(f"\n🏷️ Trending Tickers: {', '.join(f'${t}({c})' for t, c in all_tickers.most_common(10))}" if all_tickers else "")
if report["high_engagement_posts"]:
print(f"\n🔥 Top Engagement Posts:")
for p in report["high_engagement_posts"][:5]:
print(f" [{p['sentiment']}] r/{p['subreddit']} | ⬆{p['score']} 💬{p['comments']}")
print(f" {p['title'][:80]}")
print(f"\n💾 Report saved: {report_file}")
return report
if __name__ == "__main__":
if "--topic" in sys.argv:
idx = sys.argv.index("--topic")
topic = sys.argv[idx + 1] if idx + 1 < len(sys.argv) else "crypto"
posts = search_topic(topic)
for p in posts[:10]:
print(f" [{p['sentiment']}] r/{p['subreddit']} ⬆{p['score']} 💬{p['comments']}")
print(f" {p['title'][:80]}")
elif "--quick" in sys.argv:
run_full_scan(quick=True)
else:
run_full_scan()

116
tools/run_memory_indexer.py Normal file
View File

@ -0,0 +1,116 @@
#!/usr/bin/env python3
import json
import os
import sys
import glob
import subprocess
from collections import defaultdict
from datetime import datetime
def extract_text_content(content_list):
"""Extract text content from message content array, excluding thinking blocks."""
text_parts = []
for item in content_list:
if item.get('type') == 'text':
text_parts.append(item['text'])
return ' '.join(text_parts).strip()
def process_session_file(file_path):
"""Process a single JSONL session file and extract conversation turns."""
turns = []
try:
with open(file_path, 'r') as f:
lines = f.readlines()
messages = []
for line in lines:
try:
data = json.loads(line.strip())
if data.get('type') == 'message' and 'message' in data:
messages.append(data)
except json.JSONDecodeError:
continue
# Group messages into conversation turns (user -> assistant pairs)
current_user_msg = None
for msg in messages:
role = msg['message']['role']
content = msg['message'].get('content', [])
if role == 'user':
current_user_msg = extract_text_content(content)
elif role == 'assistant' and current_user_msg:
assistant_text = extract_text_content(content)
if assistant_text and current_user_msg: # Only include if both have content
turns.append({
'user': current_user_msg,
'assistant': assistant_text,
'timestamp': msg.get('timestamp')
})
current_user_msg = None
return turns
except Exception as e:
print(f"Error processing {file_path}: {e}", file=sys.stderr)
return []
def main():
sessions_dir = "/home/wdjones/.openclaw/agents/main/sessions/"
# Get all non-deleted session files
session_files = []
for file_path in glob.glob(os.path.join(sessions_dir, "*.jsonl")):
if not file_path.endswith('.deleted'):
session_files.append(file_path)
# Sort by modification time (newest first)
session_files.sort(key=os.path.getmtime, reverse=True)
all_turns = []
# Process session files until we have enough turns
for session_file in session_files:
turns = process_session_file(session_file)
all_turns.extend(turns)
if len(all_turns) >= 10:
break
# Get the last 10 turns
recent_turns = all_turns[-10:] if len(all_turns) > 10 else all_turns
print(f"Processing {len(recent_turns)} conversation turns through auto-memory indexer...", file=sys.stderr)
# Process each turn individually through the auto-memory script
success_count = 0
for i, turn in enumerate(recent_turns):
try:
turn_data = {
"user": turn['user'],
"assistant": turn['assistant'],
"agent_id": "case",
"session": "main"
}
# Run the auto-memory-hook script for each turn
result = subprocess.run([
'python3', '/home/wdjones/.openclaw/workspace/tools/auto-memory-hook.py'
], input=json.dumps(turn_data), text=True, capture_output=True)
if result.returncode == 0:
success_count += 1
print(f"Turn {i+1}/{len(recent_turns)}: Processed successfully", file=sys.stderr)
else:
print(f"Turn {i+1}/{len(recent_turns)}: Error - {result.stderr}", file=sys.stderr)
except Exception as e:
print(f"Turn {i+1}/{len(recent_turns)}: Exception - {e}", file=sys.stderr)
print(f"Auto-memory indexer completed: {success_count}/{len(recent_turns)} turns processed successfully", file=sys.stderr)
if __name__ == "__main__":
main()

239
tools/self-healing-server.py Normal file
View File

@ -0,0 +1,239 @@
#!/usr/bin/env python3
"""
Self-Healing Server — Always-on infrastructure monitoring & auto-repair.
Monitors services, disk, memory, CPU. Auto-restarts failed services.
Logs all actions. Run via cron every 5 minutes or as a daemon.
Usage:
python3 self-healing-server.py # Run once (check & heal)
python3 self-healing-server.py --daemon # Run continuously (every 5 min)
python3 self-healing-server.py --status # Show current health
"""
import json
import subprocess
import os
import sys
import time
import psutil
from datetime import datetime
from pathlib import Path
DATA_DIR = Path(__file__).parent.parent / "data" / "self-healing"
DATA_DIR.mkdir(parents=True, exist_ok=True)
LOG_FILE = DATA_DIR / "heal-log.jsonl"
CONFIG_FILE = DATA_DIR / "config.json"
DEFAULT_CONFIG = {
"monitored_services": [
"nexus",
"nginx",
"ssh"
],
"monitored_ports": {
"8000": "control-panel",
"8888": "feed-hunter",
"8889": "market-watch",
"8890": "ticker",
"80": "nginx",
"3000": "nexus"
},
"thresholds": {
"disk_percent": 90,
"memory_percent": 90,
"cpu_percent": 95,
"swap_percent": 80
},
"auto_restart": True,
"check_interval_seconds": 300
}
def load_config():
if CONFIG_FILE.exists():
return json.loads(CONFIG_FILE.read_text())
CONFIG_FILE.write_text(json.dumps(DEFAULT_CONFIG, indent=2))
return DEFAULT_CONFIG
def log_event(event_type, message, action=None, success=None):
entry = {
"ts": datetime.now().isoformat(),
"type": event_type,
"message": message,
}
if action:
entry["action"] = action
if success is not None:
entry["success"] = success
with open(LOG_FILE, "a") as f:
f.write(json.dumps(entry) + "\n")
icon = "✅" if success else ("⚠️" if success is None else "❌")
print(f" {icon} [{event_type}] {message}")
def run_cmd(cmd, timeout=10):
try:
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
return r.returncode, r.stdout.strip(), r.stderr.strip()
except subprocess.TimeoutExpired:
return -1, "", "timeout"
def check_services(config):
issues = []
for svc in config["monitored_services"]:
# Try systemd user units first, then system
code, out, _ = run_cmd(f"systemctl --user is-active {svc} 2>/dev/null || systemctl is-active {svc} 2>/dev/null")
if "active" not in out:
issues.append(("service_down", svc))
log_event("service_down", f"Service '{svc}' is not active (status: {out})")
if config.get("auto_restart"):
# Try user unit first, then system
code, _, err = run_cmd(f"systemctl --user restart {svc} 2>/dev/null || sudo systemctl restart {svc} 2>/dev/null")
success = code == 0
log_event("auto_restart", f"Attempted restart of '{svc}'", action="restart", success=success)
else:
log_event("service_ok", f"Service '{svc}' is active", success=True)
return issues
def check_ports(config):
issues = []
connections = psutil.net_connections(kind='inet')
listening_ports = {str(c.laddr.port) for c in connections if c.status == 'LISTEN'}
for port, name in config.get("monitored_ports", {}).items():
if port not in listening_ports:
issues.append(("port_down", f"{name} (:{port})"))
log_event("port_down", f"Nothing listening on port {port} ({name})")
else:
log_event("port_ok", f"Port {port} ({name}) is listening", success=True)
return issues
def check_resources(config):
issues = []
thresholds = config.get("thresholds", {})
# Disk
disk = psutil.disk_usage('/')
if disk.percent >= thresholds.get("disk_percent", 90):
issues.append(("disk_high", f"{disk.percent}%"))
log_event("disk_high", f"Disk usage at {disk.percent}% (threshold: {thresholds.get('disk_percent', 90)}%)")
# Auto-clean: journal logs, apt cache
run_cmd("sudo journalctl --vacuum-time=3d 2>/dev/null")
run_cmd("sudo apt-get autoremove -y 2>/dev/null")
log_event("auto_clean", "Ran journal vacuum and apt autoremove", action="clean", success=True)
else:
log_event("disk_ok", f"Disk usage at {disk.percent}%", success=True)
# Memory
mem = psutil.virtual_memory()
if mem.percent >= thresholds.get("memory_percent", 90):
issues.append(("memory_high", f"{mem.percent}%"))
log_event("memory_high", f"Memory usage at {mem.percent}%")
# Clear caches
run_cmd("sync && echo 3 | sudo tee /proc/sys/vm/drop_caches 2>/dev/null")
log_event("auto_clean", "Dropped filesystem caches", action="drop_caches", success=True)
else:
log_event("memory_ok", f"Memory usage at {mem.percent}%", success=True)
# CPU (1-min load avg)
load_1, _, _ = psutil.getloadavg()
cpu_count = psutil.cpu_count()
cpu_pct = (load_1 / cpu_count) * 100
if cpu_pct >= thresholds.get("cpu_percent", 95):
issues.append(("cpu_high", f"{cpu_pct:.0f}%"))
log_event("cpu_high", f"CPU load at {cpu_pct:.0f}% (load avg: {load_1})")
else:
log_event("cpu_ok", f"CPU load at {cpu_pct:.0f}%", success=True)
# Swap
swap = psutil.swap_memory()
if swap.percent >= thresholds.get("swap_percent", 80):
issues.append(("swap_high", f"{swap.percent}%"))
log_event("swap_high", f"Swap usage at {swap.percent}%")
return issues
def check_zombie_processes():
issues = []
zombies = [p for p in psutil.process_iter(['pid', 'name', 'status']) if p.info['status'] == psutil.STATUS_ZOMBIE]
if len(zombies) > 5:
issues.append(("zombies", f"{len(zombies)} zombie processes"))
log_event("zombies", f"Found {len(zombies)} zombie processes")
return issues
def run_health_check():
config = load_config()
print(f"\n🏥 Self-Healing Server Check — {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
print("=" * 60)
all_issues = []
print("\n📡 Services:")
all_issues.extend(check_services(config))
print("\n🔌 Ports:")
all_issues.extend(check_ports(config))
print("\n💾 Resources:")
all_issues.extend(check_resources(config))
print("\n👻 Processes:")
all_issues.extend(check_zombie_processes())
if not any(i[0] == "zombies" for i in all_issues):
print(" ✅ No zombie process issues")
# Summary
print(f"\n{'=' * 60}")
if all_issues:
print(f"⚠️ {len(all_issues)} issue(s) found:")
for itype, detail in all_issues:
print(f" - {itype}: {detail}")
else:
print("✅ All systems healthy!")
# Write status file
status = {
"last_check": datetime.now().isoformat(),
"healthy": len(all_issues) == 0,
"issues": [{"type": t, "detail": d} for t, d in all_issues],
"disk_pct": psutil.disk_usage('/').percent,
"mem_pct": psutil.virtual_memory().percent,
"cpu_count": psutil.cpu_count(),
}
(DATA_DIR / "status.json").write_text(json.dumps(status, indent=2))
return all_issues
def show_status():
status_file = DATA_DIR / "status.json"
if not status_file.exists():
print("No status yet. Run a health check first.")
return
status = json.loads(status_file.read_text())
print(json.dumps(status, indent=2))
if __name__ == "__main__":
if "--status" in sys.argv:
show_status()
elif "--daemon" in sys.argv:
config = load_config()
interval = config.get("check_interval_seconds", 300)
print(f"🔄 Running in daemon mode (every {interval}s). Ctrl+C to stop.")
while True:
try:
run_health_check()
time.sleep(interval)
except KeyboardInterrupt:
print("\nStopped.")
break
else:
run_health_check()

61
tools/smoke-test.sh Executable file
View File

@ -0,0 +1,61 @@
#!/bin/bash
# Infrastructure Smoke Test — run before any deploy
# Exit codes: 0 = all pass, 1 = failures found
PASS=0
FAIL=0
check() {
local name="$1"
local cmd="$2"
if eval "$cmd" > /dev/null 2>&1; then
echo "✅ $name"
((PASS++))
else
echo "🔴 $name — FAILED"
((FAIL++))
fi
}
echo "=== Infrastructure Smoke Test ==="
echo ""
# ChromaDB v2 API
check "ChromaDB v2 API" "curl -sf http://192.168.86.25:8000/api/v2/tenants/default_tenant/databases/default_database/collections"
# ChromaDB v1 should be dead (verify we're not using it)
if curl -sf http://192.168.86.25:8000/api/v1/heartbeat > /dev/null 2>&1; then
echo "⚠️ ChromaDB v1 API is responding (should be dead) — check code for v1 usage"
fi
# Ollama
check "Ollama reachable" "curl -sf http://192.168.86.40:11434/api/tags"
# Ollama embeddings
check "Ollama embeddings" "curl -sf http://192.168.86.40:11434/api/embeddings -d '{\"model\":\"nomic-embed-text\",\"prompt\":\"test\"}'"
# SSH to GPU box
check "SSH to GPU box" "ssh -o ConnectTimeout=5 -o BatchMode=yes case@192.168.86.40 'echo OK'"
# Whisper on GPU box
check "Faster Whisper importable" "ssh -o ConnectTimeout=5 case@192.168.86.40 'source whisper-env/bin/activate && python3 -c \"from faster_whisper import WhisperModel; print(True)\"'"
# yt-dlp
check "yt-dlp installed" "/home/wdjones/.local/bin/yt-dlp --version"
# Docker
check "Docker running" "docker info"
# Knowledge Builder service
check "Knowledge Builder service" "systemctl --user is-active knowledge-builder.service"
# Knowledge Builder HTTP
check "Knowledge Builder HTTP" "curl -sf http://localhost:3001"
echo ""
echo "=== Results: $PASS passed, $FAIL failed ==="
if [ $FAIL -gt 0 ]; then
exit 1
fi
exit 0

281
tools/strategy-sentinel.py Normal file
View File

@ -0,0 +1,281 @@
#!/usr/bin/env python3
"""
Strategy Performance Sentinel — Auto-run strategies, rank winners, spawn variants.
Tracks trading/prediction strategy performance over time, ranks by ROI,
and suggests parameter variations for top performers.
Usage:
python3 strategy-sentinel.py status # Show all strategy performance
python3 strategy-sentinel.py evaluate # Run evaluation cycle
python3 strategy-sentinel.py add <name> <json> # Register a strategy
python3 strategy-sentinel.py rank # Rank strategies by performance
python3 strategy-sentinel.py spawn # Auto-spawn variants of winners
"""
import json
import sys
import random
from datetime import datetime
from pathlib import Path
DATA_DIR = Path(__file__).parent.parent / "data" / "strategy-sentinel"
DATA_DIR.mkdir(parents=True, exist_ok=True)
STRATEGIES_FILE = DATA_DIR / "strategies.json"
HISTORY_FILE = DATA_DIR / "history.jsonl"
# Seed strategies based on Polymarket patterns
DEFAULT_STRATEGIES = {
"tail-conservative": {
"type": "polymarket",
"description": "Buy events under 5% probability, min $50k volume",
"params": {"threshold": 0.05, "min_volume": 50000, "position_pct": 2},
"metrics": {"trades": 0, "wins": 0, "losses": 0, "total_pnl": 0, "roi_pct": 0},
"active": True,
"created": datetime.now().isoformat(),
},
"tail-aggressive": {
"type": "polymarket",
"description": "Buy events under 15% probability, min $10k volume",
"params": {"threshold": 0.15, "min_volume": 10000, "position_pct": 5},
"metrics": {"trades": 0, "wins": 0, "losses": 0, "total_pnl": 0, "roi_pct": 0},
"active": True,
"created": datetime.now().isoformat(),
},
"momentum-high-vol": {
"type": "polymarket",
"description": "Follow momentum on high-volume markets near 50%",
"params": {"price_range": [0.40, 0.60], "min_volume": 100000, "position_pct": 3},
"metrics": {"trades": 0, "wins": 0, "losses": 0, "total_pnl": 0, "roi_pct": 0},
"active": True,
"created": datetime.now().isoformat(),
},
"crypto-sentiment-bull": {
"type": "reddit-intel",
"description": "Go long crypto when Reddit sentiment is bullish across 3+ subs",
"params": {"min_bullish_subs": 3, "hold_hours": 24},
"metrics": {"trades": 0, "wins": 0, "losses": 0, "total_pnl": 0, "roi_pct": 0},
"active": True,
"created": datetime.now().isoformat(),
},
}
def load_strategies():
if STRATEGIES_FILE.exists():
return json.loads(STRATEGIES_FILE.read_text())
save_strategies(DEFAULT_STRATEGIES)
return DEFAULT_STRATEGIES
def save_strategies(strategies):
STRATEGIES_FILE.write_text(json.dumps(strategies, indent=2))
def log_event(strategy_name, event_type, data):
entry = {
"ts": datetime.now().isoformat(),
"strategy": strategy_name,
"event": event_type,
"data": data,
}
with open(HISTORY_FILE, "a") as f:
f.write(json.dumps(entry) + "\n")
def record_trade(strategy_name, pnl, details=""):
"""Record a trade result for a strategy."""
strategies = load_strategies()
if strategy_name not in strategies:
print(f"❌ Strategy '{strategy_name}' not found")
return
s = strategies[strategy_name]
m = s["metrics"]
m["trades"] += 1
m["total_pnl"] += pnl
if pnl > 0:
m["wins"] += 1
else:
m["losses"] += 1
m["roi_pct"] = (m["total_pnl"] / max(m["trades"], 1)) * 100 # Simplified
m["win_rate"] = m["wins"] / max(m["trades"], 1)
m["last_trade"] = datetime.now().isoformat()
save_strategies(strategies)
log_event(strategy_name, "trade", {"pnl": pnl, "details": details})
print(f" 📝 Recorded: {strategy_name} → P&L: ${pnl:+.2f}")
def rank_strategies():
"""Rank all strategies by performance."""
strategies = load_strategies()
ranked = sorted(
[(name, s) for name, s in strategies.items()],
key=lambda x: x[1]["metrics"].get("roi_pct", 0),
reverse=True
)
print(f"\n🏆 Strategy Rankings — {datetime.now().strftime('%Y-%m-%d %H:%M')}")
print("=" * 70)
print(f"{'Rank':<5} {'Strategy':<25} {'Trades':<8} {'Win%':<8} {'P&L':<12} {'Status'}")
print("-" * 70)
for i, (name, s) in enumerate(ranked, 1):
m = s["metrics"]
win_rate = m.get("win_rate", 0)
status = "🟢" if s["active"] else "🔴"
medal = ["🥇", "🥈", "🥉"][i-1] if i <= 3 and m["trades"] > 0 else f" {i}."
print(f"{medal:<5} {name:<25} {m['trades']:<8} {win_rate:>5.0%} ${m['total_pnl']:>+9.2f} {status}")
if not any(s["metrics"]["trades"] > 0 for _, s in ranked):
print("\n 📭 No trades recorded yet. Strategies are seeded and ready.")
print(" Run polymarket-autopilot.py to generate trade signals.")
return ranked
def spawn_variants():
"""Auto-spawn parameter variants of top-performing strategies."""
strategies = load_strategies()
# Find strategies with trades
active_with_trades = {n: s for n, s in strategies.items()
if s["active"] and s["metrics"]["trades"] > 0}
if not active_with_trades:
# Spawn variants of seed strategies instead
print("📭 No trade data yet. Spawning variants of seed strategies...")
active_with_trades = {n: s for n, s in strategies.items() if s["active"]}
spawned = 0
for name, s in list(active_with_trades.items())[:3]:
variant_name = f"{name}-v{random.randint(100, 999)}"
if variant_name in strategies:
continue
# Mutate parameters
new_params = dict(s["params"])
for key, val in new_params.items():
if isinstance(val, (int, float)):
# Vary by ±20%
delta = val * 0.2 * random.choice([-1, 1])
new_params[key] = round(val + delta, 4)
strategies[variant_name] = {
"type": s["type"],
"description": f"Auto-variant of {name}",
"params": new_params,
"metrics": {"trades": 0, "wins": 0, "losses": 0, "total_pnl": 0, "roi_pct": 0},
"active": True,
"parent": name,
"created": datetime.now().isoformat(),
}
spawned += 1
print(f" 🧬 Spawned: {variant_name}")
print(f" Params: {json.dumps(new_params)}")
save_strategies(strategies)
print(f"\n✅ Spawned {spawned} variant(s)")
def evaluate():
"""Run evaluation cycle: check Polymarket signals against strategy params."""
strategies = load_strategies()
polymarket_data = DATA_DIR.parent / "polymarket" / "latest-signals.json"
reddit_data = DATA_DIR.parent / "reddit-intel" / "latest.json"
print(f"\n🔬 Strategy Evaluation — {datetime.now().strftime('%Y-%m-%d %H:%M')}")
print("=" * 60)
# Check Polymarket signals
if polymarket_data.exists():
signals = json.loads(polymarket_data.read_text())
print(f" 📊 Polymarket: {len(signals.get('signals', []))} signals available")
for name, s in strategies.items():
if s["type"] == "polymarket" and s["active"]:
matching = []
for sig in signals.get("signals", []):
params = s["params"]
if "threshold" in params and sig.get("price", 1) < params["threshold"]:
matching.append(sig)
if matching:
print(f" 🎯 {name}: {len(matching)} matching signal(s)")
log_event(name, "signals_matched", {"count": len(matching)})
else:
print(" ⚠️ No Polymarket data. Run polymarket-autopilot.py scan first.")
# Check Reddit sentiment
if reddit_data.exists():
intel = json.loads(reddit_data.read_text())
sentiment = intel.get("overall_market_sentiment", "neutral")
print(f" 📰 Reddit sentiment: {sentiment}")
for name, s in strategies.items():
if s["type"] == "reddit-intel" and s["active"]:
bullish_subs = sum(1 for v in intel.get("sentiment_by_sub", {}).values() if v == "bullish")
threshold = s["params"].get("min_bullish_subs", 3)
if bullish_subs >= threshold:
print(f" 🎯 {name}: SIGNAL (bullish subs: {bullish_subs}/{threshold})")
log_event(name, "signal_triggered", {"bullish_subs": bullish_subs})
else:
print(" ⚠️ No Reddit data. Run reddit-market-intel.py first.")
print("\n✅ Evaluation complete")
def show_status():
strategies = load_strategies()
print(f"\n📡 Strategy Sentinel — {len(strategies)} strategies tracked")
print("=" * 60)
for name, s in strategies.items():
status = "🟢 Active" if s["active"] else "🔴 Paused"
print(f"\n {name} [{status}]")
print(f" Type: {s['type']} | {s['description']}")
print(f" Params: {json.dumps(s['params'])}")
m = s["metrics"]
if m["trades"] > 0:
print(f" Performance: {m['trades']} trades | {m.get('win_rate', 0):.0%} win | ${m['total_pnl']:+.2f} P&L")
if s.get("parent"):
print(f" Parent: {s['parent']}")
def add_strategy(name, config_json):
strategies = load_strategies()
try:
config = json.loads(config_json)
except json.JSONDecodeError:
print(f"❌ Invalid JSON: {config_json}")
return
strategies[name] = {
"type": config.get("type", "custom"),
"description": config.get("description", "Custom strategy"),
"params": config.get("params", {}),
"metrics": {"trades": 0, "wins": 0, "losses": 0, "total_pnl": 0, "roi_pct": 0},
"active": True,
"created": datetime.now().isoformat(),
}
save_strategies(strategies)
print(f"✅ Added strategy: {name}")
if __name__ == "__main__":
cmd = sys.argv[1] if len(sys.argv) > 1 else "status"
if cmd == "status":
show_status()
elif cmd == "evaluate":
evaluate()
elif cmd == "rank":
rank_strategies()
elif cmd == "spawn":
spawn_variants()
elif cmd == "add" and len(sys.argv) >= 4:
add_strategy(sys.argv[2], sys.argv[3])
elif cmd == "record" and len(sys.argv) >= 4:
record_trade(sys.argv[2], float(sys.argv[3]), sys.argv[4] if len(sys.argv) > 4 else "")
else:
print(__doc__)

47
tools/tool-forge/README.md Normal file
View File

@ -0,0 +1,47 @@
# Tool Forge — Self-Creating Tools System
Agents detect when they need a tool that doesn't exist and write it on the fly.
RAG-backed tool discovery keeps context lean by only surfacing relevant tools.
## Architecture
```
Agent needs capability → search tool registry (RAG) → found? use it : create it → index new tool → execute
```
## Components
- **registry.py** — ChromaDB-backed tool registry with semantic search
- **forge.py** — Tool creation engine (generates Python tools from natural language specs)
- **runner.py** — Safe tool execution sandbox
- **cli.py** — CLI interface: `search`, `create`, `run`, `list`, `describe`
## Usage
```bash
# Search for an existing tool
python3 cli.py search "convert CSV to JSON"
# Create a new tool on the fly
python3 cli.py create "convert CSV to JSON" --desc "Takes a CSV file path, returns JSON array"
# Run a tool
python3 cli.py run csv_to_json --args '{"file_path": "data.csv"}'
# List all tools
python3 cli.py list
# Index existing tools from the tools/ directory
python3 cli.py index-existing
```
## For Agents
```python
from tool_forge import ToolForge
forge = ToolForge()
# Returns matching tools or creates one if none found
tool = forge.ensure_tool("convert CSV to JSON", auto_create=True)
result = tool.run(file_path="data.csv")
```

148
tools/tool-forge/forge.py Normal file
View File

@ -0,0 +1,148 @@
"""
Tool Forge — Creates new Python tools from natural language specifications.
Generates standalone, safe, importable tool modules.
"""
import os
import re
import json
import textwrap
from pathlib import Path
from registry import ToolRegistry, TOOLS_DIR
TOOL_TEMPLATE = '''\
"""
{description}
Auto-generated by Tool Forge.
Parameters: {params_doc}
"""
import json
import sys
{extra_imports}
def run({param_signature}):
"""{description}"""
{body}
def main():
"""CLI entry point."""
import argparse
parser = argparse.ArgumentParser(description="{description}")
{argparse_args}
args = parser.parse_args()
result = run(**vars(args))
if result is not None:
if isinstance(result, (dict, list)):
print(json.dumps(result, indent=2, default=str))
else:
print(result)
if __name__ == "__main__":
main()
'''
class ToolForge:
def __init__(self, registry: ToolRegistry = None):
self.registry = registry or ToolRegistry()
def create_tool(self, name: str, description: str, params: dict = None,
body: str = None, imports: list = None, tags: list = None) -> dict:
"""
Create a new tool and register it.
Args:
name: Tool name (snake_case)
description: What the tool does
params: Dict of {param_name: {"type": str, "desc": str, "default": any}}
body: Python function body (indented 4 spaces). If None, generates a stub.
imports: Extra import lines
tags: Searchable tags
"""
name = re.sub(r'[^a-z0-9_]', '_', name.lower().replace('-', '_').replace(' ', '_'))
name = re.sub(r'_+', '_', name).strip('_')
params = params or {}
tags = tags or []
imports = imports or []
# Build parameter signature
required = {k: v for k, v in params.items() if "default" not in v}
optional = {k: v for k, v in params.items() if "default" in v}
sig_parts = list(required.keys())
for k, v in optional.items():
sig_parts.append(f"{k}={repr(v['default'])}")
param_signature = ", ".join(sig_parts) if sig_parts else ""
# Params doc
params_doc = ", ".join(f"{k} ({v.get('type', 'any')}): {v.get('desc', '')}"
for k, v in params.items()) or "none"
# Argparse args
argparse_lines = []
for k, v in params.items():
type_str = v.get("type", "str")
py_type = {"str": "str", "int": "int", "float": "float", "bool": "bool"}.get(type_str, "str")
if "default" in v:
argparse_lines.append(
f' parser.add_argument("--{k}", type={py_type}, default={repr(v["default"])}, help="{v.get("desc", "")}")')
else:
argparse_lines.append(
f' parser.add_argument("{k}", type={py_type}, help="{v.get("desc", "")}")')
argparse_args = "\n".join(argparse_lines) if argparse_lines else ' pass'
# Body
if not body:
body = f' # TODO: Implement {name}\n raise NotImplementedError("Tool {name} needs implementation")'
extra_imports = "\n".join(f"import {i}" for i in imports)
source = TOOL_TEMPLATE.format(
description=description,
params_doc=params_doc,
param_signature=param_signature,
body=body,
extra_imports=extra_imports,
argparse_args=argparse_args,
)
# Write tool file
tool_path = os.path.join(TOOLS_DIR, f"{name}.py")
Path(tool_path).write_text(source)
os.chmod(tool_path, 0o755)
# Register in RAG
self.registry.register(name, description, tool_path, params, tags + ["auto-created"])
return {
"name": name,
"path": tool_path,
"description": description,
"params": params,
}
def ensure_tool(self, query: str, auto_create: bool = True,
params: dict = None, body: str = None) -> dict:
"""
Search for existing tool; create one if not found and auto_create is True.
Returns tool metadata.
"""
matches = self.registry.search(query, n=3, threshold=0.5)
if matches:
return matches[0]
if not auto_create:
return None
# Derive a name from the query
name = re.sub(r'[^a-z0-9 ]', '', query.lower())
name = '_'.join(name.split()[:5])
return self.create_tool(
name=name,
description=query,
params=params,
body=body,
)

106
tools/tool-forge/registry.py Normal file
View File

@ -0,0 +1,106 @@
"""
Tool Registry — ChromaDB-backed semantic search over tool catalog.
Keeps agent context lean by only surfacing relevant tools via RAG.
"""
import chromadb
import json
import os
from pathlib import Path
from typing import Optional
DB_PATH = os.path.join(os.path.dirname(__file__), ".chromadb")
TOOLS_DIR = os.path.join(os.path.dirname(__file__), "created_tools")
class ToolRegistry:
def __init__(self, db_path: str = DB_PATH):
self.client = chromadb.PersistentClient(path=db_path)
self.collection = self.client.get_or_create_collection(
name="tool_registry",
metadata={"hnsw:space": "cosine"}
)
os.makedirs(TOOLS_DIR, exist_ok=True)
def register(self, name: str, description: str, source_path: str,
params: dict = None, tags: list = None) -> dict:
"""Register a tool in the semantic index."""
doc = f"{name}: {description}"
if tags:
doc += f" tags: {', '.join(tags)}"
metadata = {
"name": name,
"description": description,
"source_path": source_path,
"params_json": json.dumps(params or {}),
"tags": ",".join(tags or []),
}
self.collection.upsert(
ids=[name],
documents=[doc],
metadatas=[metadata],
)
return metadata
def search(self, query: str, n: int = 5, threshold: float = 0.6) -> list[dict]:
"""Semantic search for tools. Returns matches above threshold."""
results = self.collection.query(
query_texts=[query],
n_results=min(n, max(self.collection.count(), 1)),
)
if not results["ids"][0]:
return []
tools = []
for i, id_ in enumerate(results["ids"][0]):
distance = results["distances"][0][i] if results["distances"] else 1.0
similarity = 1 - distance
if similarity >= threshold:
meta = results["metadatas"][0][i]
meta["similarity"] = round(similarity, 3)
tools.append(meta)
return tools
def get(self, name: str) -> Optional[dict]:
"""Get a specific tool by name."""
try:
result = self.collection.get(ids=[name])
if result["ids"]:
return result["metadatas"][0]
except Exception:
pass
return None
def list_all(self) -> list[dict]:
"""List all registered tools."""
result = self.collection.get()
return result["metadatas"] if result["metadatas"] else []
def delete(self, name: str):
"""Remove a tool from the registry."""
self.collection.delete(ids=[name])
def index_existing_tools(self, tools_dir: str = None) -> int:
"""Index existing Python tools from a directory by extracting docstrings."""
tools_dir = tools_dir or os.path.join(os.path.dirname(__file__), "..")
count = 0
for f in Path(tools_dir).glob("*.py"):
if f.name.startswith("_"):
continue
try:
source = f.read_text()
# Extract first docstring
desc = ""
if '"""' in source:
parts = source.split('"""')
if len(parts) >= 3:
desc = parts[1].strip().split("\n")[0]
if not desc:
desc = f"Tool: {f.stem}"
name = f.stem.replace("-", "_")
self.register(name, desc, str(f.resolve()), tags=["existing"])
count += 1
except Exception:
continue
return count

352
tools/video-to-knowledge.py Executable file
View File

@ -0,0 +1,352 @@
#!/usr/bin/env python3
"""
Video-to-Knowledge Pipeline
===========================
Extracts audio from video files, transcribes via Faster Whisper,
generates a clean markdown document, and indexes into ChromaDB for RAG.
Usage:
# Single video
python3 video-to-knowledge.py /path/to/video.mp4
# Directory (recursive)
python3 video-to-knowledge.py /path/to/course/
# Custom output dir
python3 video-to-knowledge.py /path/to/video.mp4 --output /path/to/output/
# Custom collection name
python3 video-to-knowledge.py /path/to/video.mp4 --collection "real-estate-course"
# Skip RAG indexing (just transcribe + markdown)
python3 video-to-knowledge.py /path/to/video.mp4 --no-rag
# Use a specific Whisper model size
python3 video-to-knowledge.py /path/to/video.mp4 --model large-v3
"""
import argparse
import json
import os
import re
import subprocess
import sys
import hashlib
from pathlib import Path
from datetime import timedelta
# Defaults
CHROMADB_HOST = "192.168.86.25"
CHROMADB_PORT = 8000
OLLAMA_HOST = "192.168.86.40"
OLLAMA_PORT = 11434
EMBED_MODEL = "nomic-embed-text"
DEFAULT_COLLECTION = "video-knowledge"
WHISPER_MODEL = "base.en"
CHUNK_SIZE = 1000 # chars per RAG chunk
CHUNK_OVERLAP = 200
VIDEO_EXTENSIONS = {'.mp4', '.mkv', '.avi', '.mov', '.webm', '.flv', '.wmv', '.m4v', '.ts'}
def log(msg):
print(f" → {msg}")
def extract_audio(video_path: Path, output_dir: Path) -> Path:
"""Extract audio from video as WAV (16kHz mono for Whisper)."""
audio_path = output_dir / f"{video_path.stem}.wav"
if audio_path.exists():
log(f"Audio already extracted: {audio_path.name}")
return audio_path
log(f"Extracting audio from {video_path.name}...")
subprocess.run([
"ffmpeg", "-y", "-i", str(video_path),
"-vn", "-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1",
str(audio_path)
], capture_output=True, check=True)
return audio_path
def transcribe(audio_path: Path, model_size: str = WHISPER_MODEL) -> list[dict]:
"""Transcribe audio using faster-whisper. Returns list of segments."""
log(f"Transcribing with faster-whisper ({model_size})...")
from faster_whisper import WhisperModel
model = WhisperModel(model_size, device="cpu", compute_type="int8")
segments_raw, info = model.transcribe(str(audio_path), beam_size=5)
segments = []
for seg in segments_raw:
segments.append({
"start": seg.start,
"end": seg.end,
"text": seg.text.strip()
})
log(f"Transcribed {len(segments)} segments, language: {info.language} ({info.language_probability:.0%})")
return segments
def format_timestamp(seconds: float) -> str:
"""Format seconds as HH:MM:SS."""
td = timedelta(seconds=int(seconds))
hours, remainder = divmod(td.seconds, 3600)
minutes, secs = divmod(remainder, 60)
if hours:
return f"{hours:02d}:{minutes:02d}:{secs:02d}"
return f"{minutes:02d}:{secs:02d}"
def segments_to_markdown(segments: list[dict], video_name: str, video_path: str) -> str:
"""Convert transcript segments into a clean, readable markdown document."""
lines = [
f"# {video_name}",
f"",
f"**Source:** `{video_path}` ",
f"**Segments:** {len(segments)} ",
f"**Duration:** {format_timestamp(segments[-1]['end']) if segments else 'N/A'}",
f"",
f"---",
f"",
]
# Group segments into ~2 minute chapters for readability
chapter_duration = 120 # seconds
current_chapter_start = 0
chapter_num = 1
chapter_text = []
for seg in segments:
if seg["start"] >= current_chapter_start + chapter_duration and chapter_text:
# Write chapter
ts = format_timestamp(current_chapter_start)
lines.append(f"## [{ts}] Section {chapter_num}")
lines.append("")
lines.append(" ".join(chapter_text))
lines.append("")
chapter_num += 1
current_chapter_start = seg["start"]
chapter_text = []
chapter_text.append(seg["text"])
# Final chapter
if chapter_text:
ts = format_timestamp(current_chapter_start)
lines.append(f"## [{ts}] Section {chapter_num}")
lines.append("")
lines.append(" ".join(chapter_text))
lines.append("")
return "\n".join(lines)
def chunk_text(text: str, chunk_size: int = CHUNK_SIZE, overlap: int = CHUNK_OVERLAP) -> list[str]:
"""Split text into overlapping chunks for RAG indexing."""
chunks = []
start = 0
while start < len(text):
end = start + chunk_size
chunk = text[start:end]
if chunk.strip():
chunks.append(chunk.strip())
start = end - overlap
return chunks
def get_embedding(text: str) -> list[float]:
"""Get embedding from Ollama."""
import requests
resp = requests.post(
f"http://{OLLAMA_HOST}:{OLLAMA_PORT}/api/embed",
json={"model": EMBED_MODEL, "input": text},
timeout=30
)
resp.raise_for_status()
data = resp.json()
# Handle both single and batch responses
if "embeddings" in data:
return data["embeddings"][0]
return data["embedding"]
def index_to_chromadb(chunks: list[str], video_name: str, video_path: str, collection_name: str):
"""Index text chunks into ChromaDB."""
import chromadb
log(f"Connecting to ChromaDB at {CHROMADB_HOST}:{CHROMADB_PORT}...")
client = chromadb.HttpClient(host=CHROMADB_HOST, port=CHROMADB_PORT)
collection = client.get_or_create_collection(
name=collection_name,
metadata={"hnsw:space": "cosine"}
)
# Generate a stable ID prefix from video path
video_hash = hashlib.md5(video_path.encode()).hexdigest()[:8]
log(f"Indexing {len(chunks)} chunks into collection '{collection_name}'...")
batch_size = 20
for i in range(0, len(chunks), batch_size):
batch = chunks[i:i + batch_size]
ids = [f"{video_hash}-{i + j}" for j in range(len(batch))]
embeddings = [get_embedding(chunk) for chunk in batch]
metadatas = [{
"source": video_path,
"video": video_name,
"chunk_index": i + j,
"total_chunks": len(chunks)
} for j in range(len(batch))]
collection.upsert(
ids=ids,
embeddings=embeddings,
documents=batch,
metadatas=metadatas
)
log(f" Indexed {min(i + batch_size, len(chunks))}/{len(chunks)} chunks")
log(f"✅ Indexed into '{collection_name}' (total docs: {collection.count()})")
def process_video(video_path: Path, output_dir: Path, collection: str,
model_size: str, skip_rag: bool) -> dict:
"""Full pipeline for a single video."""
video_name = video_path.stem
print(f"\n{'='*60}")
print(f"📹 Processing: {video_path.name}")
print(f"{'='*60}")
# Create output subdir mirroring source structure
vid_output = output_dir / video_name
vid_output.mkdir(parents=True, exist_ok=True)
# 1. Extract audio
audio_path = extract_audio(video_path, vid_output)
# 2. Transcribe
segments = transcribe(audio_path, model_size)
# 3. Save raw transcript JSON
transcript_path = vid_output / f"{video_name}_transcript.json"
with open(transcript_path, "w") as f:
json.dump(segments, f, indent=2)
log(f"Saved transcript: {transcript_path.name}")
# 4. Generate markdown
markdown = segments_to_markdown(segments, video_name, str(video_path))
md_path = vid_output / f"{video_name}.md"
with open(md_path, "w") as f:
f.write(markdown)
log(f"Saved markdown: {md_path.name}")
# 5. Index to RAG
if not skip_rag:
full_text = " ".join(seg["text"] for seg in segments)
chunks = chunk_text(full_text)
try:
index_to_chromadb(chunks, video_name, str(video_path), collection)
except Exception as e:
log(f"⚠️ RAG indexing failed: {e}")
log("Transcript and markdown were still saved successfully.")
else:
log("Skipping RAG indexing (--no-rag)")
# Clean up audio (large file)
if audio_path.exists():
audio_path.unlink()
log("Cleaned up extracted audio")
return {
"video": str(video_path),
"segments": len(segments),
"markdown": str(md_path),
"transcript": str(transcript_path)
}
def find_videos(path: Path) -> list[Path]:
"""Find all video files in path (recursive if directory)."""
if path.is_file():
if path.suffix.lower() in VIDEO_EXTENSIONS:
return [path]
else:
print(f"❌ Not a recognized video file: {path}")
return []
videos = []
for ext in VIDEO_EXTENSIONS:
videos.extend(path.rglob(f"*{ext}"))
videos.extend(path.rglob(f"*{ext.upper()}"))
videos = sorted(set(videos))
return videos
def main():
parser = argparse.ArgumentParser(
description="Video-to-Knowledge Pipeline: Transcribe videos → Markdown + RAG",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=__doc__
)
parser.add_argument("input", help="Video file or directory to process")
parser.add_argument("--output", "-o", help="Output directory (default: ./video-knowledge-output/)")
parser.add_argument("--collection", "-c", default=DEFAULT_COLLECTION,
help=f"ChromaDB collection name (default: {DEFAULT_COLLECTION})")
parser.add_argument("--model", "-m", default=WHISPER_MODEL,
help=f"Whisper model size (default: {WHISPER_MODEL})")
parser.add_argument("--no-rag", action="store_true",
help="Skip ChromaDB indexing (just transcribe + markdown)")
parser.add_argument("--chunk-size", type=int, default=CHUNK_SIZE,
help=f"RAG chunk size in chars (default: {CHUNK_SIZE})")
args = parser.parse_args()
input_path = Path(args.input).resolve()
if not input_path.exists():
print(f"❌ Path not found: {input_path}")
sys.exit(1)
output_dir = Path(args.output) if args.output else Path("./video-knowledge-output")
output_dir.mkdir(parents=True, exist_ok=True)
videos = find_videos(input_path)
if not videos:
print(f"❌ No video files found in: {input_path}")
sys.exit(1)
print(f"🎬 Found {len(videos)} video(s) to process")
print(f"📂 Output: {output_dir}")
print(f"🧠 Collection: {args.collection}")
print(f"🎙️ Whisper model: {args.model}")
results = []
for video in videos:
try:
result = process_video(video, output_dir, args.collection,
args.model, args.no_rag)
results.append(result)
except Exception as e:
print(f"❌ Failed on {video.name}: {e}")
results.append({"video": str(video), "error": str(e)})
# Summary
print(f"\n{'='*60}")
print(f"✅ COMPLETE — {len([r for r in results if 'error' not in r])}/{len(results)} videos processed")
print(f"{'='*60}")
for r in results:
if "error" in r:
print(f" ❌ {Path(r['video']).name}: {r['error']}")
else:
print(f" ✅ {Path(r['video']).name}: {r['segments']} segments → {r['markdown']}")
# Save manifest
manifest_path = output_dir / "manifest.json"
with open(manifest_path, "w") as f:
json.dump(results, f, indent=2)
print(f"\n📋 Manifest: {manifest_path}")
if __name__ == "__main__":
main()

18
tools/whisper-transcribe.py Executable file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env python3
"""Whisper transcription wrapper for OpenClaw voice notes.
Outputs transcript text to stdout."""
import sys
def transcribe(path):
from faster_whisper import WhisperModel
model = WhisperModel("base.en", device="cpu", compute_type="int8")
segments, _ = model.transcribe(path, beam_size=5, language="en")
text = " ".join(seg.text for seg in segments).strip()
print(text)
if __name__ == "__main__":
if len(sys.argv) < 2:
print("Usage: whisper-transcribe.py <audio-file>", file=sys.stderr)
sys.exit(1)
transcribe(sys.argv[-1])