140 lines
4.8 KiB
Python
140 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import json
|
|
import sys
|
|
import subprocess
|
|
import os
|
|
import glob
|
|
import time
|
|
|
|
def extract_text_content(content):
|
|
"""Extract text from message content array"""
|
|
if isinstance(content, list):
|
|
for item in content:
|
|
if item.get('type') == 'text':
|
|
return item.get('text', '')
|
|
return str(content) if content else ""
|
|
|
|
def get_session_files():
|
|
"""Get session files sorted by modification time"""
|
|
session_dir = "/home/wdjones/.openclaw/agents/main/sessions/"
|
|
files = glob.glob(os.path.join(session_dir, "*.jsonl"))
|
|
# Filter out lock files and deleted files
|
|
files = [f for f in files if not f.endswith('.lock') and '.deleted.' not in f]
|
|
# Sort by modification time, newest first
|
|
files.sort(key=os.path.getmtime, reverse=True)
|
|
return files
|
|
|
|
def extract_message_pairs(session_file):
|
|
"""Extract user-assistant message pairs from session file"""
|
|
messages = []
|
|
|
|
if not os.path.exists(session_file):
|
|
return []
|
|
|
|
try:
|
|
with open(session_file, 'r') as f:
|
|
for line in f:
|
|
try:
|
|
data = json.loads(line.strip())
|
|
if data.get('type') == 'message' and 'message' in data:
|
|
msg = data['message']
|
|
if msg.get('role') in ['user', 'assistant']:
|
|
messages.append({
|
|
'role': msg['role'],
|
|
'content': extract_text_content(msg.get('content', ''))
|
|
})
|
|
except json.JSONDecodeError:
|
|
continue
|
|
except Exception as e:
|
|
print(f"Error reading {session_file}: {e}")
|
|
return []
|
|
|
|
# Find user-assistant pairs
|
|
pairs = []
|
|
current_user = None
|
|
|
|
for msg in messages:
|
|
if msg['role'] == 'user':
|
|
# Skip cron/system messages
|
|
if not msg['content'].startswith('[cron:') and not msg['content'].startswith('[system:'):
|
|
current_user = msg['content']
|
|
elif msg['role'] == 'assistant' and current_user:
|
|
if msg['content'].strip(): # Skip empty responses
|
|
pairs.append({
|
|
'user': current_user,
|
|
'assistant': msg['content'],
|
|
'agent_id': 'case',
|
|
'session': 'main'
|
|
})
|
|
current_user = None # Reset after pairing
|
|
|
|
return pairs
|
|
|
|
def main():
|
|
# Get session files
|
|
session_files = get_session_files()
|
|
|
|
# Collect message pairs from recent sessions until we have at least 10
|
|
all_pairs = []
|
|
|
|
print(f"Checking {len(session_files)} session files...")
|
|
|
|
for i, session_file in enumerate(session_files[:20]): # Check up to 20 recent sessions
|
|
pairs = extract_message_pairs(session_file)
|
|
filename = os.path.basename(session_file)
|
|
|
|
if pairs:
|
|
print(f"Session {filename}: {len(pairs)} conversation pairs")
|
|
# Add only non-cron pairs
|
|
valid_pairs = [p for p in pairs if not p['user'].startswith('[cron:')]
|
|
if valid_pairs:
|
|
all_pairs.extend(valid_pairs)
|
|
print(f" Added {len(valid_pairs)} valid pairs")
|
|
else:
|
|
print(f"Session {filename}: No conversation pairs found")
|
|
|
|
if len(all_pairs) >= 10:
|
|
print(f"Found enough conversation pairs ({len(all_pairs)}), stopping search")
|
|
break
|
|
|
|
# Take the last 10 pairs
|
|
last_pairs = all_pairs[-10:] if len(all_pairs) >= 10 else all_pairs
|
|
|
|
if not last_pairs:
|
|
print("No valid conversation pairs found to process")
|
|
return
|
|
|
|
print(f"\nProcessing {len(last_pairs)} conversation pairs...")
|
|
|
|
# Process each pair through the auto-memory hook
|
|
processed = 0
|
|
for i, pair in enumerate(last_pairs):
|
|
try:
|
|
payload = json.dumps(pair)
|
|
|
|
# Pipe to the auto-memory hook script
|
|
process = subprocess.Popen(
|
|
["python3", "/home/wdjones/.openclaw/workspace/tools/auto-memory-hook.py"],
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True
|
|
)
|
|
|
|
stdout, stderr = process.communicate(input=payload)
|
|
|
|
if process.returncode == 0:
|
|
processed += 1
|
|
print(f"✓ Processed pair {i+1}/{len(last_pairs)}")
|
|
else:
|
|
print(f"✗ Error processing pair {i+1}: {stderr}")
|
|
|
|
except Exception as e:
|
|
print(f"✗ Exception processing pair {i+1}: {e}")
|
|
|
|
print(f"\nCompleted: {processed}/{len(last_pairs)} pairs processed successfully")
|
|
print(f"Check logs/auto-memory.log for processing details")
|
|
|
|
if __name__ == "__main__":
|
|
main() |