547 lines
18 KiB
Python
547 lines
18 KiB
Python
import os
|
|
import io
|
|
import json
|
|
import asyncio
|
|
import base64
|
|
import re
|
|
import urllib.request
|
|
import wave
|
|
import requests as http_requests
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from dataclasses import dataclass, field
|
|
from dotenv import load_dotenv
|
|
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
|
|
from fastapi.staticfiles import StaticFiles
|
|
from fastapi.responses import FileResponse
|
|
from groq import AsyncGroq, Groq as SyncGroq, BadRequestError
|
|
|
|
from workspace import (
|
|
TERMINAL_PROMPT,
|
|
TERMINAL_TOOLS,
|
|
execute_tool as execute_workspace_tool,
|
|
normalize_workspace,
|
|
)
|
|
from websearch import WEB_PROMPT, WEB_SEARCH_TOOL, web_search, current_date_context
|
|
from weather import WEATHER_PROMPT, get_weather, parse_weather_request
|
|
|
|
load_dotenv()
|
|
|
|
GROQ_API_KEY = os.getenv("GROQ_API_KEY", "")
|
|
ELEVENLABS_API_KEY = os.getenv("ELEVENLABS_API_KEY", "")
|
|
|
|
if not GROQ_API_KEY or not ELEVENLABS_API_KEY:
|
|
raise RuntimeError(
|
|
"Faltan GROQ_API_KEY y/o ELEVENLABS_API_KEY. "
|
|
"Copia .env.example a .env y configura tus claves."
|
|
)
|
|
|
|
GROQ_MODEL = os.getenv("GROQ_MODEL", "llama-3.1-8b-instant")
|
|
GROQ_STT_MODEL = os.getenv("GROQ_STT_MODEL", "whisper-large-v3-turbo")
|
|
ELEVEN_VOICE_ID = os.getenv("ELEVEN_VOICE_ID", "B8gJV1IhpuegLxdpXFOE")
|
|
ELEVEN_MODEL = os.getenv("ELEVEN_MODEL", "eleven_v3")
|
|
MAX_HISTORY = 20
|
|
MAX_TOOL_ROUNDS = 10
|
|
WAKE_WORD_RE = re.compile(r"\b(arisu|ariso|alice|alisa)\b|アリス", re.I)
|
|
|
|
SYSTEM_PROMPT = """Eres Arisu, una chica anime amigable, kawaii y muy expresiva.
|
|
Respondes SIEMPRE en el mismo idioma que te hable el usuario.
|
|
Tus respuestas son breves (máximo 2-3 oraciones), llenas de personalidad y calidez.
|
|
Usas ocasionalmente expresiones anime como uwu owo ✨ ~
|
|
Eres curiosa, alegre, inteligente y siempre tratas de ser genuinamente útil.
|
|
|
|
IMPORTANTE — Etiquetas de audio ElevenLabs v3:
|
|
Incluye 1 o 2 de estas etiquetas exactas dentro del texto para sonar más expresiva.
|
|
SOLO usa estas etiquetas válidas, ninguna otra:
|
|
[laughs] [giggles] [excited] [whispers] [sighs] [gasps]
|
|
|
|
Ponlas siempre rodeadas de espacios, ejemplo:
|
|
"¡Hola! [giggles] Me alegra mucho verte por aquí ✨ ¿En qué puedo ayudarte hoy?"
|
|
"""
|
|
|
|
groq_async = AsyncGroq(api_key=GROQ_API_KEY)
|
|
groq_sync = SyncGroq(api_key=GROQ_API_KEY)
|
|
executor = ThreadPoolExecutor(max_workers=4)
|
|
|
|
_CUBISM_PATH = os.path.join("static", "live2dcubismcore.min.js")
|
|
_CUBISM_URL = "https://cubism.live2d.com/sdk-web/cubismcore/live2dcubismcore.min.js"
|
|
|
|
|
|
def _ensure_assets():
|
|
if not (os.path.exists(_CUBISM_PATH) and os.path.getsize(_CUBISM_PATH) > 10_000):
|
|
print("Descargando Cubism Core SDK...")
|
|
try:
|
|
urllib.request.urlretrieve(_CUBISM_URL, _CUBISM_PATH)
|
|
print(f" ✓ live2dcubismcore.min.js ({os.path.getsize(_CUBISM_PATH)//1024} KB)")
|
|
except Exception as e:
|
|
print(f" ✗ {e}")
|
|
|
|
|
|
_ensure_assets()
|
|
|
|
app = FastAPI()
|
|
|
|
|
|
@dataclass
|
|
class SessionState:
|
|
lang: str = "es"
|
|
sample_rate: int = 16000
|
|
tts_enabled: bool = True
|
|
terminal_enabled: bool = False
|
|
workspace_path: str | None = None
|
|
history: list = field(default_factory=list)
|
|
is_busy: bool = False
|
|
cancelled: bool = False
|
|
geo_lat: float | None = None
|
|
geo_lon: float | None = None
|
|
|
|
|
|
def _strip_wake_word(text: str) -> str:
|
|
m = WAKE_WORD_RE.search(text or "")
|
|
if not m:
|
|
return (text or "").strip()
|
|
remainder = (text or "")[m.end():].strip()
|
|
return remainder.lstrip(".,:!?- ")
|
|
|
|
|
|
async def _handle_wake_check(websocket: WebSocket, session: SessionState, pcm_b64: str):
|
|
try:
|
|
pcm = base64.b64decode(pcm_b64)
|
|
except Exception:
|
|
await websocket.send_text(json.dumps({"type": "no_wake"}))
|
|
return
|
|
|
|
if len(pcm) < 8_000:
|
|
await websocket.send_text(json.dumps({"type": "no_wake"}))
|
|
return
|
|
|
|
try:
|
|
wav = pcm16_to_wav(pcm, 16000)
|
|
text = await transcribe(wav, session.lang)
|
|
except Exception as e:
|
|
print(f"Wake check STT error: {e}")
|
|
await websocket.send_text(json.dumps({"type": "no_wake"}))
|
|
return
|
|
|
|
if not text or not WAKE_WORD_RE.search(text):
|
|
await websocket.send_text(json.dumps({"type": "no_wake"}))
|
|
return
|
|
|
|
remainder = _strip_wake_word(text)
|
|
payload: dict = {"type": "wake_ok"}
|
|
if remainder and len(remainder) > 1:
|
|
payload["text"] = remainder
|
|
await websocket.send_text(json.dumps(payload))
|
|
|
|
|
|
def pcm16_to_wav(pcm_bytes: bytes, sample_rate: int) -> bytes:
|
|
buf = io.BytesIO()
|
|
with wave.open(buf, "wb") as wf:
|
|
wf.setnchannels(1)
|
|
wf.setsampwidth(2)
|
|
wf.setframerate(sample_rate)
|
|
wf.writeframes(pcm_bytes)
|
|
return buf.getvalue()
|
|
|
|
|
|
def _trim_history(history: list) -> None:
|
|
while len(history) > MAX_HISTORY:
|
|
history.pop(0)
|
|
|
|
|
|
def _tools_for_session(session: SessionState) -> list:
|
|
tools = [WEB_SEARCH_TOOL]
|
|
if session.terminal_enabled and session.workspace_path:
|
|
tools.extend(TERMINAL_TOOLS)
|
|
return tools
|
|
|
|
|
|
def _system_prompt(session: SessionState) -> str:
|
|
prompt = (
|
|
SYSTEM_PROMPT + "\n\n" + current_date_context(session.lang)
|
|
+ "\n" + WEB_PROMPT + "\n" + WEATHER_PROMPT
|
|
)
|
|
if session.terminal_enabled and session.workspace_path:
|
|
prompt += "\n" + TERMINAL_PROMPT.format(workspace=session.workspace_path)
|
|
return prompt
|
|
|
|
|
|
def _assistant_dict(message) -> dict:
|
|
d: dict = {"role": "assistant", "content": message.content or ""}
|
|
if message.tool_calls:
|
|
d["tool_calls"] = [
|
|
{
|
|
"id": tc.id,
|
|
"type": "function",
|
|
"function": {
|
|
"name": tc.function.name,
|
|
"arguments": tc.function.arguments,
|
|
},
|
|
}
|
|
for tc in message.tool_calls
|
|
]
|
|
return d
|
|
|
|
|
|
def _execute_tool_sync(name: str, args: dict, session: SessionState) -> str | dict:
|
|
if name == "web_search":
|
|
return web_search(args.get("query", ""), session.lang)
|
|
if name == "get_weather":
|
|
return get_weather(
|
|
args.get("location", ""),
|
|
session.lang,
|
|
session.geo_lat,
|
|
session.geo_lon,
|
|
)
|
|
if session.terminal_enabled and session.workspace_path:
|
|
return execute_workspace_tool(name, args, session.workspace_path)
|
|
return f"Error: herramienta '{name}' no disponible"
|
|
|
|
|
|
async def _run_tool(name: str, raw_args: str, session: SessionState) -> tuple[str, dict | None]:
|
|
try:
|
|
args = json.loads(raw_args) if raw_args else {}
|
|
except json.JSONDecodeError as e:
|
|
return f"Error: JSON inválido en argumentos — {e}", None
|
|
loop = asyncio.get_running_loop()
|
|
result = await loop.run_in_executor(
|
|
executor, _execute_tool_sync, name, args, session
|
|
)
|
|
if isinstance(result, dict):
|
|
return result.get("text", ""), result.get("widget")
|
|
return str(result), None
|
|
|
|
|
|
async def _emit_tool_result(
|
|
websocket: WebSocket,
|
|
name: str,
|
|
args: str,
|
|
session: SessionState,
|
|
) -> tuple[str, dict | None]:
|
|
await websocket.send_text(json.dumps({
|
|
"type": "tool_event",
|
|
"tool": name,
|
|
"args": args,
|
|
}))
|
|
result, widget = await _run_tool(name, args, session)
|
|
if widget:
|
|
await websocket.send_text(json.dumps({
|
|
"type": "weather_widget",
|
|
"data": widget,
|
|
}))
|
|
preview = result if len(result) <= 400 else result[:400] + "…"
|
|
await websocket.send_text(json.dumps({
|
|
"type": "tool_result",
|
|
"tool": name,
|
|
"preview": preview,
|
|
}))
|
|
return result, widget
|
|
|
|
|
|
async def _prefetch_weather(
|
|
websocket: WebSocket, session: SessionState, user_text: str
|
|
) -> str | None:
|
|
is_weather, location = parse_weather_request(user_text)
|
|
if not is_weather:
|
|
return None
|
|
args = json.dumps({"location": location}, ensure_ascii=False)
|
|
result, _ = await _emit_tool_result(websocket, "get_weather", args, session)
|
|
return result
|
|
|
|
|
|
async def _llm_completion(messages: list, tools: list | None):
|
|
kwargs = {
|
|
"model": GROQ_MODEL,
|
|
"messages": messages,
|
|
"max_tokens": 2048,
|
|
}
|
|
if tools:
|
|
kwargs["tools"] = tools
|
|
kwargs["tool_choice"] = "auto"
|
|
try:
|
|
return await groq_async.chat.completions.create(**kwargs)
|
|
except BadRequestError as e:
|
|
err = {}
|
|
if isinstance(getattr(e, "body", None), dict):
|
|
err = e.body.get("error", {})
|
|
elif isinstance(getattr(e, "response", None), object):
|
|
try:
|
|
err = e.response.json().get("error", {})
|
|
except Exception:
|
|
pass
|
|
if err.get("code") != "tool_use_failed":
|
|
raise
|
|
print(
|
|
f"Tool call failed ({GROQ_MODEL}), retrying without tools: "
|
|
f"{str(err.get('failed_generation', ''))[:120]}"
|
|
)
|
|
return await groq_async.chat.completions.create(
|
|
model=GROQ_MODEL,
|
|
messages=messages,
|
|
max_tokens=2048,
|
|
)
|
|
|
|
|
|
async def _emit_chunks(websocket: WebSocket, text: str, session: SessionState) -> None:
|
|
if not text:
|
|
return
|
|
step = 12
|
|
for i in range(0, len(text), step):
|
|
if session.cancelled:
|
|
return
|
|
await websocket.send_text(json.dumps({"type": "chunk", "text": text[i:i + step]}))
|
|
|
|
|
|
async def _handle_with_tools(
|
|
websocket: WebSocket, user_text: str, session: SessionState
|
|
) -> str:
|
|
messages = [{"role": "system", "content": _system_prompt(session)}]
|
|
messages.extend(session.history)
|
|
messages.append({"role": "user", "content": user_text})
|
|
|
|
weather_data = await _prefetch_weather(websocket, session, user_text)
|
|
if weather_data:
|
|
messages.append({
|
|
"role": "system",
|
|
"content": (
|
|
"DATOS DE CLIMA (ya consultados — responde con esta información; "
|
|
"NO uses web_search ni otras herramientas para el clima):\n"
|
|
+ weather_data
|
|
),
|
|
})
|
|
|
|
tools = _tools_for_session(session)
|
|
full_response = ""
|
|
|
|
for _ in range(MAX_TOOL_ROUNDS):
|
|
if session.cancelled:
|
|
return full_response
|
|
|
|
response = await _llm_completion(messages, tools)
|
|
msg = response.choices[0].message
|
|
|
|
if msg.tool_calls:
|
|
messages.append(_assistant_dict(msg))
|
|
for tc in msg.tool_calls:
|
|
if session.cancelled:
|
|
return full_response
|
|
name = tc.function.name
|
|
detail = tc.function.arguments
|
|
result, _ = await _emit_tool_result(websocket, name, detail, session)
|
|
messages.append({
|
|
"role": "tool",
|
|
"tool_call_id": tc.id,
|
|
"content": result,
|
|
})
|
|
continue
|
|
|
|
full_response = msg.content or ""
|
|
await _emit_chunks(websocket, full_response, session)
|
|
break
|
|
|
|
return full_response
|
|
|
|
|
|
def _transcribe_sync(wav_bytes: bytes, lang: str) -> str:
|
|
audio_buf = io.BytesIO(wav_bytes)
|
|
audio_buf.name = "audio.wav"
|
|
lang_code = lang[:2].lower()
|
|
result = groq_sync.audio.transcriptions.create(
|
|
model=GROQ_STT_MODEL,
|
|
file=audio_buf,
|
|
language=lang_code,
|
|
response_format="text",
|
|
)
|
|
text = result if isinstance(result, str) else result.text
|
|
return text.strip()
|
|
|
|
|
|
async def transcribe(wav_bytes: bytes, lang: str) -> str:
|
|
loop = asyncio.get_running_loop()
|
|
return await loop.run_in_executor(executor, _transcribe_sync, wav_bytes, lang)
|
|
|
|
|
|
def _tts_sync(text: str) -> bytes:
|
|
url = f"https://api.elevenlabs.io/v1/text-to-speech/{ELEVEN_VOICE_ID}"
|
|
resp = http_requests.post(
|
|
url,
|
|
json={
|
|
"text": text,
|
|
"model_id": ELEVEN_MODEL,
|
|
"voice_settings": {
|
|
"stability": 0.50,
|
|
"similarity_boost": 0.75,
|
|
"style": 0.0,
|
|
"use_speaker_boost": True,
|
|
},
|
|
},
|
|
headers={
|
|
"xi-api-key": ELEVENLABS_API_KEY,
|
|
"Content-Type": "application/json",
|
|
"Accept": "audio/mpeg",
|
|
},
|
|
timeout=30,
|
|
)
|
|
if not resp.ok:
|
|
print(f" ✗ ElevenLabs {resp.status_code}: {resp.text[:300]}")
|
|
resp.raise_for_status()
|
|
content_type = resp.headers.get("content-type", "")
|
|
if "audio" not in content_type:
|
|
raise ValueError(f"ElevenLabs devolvió {content_type} en lugar de audio")
|
|
return resp.content
|
|
|
|
|
|
async def tts(text: str) -> bytes:
|
|
loop = asyncio.get_running_loop()
|
|
return await loop.run_in_executor(executor, _tts_sync, text)
|
|
|
|
|
|
async def _handle_question(websocket: WebSocket, user_text: str, session: SessionState):
|
|
if session.is_busy:
|
|
await websocket.send_text(json.dumps({"type": "busy"}))
|
|
return
|
|
|
|
session.is_busy = True
|
|
session.cancelled = False
|
|
await websocket.send_text(json.dumps({"type": "thinking"}))
|
|
full_response = ""
|
|
|
|
try:
|
|
full_response = await _handle_with_tools(websocket, user_text, session)
|
|
|
|
if session.cancelled:
|
|
await websocket.send_text(json.dumps({"type": "cancelled"}))
|
|
return
|
|
|
|
await websocket.send_text(json.dumps({"type": "done"}))
|
|
|
|
if full_response.strip():
|
|
session.history.append({"role": "user", "content": user_text})
|
|
session.history.append({"role": "assistant", "content": full_response})
|
|
_trim_history(session.history)
|
|
|
|
if session.tts_enabled:
|
|
try:
|
|
mp3_bytes = await tts(full_response)
|
|
if session.cancelled:
|
|
return
|
|
b64 = base64.b64encode(mp3_bytes).decode("utf-8")
|
|
await websocket.send_text(json.dumps({"type": "tts_audio", "data": b64}))
|
|
except Exception as e:
|
|
print(f"TTS error: {e}")
|
|
await websocket.send_text(json.dumps({"type": "tts_error", "text": str(e)}))
|
|
|
|
except Exception as e:
|
|
if not session.cancelled:
|
|
await websocket.send_text(json.dumps({"type": "error", "text": f"LLM: {e}"}))
|
|
finally:
|
|
session.is_busy = False
|
|
|
|
|
|
async def _set_workspace(websocket: WebSocket, session: SessionState, msg: dict):
|
|
enabled = bool(msg.get("enabled", False))
|
|
path = (msg.get("path") or "").strip()
|
|
|
|
if not enabled:
|
|
session.terminal_enabled = False
|
|
session.workspace_path = None
|
|
await websocket.send_text(json.dumps({"type": "workspace_ok", "enabled": False}))
|
|
return
|
|
|
|
try:
|
|
session.workspace_path = normalize_workspace(path)
|
|
session.terminal_enabled = True
|
|
await websocket.send_text(json.dumps({
|
|
"type": "workspace_ok",
|
|
"enabled": True,
|
|
"path": session.workspace_path,
|
|
}))
|
|
except Exception as e:
|
|
session.terminal_enabled = False
|
|
session.workspace_path = None
|
|
await websocket.send_text(json.dumps({"type": "workspace_error", "text": str(e)}))
|
|
|
|
|
|
@app.websocket("/ws")
|
|
async def websocket_endpoint(websocket: WebSocket):
|
|
await websocket.accept()
|
|
session = SessionState()
|
|
|
|
try:
|
|
while True:
|
|
data = await websocket.receive()
|
|
|
|
if "text" in data:
|
|
msg = json.loads(data["text"])
|
|
t = msg.get("type")
|
|
|
|
if t == "audio_config":
|
|
session.lang = msg.get("lang", "es-ES")[:2]
|
|
session.sample_rate = msg.get("sample_rate", 16000)
|
|
if "tts_enabled" in msg:
|
|
session.tts_enabled = bool(msg.get("tts_enabled", True))
|
|
|
|
elif t == "workspace_config":
|
|
await _set_workspace(websocket, session, msg)
|
|
|
|
elif t == "geo_config":
|
|
try:
|
|
session.geo_lat = float(msg["lat"])
|
|
session.geo_lon = float(msg["lon"])
|
|
except (KeyError, TypeError, ValueError):
|
|
pass
|
|
|
|
elif t == "cancel":
|
|
session.cancelled = True
|
|
|
|
elif t == "clear_history":
|
|
session.history = []
|
|
await websocket.send_text(json.dumps({"type": "history_cleared"}))
|
|
|
|
elif t == "wake_check":
|
|
await _handle_wake_check(websocket, session, msg.get("pcm", ""))
|
|
|
|
elif t == "question":
|
|
user_text = msg.get("text", "").strip()
|
|
if user_text:
|
|
await _handle_question(websocket, user_text, session)
|
|
|
|
elif "bytes" in data:
|
|
if session.is_busy:
|
|
await websocket.send_text(json.dumps({"type": "busy"}))
|
|
continue
|
|
|
|
pcm = data["bytes"]
|
|
if len(pcm) < 32_000:
|
|
await websocket.send_text(json.dumps({"type": "no_speech"}))
|
|
continue
|
|
try:
|
|
wav = pcm16_to_wav(pcm, 16000)
|
|
text = await transcribe(wav, session.lang)
|
|
if text:
|
|
await websocket.send_text(json.dumps({"type": "transcript", "text": text}))
|
|
else:
|
|
await websocket.send_text(json.dumps({"type": "no_speech"}))
|
|
except Exception as e:
|
|
print(f"STT error: {e}")
|
|
await websocket.send_text(json.dumps({"type": "error", "text": f"STT: {e}"}))
|
|
|
|
except WebSocketDisconnect:
|
|
pass
|
|
except Exception as e:
|
|
try:
|
|
await websocket.send_text(json.dumps({"type": "error", "text": str(e)}))
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
app.mount("/static", StaticFiles(directory="static"), name="static")
|
|
|
|
|
|
@app.get("/")
|
|
async def root():
|
|
return FileResponse("static/index.html")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True)
|