Initial Commit

.gitignore

@@ -0,0 +1 @@
+.env

README.md

@@ -0,0 +1,3 @@
+# ፨  Talk to Graphs
+
+Template for Voice agents using graphai & Livekit

backend/.gitignore

@@ -0,0 +1,174 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc

backend/.python-version

@@ -0,0 +1 @@
+3.13

backend/main.py

@@ -0,0 +1,5 @@
+from src.db import AsyncDB
+
+if __name__ == "__main__":
+    db = AsyncDB()
+    db.init_db()

backend/pyproject.toml

@@ -0,0 +1,54 @@
+[project]
+name = "voice-assistant-demo"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "aiosqlite>=0.21.0",
+    "fastapi>=0.115.12",
+    "graphai-lib>=0.0.5",
+    "greenlet>=3.1.1",
+    "hypercorn>=0.17.3",
+    "livekit>=1.0.5",
+    "livekit-agents>=1.0.11",
+    "livekit-api>=1.0.2",
+    "livekit-plugins-elevenlabs>=1.0.11",
+    "livekit-plugins-groq>=1.0.11",
+    "livekit-plugins-noise-cancellation>=0.2.1",
+    "livekit-plugins-openai>=1.0.11",
+    "livekit-plugins-silero>=1.0.11",
+    "livekit-plugins-turn-detector>=1.0.11",
+    "pydantic>=2.11.3",
+    "pydantic-settings>=2.8.1",
+    "semantic-router>=0.1.7",
+    "sqlalchemy>=2.0.40",
+    "sqlmodel>=0.0.24",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py312"
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "F",
+    "UP",
+    "I",
+    "B006",
+]
+ignore = ["E501"]
+fixable = ["ALL"]
+exclude = ["*.ipynb"]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+skip-magic-trailing-comma = false
+line-ending = "auto"
+docstring-code-line-length = "dynamic"
+exclude = ["*.ipynb"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"

backend/src/agent.py

@@ -0,0 +1,254 @@
+import logging
+from dataclasses import dataclass, field
+from typing import Any
+
+from livekit.agents import (
+    Agent,
+    AgentSession,
+    JobContext,
+    JobProcess,
+    RunContext,
+    WorkerOptions,
+    cli,
+    function_tool,
+)
+from livekit.plugins import (
+    elevenlabs,
+    groq,
+    openai,
+    silero,
+)
+
+from .db import AsyncDB, CustomerRepository, MaintenanceRepository
+from .settings import settings
+from .utils import load_prompt
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class UserData:
+    personas: dict[str, Any] = field(default_factory=dict)
+    prev_agent: Agent | None = None
+    ctx: JobContext | None = None
+    db: AsyncDB | None = None
+
+    # Customer information
+    customer_id: int | None = None
+    first_name: str | None = None
+    last_name: str | None = None
+
+    def is_identified(self) -> bool:
+        return self.first_name is not None and self.last_name is not None
+
+    def is_registered(self) -> bool:
+        return self.customer_id is not None
+
+    def reset(self) -> None:
+        self.first_name = None
+        self.last_name = None
+        self.customer_id = None
+
+    def summarize(self) -> str:
+        return "User data: Medical office triage system"
+
+
+RunContext_T = RunContext[UserData]
+
+
+class BaseAgent(Agent):
+    async def on_enter(self) -> None:
+        agent_name = self.__class__.__name__
+        logger.info(f"Entering {agent_name} context...")
+
+        userdata: UserData = self.session.userdata
+
+        if userdata.ctx and userdata.ctx.room:
+            await userdata.ctx.room.local_participant.set_attributes({"agent": agent_name})
+
+        chat_ctx = self.chat_ctx.copy()
+
+        chat_ctx.add_message(
+            role="system", content=f"You are the {agent_name}. {userdata.summarize()}"
+        )
+        await self.update_chat_ctx(chat_ctx)
+        self.session.generate_reply()
+
+    def _truncate_chat_ctx(
+        self,
+        items: list,
+        keep_last_n_messages: int = 6,
+        keep_system_message: bool = False,
+        keep_function_call: bool = False,
+    ) -> list:
+        def _valid_item(items) -> bool:
+            if not keep_system_message and item.type == "message" and item.role == "system":
+                return False
+            if not keep_function_call and item.type in ["function_call", "function_call_output"]:
+                return False
+            return True
+
+        new_items = []
+        for item in reversed(items):
+            if _valid_item(item):
+                new_items.append(item)
+            if len(new_items) >= keep_last_n_messages:
+                break
+
+        while new_items and new_items[0].type in ["function_call", "function_call_output"]:
+            new_items.pop(0)
+
+        return new_items
+
+    async def _transfer_to_agent(self, name: str, context: RunContext_T) -> Agent:
+        userdata = context.userdata
+        current_agent = context.session.current_agent
+        next_agent = userdata.personas[name]
+        userdata.prev_agent = current_agent
+
+        return next_agent
+
+
+class TriageAgent(BaseAgent):
+    def __init__(
+        self,
+    ) -> None:
+        super().__init__(
+            instructions="Greet the user",
+            stt=groq.STT(api_key=settings.groq.api_key.get_secret_value()),
+            llm=openai.LLM(
+                base_url="https://openrouter.ai/api/v1",
+                api_key=settings.openrouter.api_key.get_secret_value(),
+                model=settings.openrouter.model,
+            ),
+            tts=elevenlabs.TTS(
+                api_key=settings.elevenlabs.api_key.get_secret_value(),
+                voice_id=settings.elevenlabs.voice_id,
+            ),
+            vad=silero.VAD.load(),
+        )
+
+    @function_tool
+    async def transfer_to_inquiry(self, context: RunContext_T) -> Agent:
+        userdata = self.session.userdata
+        if userdata.is_identified():
+            message = f"Thank you, {userdata.first_name}. Transferring you to our Agent that can help you with further queries"
+        else:
+            message = "I'll transfer you to our Agent who can help you with your inquiry."
+
+        await self.session.say(message)
+        return await self._transfer_to_agent("inquiry", context)
+
+
+class InquiryAgent(BaseAgent):
+    def __init__(self) -> None:
+        super().__init__(
+            instructions=load_prompt("inquiry_prompt.yaml"),
+            stt=groq.STT(api_key=settings.groq.api_key.get_secret_value()),
+            llm=openai.LLM(
+                model=settings.openrouter.model,
+                api_key=settings.openrouter.api_key.get_secret_value(),
+                base_url="https://openrouter.ai/api/v1",
+            ),
+            tts=elevenlabs.TTS(
+                voice_id=settings.elevenlabs.voice_id,
+                api_key=settings.elevenlabs.api_key.get_secret_value(),
+            ),
+            vad=silero.VAD.load(),
+        )
+
+    @function_tool
+    async def identify_customer(self, first_name: str, last_name: str):
+        userdata: UserData = self.session.userdata
+        userdata.first_name = first_name
+        userdata.last_name = last_name
+
+        if not userdata.db:
+            return "It seems I'm having trouble connecting with the database"
+
+        async with userdata.db.db_session() as session:
+            async with CustomerRepository(session) as repository:
+                customer = await repository.get_by_name(first_name=first_name, last_name=last_name)
+                if not customer:
+                    return (
+                        f"I'm sorry, {first_name}, it doesn't seem like your name is in the system"
+                    )
+                userdata.customer_id = customer.id
+
+        return f"Thank you, {first_name}. I've found your customer data"
+
+    @function_tool
+    async def get_maintenance_history(self):
+        userdata: UserData = self.session.userdata
+        if not userdata.is_identified():
+            return "Please identify the current customer using the identify_customer function"
+
+        if not userdata.is_registered():
+            return "The current customer does not exist in our records, please relay this to them."
+
+        if not userdata.db:
+            return "It seems I'm having trouble connecting with the database"
+
+        async with userdata.db.db_session() as session:
+            async with MaintenanceRepository(session) as repository:
+                history = await repository.get_maintenance_history(customer_id=userdata.customer_id)
+
+        return history
+
+    @function_tool
+    async def transfer_to_triage(self, context: RunContext_T) -> Agent:
+        userdata: UserData = self.session.userdata
+        if userdata.is_identified():
+            message = f"Thank you, {userdata.first_name}. I'll transfer you back to our Triage agent who can better direct your inquiry."
+        else:
+            message = (
+                "I'll transfer you back to our Triage agent who can better direct your inquiry."
+            )
+
+        await self.session.say(message)
+        return await self._transfer_to_agent("triage", context)
+
+
+def prewarm(proc: JobProcess):
+    proc.userdata["db"] = AsyncDB()
+
+
+async def entrypoint(ctx: JobContext):
+    await ctx.connect()
+
+    userdata = UserData(ctx=ctx)
+
+    triage_agent = TriageAgent()
+    inquiry_agent = InquiryAgent()
+
+    userdata.personas.update(
+        {
+            "triage": triage_agent,
+            "inquiry": inquiry_agent,
+        }
+    )
+
+    session = AgentSession[UserData](userdata=userdata)
+
+    await session.start(
+        agent=triage_agent,
+        room=ctx.room,
+    )
+
+    async def db_dispose():
+        if hasattr(ctx.proc.userdata, "db") and ctx.proc.userdata["db"] is not None:
+            await ctx.proc.userdata["db"].close()
+
+    ctx.add_shutdown_callback(db_dispose)
+
+
+if __name__ == "__main__":
+    cli.run_app(
+        WorkerOptions(
+            prewarm_fnc=prewarm,
+            entrypoint_fnc=entrypoint,
+            ws_url=settings.livekit.url,
+            api_key=settings.livekit.api_key.get_secret_value(),
+            api_secret=settings.livekit.api_secret.get_secret_value(),
+        )
+    )

backend/src/app.py

@@ -0,0 +1,34 @@
+from uuid import uuid4
+
+from fastapi import FastAPI
+from livekit import api
+
+from settings import settings
+
+app = FastAPI()
+
+
+# @asynccontextmanager
+# async def lifespan(app: FastAPI): ...
+
+
+@app.post("/token")
+async def create_lk_token():
+    room_name = f"room-test--{uuid4()}"
+
+    token = (
+        api.AccessToken(
+            api_key=settings.livekit.api_key.get_secret_value(),
+            api_secret=settings.livekit.api_secret.get_secret_value(),
+        )
+        .with_identity("test")
+        .with_name(room_name)
+        .with_grants(
+            api.VideoGrants(
+                room_join=True,
+                room=room_name,
+            )
+        )
+    )
+
+    return {"participantToken": token.to_jwt(), "serverUrl": settings.livekit.url}

backend/src/db.py

@@ -0,0 +1,93 @@
+import logging
+from contextlib import asynccontextmanager
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+from sqlmodel import Field, MetaData, SQLModel, create_engine
+
+from .settings import settings
+
+logger = logging.getLogger(__name__)
+
+SQLITE_NAMING_CONVENTION = {
+    "ix": "%(column_0_label)s_idx",
+    "uq": "%(table_name)s_%(column_0_name)s_key",
+    "ck": "%(table_name)s_%(constraint_name)s_check",
+    "fk": "%(table_name)s_%(column_0_name)s_fkey",
+    "pk": "%(table_name)s_pkey",
+}
+
+metadata = MetaData(naming_convention=SQLITE_NAMING_CONVENTION)
+
+
+class AsyncDB:
+    def __init__(self) -> None:
+        try:
+            self.engine = create_async_engine(
+                url=settings.db.async_dsn,
+            )
+            self.Session = async_sessionmaker(
+                bind=self.engine, expire_on_commit=False, class_=AsyncSession
+            )
+        except Exception as e:
+            logger.error(f"SQLite: Error initialising the database: {e}")
+
+    @asynccontextmanager
+    async def db_session(self):
+        db = self.Session()
+        try:
+            yield db
+        except Exception as e:
+            logger.error(f"SQLite: Error getting session: {e}")
+            raise e
+        finally:
+            await db.close()
+
+    def init_db(self):
+        engine = create_engine(url=settings.db.dsn)
+        SQLModel.metadata.create_all(bind=engine)
+        engine.dispose()
+
+    async def close(self):
+        await self.engine.dispose()
+        logger.info("SQLite: Database connection closed")
+
+
+class MaintenanceRepository:
+    def __init__(self, session: AsyncSession) -> None:
+        self.session = session
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, exc_traceback):
+        await self.session.close()
+
+    async def get_maintenance_history(self, customer_id: int):
+        result = await self.session.execute(select().filter_by(customer_id=customer_id))
+        maintenance_history = result.scalars().all()
+        return maintenance_history
+
+
+class Customer(SQLModel, table=True):
+    id: int = Field(default=None, primary_key=True, sa_column_kwargs={"autoincrement": True})
+    first_message: str
+    last_name: str
+
+
+class CustomerRepository:
+    def __init__(self, session: AsyncSession) -> None:
+        self.session = session
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, exc_traceback):
+        await self.session.close()
+
+    async def get_by_name(self, first_name: str, last_name: str) -> Customer | None:
+        result = await self.session.execute(
+            select(Customer).filter_by(first_name=first_name, last_name=last_name)
+        )
+        user = result.scalar()
+        return user

backend/src/settings.py

@@ -0,0 +1,54 @@
+from pydantic import BaseModel, SecretStr
+from pydantic_settings import BaseSettings
+
+
+class DatabaseModel(BaseModel):
+    sqlite_file: str
+
+    @property
+    def dsn(self):
+        return f"sqlite:////{self.sqlite_file}"
+
+    @property
+    def async_dsn(self):
+        return f"sqlite+aiosqlite:////{self.sqlite_file}"
+
+    model_config = {"extra": "allow"}
+
+
+class LivekitModel(BaseModel):
+    api_key: SecretStr
+    api_secret: SecretStr
+    url: str
+
+
+class OpenrouterModel(BaseModel):
+    api_key: SecretStr
+    model: str
+
+
+class GroqModel(BaseModel):
+    api_key: SecretStr
+    model: str | None = None
+
+
+class ElevenlabsModel(BaseModel):
+    api_key: SecretStr
+    voice_id: str
+
+
+class Settings(BaseSettings):
+    db: DatabaseModel
+    livekit: LivekitModel
+    openrouter: OpenrouterModel
+    groq: GroqModel
+    elevenlabs: ElevenlabsModel
+
+    model_config = {
+        "env_file": ".env",
+        "env_file_encoding": "utf-8",
+        "env_nested_delimiter": "__",
+    }
+
+
+settings = Settings()

backend/src/utils.py

@@ -0,0 +1,17 @@
+import os
+
+import yaml
+
+
+def load_prompt(filename):
+    """Load a prompt from a YAML file."""
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    prompt_path = os.path.join(script_dir, "prompts", filename)
+
+    try:
+        with open(prompt_path) as file:
+            prompt_data = yaml.safe_load(file)
+            return prompt_data.get("instructions", "")
+    except (FileNotFoundError, yaml.YAMLError) as e:
+        print(f"Error loading prompt file {filename}: {e}")
+        return ""

frontend/.gitignore

@@ -0,0 +1,36 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+.yarn/install-state.gz
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

frontend/README.md

@@ -0,0 +1,36 @@
+This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
+
+## Getting Started
+
+First, run the development server:
+
+```bash
+npm run dev
+# or
+yarn dev
+# or
+pnpm dev
+# or
+bun dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
+
+This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
+
+## Learn More
+
+To learn more about Next.js, take a look at the following resources:
+
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
+- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+
+You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
+
+## Deploy on Vercel
+
+The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
+
+Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.

frontend/next.config.mjs

@@ -0,0 +1,14 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  async rewrites() {
+    return [
+      {
+        // matching all API routes
+        source: "/api/:path*",
+        destination: "http://127.0.0.1:8000/:path*"
+      }
+    ]
+  }
+};
+
+export default nextConfig;

frontend/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "frontend",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint"
+  },
+  "dependencies": {
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1",
+    "next": "14.2.7",
+    "@livekit/components-react": "^2.7.0",
+    "@livekit/components-styles": "^1.1.4",
+    "@livekit/krisp-noise-filter": "^0.2.14",
+    "framer-motion": "^11.18.0",
+    "livekit-client": "^2.8.0",
+    "livekit-server-sdk": "^2.9.7"
+  },
+  "devDependencies": {
+    "@trivago/prettier-plugin-sort-imports": "^5.2.2",
+    "@types/node": "^20",
+    "@types/react": "^18",
+    "@types/react-dom": "^18",
+    "eslint": "^8.57.1",
+    "eslint-config-next": "14.2.28",
+    "eslint-config-prettier": "9.1.0",
+    "next": "14",
+    "postcss": "^8",
+    "prettier": "^3.4.2",
+    "tailwindcss": "^3.4.1",
+    "typescript": "^5"
+  }
+}

frontend/postcss.config.mjs

@@ -0,0 +1,8 @@
+/** @type {import('postcss-load-config').Config} */
+const config = {
+  plugins: {
+    tailwindcss: {},
+  },
+};
+
+export default config;

frontend/public/next.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>

frontend/public/vercel.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 283 64"><path fill="black" d="M141 16c-11 0-19 7-19 18s9 18 20 18c7 0 13-3 16-7l-7-5c-2 3-6 4-9 4-5 0-9-3-10-7h28v-3c0-11-8-18-19-18zm-9 15c1-4 4-7 9-7s8 3 9 7h-18zm117-15c-11 0-19 7-19 18s9 18 20 18c6 0 12-3 16-7l-8-5c-2 3-5 4-8 4-5 0-9-3-11-7h28l1-3c0-11-8-18-19-18zm-10 15c2-4 5-7 10-7s8 3 9 7h-19zm-39 3c0 6 4 10 10 10 4 0 7-2 9-5l8 5c-3 5-9 8-17 8-11 0-19-7-19-18s8-18 19-18c8 0 14 3 17 8l-8 5c-2-3-5-5-9-5-6 0-10 4-10 10zm83-29v46h-9V5h9zM37 0l37 64H0L37 0zm92 5-27 48L74 5h10l18 30 17-30h10zm59 12v10l-3-1c-6 0-10 4-10 10v15h-9V17h9v9c0-5 6-9 13-9z"/></svg>

frontend/src/app/globals.css

@@ -0,0 +1,20 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+:root {
+  --lk-va-bar-width: 72px;
+  --lk-control-bar-height: unset;
+}
+
+.agent-visualizer > .lk-audio-bar {
+  width: 72px;
+}
+
+.lk-agent-control-bar {
+  @apply border-t-0 p-0 h-min mr-4;
+}
+
+.lk-disconnect-button {
+  @apply h-[36px] hover:bg-[#6b221a] hover:text-[white] bg-[#31100c] border-[#6b221a];
+}

frontend/src/app/layout.tsx

@@ -0,0 +1,20 @@
+import "@livekit/components-styles";
+import { Public_Sans } from "next/font/google";
+import "./globals.css";
+
+const publicSans400 = Public_Sans({
+  weight: "400",
+  subsets: ["latin"],
+});
+
+export default function RootLayout({
+  children,
+}: Readonly<{
+  children: React.ReactNode;
+}>) {
+  return (
+    <html lang="en" className={`h-full ${publicSans400.className}`}>
+      <body className="h-full">{children}</body>
+    </html>
+  );
+}

frontend/src/app/page.tsx

@@ -0,0 +1,143 @@
+"use client";
+
+import { CloseIcon } from "@/components/CloseIcon";
+import { NoAgentNotification } from "@/components/NoAgentNotification";
+import TranscriptionView from "@/components/TranscriptionView";
+import {
+  BarVisualizer,
+  DisconnectButton,
+  RoomAudioRenderer,
+  RoomContext,
+  VoiceAssistantControlBar,
+  useVoiceAssistant,
+} from "@livekit/components-react";
+import { useKrispNoiseFilter } from "@livekit/components-react/krisp";
+import { AnimatePresence, motion } from "framer-motion";
+import { Room, RoomEvent } from "livekit-client";
+import { useCallback, useEffect, useState } from "react";
+
+export default function Page() {
+  const [room] = useState(new Room());
+
+  const onConnectButtonClicked = useCallback(async () => {
+    // Generate room connection details, including:
+    //   - A random Room name
+    //   - A random Participant name
+    //   - An Access Token to permit the participant to join the room
+    //   - The URL of the LiveKit server to connect to
+    //
+    // In real-world application, you would likely allow the user to specify their
+    // own participant name, and possibly to choose from existing rooms to join.
+
+    const resp = await fetch("/api/token", {
+      method: "POST",
+    })
+    const { participantToken, serverUrl } = await resp.json();
+
+    if (!serverUrl) {
+      throw new Error("Server misconfigured");
+    }
+
+    await room.connect(serverUrl, participantToken);
+    await room.localParticipant.setMicrophoneEnabled(true);
+  }, [room]);
+
+  useEffect(() => {
+    room.on(RoomEvent.MediaDevicesError, onDeviceFailure);
+
+    return () => {
+      room.off(RoomEvent.MediaDevicesError, onDeviceFailure);
+    };
+  }, [room]);
+
+  return (
+    <main data-lk-theme="default" className="h-full grid content-center bg-[var(--lk-bg)]">
+      <RoomContext.Provider value={room}>
+        <div className="lk-room-container max-h-[90vh]">
+          <SimpleVoiceAssistant onConnectButtonClicked={onConnectButtonClicked} />
+        </div>
+      </RoomContext.Provider>
+    </main>
+  );
+}
+
+function SimpleVoiceAssistant(props: { onConnectButtonClicked: () => void }) {
+  const { state: agentState } = useVoiceAssistant();
+  return (
+    <>
+      <AnimatePresence>
+        {agentState === "disconnected" && (
+          <motion.button
+            initial={{ opacity: 0, top: 0 }}
+            animate={{ opacity: 1 }}
+            exit={{ opacity: 0, top: "-10px" }}
+            transition={{ duration: 1, ease: [0.09, 1.04, 0.245, 1.055] }}
+            className="uppercase absolute left-1/2 -translate-x-1/2 px-4 py-2 bg-white text-black rounded-md"
+            onClick={() => props.onConnectButtonClicked()}
+          >
+            Start a conversation
+          </motion.button>
+        )}
+        <div className="w-3/4 lg:w-1/2 mx-auto h-full">
+          <TranscriptionView />
+        </div>
+      </AnimatePresence>
+
+      <RoomAudioRenderer />
+      <NoAgentNotification state={agentState} />
+      <div className="fixed bottom-0 w-full px-4 py-2">
+        <ControlBar />
+      </div>
+    </>
+  );
+}
+
+function ControlBar() {
+  /**
+   * Use Krisp background noise reduction when available.
+   * Note: This is only available on Scale plan, see {@link https://livekit.io/pricing | LiveKit Pricing} for more details.
+   */
+  const krisp = useKrispNoiseFilter();
+  useEffect(() => {
+    krisp.setNoiseFilterEnabled(true);
+  }, []);
+
+  const { state: agentState, audioTrack } = useVoiceAssistant();
+
+  return (
+    <div className="relative h-[100px]">
+      <AnimatePresence>
+        {agentState !== "disconnected" && agentState !== "connecting" && (
+          <motion.div
+            initial={{ opacity: 0, top: "10px" }}
+            animate={{ opacity: 1, top: 0 }}
+            exit={{ opacity: 0, top: "-10px" }}
+            transition={{ duration: 0.4, ease: [0.09, 1.04, 0.245, 1.055] }}
+            className="flex absolute w-full h-full justify-between px-8 sm:px-4"
+          >
+            <BarVisualizer
+              state={agentState}
+              barCount={5}
+              trackRef={audioTrack}
+              className="agent-visualizer w-24 gap-2"
+              options={{ minHeight: 12 }}
+            />
+            <div className="flex items-center">
+              <VoiceAssistantControlBar controls={{ leave: false }} />
+              <DisconnectButton>
+                <CloseIcon />
+              </DisconnectButton>
+            </div>
+          </motion.div>
+        )}
+      </AnimatePresence>
+    </div>
+  );
+}
+
+function onDeviceFailure(error: Error) {
+  console.error(error);
+  alert(
+    "Error acquiring camera or microphone permissions. Please make sure you grant the necessary permissions in your browser and reload the tab"
+  );
+}

frontend/src/components/CloseIcon.tsx

@@ -0,0 +1,12 @@
+export function CloseIcon() {
+  return (
+    <svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M3.33398 3.33334L12.6673 12.6667M12.6673 3.33334L3.33398 12.6667"
+        stroke="currentColor"
+        stroke-width="2"
+        stroke-linecap="square"
+      />
+    </svg>
+  );
+}

frontend/src/components/NoAgentNotification.tsx

@@ -0,0 +1,99 @@
+import type { AgentState } from "@livekit/components-react";
+import { useEffect, useRef, useState } from "react";
+
+interface NoAgentNotificationProps extends React.PropsWithChildren<object> {
+  state: AgentState;
+}
+
+/**
+ * Renders some user info when no agent connects to the room after a certain time.
+ */
+export function NoAgentNotification(props: NoAgentNotificationProps) {
+  const timeToWaitMs = 10_000;
+  const timeoutRef = useRef<number | null>(null);
+  const [showNotification, setShowNotification] = useState(false);
+  const agentHasConnected = useRef(false);
+
+  // If the agent has connected, we don't need to show the notification.
+  if (
+    ["listening", "thinking", "speaking"].includes(props.state) &&
+    agentHasConnected.current == false
+  ) {
+    agentHasConnected.current = true;
+  }
+
+  useEffect(() => {
+    if (props.state === "connecting") {
+      timeoutRef.current = window.setTimeout(() => {
+        if (props.state === "connecting" && agentHasConnected.current === false) {
+          setShowNotification(true);
+        }
+      }, timeToWaitMs);
+    } else {
+      if (timeoutRef.current) {
+        window.clearTimeout(timeoutRef.current);
+      }
+      setShowNotification(false);
+    }
+
+    return () => {
+      if (timeoutRef.current) {
+        window.clearTimeout(timeoutRef.current);
+      }
+    };
+  }, [props.state]);
+
+  return (
+    <>
+      {showNotification ? (
+        <div className="fixed text-sm left-1/2 max-w-[90vw] -translate-x-1/2 flex top-6 items-center gap-4 bg-[#1F1F1F] px-4 py-3 rounded-lg">
+          <div>
+            {/* Warning Icon */}
+            <svg
+              width="24"
+              height="24"
+              viewBox="0 0 24 24"
+              fill="none"
+              xmlns="http://www.w3.org/2000/svg"
+            >
+              <path
+                fillRule="evenodd"
+                clipRule="evenodd"
+                d="M9.85068 3.63564C10.8197 2.00589 13.1793 2.00589 14.1484 3.63564L21.6323 16.2223C22.6232 17.8888 21.4223 20 19.4835 20H4.51555C2.57676 20 1.37584 17.8888 2.36671 16.2223L9.85068 3.63564ZM12 8.5C12.2761 8.5 12.5 8.72386 12.5 9V13.5C12.5 13.7761 12.2761 14 12 14C11.7239 14 11.5 13.7761 11.5 13.5V9C11.5 8.72386 11.7239 8.5 12 8.5ZM12.75 16C12.75 16.4142 12.4142 16.75 12 16.75C11.5858 16.75 11.25 16.4142 11.25 16C11.25 15.5858 11.5858 15.25 12 15.25C12.4142 15.25 12.75 15.5858 12.75 16Z"
+                fill="#666666"
+              />
+            </svg>
+          </div>
+          <p className="text-pretty w-max">
+            It&apos;s quiet... too quiet. Is your agent lost? Ensure your agent is properly
+            configured and running on your machine.
+          </p>
+          <a
+            href="https://docs.livekit.io/agents/quickstarts/s2s/"
+            target="_blank"
+            className="underline whitespace-nowrap"
+          >
+            View guide
+          </a>
+          <button onClick={() => setShowNotification(false)}>
+            {/* Close Icon */}
+            <svg
+              width="16"
+              height="16"
+              viewBox="0 0 16 16"
+              fill="none"
+              xmlns="http://www.w3.org/2000/svg"
+            >
+              <path
+                d="M3.16602 3.16666L12.8327 12.8333M12.8327 3.16666L3.16602 12.8333"
+                stroke="#999999"
+                strokeWidth="1.5"
+                strokeLinecap="square"
+              />
+            </svg>
+          </button>
+        </div>
+      ) : null}
+    </>
+  );
+}

frontend/src/components/TranscriptionView.tsx

@@ -0,0 +1,35 @@
+import useCombinedTranscriptions from "@/hooks/useCombinedTranscriptions";
+import * as React from "react";
+
+export default function TranscriptionView() {
+  const combinedTranscriptions = useCombinedTranscriptions();
+
+  // scroll to bottom when new transcription is added
+  React.useEffect(() => {
+    const transcription = combinedTranscriptions[combinedTranscriptions.length - 1];
+    if (transcription) {
+      const transcriptionElement = document.getElementById(transcription.id);
+      if (transcriptionElement) {
+        transcriptionElement.scrollIntoView({ behavior: "smooth" });
+      }
+    }
+  }, [combinedTranscriptions]);
+
+  return (
+    <div className="h-full flex flex-col gap-2 overflow-y-auto">
+      {combinedTranscriptions.map((segment) => (
+        <div
+          id={segment.id}
+          key={segment.id}
+          className={
+            segment.role === "assistant"
+              ? "p-2 self-start fit-content"
+              : "bg-gray-800 rounded-md p-2 self-end fit-content"
+          }
+        >
+          {segment.text}
+        </div>
+      ))}
+    </div>
+  );
+}

frontend/src/hooks/useCombinedTranscriptions.ts

@@ -0,0 +1,23 @@
+import { useTrackTranscription, useVoiceAssistant } from "@livekit/components-react";
+import { useMemo } from "react";
+import useLocalMicTrack from "./useLocalMicTrack";
+
+export default function useCombinedTranscriptions() {
+  const { agentTranscriptions } = useVoiceAssistant();
+
+  const micTrackRef = useLocalMicTrack();
+  const { segments: userTranscriptions } = useTrackTranscription(micTrackRef);
+
+  const combinedTranscriptions = useMemo(() => {
+    return [
+      ...agentTranscriptions.map((val) => {
+        return { ...val, role: "assistant" };
+      }),
+      ...userTranscriptions.map((val) => {
+        return { ...val, role: "user" };
+      }),
+    ].sort((a, b) => a.firstReceivedTime - b.firstReceivedTime);
+  }, [agentTranscriptions, userTranscriptions]);
+
+  return combinedTranscriptions;
+}

frontend/src/hooks/useLocalMicTrack.ts

@@ -0,0 +1,17 @@
+import { TrackReferenceOrPlaceholder, useLocalParticipant } from "@livekit/components-react";
+import { Track } from "livekit-client";
+import { useMemo } from "react";
+
+export default function useLocalMicTrack() {
+  const { microphoneTrack, localParticipant } = useLocalParticipant();
+
+  const micTrackRef: TrackReferenceOrPlaceholder = useMemo(() => {
+    return {
+      participant: localParticipant,
+      source: Track.Source.Microphone,
+      publication: microphoneTrack,
+    };
+  }, [localParticipant, microphoneTrack]);
+
+  return micTrackRef;
+}

frontend/tailwind.config.ts

@@ -0,0 +1,20 @@
+import type { Config } from "tailwindcss";
+
+const config: Config = {
+  content: [
+    "./src/pages/**/*.{js,ts,jsx,tsx,mdx}",
+    "./src/components/**/*.{js,ts,jsx,tsx,mdx}",
+    "./src/app/**/*.{js,ts,jsx,tsx,mdx}",
+  ],
+  theme: {
+    extend: {
+      backgroundImage: {
+        "gradient-radial": "radial-gradient(var(--tw-gradient-stops))",
+        "gradient-conic":
+          "conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))",
+      },
+    },
+  },
+  plugins: [],
+};
+export default config;

frontend/tsconfig.json

@@ -0,0 +1,26 @@
+{
+  "compilerOptions": {
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/*": ["./src/*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}