eric/joel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: joel voice

Browse Source
This commit is contained in:
eric
2026-02-05 14:07:58 +01:00
parent 172c5decd3
commit 24b4c12e7d
19 changed files with 627 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
src/commands/definitions/voiceover.ts Normal file
View File

@@ -0,0 +1,94 @@
/**
* Voiceover command - generate audio with ElevenLabs
*/
import { SlashCommandBuilder } from "discord.js";
import type { Command } from "../types";
import { createLogger } from "../../core/logger";
import { config } from "../../core/config";
import { getVoiceoverService } from "../../services/ai/voiceover";
const logger = createLogger("Commands:Voiceover");
const MAX_TEXT_LENGTH = 1000;
const command: Command = {
data: new SlashCommandBuilder()
.setName("voiceover")
.setDescription("Generate a voiceover using ElevenLabs")
.addStringOption((option) =>
option
.setName("text")
.setDescription("Text to speak")
.setRequired(true)
)
.addStringOption((option) =>
option
.setName("voice")
.setDescription("Optional ElevenLabs voice ID")
.setRequired(false)
) as SlashCommandBuilder,
category: "ai",
execute: async (interaction) => {
const text = interaction.options.getString("text", true).trim();
const voiceId = interaction.options.getString("voice") || undefined;
if (!config.elevenlabs.apiKey) {
await interaction.reply({
content: "Voiceover is not configured (missing ELEVENLABS_API_KEY).",
ephemeral: true,
});
return;
}
if (text.length === 0) {
await interaction.reply({
content: "Please provide text to speak.",
ephemeral: true,
});
return;
}
if (text.length > MAX_TEXT_LENGTH) {
await interaction.reply({
content: `Text is too long. Max length is ${MAX_TEXT_LENGTH} characters.`,
ephemeral: true,
});
return;
}
if (!voiceId && !config.elevenlabs.voiceId) {
await interaction.reply({
content: "Voiceover needs a voice ID (set ELEVENLABS_VOICE_ID or pass one).",
ephemeral: true,
});
return;
}
await interaction.deferReply();
try {
const voiceover = getVoiceoverService();
const audio = await voiceover.generate({ text, voiceId });
await interaction.editReply({
content: "Here is your voiceover:",
files: [
{
attachment: audio,
name: "voiceover.mp3",
},
],
});
} catch (error) {
logger.error("Voiceover generation failed", error);
const message = error instanceof Error ? error.message : "Unknown error";
await interaction.editReply({
content: `Voiceover failed: ${message}`,
});
}
},
};
export default command;

10
src/core/config.ts
View File

@@ -25,6 +25,11 @@ interface BotConfig {
klipy: {
apiKey: string;
};
elevenlabs: {
apiKey: string;
voiceId: string;
modelId: string;
};
bot: {
/** Chance of Joel responding without being mentioned (0-1) */
freeWillChance: number;
@@ -82,6 +87,11 @@ export const config: BotConfig = {
klipy: {
apiKey: getEnvOrDefault("KLIPY_API_KEY", ""),
},
elevenlabs: {
apiKey: getEnvOrDefault("ELEVENLABS_API_KEY", ""),
voiceId: getEnvOrDefault("ELEVENLABS_VOICE_ID", ""),
modelId: getEnvOrDefault("ELEVENLABS_MODEL", "eleven_multilingual_v2"),
},
bot: {
freeWillChance: 0.02,
memoryChance: 0.3,

BIN
src/database/db.sqlite3
View File

Binary file not shown.

31
src/database/repositories/memory.repository.ts
View File

@@ -80,11 +80,11 @@ export const memoryRepository = {
/**
* Find memories by user ID, sorted by importance then recency
*/
async findByUserId(userId: string, limit = 10): Promise<Memory[]> {
async findByUserId(userId: string, guildId: string, limit = 10): Promise<Memory[]> {
const results = await db
.select()
.from(memories)
.where(eq(memories.user_id, userId))
.where(and(eq(memories.user_id, userId), eq(memories.guild_id, guildId)))
.orderBy(desc(memories.importance), desc(memories.created_at))
.limit(limit);
@@ -101,6 +101,7 @@ export const memoryRepository = {
*/
async findByCategory(
userId: string,
guildId: string,
category: MemoryCategory,
limit = 10
): Promise<Memory[]> {
@@ -110,6 +111,7 @@ export const memoryRepository = {
.where(
and(
eq(memories.user_id, userId),
eq(memories.guild_id, guildId),
eq(memories.category, category)
)
)
@@ -159,11 +161,11 @@ export const memoryRepository = {
/**
* Get the most important memories for a user
*/
async getMostImportant(userId: string, limit = 5): Promise<Memory[]> {
async getMostImportant(userId: string, guildId: string, limit = 5): Promise<Memory[]> {
return db
.select()
.from(memories)
.where(eq(memories.user_id, userId))
.where(and(eq(memories.user_id, userId), eq(memories.guild_id, guildId)))
.orderBy(desc(memories.importance), desc(memories.access_count))
.limit(limit);
},
@@ -171,11 +173,11 @@ export const memoryRepository = {
/**
* Get frequently accessed memories (likely most useful)
*/
async getMostAccessed(userId: string, limit = 5): Promise<Memory[]> {
async getMostAccessed(userId: string, guildId: string, limit = 5): Promise<Memory[]> {
return db
.select()
.from(memories)
.where(eq(memories.user_id, userId))
.where(and(eq(memories.user_id, userId), eq(memories.guild_id, guildId)))
.orderBy(desc(memories.access_count), desc(memories.importance))
.limit(limit);
},
@@ -184,7 +186,12 @@ export const memoryRepository = {
* Check for duplicate or similar memories using embedding similarity
* Falls back to substring matching if embeddings are unavailable
*/
async findSimilar(userId: string, content: string, threshold = 0.85): Promise<Memory[]> {
async findSimilar(
userId: string,
guildId: string,
content: string,
threshold = 0.85
): Promise<Memory[]> {
const embeddingService = getEmbeddingService();
// Try embedding-based similarity first
@@ -199,6 +206,7 @@ export const memoryRepository = {
.where(
and(
eq(memories.user_id, userId),
eq(memories.guild_id, guildId),
sql`${memories.embedding} IS NOT NULL`
)
);
@@ -236,6 +244,7 @@ export const memoryRepository = {
.where(
and(
eq(memories.user_id, userId),
eq(memories.guild_id, guildId),
like(sql`lower(${memories.content})`, `%${searchTerm}%`)
)
)
@@ -331,10 +340,10 @@ export const memoryRepository = {
/**
* Delete all memories for a user
*/
async deleteByUserId(userId: string): Promise<number> {
async deleteByUserId(userId: string, guildId: string): Promise<number> {
const result = await db
.delete(memories)
.where(eq(memories.user_id, userId))
.where(and(eq(memories.user_id, userId), eq(memories.guild_id, guildId)))
.returning({ id: memories.id });
return result.length;
@@ -369,7 +378,7 @@ export const memoryRepository = {
/**
* Get memory statistics for a user
*/
async getStats(userId: string): Promise<{
async getStats(userId: string, guildId: string): Promise<{
total: number;
byCategory: Record<string, number>;
avgImportance: number;
@@ -377,7 +386,7 @@ export const memoryRepository = {
const allMemories = await db
.select()
.from(memories)
.where(eq(memories.user_id, userId));
.where(and(eq(memories.user_id, userId), eq(memories.guild_id, guildId)));
const byCategory: Record<string, number> = {};
let totalImportance = 0;

4
src/features/joel/responder.ts
View File

@@ -12,6 +12,7 @@ import { personalities, botOptions } from "../../database/schema";
import { eq } from "drizzle-orm";
import { buildStyledPrompt, STYLE_MODIFIERS } from "./personalities";
import { getRandomMention } from "./mentions";
import { speakVoiceover } from "./voice";
import { TypingIndicator } from "./typing";
const logger = createLogger("Features:Joel");
@@ -115,6 +116,9 @@ export const joelResponder = {
const fullResponse = response + mention;
await this.sendResponse(message, fullResponse);
speakVoiceover(message, fullResponse).catch((error) => {
logger.error("Failed to play voiceover", error);
});
} catch (error) {
logger.error("Failed to respond", error);
await this.handleError(message, error);

168
src/features/joel/voice.ts Normal file
View File

@@ -0,0 +1,168 @@
/**
* Voiceover playback for Joel responses
*/
import {
AudioPlayerStatus,
VoiceConnectionStatus,
createAudioPlayer,
createAudioResource,
entersState,
getVoiceConnection,
joinVoiceChannel,
StreamType,
type DiscordGatewayAdapterCreator,
} from "@discordjs/voice";
import type { Message } from "discord.js";
import { Readable } from "node:stream";
import { config } from "../../core/config";
import { createLogger } from "../../core/logger";
import { getVoiceoverService } from "../../services/ai/voiceover";
const logger = createLogger("Features:Joel:Voice");
const MAX_VOICE_TEXT_LENGTH = 800;
const PLAYBACK_TIMEOUT_MS = 60_000;
const READY_TIMEOUT_MS = 15_000;
function isAbortError(error: unknown): boolean {
return error instanceof Error && error.name === "AbortError";
}
function sanitizeForVoiceover(content: string): string {
let text = content.replace(/```[\s\S]*?```/g, " ");
text = text.replace(/`([^`]+)`/g, "$1");
text = text.replace(/\s+/g, " ").trim();
if (text.length > MAX_VOICE_TEXT_LENGTH) {
text = `${text.slice(0, MAX_VOICE_TEXT_LENGTH - 3).trimEnd()}...`;
}
return text;
}
async function getOrCreateConnection(message: Message<true>) {
const voiceChannel = message.member?.voice.channel;
if (!voiceChannel) {
logger.debug("No voice channel for author", {
userId: message.author.id,
guildId: message.guildId,
});
return null;
}
const me = message.guild.members.me ?? (await message.guild.members.fetchMe());
const permissions = voiceChannel.permissionsFor(me);
if (!permissions?.has(["Connect", "Speak"])) {
logger.debug("Missing voice permissions", {
guildId: message.guildId,
channelId: voiceChannel.id,
});
return null;
}
const existing = getVoiceConnection(message.guildId);
if (existing && existing.joinConfig.channelId === voiceChannel.id) {
logger.debug("Reusing existing voice connection", {
guildId: message.guildId,
channelId: voiceChannel.id,
});
return existing;
}
if (existing) {
existing.destroy();
}
logger.debug("Joining voice channel", {
guildId: message.guildId,
channelId: voiceChannel.id,
});
const connection = joinVoiceChannel({
channelId: voiceChannel.id,
guildId: voiceChannel.guild.id,
adapterCreator: voiceChannel.guild.voiceAdapterCreator as unknown as DiscordGatewayAdapterCreator,
selfDeaf: false,
});
try {
await entersState(connection, VoiceConnectionStatus.Ready, READY_TIMEOUT_MS);
logger.debug("Voice connection ready", {
guildId: message.guildId,
channelId: voiceChannel.id,
});
return connection;
} catch (error) {
if (isAbortError(error)) {
logger.debug("Voice connection ready timeout", {
guildId: message.guildId,
channelId: voiceChannel.id,
status: connection.state.status,
});
} else {
logger.error("Voice connection failed to become ready", error);
}
connection.destroy();
return null;
}
}
export async function speakVoiceover(message: Message<true>, content: string): Promise<void> {
if (!config.elevenlabs.apiKey || !config.elevenlabs.voiceId) {
logger.debug("Voiceover disabled (missing config)");
return;
}
const text = sanitizeForVoiceover(content);
if (!text) {
logger.debug("Voiceover skipped (empty text after sanitize)");
return;
}
const connection = await getOrCreateConnection(message);
if (!connection) {
logger.debug("Voiceover skipped (no connection)", {
guildId: message.guildId,
authorId: message.author.id,
});
return;
}
try {
const voiceover = getVoiceoverService();
logger.debug("Requesting ElevenLabs voiceover", { textLength: text.length });
const audio = await voiceover.generate({ text });
logger.debug("Voiceover audio received", { bytes: audio.length });
const player = createAudioPlayer();
const resource = createAudioResource(Readable.from(audio), {
inputType: StreamType.Arbitrary,
});
player.on("error", (error) => {
logger.error("Audio player error", error);
});
player.on(AudioPlayerStatus.Playing, () => {
logger.debug("Audio player started", { guildId: message.guildId });
});
player.on(AudioPlayerStatus.Idle, () => {
logger.debug("Audio player idle", { guildId: message.guildId });
});
connection.subscribe(player);
player.play(resource);
await entersState(player, AudioPlayerStatus.Playing, 5_000).catch(() => undefined);
await entersState(player, AudioPlayerStatus.Idle, PLAYBACK_TIMEOUT_MS);
} catch (error) {
if (!isAbortError(error)) {
logger.error("Voiceover playback failed", error);
}
} finally {
if (connection.state.status !== VoiceConnectionStatus.Destroyed) {
connection.destroy();
}
}
}

2
src/index.ts
View File

@@ -29,6 +29,8 @@ const client = new BotClient({
GatewayIntentBits.GuildMessages,
GatewayIntentBits.GuildModeration,
GatewayIntentBits.GuildMembers,
GatewayIntentBits.GuildVoiceStates,
],
});

14
src/services/ai/tool-handlers.ts
View File

@@ -27,9 +27,9 @@ const toolHandlers: Record<string, ToolHandler> = {
let userMemories;
if (category) {
userMemories = await memoryRepository.findByCategory(userId, category, limit);
userMemories = await memoryRepository.findByCategory(userId, context.guildId, category, limit);
} else {
userMemories = await memoryRepository.findByUserId(userId, limit);
userMemories = await memoryRepository.findByUserId(userId, context.guildId, limit);
}
if (userMemories.length === 0) {
@@ -61,7 +61,7 @@ const toolHandlers: Record<string, ToolHandler> = {
}
// Check for duplicate memories using new similarity check
const similar = await memoryRepository.findSimilar(userId, content);
const similar = await memoryRepository.findSimilar(userId, context.guildId, content);
if (similar.length > 0) {
return "Already knew something similar. Memory not saved (duplicate).";
}
@@ -89,7 +89,7 @@ const toolHandlers: Record<string, ToolHandler> = {
*/
async search_memories(args, context): Promise<string> {
const query = args.query as string;
const guildId = args.guild_id as string | undefined;
const guildId = (args.guild_id as string | undefined) ?? context.guildId;
const category = args.category as MemoryCategory | undefined;
const minImportance = args.min_importance as number | undefined;
@@ -138,7 +138,7 @@ const toolHandlers: Record<string, ToolHandler> = {
logger.warn("Forgetting user memories", { userId, requestedBy: context.userId });
const deleted = await memoryRepository.deleteByUserId(userId);
const deleted = await memoryRepository.deleteByUserId(userId, context.guildId);
return `Deleted ${deleted} memories about user ${userId}.`;
},
@@ -157,7 +157,7 @@ const toolHandlers: Record<string, ToolHandler> = {
}
// Check for duplicates
const similar = await memoryRepository.findSimilar(context.userId, content);
const similar = await memoryRepository.findSimilar(context.userId, context.guildId, content);
if (similar.length > 0) {
return "Similar memory already exists. Skipped.";
}
@@ -185,7 +185,7 @@ const toolHandlers: Record<string, ToolHandler> = {
async get_memory_stats(args, context): Promise<string> {
const userId = (args.user_id as string) || context.userId;
const stats = await memoryRepository.getStats(userId);
const stats = await memoryRepository.getStats(userId, context.guildId);
if (stats.total === 0) {
return `No memories stored for this user.`;

104
src/services/ai/voiceover.ts Normal file
View File

@@ -0,0 +1,104 @@
/**
* ElevenLabs voiceover service
*/
import { config } from "../../core/config";
import { createLogger } from "../../core/logger";
const logger = createLogger("AI:Voiceover");
const DEFAULT_OUTPUT_FORMAT = "mp3_44100_128" as const;
const DEFAULT_STABILITY = 0.1;
const DEFAULT_SIMILARITY = 0.90;
const DEFAULT_STYLE = 0.25;
const DEFAULT_SPEED = 1.20
function clamp01(value: number): number {
return Math.max(0, Math.min(1, value));
}
export interface VoiceoverOptions {
text: string;
voiceId?: string;
modelId?: string;
stability?: number;
similarityBoost?: number;
style?: number;
speakerBoost?: boolean;
}
export class VoiceoverService {
async generate(options: VoiceoverOptions): Promise<Buffer> {
const apiKey = config.elevenlabs.apiKey;
if (!apiKey) {
throw new Error("Voiceover is not configured (missing ELEVENLABS_API_KEY).");
}
const voiceId = options.voiceId || config.elevenlabs.voiceId;
if (!voiceId) {
throw new Error("Voiceover is missing a voice ID (set ELEVENLABS_VOICE_ID or pass one).");
}
const text = options.text.trim();
if (!text) {
throw new Error("Voiceover text is empty.");
}
const modelId = options.modelId || config.elevenlabs.modelId;
const voiceSettings = {
stability: clamp01(options.stability ?? DEFAULT_STABILITY),
similarity_boost: clamp01(options.similarityBoost ?? DEFAULT_SIMILARITY),
style: clamp01(options.style ?? DEFAULT_STYLE),
use_speaker_boost: options.speakerBoost ?? true,
};
const url = new URL(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream`);
url.searchParams.set("output_format", DEFAULT_OUTPUT_FORMAT);
logger.debug("Generating voiceover", {
textLength: text.length,
voiceId,
modelId,
});
const response = await fetch(url.toString(), {
method: "POST",
headers: {
"xi-api-key": apiKey,
"Content-Type": "application/json",
"Accept": "audio/mpeg",
},
body: JSON.stringify({
text,
model_id: modelId,
voice_settings: voiceSettings,
}),
});
if (!response.ok) {
const errorBody = await response.text();
logger.error("ElevenLabs API error", {
status: response.status,
body: errorBody.slice(0, 300),
});
throw new Error(`ElevenLabs API error (HTTP ${response.status}).`);
}
const audioBuffer = await response.arrayBuffer();
return Buffer.from(audioBuffer);
}
async health(): Promise<boolean> {
return !!config.elevenlabs.apiKey;
}
}
let voiceoverService: VoiceoverService | null = null;
export function getVoiceoverService(): VoiceoverService {
if (!voiceoverService) {
voiceoverService = new VoiceoverService();
}
return voiceoverService;
}