feat: joel voice
This commit is contained in:
19
.direnv/bin/nix-direnv-reload
Executable file
19
.direnv/bin/nix-direnv-reload
Executable file
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
if [[ ! -d "/Users/eric/Projects/joel-discord" ]]; then
|
||||
echo "Cannot find source directory; Did you move it?"
|
||||
echo "(Looking for "/Users/eric/Projects/joel-discord")"
|
||||
echo 'Cannot force reload with this script - use "direnv reload" manually and then try again'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# rebuild the cache forcefully
|
||||
_nix_direnv_force_reload=1 direnv exec "/Users/eric/Projects/joel-discord" true
|
||||
|
||||
# Update the mtime for .envrc.
|
||||
# This will cause direnv to reload again - but without re-building.
|
||||
touch "/Users/eric/Projects/joel-discord/.envrc"
|
||||
|
||||
# Also update the timestamp of whatever profile_rc we have.
|
||||
# This makes sure that we know we are up to date.
|
||||
touch -r "/Users/eric/Projects/joel-discord/.envrc" "/Users/eric/Projects/joel-discord/.direnv"/*.rc
|
||||
@@ -7,6 +7,9 @@ HF_TOKEN=""
|
||||
OPENAI_API_KEY=""
|
||||
OPENROUTER_API_KEY=""
|
||||
REPLICATE_API_TOKEN=""
|
||||
ELEVENLABS_API_KEY=""
|
||||
ELEVENLABS_VOICE_ID=""
|
||||
ELEVENLABS_MODEL="eleven_multilingual_v2"
|
||||
WEB_PORT="3000"
|
||||
WEB_BASE_URL="http://localhost:3000"
|
||||
SESSION_SECRET=""
|
||||
2
.gitignore copy
Normal file
2
.gitignore copy
Normal file
@@ -0,0 +1,2 @@
|
||||
.direnv/
|
||||
.pre-commit-config.yaml
|
||||
24
LICENSE
Normal file
24
LICENSE
Normal file
@@ -0,0 +1,24 @@
|
||||
This is free and unencumbered software released into the public domain.
|
||||
|
||||
Anyone is free to copy, modify, publish, use, compile, sell, or
|
||||
distribute this software, either in source code form or as a compiled
|
||||
binary, for any purpose, commercial or non-commercial, and by any
|
||||
means.
|
||||
|
||||
In jurisdictions that recognize copyright laws, the author or authors
|
||||
of this software dedicate any and all copyright interest in the
|
||||
software to the public domain. We make this dedication for the benefit
|
||||
of the public at large and to the detriment of our heirs and
|
||||
successors. We intend this dedication to be an overt act of
|
||||
relinquishment in perpetuity of all present and future rights to this
|
||||
software under copyright law.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
For more information, please refer to <https://unlicense.org>
|
||||
@@ -43,6 +43,9 @@ src/
|
||||
| `DISCORD_CLIENT_SECRET` | Discord application client secret |
|
||||
| `OPENROUTER_API_KEY` | OpenRouter API key for AI |
|
||||
| `KLIPY_API_KEY` | Klipy API key for GIF search (optional) |
|
||||
| `ELEVENLABS_API_KEY` | ElevenLabs API key for voiceover |
|
||||
| `ELEVENLABS_VOICE_ID` | Default ElevenLabs voice ID (optional) |
|
||||
| `ELEVENLABS_MODEL` | ElevenLabs model ID (default: `eleven_multilingual_v2`) |
|
||||
| `WEB_PORT` | Port for web dashboard (default: 3000) |
|
||||
| `WEB_BASE_URL` | Base URL for web dashboard |
|
||||
| `SESSION_SECRET` | Secret for session encryption |
|
||||
|
||||
101
flake.lock
generated
Normal file
101
flake.lock
generated
Normal file
@@ -0,0 +1,101 @@
|
||||
{
|
||||
"nodes": {
|
||||
"flake-compat": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1767039857,
|
||||
"narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
|
||||
"owner": "NixOS",
|
||||
"repo": "flake-compat",
|
||||
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"repo": "flake-compat",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"git-hooks": {
|
||||
"inputs": {
|
||||
"flake-compat": "flake-compat",
|
||||
"gitignore": "gitignore",
|
||||
"nixpkgs": "nixpkgs"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1769939035,
|
||||
"narHash": "sha256-Fok2AmefgVA0+eprw2NDwqKkPGEI5wvR+twiZagBvrg=",
|
||||
"owner": "cachix",
|
||||
"repo": "git-hooks.nix",
|
||||
"rev": "a8ca480175326551d6c4121498316261cbb5b260",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "cachix",
|
||||
"repo": "git-hooks.nix",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"gitignore": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"git-hooks",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1709087332,
|
||||
"narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
|
||||
"owner": "hercules-ci",
|
||||
"repo": "gitignore.nix",
|
||||
"rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "hercules-ci",
|
||||
"repo": "gitignore.nix",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1764947035,
|
||||
"narHash": "sha256-EYHSjVM4Ox4lvCXUMiKKs2vETUSL5mx+J2FfutM7T9w=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "a672be65651c80d3f592a89b3945466584a22069",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixpkgs-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs_2": {
|
||||
"locked": {
|
||||
"lastModified": 1770115704,
|
||||
"narHash": "sha256-KHFT9UWOF2yRPlAnSXQJh6uVcgNcWlFqqiAZ7OVlHNc=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "e6eae2ee2110f3d31110d5c222cd395303343b08",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nixos",
|
||||
"ref": "nixos-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"git-hooks": "git-hooks",
|
||||
"nixpkgs": "nixpkgs_2"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
64
flake.nix
Normal file
64
flake.nix
Normal file
@@ -0,0 +1,64 @@
|
||||
{
|
||||
description = "A very basic flake";
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-unstable";
|
||||
git-hooks.url = "github:cachix/git-hooks.nix";
|
||||
};
|
||||
|
||||
outputs =
|
||||
{ self, nixpkgs, ... }@inputs:
|
||||
let
|
||||
supportedSystems = [
|
||||
"x86_64-linux"
|
||||
"aarch64-linux"
|
||||
"x86_64-darwin"
|
||||
"aarch64-darwin"
|
||||
];
|
||||
forAllSystems = nixpkgs.lib.genAttrs supportedSystems;
|
||||
in
|
||||
{
|
||||
# Run the hooks with `nix fmt`.
|
||||
formatter = forAllSystems (
|
||||
system:
|
||||
let
|
||||
pkgs = nixpkgs.legacyPackages.${system};
|
||||
config = self.checks.${system}.pre-commit-check.config;
|
||||
inherit (config) package configFile;
|
||||
script = ''
|
||||
${pkgs.lib.getExe package} run --all-files --config ${configFile}
|
||||
'';
|
||||
in
|
||||
pkgs.writeShellScriptBin "pre-commit-run" script
|
||||
);
|
||||
|
||||
checks = forAllSystems (system: {
|
||||
pre-commit-check = inputs.git-hooks.lib.${system}.run {
|
||||
src = ./.;
|
||||
hooks = {
|
||||
nixfmt.enable = true;
|
||||
};
|
||||
};
|
||||
});
|
||||
|
||||
devShells = forAllSystems (
|
||||
system:
|
||||
let
|
||||
pkgs = import nixpkgs { inherit system; };
|
||||
inherit (self.checks.${system}.pre-commit-check) shellHook enabledPackages;
|
||||
in
|
||||
{
|
||||
default = pkgs.mkShell {
|
||||
packages = with pkgs; [
|
||||
ffmpeg
|
||||
bun
|
||||
gitlint
|
||||
];
|
||||
|
||||
inherit shellHook;
|
||||
buildInputs = enabledPackages;
|
||||
};
|
||||
}
|
||||
);
|
||||
};
|
||||
}
|
||||
@@ -24,6 +24,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/openai": "^0.0.13",
|
||||
"@discordjs/voice": "^0.18.0",
|
||||
"@fal-ai/client": "^1.8.4",
|
||||
"@huggingface/inference": "^4.13.10",
|
||||
"@libsql/client": "^0.17.0",
|
||||
|
||||
94
src/commands/definitions/voiceover.ts
Normal file
94
src/commands/definitions/voiceover.ts
Normal file
@@ -0,0 +1,94 @@
|
||||
/**
|
||||
* Voiceover command - generate audio with ElevenLabs
|
||||
*/
|
||||
|
||||
import { SlashCommandBuilder } from "discord.js";
|
||||
import type { Command } from "../types";
|
||||
import { createLogger } from "../../core/logger";
|
||||
import { config } from "../../core/config";
|
||||
import { getVoiceoverService } from "../../services/ai/voiceover";
|
||||
|
||||
const logger = createLogger("Commands:Voiceover");
|
||||
|
||||
const MAX_TEXT_LENGTH = 1000;
|
||||
|
||||
const command: Command = {
|
||||
data: new SlashCommandBuilder()
|
||||
.setName("voiceover")
|
||||
.setDescription("Generate a voiceover using ElevenLabs")
|
||||
.addStringOption((option) =>
|
||||
option
|
||||
.setName("text")
|
||||
.setDescription("Text to speak")
|
||||
.setRequired(true)
|
||||
)
|
||||
.addStringOption((option) =>
|
||||
option
|
||||
.setName("voice")
|
||||
.setDescription("Optional ElevenLabs voice ID")
|
||||
.setRequired(false)
|
||||
) as SlashCommandBuilder,
|
||||
category: "ai",
|
||||
execute: async (interaction) => {
|
||||
const text = interaction.options.getString("text", true).trim();
|
||||
const voiceId = interaction.options.getString("voice") || undefined;
|
||||
|
||||
if (!config.elevenlabs.apiKey) {
|
||||
await interaction.reply({
|
||||
content: "Voiceover is not configured (missing ELEVENLABS_API_KEY).",
|
||||
ephemeral: true,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (text.length === 0) {
|
||||
await interaction.reply({
|
||||
content: "Please provide text to speak.",
|
||||
ephemeral: true,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (text.length > MAX_TEXT_LENGTH) {
|
||||
await interaction.reply({
|
||||
content: `Text is too long. Max length is ${MAX_TEXT_LENGTH} characters.`,
|
||||
ephemeral: true,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (!voiceId && !config.elevenlabs.voiceId) {
|
||||
await interaction.reply({
|
||||
content: "Voiceover needs a voice ID (set ELEVENLABS_VOICE_ID or pass one).",
|
||||
ephemeral: true,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await interaction.deferReply();
|
||||
|
||||
try {
|
||||
const voiceover = getVoiceoverService();
|
||||
const audio = await voiceover.generate({ text, voiceId });
|
||||
|
||||
await interaction.editReply({
|
||||
content: "Here is your voiceover:",
|
||||
files: [
|
||||
{
|
||||
attachment: audio,
|
||||
name: "voiceover.mp3",
|
||||
},
|
||||
],
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error("Voiceover generation failed", error);
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
|
||||
await interaction.editReply({
|
||||
content: `Voiceover failed: ${message}`,
|
||||
});
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
export default command;
|
||||
@@ -25,6 +25,11 @@ interface BotConfig {
|
||||
klipy: {
|
||||
apiKey: string;
|
||||
};
|
||||
elevenlabs: {
|
||||
apiKey: string;
|
||||
voiceId: string;
|
||||
modelId: string;
|
||||
};
|
||||
bot: {
|
||||
/** Chance of Joel responding without being mentioned (0-1) */
|
||||
freeWillChance: number;
|
||||
@@ -82,6 +87,11 @@ export const config: BotConfig = {
|
||||
klipy: {
|
||||
apiKey: getEnvOrDefault("KLIPY_API_KEY", ""),
|
||||
},
|
||||
elevenlabs: {
|
||||
apiKey: getEnvOrDefault("ELEVENLABS_API_KEY", ""),
|
||||
voiceId: getEnvOrDefault("ELEVENLABS_VOICE_ID", ""),
|
||||
modelId: getEnvOrDefault("ELEVENLABS_MODEL", "eleven_multilingual_v2"),
|
||||
},
|
||||
bot: {
|
||||
freeWillChance: 0.02,
|
||||
memoryChance: 0.3,
|
||||
|
||||
Binary file not shown.
@@ -80,11 +80,11 @@ export const memoryRepository = {
|
||||
/**
|
||||
* Find memories by user ID, sorted by importance then recency
|
||||
*/
|
||||
async findByUserId(userId: string, limit = 10): Promise<Memory[]> {
|
||||
async findByUserId(userId: string, guildId: string, limit = 10): Promise<Memory[]> {
|
||||
const results = await db
|
||||
.select()
|
||||
.from(memories)
|
||||
.where(eq(memories.user_id, userId))
|
||||
.where(and(eq(memories.user_id, userId), eq(memories.guild_id, guildId)))
|
||||
.orderBy(desc(memories.importance), desc(memories.created_at))
|
||||
.limit(limit);
|
||||
|
||||
@@ -101,6 +101,7 @@ export const memoryRepository = {
|
||||
*/
|
||||
async findByCategory(
|
||||
userId: string,
|
||||
guildId: string,
|
||||
category: MemoryCategory,
|
||||
limit = 10
|
||||
): Promise<Memory[]> {
|
||||
@@ -110,6 +111,7 @@ export const memoryRepository = {
|
||||
.where(
|
||||
and(
|
||||
eq(memories.user_id, userId),
|
||||
eq(memories.guild_id, guildId),
|
||||
eq(memories.category, category)
|
||||
)
|
||||
)
|
||||
@@ -159,11 +161,11 @@ export const memoryRepository = {
|
||||
/**
|
||||
* Get the most important memories for a user
|
||||
*/
|
||||
async getMostImportant(userId: string, limit = 5): Promise<Memory[]> {
|
||||
async getMostImportant(userId: string, guildId: string, limit = 5): Promise<Memory[]> {
|
||||
return db
|
||||
.select()
|
||||
.from(memories)
|
||||
.where(eq(memories.user_id, userId))
|
||||
.where(and(eq(memories.user_id, userId), eq(memories.guild_id, guildId)))
|
||||
.orderBy(desc(memories.importance), desc(memories.access_count))
|
||||
.limit(limit);
|
||||
},
|
||||
@@ -171,11 +173,11 @@ export const memoryRepository = {
|
||||
/**
|
||||
* Get frequently accessed memories (likely most useful)
|
||||
*/
|
||||
async getMostAccessed(userId: string, limit = 5): Promise<Memory[]> {
|
||||
async getMostAccessed(userId: string, guildId: string, limit = 5): Promise<Memory[]> {
|
||||
return db
|
||||
.select()
|
||||
.from(memories)
|
||||
.where(eq(memories.user_id, userId))
|
||||
.where(and(eq(memories.user_id, userId), eq(memories.guild_id, guildId)))
|
||||
.orderBy(desc(memories.access_count), desc(memories.importance))
|
||||
.limit(limit);
|
||||
},
|
||||
@@ -184,7 +186,12 @@ export const memoryRepository = {
|
||||
* Check for duplicate or similar memories using embedding similarity
|
||||
* Falls back to substring matching if embeddings are unavailable
|
||||
*/
|
||||
async findSimilar(userId: string, content: string, threshold = 0.85): Promise<Memory[]> {
|
||||
async findSimilar(
|
||||
userId: string,
|
||||
guildId: string,
|
||||
content: string,
|
||||
threshold = 0.85
|
||||
): Promise<Memory[]> {
|
||||
const embeddingService = getEmbeddingService();
|
||||
|
||||
// Try embedding-based similarity first
|
||||
@@ -199,6 +206,7 @@ export const memoryRepository = {
|
||||
.where(
|
||||
and(
|
||||
eq(memories.user_id, userId),
|
||||
eq(memories.guild_id, guildId),
|
||||
sql`${memories.embedding} IS NOT NULL`
|
||||
)
|
||||
);
|
||||
@@ -236,6 +244,7 @@ export const memoryRepository = {
|
||||
.where(
|
||||
and(
|
||||
eq(memories.user_id, userId),
|
||||
eq(memories.guild_id, guildId),
|
||||
like(sql`lower(${memories.content})`, `%${searchTerm}%`)
|
||||
)
|
||||
)
|
||||
@@ -331,10 +340,10 @@ export const memoryRepository = {
|
||||
/**
|
||||
* Delete all memories for a user
|
||||
*/
|
||||
async deleteByUserId(userId: string): Promise<number> {
|
||||
async deleteByUserId(userId: string, guildId: string): Promise<number> {
|
||||
const result = await db
|
||||
.delete(memories)
|
||||
.where(eq(memories.user_id, userId))
|
||||
.where(and(eq(memories.user_id, userId), eq(memories.guild_id, guildId)))
|
||||
.returning({ id: memories.id });
|
||||
|
||||
return result.length;
|
||||
@@ -369,7 +378,7 @@ export const memoryRepository = {
|
||||
/**
|
||||
* Get memory statistics for a user
|
||||
*/
|
||||
async getStats(userId: string): Promise<{
|
||||
async getStats(userId: string, guildId: string): Promise<{
|
||||
total: number;
|
||||
byCategory: Record<string, number>;
|
||||
avgImportance: number;
|
||||
@@ -377,7 +386,7 @@ export const memoryRepository = {
|
||||
const allMemories = await db
|
||||
.select()
|
||||
.from(memories)
|
||||
.where(eq(memories.user_id, userId));
|
||||
.where(and(eq(memories.user_id, userId), eq(memories.guild_id, guildId)));
|
||||
|
||||
const byCategory: Record<string, number> = {};
|
||||
let totalImportance = 0;
|
||||
|
||||
@@ -12,6 +12,7 @@ import { personalities, botOptions } from "../../database/schema";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { buildStyledPrompt, STYLE_MODIFIERS } from "./personalities";
|
||||
import { getRandomMention } from "./mentions";
|
||||
import { speakVoiceover } from "./voice";
|
||||
import { TypingIndicator } from "./typing";
|
||||
|
||||
const logger = createLogger("Features:Joel");
|
||||
@@ -115,6 +116,9 @@ export const joelResponder = {
|
||||
const fullResponse = response + mention;
|
||||
|
||||
await this.sendResponse(message, fullResponse);
|
||||
speakVoiceover(message, fullResponse).catch((error) => {
|
||||
logger.error("Failed to play voiceover", error);
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error("Failed to respond", error);
|
||||
await this.handleError(message, error);
|
||||
|
||||
168
src/features/joel/voice.ts
Normal file
168
src/features/joel/voice.ts
Normal file
@@ -0,0 +1,168 @@
|
||||
/**
|
||||
* Voiceover playback for Joel responses
|
||||
*/
|
||||
|
||||
import {
|
||||
AudioPlayerStatus,
|
||||
VoiceConnectionStatus,
|
||||
createAudioPlayer,
|
||||
createAudioResource,
|
||||
entersState,
|
||||
getVoiceConnection,
|
||||
joinVoiceChannel,
|
||||
StreamType,
|
||||
type DiscordGatewayAdapterCreator,
|
||||
} from "@discordjs/voice";
|
||||
import type { Message } from "discord.js";
|
||||
import { Readable } from "node:stream";
|
||||
import { config } from "../../core/config";
|
||||
import { createLogger } from "../../core/logger";
|
||||
import { getVoiceoverService } from "../../services/ai/voiceover";
|
||||
|
||||
const logger = createLogger("Features:Joel:Voice");
|
||||
|
||||
const MAX_VOICE_TEXT_LENGTH = 800;
|
||||
const PLAYBACK_TIMEOUT_MS = 60_000;
|
||||
const READY_TIMEOUT_MS = 15_000;
|
||||
|
||||
function isAbortError(error: unknown): boolean {
|
||||
return error instanceof Error && error.name === "AbortError";
|
||||
}
|
||||
|
||||
function sanitizeForVoiceover(content: string): string {
|
||||
let text = content.replace(/```[\s\S]*?```/g, " ");
|
||||
text = text.replace(/`([^`]+)`/g, "$1");
|
||||
text = text.replace(/\s+/g, " ").trim();
|
||||
|
||||
if (text.length > MAX_VOICE_TEXT_LENGTH) {
|
||||
text = `${text.slice(0, MAX_VOICE_TEXT_LENGTH - 3).trimEnd()}...`;
|
||||
}
|
||||
|
||||
return text;
|
||||
}
|
||||
|
||||
async function getOrCreateConnection(message: Message<true>) {
|
||||
const voiceChannel = message.member?.voice.channel;
|
||||
if (!voiceChannel) {
|
||||
logger.debug("No voice channel for author", {
|
||||
userId: message.author.id,
|
||||
guildId: message.guildId,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const me = message.guild.members.me ?? (await message.guild.members.fetchMe());
|
||||
const permissions = voiceChannel.permissionsFor(me);
|
||||
if (!permissions?.has(["Connect", "Speak"])) {
|
||||
logger.debug("Missing voice permissions", {
|
||||
guildId: message.guildId,
|
||||
channelId: voiceChannel.id,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const existing = getVoiceConnection(message.guildId);
|
||||
if (existing && existing.joinConfig.channelId === voiceChannel.id) {
|
||||
logger.debug("Reusing existing voice connection", {
|
||||
guildId: message.guildId,
|
||||
channelId: voiceChannel.id,
|
||||
});
|
||||
return existing;
|
||||
}
|
||||
|
||||
if (existing) {
|
||||
existing.destroy();
|
||||
}
|
||||
|
||||
logger.debug("Joining voice channel", {
|
||||
guildId: message.guildId,
|
||||
channelId: voiceChannel.id,
|
||||
});
|
||||
|
||||
const connection = joinVoiceChannel({
|
||||
channelId: voiceChannel.id,
|
||||
guildId: voiceChannel.guild.id,
|
||||
adapterCreator: voiceChannel.guild.voiceAdapterCreator as unknown as DiscordGatewayAdapterCreator,
|
||||
selfDeaf: false,
|
||||
});
|
||||
|
||||
try {
|
||||
await entersState(connection, VoiceConnectionStatus.Ready, READY_TIMEOUT_MS);
|
||||
logger.debug("Voice connection ready", {
|
||||
guildId: message.guildId,
|
||||
channelId: voiceChannel.id,
|
||||
});
|
||||
return connection;
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) {
|
||||
logger.debug("Voice connection ready timeout", {
|
||||
guildId: message.guildId,
|
||||
channelId: voiceChannel.id,
|
||||
status: connection.state.status,
|
||||
});
|
||||
} else {
|
||||
logger.error("Voice connection failed to become ready", error);
|
||||
}
|
||||
connection.destroy();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function speakVoiceover(message: Message<true>, content: string): Promise<void> {
|
||||
if (!config.elevenlabs.apiKey || !config.elevenlabs.voiceId) {
|
||||
logger.debug("Voiceover disabled (missing config)");
|
||||
return;
|
||||
}
|
||||
|
||||
const text = sanitizeForVoiceover(content);
|
||||
if (!text) {
|
||||
logger.debug("Voiceover skipped (empty text after sanitize)");
|
||||
return;
|
||||
}
|
||||
|
||||
const connection = await getOrCreateConnection(message);
|
||||
if (!connection) {
|
||||
logger.debug("Voiceover skipped (no connection)", {
|
||||
guildId: message.guildId,
|
||||
authorId: message.author.id,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const voiceover = getVoiceoverService();
|
||||
logger.debug("Requesting ElevenLabs voiceover", { textLength: text.length });
|
||||
const audio = await voiceover.generate({ text });
|
||||
logger.debug("Voiceover audio received", { bytes: audio.length });
|
||||
const player = createAudioPlayer();
|
||||
const resource = createAudioResource(Readable.from(audio), {
|
||||
inputType: StreamType.Arbitrary,
|
||||
});
|
||||
|
||||
player.on("error", (error) => {
|
||||
logger.error("Audio player error", error);
|
||||
});
|
||||
|
||||
player.on(AudioPlayerStatus.Playing, () => {
|
||||
logger.debug("Audio player started", { guildId: message.guildId });
|
||||
});
|
||||
|
||||
player.on(AudioPlayerStatus.Idle, () => {
|
||||
logger.debug("Audio player idle", { guildId: message.guildId });
|
||||
});
|
||||
|
||||
connection.subscribe(player);
|
||||
player.play(resource);
|
||||
|
||||
await entersState(player, AudioPlayerStatus.Playing, 5_000).catch(() => undefined);
|
||||
await entersState(player, AudioPlayerStatus.Idle, PLAYBACK_TIMEOUT_MS);
|
||||
} catch (error) {
|
||||
if (!isAbortError(error)) {
|
||||
logger.error("Voiceover playback failed", error);
|
||||
}
|
||||
} finally {
|
||||
if (connection.state.status !== VoiceConnectionStatus.Destroyed) {
|
||||
connection.destroy();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,8 @@ const client = new BotClient({
|
||||
GatewayIntentBits.GuildMessages,
|
||||
GatewayIntentBits.GuildModeration,
|
||||
GatewayIntentBits.GuildMembers,
|
||||
GatewayIntentBits.GuildVoiceStates,
|
||||
|
||||
],
|
||||
});
|
||||
|
||||
|
||||
@@ -27,9 +27,9 @@ const toolHandlers: Record<string, ToolHandler> = {
|
||||
|
||||
let userMemories;
|
||||
if (category) {
|
||||
userMemories = await memoryRepository.findByCategory(userId, category, limit);
|
||||
userMemories = await memoryRepository.findByCategory(userId, context.guildId, category, limit);
|
||||
} else {
|
||||
userMemories = await memoryRepository.findByUserId(userId, limit);
|
||||
userMemories = await memoryRepository.findByUserId(userId, context.guildId, limit);
|
||||
}
|
||||
|
||||
if (userMemories.length === 0) {
|
||||
@@ -61,7 +61,7 @@ const toolHandlers: Record<string, ToolHandler> = {
|
||||
}
|
||||
|
||||
// Check for duplicate memories using new similarity check
|
||||
const similar = await memoryRepository.findSimilar(userId, content);
|
||||
const similar = await memoryRepository.findSimilar(userId, context.guildId, content);
|
||||
if (similar.length > 0) {
|
||||
return "Already knew something similar. Memory not saved (duplicate).";
|
||||
}
|
||||
@@ -89,7 +89,7 @@ const toolHandlers: Record<string, ToolHandler> = {
|
||||
*/
|
||||
async search_memories(args, context): Promise<string> {
|
||||
const query = args.query as string;
|
||||
const guildId = args.guild_id as string | undefined;
|
||||
const guildId = (args.guild_id as string | undefined) ?? context.guildId;
|
||||
const category = args.category as MemoryCategory | undefined;
|
||||
const minImportance = args.min_importance as number | undefined;
|
||||
|
||||
@@ -138,7 +138,7 @@ const toolHandlers: Record<string, ToolHandler> = {
|
||||
|
||||
logger.warn("Forgetting user memories", { userId, requestedBy: context.userId });
|
||||
|
||||
const deleted = await memoryRepository.deleteByUserId(userId);
|
||||
const deleted = await memoryRepository.deleteByUserId(userId, context.guildId);
|
||||
|
||||
return `Deleted ${deleted} memories about user ${userId}.`;
|
||||
},
|
||||
@@ -157,7 +157,7 @@ const toolHandlers: Record<string, ToolHandler> = {
|
||||
}
|
||||
|
||||
// Check for duplicates
|
||||
const similar = await memoryRepository.findSimilar(context.userId, content);
|
||||
const similar = await memoryRepository.findSimilar(context.userId, context.guildId, content);
|
||||
if (similar.length > 0) {
|
||||
return "Similar memory already exists. Skipped.";
|
||||
}
|
||||
@@ -185,7 +185,7 @@ const toolHandlers: Record<string, ToolHandler> = {
|
||||
async get_memory_stats(args, context): Promise<string> {
|
||||
const userId = (args.user_id as string) || context.userId;
|
||||
|
||||
const stats = await memoryRepository.getStats(userId);
|
||||
const stats = await memoryRepository.getStats(userId, context.guildId);
|
||||
|
||||
if (stats.total === 0) {
|
||||
return `No memories stored for this user.`;
|
||||
|
||||
104
src/services/ai/voiceover.ts
Normal file
104
src/services/ai/voiceover.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
/**
|
||||
* ElevenLabs voiceover service
|
||||
*/
|
||||
|
||||
import { config } from "../../core/config";
|
||||
import { createLogger } from "../../core/logger";
|
||||
|
||||
const logger = createLogger("AI:Voiceover");
|
||||
|
||||
const DEFAULT_OUTPUT_FORMAT = "mp3_44100_128" as const;
|
||||
const DEFAULT_STABILITY = 0.1;
|
||||
const DEFAULT_SIMILARITY = 0.90;
|
||||
const DEFAULT_STYLE = 0.25;
|
||||
const DEFAULT_SPEED = 1.20
|
||||
|
||||
function clamp01(value: number): number {
|
||||
return Math.max(0, Math.min(1, value));
|
||||
}
|
||||
|
||||
export interface VoiceoverOptions {
|
||||
text: string;
|
||||
voiceId?: string;
|
||||
modelId?: string;
|
||||
stability?: number;
|
||||
similarityBoost?: number;
|
||||
style?: number;
|
||||
speakerBoost?: boolean;
|
||||
}
|
||||
|
||||
export class VoiceoverService {
|
||||
async generate(options: VoiceoverOptions): Promise<Buffer> {
|
||||
const apiKey = config.elevenlabs.apiKey;
|
||||
if (!apiKey) {
|
||||
throw new Error("Voiceover is not configured (missing ELEVENLABS_API_KEY).");
|
||||
}
|
||||
|
||||
const voiceId = options.voiceId || config.elevenlabs.voiceId;
|
||||
if (!voiceId) {
|
||||
throw new Error("Voiceover is missing a voice ID (set ELEVENLABS_VOICE_ID or pass one).");
|
||||
}
|
||||
|
||||
const text = options.text.trim();
|
||||
if (!text) {
|
||||
throw new Error("Voiceover text is empty.");
|
||||
}
|
||||
|
||||
const modelId = options.modelId || config.elevenlabs.modelId;
|
||||
|
||||
const voiceSettings = {
|
||||
stability: clamp01(options.stability ?? DEFAULT_STABILITY),
|
||||
similarity_boost: clamp01(options.similarityBoost ?? DEFAULT_SIMILARITY),
|
||||
style: clamp01(options.style ?? DEFAULT_STYLE),
|
||||
use_speaker_boost: options.speakerBoost ?? true,
|
||||
};
|
||||
|
||||
const url = new URL(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream`);
|
||||
url.searchParams.set("output_format", DEFAULT_OUTPUT_FORMAT);
|
||||
|
||||
logger.debug("Generating voiceover", {
|
||||
textLength: text.length,
|
||||
voiceId,
|
||||
modelId,
|
||||
});
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"xi-api-key": apiKey,
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "audio/mpeg",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
text,
|
||||
model_id: modelId,
|
||||
voice_settings: voiceSettings,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorBody = await response.text();
|
||||
logger.error("ElevenLabs API error", {
|
||||
status: response.status,
|
||||
body: errorBody.slice(0, 300),
|
||||
});
|
||||
throw new Error(`ElevenLabs API error (HTTP ${response.status}).`);
|
||||
}
|
||||
|
||||
const audioBuffer = await response.arrayBuffer();
|
||||
return Buffer.from(audioBuffer);
|
||||
}
|
||||
|
||||
async health(): Promise<boolean> {
|
||||
return !!config.elevenlabs.apiKey;
|
||||
}
|
||||
}
|
||||
|
||||
let voiceoverService: VoiceoverService | null = null;
|
||||
|
||||
export function getVoiceoverService(): VoiceoverService {
|
||||
if (!voiceoverService) {
|
||||
voiceoverService = new VoiceoverService();
|
||||
}
|
||||
return voiceoverService;
|
||||
}
|
||||
Reference in New Issue
Block a user