feat: joel voice
This commit is contained in:
19
.direnv/bin/nix-direnv-reload
Executable file
19
.direnv/bin/nix-direnv-reload
Executable file
@@ -0,0 +1,19 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -e
|
||||||
|
if [[ ! -d "/Users/eric/Projects/joel-discord" ]]; then
|
||||||
|
echo "Cannot find source directory; Did you move it?"
|
||||||
|
echo "(Looking for "/Users/eric/Projects/joel-discord")"
|
||||||
|
echo 'Cannot force reload with this script - use "direnv reload" manually and then try again'
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# rebuild the cache forcefully
|
||||||
|
_nix_direnv_force_reload=1 direnv exec "/Users/eric/Projects/joel-discord" true
|
||||||
|
|
||||||
|
# Update the mtime for .envrc.
|
||||||
|
# This will cause direnv to reload again - but without re-building.
|
||||||
|
touch "/Users/eric/Projects/joel-discord/.envrc"
|
||||||
|
|
||||||
|
# Also update the timestamp of whatever profile_rc we have.
|
||||||
|
# This makes sure that we know we are up to date.
|
||||||
|
touch -r "/Users/eric/Projects/joel-discord/.envrc" "/Users/eric/Projects/joel-discord/.direnv"/*.rc
|
||||||
@@ -7,6 +7,9 @@ HF_TOKEN=""
|
|||||||
OPENAI_API_KEY=""
|
OPENAI_API_KEY=""
|
||||||
OPENROUTER_API_KEY=""
|
OPENROUTER_API_KEY=""
|
||||||
REPLICATE_API_TOKEN=""
|
REPLICATE_API_TOKEN=""
|
||||||
|
ELEVENLABS_API_KEY=""
|
||||||
|
ELEVENLABS_VOICE_ID=""
|
||||||
|
ELEVENLABS_MODEL="eleven_multilingual_v2"
|
||||||
WEB_PORT="3000"
|
WEB_PORT="3000"
|
||||||
WEB_BASE_URL="http://localhost:3000"
|
WEB_BASE_URL="http://localhost:3000"
|
||||||
SESSION_SECRET=""
|
SESSION_SECRET=""
|
||||||
2
.gitignore copy
Normal file
2
.gitignore copy
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
.direnv/
|
||||||
|
.pre-commit-config.yaml
|
||||||
24
LICENSE
Normal file
24
LICENSE
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
This is free and unencumbered software released into the public domain.
|
||||||
|
|
||||||
|
Anyone is free to copy, modify, publish, use, compile, sell, or
|
||||||
|
distribute this software, either in source code form or as a compiled
|
||||||
|
binary, for any purpose, commercial or non-commercial, and by any
|
||||||
|
means.
|
||||||
|
|
||||||
|
In jurisdictions that recognize copyright laws, the author or authors
|
||||||
|
of this software dedicate any and all copyright interest in the
|
||||||
|
software to the public domain. We make this dedication for the benefit
|
||||||
|
of the public at large and to the detriment of our heirs and
|
||||||
|
successors. We intend this dedication to be an overt act of
|
||||||
|
relinquishment in perpetuity of all present and future rights to this
|
||||||
|
software under copyright law.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||||
|
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||||
|
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||||
|
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||||
|
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||||
|
OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
|
For more information, please refer to <https://unlicense.org>
|
||||||
@@ -43,6 +43,9 @@ src/
|
|||||||
| `DISCORD_CLIENT_SECRET` | Discord application client secret |
|
| `DISCORD_CLIENT_SECRET` | Discord application client secret |
|
||||||
| `OPENROUTER_API_KEY` | OpenRouter API key for AI |
|
| `OPENROUTER_API_KEY` | OpenRouter API key for AI |
|
||||||
| `KLIPY_API_KEY` | Klipy API key for GIF search (optional) |
|
| `KLIPY_API_KEY` | Klipy API key for GIF search (optional) |
|
||||||
|
| `ELEVENLABS_API_KEY` | ElevenLabs API key for voiceover |
|
||||||
|
| `ELEVENLABS_VOICE_ID` | Default ElevenLabs voice ID (optional) |
|
||||||
|
| `ELEVENLABS_MODEL` | ElevenLabs model ID (default: `eleven_multilingual_v2`) |
|
||||||
| `WEB_PORT` | Port for web dashboard (default: 3000) |
|
| `WEB_PORT` | Port for web dashboard (default: 3000) |
|
||||||
| `WEB_BASE_URL` | Base URL for web dashboard |
|
| `WEB_BASE_URL` | Base URL for web dashboard |
|
||||||
| `SESSION_SECRET` | Secret for session encryption |
|
| `SESSION_SECRET` | Secret for session encryption |
|
||||||
|
|||||||
101
flake.lock
generated
Normal file
101
flake.lock
generated
Normal file
@@ -0,0 +1,101 @@
|
|||||||
|
{
|
||||||
|
"nodes": {
|
||||||
|
"flake-compat": {
|
||||||
|
"flake": false,
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1767039857,
|
||||||
|
"narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "flake-compat",
|
||||||
|
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "flake-compat",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"git-hooks": {
|
||||||
|
"inputs": {
|
||||||
|
"flake-compat": "flake-compat",
|
||||||
|
"gitignore": "gitignore",
|
||||||
|
"nixpkgs": "nixpkgs"
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1769939035,
|
||||||
|
"narHash": "sha256-Fok2AmefgVA0+eprw2NDwqKkPGEI5wvR+twiZagBvrg=",
|
||||||
|
"owner": "cachix",
|
||||||
|
"repo": "git-hooks.nix",
|
||||||
|
"rev": "a8ca480175326551d6c4121498316261cbb5b260",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "cachix",
|
||||||
|
"repo": "git-hooks.nix",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"gitignore": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": [
|
||||||
|
"git-hooks",
|
||||||
|
"nixpkgs"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1709087332,
|
||||||
|
"narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
|
||||||
|
"owner": "hercules-ci",
|
||||||
|
"repo": "gitignore.nix",
|
||||||
|
"rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "hercules-ci",
|
||||||
|
"repo": "gitignore.nix",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nixpkgs": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1764947035,
|
||||||
|
"narHash": "sha256-EYHSjVM4Ox4lvCXUMiKKs2vETUSL5mx+J2FfutM7T9w=",
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"rev": "a672be65651c80d3f592a89b3945466584a22069",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "NixOS",
|
||||||
|
"ref": "nixpkgs-unstable",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nixpkgs_2": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1770115704,
|
||||||
|
"narHash": "sha256-KHFT9UWOF2yRPlAnSXQJh6uVcgNcWlFqqiAZ7OVlHNc=",
|
||||||
|
"owner": "nixos",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"rev": "e6eae2ee2110f3d31110d5c222cd395303343b08",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "nixos",
|
||||||
|
"ref": "nixos-unstable",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": {
|
||||||
|
"inputs": {
|
||||||
|
"git-hooks": "git-hooks",
|
||||||
|
"nixpkgs": "nixpkgs_2"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": "root",
|
||||||
|
"version": 7
|
||||||
|
}
|
||||||
64
flake.nix
Normal file
64
flake.nix
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
{
|
||||||
|
description = "A very basic flake";
|
||||||
|
|
||||||
|
inputs = {
|
||||||
|
nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-unstable";
|
||||||
|
git-hooks.url = "github:cachix/git-hooks.nix";
|
||||||
|
};
|
||||||
|
|
||||||
|
outputs =
|
||||||
|
{ self, nixpkgs, ... }@inputs:
|
||||||
|
let
|
||||||
|
supportedSystems = [
|
||||||
|
"x86_64-linux"
|
||||||
|
"aarch64-linux"
|
||||||
|
"x86_64-darwin"
|
||||||
|
"aarch64-darwin"
|
||||||
|
];
|
||||||
|
forAllSystems = nixpkgs.lib.genAttrs supportedSystems;
|
||||||
|
in
|
||||||
|
{
|
||||||
|
# Run the hooks with `nix fmt`.
|
||||||
|
formatter = forAllSystems (
|
||||||
|
system:
|
||||||
|
let
|
||||||
|
pkgs = nixpkgs.legacyPackages.${system};
|
||||||
|
config = self.checks.${system}.pre-commit-check.config;
|
||||||
|
inherit (config) package configFile;
|
||||||
|
script = ''
|
||||||
|
${pkgs.lib.getExe package} run --all-files --config ${configFile}
|
||||||
|
'';
|
||||||
|
in
|
||||||
|
pkgs.writeShellScriptBin "pre-commit-run" script
|
||||||
|
);
|
||||||
|
|
||||||
|
checks = forAllSystems (system: {
|
||||||
|
pre-commit-check = inputs.git-hooks.lib.${system}.run {
|
||||||
|
src = ./.;
|
||||||
|
hooks = {
|
||||||
|
nixfmt.enable = true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
devShells = forAllSystems (
|
||||||
|
system:
|
||||||
|
let
|
||||||
|
pkgs = import nixpkgs { inherit system; };
|
||||||
|
inherit (self.checks.${system}.pre-commit-check) shellHook enabledPackages;
|
||||||
|
in
|
||||||
|
{
|
||||||
|
default = pkgs.mkShell {
|
||||||
|
packages = with pkgs; [
|
||||||
|
ffmpeg
|
||||||
|
bun
|
||||||
|
gitlint
|
||||||
|
];
|
||||||
|
|
||||||
|
inherit shellHook;
|
||||||
|
buildInputs = enabledPackages;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
);
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -24,6 +24,7 @@
|
|||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@ai-sdk/openai": "^0.0.13",
|
"@ai-sdk/openai": "^0.0.13",
|
||||||
|
"@discordjs/voice": "^0.18.0",
|
||||||
"@fal-ai/client": "^1.8.4",
|
"@fal-ai/client": "^1.8.4",
|
||||||
"@huggingface/inference": "^4.13.10",
|
"@huggingface/inference": "^4.13.10",
|
||||||
"@libsql/client": "^0.17.0",
|
"@libsql/client": "^0.17.0",
|
||||||
|
|||||||
94
src/commands/definitions/voiceover.ts
Normal file
94
src/commands/definitions/voiceover.ts
Normal file
@@ -0,0 +1,94 @@
|
|||||||
|
/**
|
||||||
|
* Voiceover command - generate audio with ElevenLabs
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { SlashCommandBuilder } from "discord.js";
|
||||||
|
import type { Command } from "../types";
|
||||||
|
import { createLogger } from "../../core/logger";
|
||||||
|
import { config } from "../../core/config";
|
||||||
|
import { getVoiceoverService } from "../../services/ai/voiceover";
|
||||||
|
|
||||||
|
const logger = createLogger("Commands:Voiceover");
|
||||||
|
|
||||||
|
const MAX_TEXT_LENGTH = 1000;
|
||||||
|
|
||||||
|
const command: Command = {
|
||||||
|
data: new SlashCommandBuilder()
|
||||||
|
.setName("voiceover")
|
||||||
|
.setDescription("Generate a voiceover using ElevenLabs")
|
||||||
|
.addStringOption((option) =>
|
||||||
|
option
|
||||||
|
.setName("text")
|
||||||
|
.setDescription("Text to speak")
|
||||||
|
.setRequired(true)
|
||||||
|
)
|
||||||
|
.addStringOption((option) =>
|
||||||
|
option
|
||||||
|
.setName("voice")
|
||||||
|
.setDescription("Optional ElevenLabs voice ID")
|
||||||
|
.setRequired(false)
|
||||||
|
) as SlashCommandBuilder,
|
||||||
|
category: "ai",
|
||||||
|
execute: async (interaction) => {
|
||||||
|
const text = interaction.options.getString("text", true).trim();
|
||||||
|
const voiceId = interaction.options.getString("voice") || undefined;
|
||||||
|
|
||||||
|
if (!config.elevenlabs.apiKey) {
|
||||||
|
await interaction.reply({
|
||||||
|
content: "Voiceover is not configured (missing ELEVENLABS_API_KEY).",
|
||||||
|
ephemeral: true,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (text.length === 0) {
|
||||||
|
await interaction.reply({
|
||||||
|
content: "Please provide text to speak.",
|
||||||
|
ephemeral: true,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (text.length > MAX_TEXT_LENGTH) {
|
||||||
|
await interaction.reply({
|
||||||
|
content: `Text is too long. Max length is ${MAX_TEXT_LENGTH} characters.`,
|
||||||
|
ephemeral: true,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!voiceId && !config.elevenlabs.voiceId) {
|
||||||
|
await interaction.reply({
|
||||||
|
content: "Voiceover needs a voice ID (set ELEVENLABS_VOICE_ID or pass one).",
|
||||||
|
ephemeral: true,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await interaction.deferReply();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const voiceover = getVoiceoverService();
|
||||||
|
const audio = await voiceover.generate({ text, voiceId });
|
||||||
|
|
||||||
|
await interaction.editReply({
|
||||||
|
content: "Here is your voiceover:",
|
||||||
|
files: [
|
||||||
|
{
|
||||||
|
attachment: audio,
|
||||||
|
name: "voiceover.mp3",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("Voiceover generation failed", error);
|
||||||
|
const message = error instanceof Error ? error.message : "Unknown error";
|
||||||
|
|
||||||
|
await interaction.editReply({
|
||||||
|
content: `Voiceover failed: ${message}`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
export default command;
|
||||||
@@ -25,6 +25,11 @@ interface BotConfig {
|
|||||||
klipy: {
|
klipy: {
|
||||||
apiKey: string;
|
apiKey: string;
|
||||||
};
|
};
|
||||||
|
elevenlabs: {
|
||||||
|
apiKey: string;
|
||||||
|
voiceId: string;
|
||||||
|
modelId: string;
|
||||||
|
};
|
||||||
bot: {
|
bot: {
|
||||||
/** Chance of Joel responding without being mentioned (0-1) */
|
/** Chance of Joel responding without being mentioned (0-1) */
|
||||||
freeWillChance: number;
|
freeWillChance: number;
|
||||||
@@ -82,6 +87,11 @@ export const config: BotConfig = {
|
|||||||
klipy: {
|
klipy: {
|
||||||
apiKey: getEnvOrDefault("KLIPY_API_KEY", ""),
|
apiKey: getEnvOrDefault("KLIPY_API_KEY", ""),
|
||||||
},
|
},
|
||||||
|
elevenlabs: {
|
||||||
|
apiKey: getEnvOrDefault("ELEVENLABS_API_KEY", ""),
|
||||||
|
voiceId: getEnvOrDefault("ELEVENLABS_VOICE_ID", ""),
|
||||||
|
modelId: getEnvOrDefault("ELEVENLABS_MODEL", "eleven_multilingual_v2"),
|
||||||
|
},
|
||||||
bot: {
|
bot: {
|
||||||
freeWillChance: 0.02,
|
freeWillChance: 0.02,
|
||||||
memoryChance: 0.3,
|
memoryChance: 0.3,
|
||||||
|
|||||||
Binary file not shown.
@@ -80,11 +80,11 @@ export const memoryRepository = {
|
|||||||
/**
|
/**
|
||||||
* Find memories by user ID, sorted by importance then recency
|
* Find memories by user ID, sorted by importance then recency
|
||||||
*/
|
*/
|
||||||
async findByUserId(userId: string, limit = 10): Promise<Memory[]> {
|
async findByUserId(userId: string, guildId: string, limit = 10): Promise<Memory[]> {
|
||||||
const results = await db
|
const results = await db
|
||||||
.select()
|
.select()
|
||||||
.from(memories)
|
.from(memories)
|
||||||
.where(eq(memories.user_id, userId))
|
.where(and(eq(memories.user_id, userId), eq(memories.guild_id, guildId)))
|
||||||
.orderBy(desc(memories.importance), desc(memories.created_at))
|
.orderBy(desc(memories.importance), desc(memories.created_at))
|
||||||
.limit(limit);
|
.limit(limit);
|
||||||
|
|
||||||
@@ -101,6 +101,7 @@ export const memoryRepository = {
|
|||||||
*/
|
*/
|
||||||
async findByCategory(
|
async findByCategory(
|
||||||
userId: string,
|
userId: string,
|
||||||
|
guildId: string,
|
||||||
category: MemoryCategory,
|
category: MemoryCategory,
|
||||||
limit = 10
|
limit = 10
|
||||||
): Promise<Memory[]> {
|
): Promise<Memory[]> {
|
||||||
@@ -110,6 +111,7 @@ export const memoryRepository = {
|
|||||||
.where(
|
.where(
|
||||||
and(
|
and(
|
||||||
eq(memories.user_id, userId),
|
eq(memories.user_id, userId),
|
||||||
|
eq(memories.guild_id, guildId),
|
||||||
eq(memories.category, category)
|
eq(memories.category, category)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -159,11 +161,11 @@ export const memoryRepository = {
|
|||||||
/**
|
/**
|
||||||
* Get the most important memories for a user
|
* Get the most important memories for a user
|
||||||
*/
|
*/
|
||||||
async getMostImportant(userId: string, limit = 5): Promise<Memory[]> {
|
async getMostImportant(userId: string, guildId: string, limit = 5): Promise<Memory[]> {
|
||||||
return db
|
return db
|
||||||
.select()
|
.select()
|
||||||
.from(memories)
|
.from(memories)
|
||||||
.where(eq(memories.user_id, userId))
|
.where(and(eq(memories.user_id, userId), eq(memories.guild_id, guildId)))
|
||||||
.orderBy(desc(memories.importance), desc(memories.access_count))
|
.orderBy(desc(memories.importance), desc(memories.access_count))
|
||||||
.limit(limit);
|
.limit(limit);
|
||||||
},
|
},
|
||||||
@@ -171,11 +173,11 @@ export const memoryRepository = {
|
|||||||
/**
|
/**
|
||||||
* Get frequently accessed memories (likely most useful)
|
* Get frequently accessed memories (likely most useful)
|
||||||
*/
|
*/
|
||||||
async getMostAccessed(userId: string, limit = 5): Promise<Memory[]> {
|
async getMostAccessed(userId: string, guildId: string, limit = 5): Promise<Memory[]> {
|
||||||
return db
|
return db
|
||||||
.select()
|
.select()
|
||||||
.from(memories)
|
.from(memories)
|
||||||
.where(eq(memories.user_id, userId))
|
.where(and(eq(memories.user_id, userId), eq(memories.guild_id, guildId)))
|
||||||
.orderBy(desc(memories.access_count), desc(memories.importance))
|
.orderBy(desc(memories.access_count), desc(memories.importance))
|
||||||
.limit(limit);
|
.limit(limit);
|
||||||
},
|
},
|
||||||
@@ -184,7 +186,12 @@ export const memoryRepository = {
|
|||||||
* Check for duplicate or similar memories using embedding similarity
|
* Check for duplicate or similar memories using embedding similarity
|
||||||
* Falls back to substring matching if embeddings are unavailable
|
* Falls back to substring matching if embeddings are unavailable
|
||||||
*/
|
*/
|
||||||
async findSimilar(userId: string, content: string, threshold = 0.85): Promise<Memory[]> {
|
async findSimilar(
|
||||||
|
userId: string,
|
||||||
|
guildId: string,
|
||||||
|
content: string,
|
||||||
|
threshold = 0.85
|
||||||
|
): Promise<Memory[]> {
|
||||||
const embeddingService = getEmbeddingService();
|
const embeddingService = getEmbeddingService();
|
||||||
|
|
||||||
// Try embedding-based similarity first
|
// Try embedding-based similarity first
|
||||||
@@ -199,6 +206,7 @@ export const memoryRepository = {
|
|||||||
.where(
|
.where(
|
||||||
and(
|
and(
|
||||||
eq(memories.user_id, userId),
|
eq(memories.user_id, userId),
|
||||||
|
eq(memories.guild_id, guildId),
|
||||||
sql`${memories.embedding} IS NOT NULL`
|
sql`${memories.embedding} IS NOT NULL`
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
@@ -236,6 +244,7 @@ export const memoryRepository = {
|
|||||||
.where(
|
.where(
|
||||||
and(
|
and(
|
||||||
eq(memories.user_id, userId),
|
eq(memories.user_id, userId),
|
||||||
|
eq(memories.guild_id, guildId),
|
||||||
like(sql`lower(${memories.content})`, `%${searchTerm}%`)
|
like(sql`lower(${memories.content})`, `%${searchTerm}%`)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -331,10 +340,10 @@ export const memoryRepository = {
|
|||||||
/**
|
/**
|
||||||
* Delete all memories for a user
|
* Delete all memories for a user
|
||||||
*/
|
*/
|
||||||
async deleteByUserId(userId: string): Promise<number> {
|
async deleteByUserId(userId: string, guildId: string): Promise<number> {
|
||||||
const result = await db
|
const result = await db
|
||||||
.delete(memories)
|
.delete(memories)
|
||||||
.where(eq(memories.user_id, userId))
|
.where(and(eq(memories.user_id, userId), eq(memories.guild_id, guildId)))
|
||||||
.returning({ id: memories.id });
|
.returning({ id: memories.id });
|
||||||
|
|
||||||
return result.length;
|
return result.length;
|
||||||
@@ -369,7 +378,7 @@ export const memoryRepository = {
|
|||||||
/**
|
/**
|
||||||
* Get memory statistics for a user
|
* Get memory statistics for a user
|
||||||
*/
|
*/
|
||||||
async getStats(userId: string): Promise<{
|
async getStats(userId: string, guildId: string): Promise<{
|
||||||
total: number;
|
total: number;
|
||||||
byCategory: Record<string, number>;
|
byCategory: Record<string, number>;
|
||||||
avgImportance: number;
|
avgImportance: number;
|
||||||
@@ -377,7 +386,7 @@ export const memoryRepository = {
|
|||||||
const allMemories = await db
|
const allMemories = await db
|
||||||
.select()
|
.select()
|
||||||
.from(memories)
|
.from(memories)
|
||||||
.where(eq(memories.user_id, userId));
|
.where(and(eq(memories.user_id, userId), eq(memories.guild_id, guildId)));
|
||||||
|
|
||||||
const byCategory: Record<string, number> = {};
|
const byCategory: Record<string, number> = {};
|
||||||
let totalImportance = 0;
|
let totalImportance = 0;
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import { personalities, botOptions } from "../../database/schema";
|
|||||||
import { eq } from "drizzle-orm";
|
import { eq } from "drizzle-orm";
|
||||||
import { buildStyledPrompt, STYLE_MODIFIERS } from "./personalities";
|
import { buildStyledPrompt, STYLE_MODIFIERS } from "./personalities";
|
||||||
import { getRandomMention } from "./mentions";
|
import { getRandomMention } from "./mentions";
|
||||||
|
import { speakVoiceover } from "./voice";
|
||||||
import { TypingIndicator } from "./typing";
|
import { TypingIndicator } from "./typing";
|
||||||
|
|
||||||
const logger = createLogger("Features:Joel");
|
const logger = createLogger("Features:Joel");
|
||||||
@@ -115,6 +116,9 @@ export const joelResponder = {
|
|||||||
const fullResponse = response + mention;
|
const fullResponse = response + mention;
|
||||||
|
|
||||||
await this.sendResponse(message, fullResponse);
|
await this.sendResponse(message, fullResponse);
|
||||||
|
speakVoiceover(message, fullResponse).catch((error) => {
|
||||||
|
logger.error("Failed to play voiceover", error);
|
||||||
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error("Failed to respond", error);
|
logger.error("Failed to respond", error);
|
||||||
await this.handleError(message, error);
|
await this.handleError(message, error);
|
||||||
|
|||||||
168
src/features/joel/voice.ts
Normal file
168
src/features/joel/voice.ts
Normal file
@@ -0,0 +1,168 @@
|
|||||||
|
/**
|
||||||
|
* Voiceover playback for Joel responses
|
||||||
|
*/
|
||||||
|
|
||||||
|
import {
|
||||||
|
AudioPlayerStatus,
|
||||||
|
VoiceConnectionStatus,
|
||||||
|
createAudioPlayer,
|
||||||
|
createAudioResource,
|
||||||
|
entersState,
|
||||||
|
getVoiceConnection,
|
||||||
|
joinVoiceChannel,
|
||||||
|
StreamType,
|
||||||
|
type DiscordGatewayAdapterCreator,
|
||||||
|
} from "@discordjs/voice";
|
||||||
|
import type { Message } from "discord.js";
|
||||||
|
import { Readable } from "node:stream";
|
||||||
|
import { config } from "../../core/config";
|
||||||
|
import { createLogger } from "../../core/logger";
|
||||||
|
import { getVoiceoverService } from "../../services/ai/voiceover";
|
||||||
|
|
||||||
|
const logger = createLogger("Features:Joel:Voice");
|
||||||
|
|
||||||
|
const MAX_VOICE_TEXT_LENGTH = 800;
|
||||||
|
const PLAYBACK_TIMEOUT_MS = 60_000;
|
||||||
|
const READY_TIMEOUT_MS = 15_000;
|
||||||
|
|
||||||
|
function isAbortError(error: unknown): boolean {
|
||||||
|
return error instanceof Error && error.name === "AbortError";
|
||||||
|
}
|
||||||
|
|
||||||
|
function sanitizeForVoiceover(content: string): string {
|
||||||
|
let text = content.replace(/```[\s\S]*?```/g, " ");
|
||||||
|
text = text.replace(/`([^`]+)`/g, "$1");
|
||||||
|
text = text.replace(/\s+/g, " ").trim();
|
||||||
|
|
||||||
|
if (text.length > MAX_VOICE_TEXT_LENGTH) {
|
||||||
|
text = `${text.slice(0, MAX_VOICE_TEXT_LENGTH - 3).trimEnd()}...`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getOrCreateConnection(message: Message<true>) {
|
||||||
|
const voiceChannel = message.member?.voice.channel;
|
||||||
|
if (!voiceChannel) {
|
||||||
|
logger.debug("No voice channel for author", {
|
||||||
|
userId: message.author.id,
|
||||||
|
guildId: message.guildId,
|
||||||
|
});
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const me = message.guild.members.me ?? (await message.guild.members.fetchMe());
|
||||||
|
const permissions = voiceChannel.permissionsFor(me);
|
||||||
|
if (!permissions?.has(["Connect", "Speak"])) {
|
||||||
|
logger.debug("Missing voice permissions", {
|
||||||
|
guildId: message.guildId,
|
||||||
|
channelId: voiceChannel.id,
|
||||||
|
});
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const existing = getVoiceConnection(message.guildId);
|
||||||
|
if (existing && existing.joinConfig.channelId === voiceChannel.id) {
|
||||||
|
logger.debug("Reusing existing voice connection", {
|
||||||
|
guildId: message.guildId,
|
||||||
|
channelId: voiceChannel.id,
|
||||||
|
});
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (existing) {
|
||||||
|
existing.destroy();
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug("Joining voice channel", {
|
||||||
|
guildId: message.guildId,
|
||||||
|
channelId: voiceChannel.id,
|
||||||
|
});
|
||||||
|
|
||||||
|
const connection = joinVoiceChannel({
|
||||||
|
channelId: voiceChannel.id,
|
||||||
|
guildId: voiceChannel.guild.id,
|
||||||
|
adapterCreator: voiceChannel.guild.voiceAdapterCreator as unknown as DiscordGatewayAdapterCreator,
|
||||||
|
selfDeaf: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await entersState(connection, VoiceConnectionStatus.Ready, READY_TIMEOUT_MS);
|
||||||
|
logger.debug("Voice connection ready", {
|
||||||
|
guildId: message.guildId,
|
||||||
|
channelId: voiceChannel.id,
|
||||||
|
});
|
||||||
|
return connection;
|
||||||
|
} catch (error) {
|
||||||
|
if (isAbortError(error)) {
|
||||||
|
logger.debug("Voice connection ready timeout", {
|
||||||
|
guildId: message.guildId,
|
||||||
|
channelId: voiceChannel.id,
|
||||||
|
status: connection.state.status,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
logger.error("Voice connection failed to become ready", error);
|
||||||
|
}
|
||||||
|
connection.destroy();
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function speakVoiceover(message: Message<true>, content: string): Promise<void> {
|
||||||
|
if (!config.elevenlabs.apiKey || !config.elevenlabs.voiceId) {
|
||||||
|
logger.debug("Voiceover disabled (missing config)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const text = sanitizeForVoiceover(content);
|
||||||
|
if (!text) {
|
||||||
|
logger.debug("Voiceover skipped (empty text after sanitize)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const connection = await getOrCreateConnection(message);
|
||||||
|
if (!connection) {
|
||||||
|
logger.debug("Voiceover skipped (no connection)", {
|
||||||
|
guildId: message.guildId,
|
||||||
|
authorId: message.author.id,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const voiceover = getVoiceoverService();
|
||||||
|
logger.debug("Requesting ElevenLabs voiceover", { textLength: text.length });
|
||||||
|
const audio = await voiceover.generate({ text });
|
||||||
|
logger.debug("Voiceover audio received", { bytes: audio.length });
|
||||||
|
const player = createAudioPlayer();
|
||||||
|
const resource = createAudioResource(Readable.from(audio), {
|
||||||
|
inputType: StreamType.Arbitrary,
|
||||||
|
});
|
||||||
|
|
||||||
|
player.on("error", (error) => {
|
||||||
|
logger.error("Audio player error", error);
|
||||||
|
});
|
||||||
|
|
||||||
|
player.on(AudioPlayerStatus.Playing, () => {
|
||||||
|
logger.debug("Audio player started", { guildId: message.guildId });
|
||||||
|
});
|
||||||
|
|
||||||
|
player.on(AudioPlayerStatus.Idle, () => {
|
||||||
|
logger.debug("Audio player idle", { guildId: message.guildId });
|
||||||
|
});
|
||||||
|
|
||||||
|
connection.subscribe(player);
|
||||||
|
player.play(resource);
|
||||||
|
|
||||||
|
await entersState(player, AudioPlayerStatus.Playing, 5_000).catch(() => undefined);
|
||||||
|
await entersState(player, AudioPlayerStatus.Idle, PLAYBACK_TIMEOUT_MS);
|
||||||
|
} catch (error) {
|
||||||
|
if (!isAbortError(error)) {
|
||||||
|
logger.error("Voiceover playback failed", error);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
if (connection.state.status !== VoiceConnectionStatus.Destroyed) {
|
||||||
|
connection.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -29,6 +29,8 @@ const client = new BotClient({
|
|||||||
GatewayIntentBits.GuildMessages,
|
GatewayIntentBits.GuildMessages,
|
||||||
GatewayIntentBits.GuildModeration,
|
GatewayIntentBits.GuildModeration,
|
||||||
GatewayIntentBits.GuildMembers,
|
GatewayIntentBits.GuildMembers,
|
||||||
|
GatewayIntentBits.GuildVoiceStates,
|
||||||
|
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -27,9 +27,9 @@ const toolHandlers: Record<string, ToolHandler> = {
|
|||||||
|
|
||||||
let userMemories;
|
let userMemories;
|
||||||
if (category) {
|
if (category) {
|
||||||
userMemories = await memoryRepository.findByCategory(userId, category, limit);
|
userMemories = await memoryRepository.findByCategory(userId, context.guildId, category, limit);
|
||||||
} else {
|
} else {
|
||||||
userMemories = await memoryRepository.findByUserId(userId, limit);
|
userMemories = await memoryRepository.findByUserId(userId, context.guildId, limit);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (userMemories.length === 0) {
|
if (userMemories.length === 0) {
|
||||||
@@ -61,7 +61,7 @@ const toolHandlers: Record<string, ToolHandler> = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Check for duplicate memories using new similarity check
|
// Check for duplicate memories using new similarity check
|
||||||
const similar = await memoryRepository.findSimilar(userId, content);
|
const similar = await memoryRepository.findSimilar(userId, context.guildId, content);
|
||||||
if (similar.length > 0) {
|
if (similar.length > 0) {
|
||||||
return "Already knew something similar. Memory not saved (duplicate).";
|
return "Already knew something similar. Memory not saved (duplicate).";
|
||||||
}
|
}
|
||||||
@@ -89,7 +89,7 @@ const toolHandlers: Record<string, ToolHandler> = {
|
|||||||
*/
|
*/
|
||||||
async search_memories(args, context): Promise<string> {
|
async search_memories(args, context): Promise<string> {
|
||||||
const query = args.query as string;
|
const query = args.query as string;
|
||||||
const guildId = args.guild_id as string | undefined;
|
const guildId = (args.guild_id as string | undefined) ?? context.guildId;
|
||||||
const category = args.category as MemoryCategory | undefined;
|
const category = args.category as MemoryCategory | undefined;
|
||||||
const minImportance = args.min_importance as number | undefined;
|
const minImportance = args.min_importance as number | undefined;
|
||||||
|
|
||||||
@@ -138,7 +138,7 @@ const toolHandlers: Record<string, ToolHandler> = {
|
|||||||
|
|
||||||
logger.warn("Forgetting user memories", { userId, requestedBy: context.userId });
|
logger.warn("Forgetting user memories", { userId, requestedBy: context.userId });
|
||||||
|
|
||||||
const deleted = await memoryRepository.deleteByUserId(userId);
|
const deleted = await memoryRepository.deleteByUserId(userId, context.guildId);
|
||||||
|
|
||||||
return `Deleted ${deleted} memories about user ${userId}.`;
|
return `Deleted ${deleted} memories about user ${userId}.`;
|
||||||
},
|
},
|
||||||
@@ -157,7 +157,7 @@ const toolHandlers: Record<string, ToolHandler> = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Check for duplicates
|
// Check for duplicates
|
||||||
const similar = await memoryRepository.findSimilar(context.userId, content);
|
const similar = await memoryRepository.findSimilar(context.userId, context.guildId, content);
|
||||||
if (similar.length > 0) {
|
if (similar.length > 0) {
|
||||||
return "Similar memory already exists. Skipped.";
|
return "Similar memory already exists. Skipped.";
|
||||||
}
|
}
|
||||||
@@ -185,7 +185,7 @@ const toolHandlers: Record<string, ToolHandler> = {
|
|||||||
async get_memory_stats(args, context): Promise<string> {
|
async get_memory_stats(args, context): Promise<string> {
|
||||||
const userId = (args.user_id as string) || context.userId;
|
const userId = (args.user_id as string) || context.userId;
|
||||||
|
|
||||||
const stats = await memoryRepository.getStats(userId);
|
const stats = await memoryRepository.getStats(userId, context.guildId);
|
||||||
|
|
||||||
if (stats.total === 0) {
|
if (stats.total === 0) {
|
||||||
return `No memories stored for this user.`;
|
return `No memories stored for this user.`;
|
||||||
|
|||||||
104
src/services/ai/voiceover.ts
Normal file
104
src/services/ai/voiceover.ts
Normal file
@@ -0,0 +1,104 @@
|
|||||||
|
/**
|
||||||
|
* ElevenLabs voiceover service
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { config } from "../../core/config";
|
||||||
|
import { createLogger } from "../../core/logger";
|
||||||
|
|
||||||
|
const logger = createLogger("AI:Voiceover");
|
||||||
|
|
||||||
|
const DEFAULT_OUTPUT_FORMAT = "mp3_44100_128" as const;
|
||||||
|
const DEFAULT_STABILITY = 0.1;
|
||||||
|
const DEFAULT_SIMILARITY = 0.90;
|
||||||
|
const DEFAULT_STYLE = 0.25;
|
||||||
|
const DEFAULT_SPEED = 1.20
|
||||||
|
|
||||||
|
function clamp01(value: number): number {
|
||||||
|
return Math.max(0, Math.min(1, value));
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VoiceoverOptions {
|
||||||
|
text: string;
|
||||||
|
voiceId?: string;
|
||||||
|
modelId?: string;
|
||||||
|
stability?: number;
|
||||||
|
similarityBoost?: number;
|
||||||
|
style?: number;
|
||||||
|
speakerBoost?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class VoiceoverService {
|
||||||
|
async generate(options: VoiceoverOptions): Promise<Buffer> {
|
||||||
|
const apiKey = config.elevenlabs.apiKey;
|
||||||
|
if (!apiKey) {
|
||||||
|
throw new Error("Voiceover is not configured (missing ELEVENLABS_API_KEY).");
|
||||||
|
}
|
||||||
|
|
||||||
|
const voiceId = options.voiceId || config.elevenlabs.voiceId;
|
||||||
|
if (!voiceId) {
|
||||||
|
throw new Error("Voiceover is missing a voice ID (set ELEVENLABS_VOICE_ID or pass one).");
|
||||||
|
}
|
||||||
|
|
||||||
|
const text = options.text.trim();
|
||||||
|
if (!text) {
|
||||||
|
throw new Error("Voiceover text is empty.");
|
||||||
|
}
|
||||||
|
|
||||||
|
const modelId = options.modelId || config.elevenlabs.modelId;
|
||||||
|
|
||||||
|
const voiceSettings = {
|
||||||
|
stability: clamp01(options.stability ?? DEFAULT_STABILITY),
|
||||||
|
similarity_boost: clamp01(options.similarityBoost ?? DEFAULT_SIMILARITY),
|
||||||
|
style: clamp01(options.style ?? DEFAULT_STYLE),
|
||||||
|
use_speaker_boost: options.speakerBoost ?? true,
|
||||||
|
};
|
||||||
|
|
||||||
|
const url = new URL(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream`);
|
||||||
|
url.searchParams.set("output_format", DEFAULT_OUTPUT_FORMAT);
|
||||||
|
|
||||||
|
logger.debug("Generating voiceover", {
|
||||||
|
textLength: text.length,
|
||||||
|
voiceId,
|
||||||
|
modelId,
|
||||||
|
});
|
||||||
|
|
||||||
|
const response = await fetch(url.toString(), {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"xi-api-key": apiKey,
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Accept": "audio/mpeg",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
text,
|
||||||
|
model_id: modelId,
|
||||||
|
voice_settings: voiceSettings,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorBody = await response.text();
|
||||||
|
logger.error("ElevenLabs API error", {
|
||||||
|
status: response.status,
|
||||||
|
body: errorBody.slice(0, 300),
|
||||||
|
});
|
||||||
|
throw new Error(`ElevenLabs API error (HTTP ${response.status}).`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const audioBuffer = await response.arrayBuffer();
|
||||||
|
return Buffer.from(audioBuffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
async health(): Promise<boolean> {
|
||||||
|
return !!config.elevenlabs.apiKey;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let voiceoverService: VoiceoverService | null = null;
|
||||||
|
|
||||||
|
export function getVoiceoverService(): VoiceoverService {
|
||||||
|
if (!voiceoverService) {
|
||||||
|
voiceoverService = new VoiceoverService();
|
||||||
|
}
|
||||||
|
return voiceoverService;
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user