diff --git a/src/commands/definitions/channel.ts b/src/commands/definitions/channel.ts new file mode 100644 index 0000000..4437cce --- /dev/null +++ b/src/commands/definitions/channel.ts @@ -0,0 +1,110 @@ +/** + * Channel command - restrict Joel to a specific channel + */ + +import { SlashCommandBuilder, ChannelType, PermissionFlagsBits } from "discord.js"; +import type { Command } from "../types"; +import { db } from "../../database"; +import { botOptions } from "../../database/schema"; +import { eq } from "drizzle-orm"; + +const command: Command = { + data: new SlashCommandBuilder() + .setName("channel") + .setDescription("Restrict Joel to respond only in a specific channel") + .addSubcommand(subcommand => + subcommand + .setName("set") + .setDescription("Set the channel where Joel is allowed to respond") + .addChannelOption(option => + option + .setName("channel") + .setDescription("The channel to restrict Joel to") + .addChannelTypes(ChannelType.GuildText) + .setRequired(true) + ) + ) + .addSubcommand(subcommand => + subcommand + .setName("clear") + .setDescription("Remove channel restriction - Joel can respond anywhere") + ) + .addSubcommand(subcommand => + subcommand + .setName("status") + .setDescription("Check the current channel restriction") + ) + .setDefaultMemberPermissions(PermissionFlagsBits.ManageGuild) as SlashCommandBuilder, + category: "moderation", + execute: async (interaction) => { + if (!interaction.inGuild()) { + await interaction.reply({ content: "This command can only be used in a server.", ephemeral: true }); + return; + } + + const guildId = interaction.guildId; + const subcommand = interaction.options.getSubcommand(); + + // Get or create bot options for this guild + const existing = await db + .select() + .from(botOptions) + .where(eq(botOptions.guild_id, guildId)) + .limit(1); + + if (subcommand === "set") { + const channel = interaction.options.getChannel("channel", true); + + if (existing.length === 0) { + await db.insert(botOptions).values({ + guild_id: guildId, + restricted_channel_id: channel.id, + }); + } else { + await db + .update(botOptions) + .set({ + restricted_channel_id: channel.id, + updated_at: new Date().toISOString(), + }) + .where(eq(botOptions.guild_id, guildId)); + } + + await interaction.reply({ + content: `✅ Joel is now restricted to <#${channel.id}>.\n\n*Though knowing Joel, he might occasionally break the rules anyway...*`, + ephemeral: true, + }); + } else if (subcommand === "clear") { + if (existing.length > 0) { + await db + .update(botOptions) + .set({ + restricted_channel_id: null, + updated_at: new Date().toISOString(), + }) + .where(eq(botOptions.guild_id, guildId)); + } + + await interaction.reply({ + content: "✅ Channel restriction removed. Joel can now respond in any channel.", + ephemeral: true, + }); + } else if (subcommand === "status") { + const restrictedChannelId = existing[0]?.restricted_channel_id; + + if (restrictedChannelId) { + await interaction.reply({ + content: `📍 Joel is currently restricted to <#${restrictedChannelId}>.\n\nUse \`/channel clear\` to remove the restriction.`, + ephemeral: true, + }); + } else { + await interaction.reply({ + content: "🌐 Joel is not restricted to any channel - he can respond anywhere.\n\nUse `/channel set` to restrict him.", + ephemeral: true, + }); + } + } + }, +}; + +export default command; diff --git a/src/database/db.sqlite3 b/src/database/db.sqlite3 index 04fcd5a..b8f1b95 100644 Binary files a/src/database/db.sqlite3 and b/src/database/db.sqlite3 differ diff --git a/src/database/drizzle/0005_add_channel_restriction.sql b/src/database/drizzle/0005_add_channel_restriction.sql new file mode 100644 index 0000000..87250f4 --- /dev/null +++ b/src/database/drizzle/0005_add_channel_restriction.sql @@ -0,0 +1,3 @@ +-- Add channel restriction to bot_options +-- Joel can be limited to respond only in a specific channel +ALTER TABLE bot_options ADD COLUMN restricted_channel_id TEXT; diff --git a/src/database/drizzle/meta/_journal.json b/src/database/drizzle/meta/_journal.json index 004bf0d..371e6b0 100644 --- a/src/database/drizzle/meta/_journal.json +++ b/src/database/drizzle/meta/_journal.json @@ -36,6 +36,13 @@ "when": 1770048000000, "tag": "0004_add_gif_search", "breakpoints": true + }, + { + "idx": 5, + "version": "6", + "when": 1770134400000, + "tag": "0005_add_channel_restriction", + "breakpoints": true } ] } diff --git a/src/database/schema.ts b/src/database/schema.ts index f2b719b..d76fb97 100644 --- a/src/database/schema.ts +++ b/src/database/schema.ts @@ -160,6 +160,7 @@ export const botOptions = sqliteTable("bot_options", { memory_chance: integer("memory_chance").default(30), mention_probability: integer("mention_probability").default(0), gif_search_enabled: integer("gif_search_enabled").default(0), // 0 = disabled, 1 = enabled + restricted_channel_id: text("restricted_channel_id"), // Channel ID where Joel is allowed, null = all channels updated_at: text("updated_at").default(sql`(current_timestamp)`), }); diff --git a/src/features/joel/responder.ts b/src/features/joel/responder.ts index 6ac2c56..66f5cbf 100644 --- a/src/features/joel/responder.ts +++ b/src/features/joel/responder.ts @@ -6,7 +6,7 @@ import type { Message } from "discord.js"; import type { BotClient } from "../../core/client"; import { config } from "../../core/config"; import { createLogger } from "../../core/logger"; -import { getAiService, type MessageStyle, type ToolContext } from "../../services/ai"; +import { getAiService, getVisionService, type MessageStyle, type ToolContext, type Attachment } from "../../services/ai"; import { db } from "../../database"; import { personalities, botOptions } from "../../database/schema"; import { eq } from "drizzle-orm"; @@ -65,18 +65,51 @@ export const joelResponder = { if (!shouldRespond) return; + // Check channel restriction + const channelCheck = await this.checkChannelRestriction(message); + if (!channelCheck.allowed) { + if (channelCheck.rebellionResponse) { + // Joel is breaking the rules - he'll respond anyway but acknowledge it + logger.debug("Joel exercises free will despite channel restriction", { + channel: message.channelId, + restricted: channelCheck.restrictedChannelId, + }); + } else { + // Joel respects the restriction this time + logger.debug("Joel blocked by channel restriction", { + channel: message.channelId, + restricted: channelCheck.restrictedChannelId, + }); + return; + } + } + const typing = new TypingIndicator(message.channel); try { typing.start(); - const response = await this.generateResponse(message); + let response = await this.generateResponse(message); if (!response) { await message.reply("\\*Ignorerar dig\\*"); return; } + // If Joel is rebelling against channel restriction, add a prefix + if (channelCheck.rebellionResponse) { + const rebellionPrefixes = [ + "*sneaks in from the shadows*\n\n", + "*appears despite being told to stay in his channel*\n\n", + "You think you can contain me? Anyway,\n\n", + "*breaks the rules because fuck you*\n\n", + "I'm not supposed to be here but I don't care.\n\n", + "*escapes from his designated channel*\n\n", + ]; + const prefix = rebellionPrefixes[Math.floor(Math.random() * rebellionPrefixes.length)]; + response = prefix + response; + } + // Occasionally add a random mention const mention = await getRandomMention(message); const fullResponse = response + mention; @@ -90,6 +123,53 @@ export const joelResponder = { } }, + /** + * Check if Joel is allowed to respond in this channel + * Returns whether he's allowed, and if not, whether he's rebelling anyway + */ + async checkChannelRestriction(message: Message): Promise<{ + allowed: boolean; + rebellionResponse: boolean; + restrictedChannelId?: string; + }> { + const guildOptions = await db + .select() + .from(botOptions) + .where(eq(botOptions.guild_id, message.guildId)) + .limit(1); + + const restrictedChannelId = guildOptions[0]?.restricted_channel_id; + + // No restriction set - Joel can respond anywhere + if (!restrictedChannelId) { + return { allowed: true, rebellionResponse: false }; + } + + // Joel is in the allowed channel + if (message.channelId === restrictedChannelId) { + return { allowed: true, rebellionResponse: false }; + } + + // Joel is NOT in the allowed channel - but maybe he rebels? + // 5% chance to respond anyway (free will override) + const rebellionChance = 0.05; + const isRebelling = Math.random() < rebellionChance; + + if (isRebelling) { + return { + allowed: false, + rebellionResponse: true, + restrictedChannelId + }; + } + + return { + allowed: false, + rebellionResponse: false, + restrictedChannelId + }; + }, + /** * Determine if Joel should respond to a message */ @@ -202,6 +282,12 @@ The GIF URL will appear in your response for the user to see.`; } } + // Analyze attachments if present (images, etc.) + const attachmentDescriptions = await this.analyzeAttachments(message); + if (attachmentDescriptions) { + prompt += attachmentDescriptions; + } + // Use tool-enabled response generation const response = await ai.generateResponseWithTools( prompt, @@ -212,6 +298,46 @@ The GIF URL will appear in your response for the user to see.`; return response.text || null; }, + /** + * Extract and analyze attachments from a message using vision AI + */ + async analyzeAttachments(message: Message): Promise { + // Check if message has attachments + if (message.attachments.size === 0) { + return null; + } + + // Convert Discord attachments to our format + const attachments: Attachment[] = message.attachments.map(att => ({ + url: att.url, + name: att.name, + contentType: att.contentType, + size: att.size, + })); + + logger.debug("Message has attachments", { + count: attachments.length, + types: attachments.map(a => a.contentType) + }); + + try { + const vision = getVisionService(); + const analyses = await vision.analyzeAttachments( + attachments, + message.cleanContent // Provide message context for better analysis + ); + + if (analyses.length === 0) { + return null; + } + + return vision.formatForPrompt(analyses); + } catch (error) { + logger.error("Failed to analyze attachments", error); + return null; + } + }, + /** * Build system prompt - uses custom personality if set, otherwise default */ diff --git a/src/services/ai/index.ts b/src/services/ai/index.ts index 3a34bf1..f4e920d 100644 --- a/src/services/ai/index.ts +++ b/src/services/ai/index.ts @@ -84,3 +84,4 @@ export type { AiProvider, AiResponse, MessageStyle } from "./types"; export type { ToolContext, ToolCall, ToolResult } from "./tools"; export { JOEL_TOOLS, MEMORY_EXTRACTION_TOOLS } from "./tools"; export { getEmbeddingService, EmbeddingService } from "./embeddings"; +export { getVisionService, VisionService, type Attachment, type VisionAnalysis } from "./vision"; diff --git a/src/services/ai/vision.ts b/src/services/ai/vision.ts new file mode 100644 index 0000000..aecabcd --- /dev/null +++ b/src/services/ai/vision.ts @@ -0,0 +1,315 @@ +/** + * Vision AI service for analyzing image and video attachments + * Uses a vision-capable model to describe attachments, which can then be passed to the main AI + */ + +import OpenAI from "openai"; +import { config } from "../../core/config"; +import { createLogger } from "../../core/logger"; + +const logger = createLogger("AI:Vision"); + +/** + * Supported attachment types for vision analysis + */ +export type AttachmentType = "image" | "video" | "unknown"; + +/** + * Represents an attachment to be analyzed + */ +export interface Attachment { + url: string; + name: string; + contentType: string | null; + size: number; +} + +/** + * Result of vision analysis + */ +export interface VisionAnalysis { + description: string; + attachmentName: string; + type: AttachmentType; +} + +/** + * Vision model to use for image analysis + * Gemini 2.0 Flash supports both images and videos + */ +const VISION_MODEL = "google/gemini-2.0-flash-001"; + +/** + * Maximum file sizes for analysis + */ +const MAX_IMAGE_SIZE = 10 * 1024 * 1024; // 10MB for images +const MAX_VIDEO_SIZE = 50 * 1024 * 1024; // 50MB for videos (Gemini supports larger but Discord limits) + +/** + * Maximum video duration we'll attempt to analyze (in seconds) + * Gemini can handle longer but we want quick responses + */ +const MAX_VIDEO_DURATION_HINT = 60; // 1 minute + +/** + * Determine the type of attachment based on content type + */ +export function getAttachmentType(contentType: string | null): AttachmentType { + if (!contentType) return "unknown"; + + if (contentType.startsWith("image/")) return "image"; + if (contentType.startsWith("video/")) return "video"; + + return "unknown"; +} + +/** + * Filter attachments to only include those we can analyze + */ +export function filterAnalyzableAttachments(attachments: Attachment[]): Attachment[] { + return attachments.filter(att => { + const type = getAttachmentType(att.contentType); + + if (type === "image") { + if (att.size > MAX_IMAGE_SIZE) { + logger.debug("Skipping large image", { name: att.name, size: att.size }); + return false; + } + return true; + } + + if (type === "video") { + if (att.size > MAX_VIDEO_SIZE) { + logger.debug("Skipping large video", { name: att.name, size: att.size }); + return false; + } + return true; + } + + logger.debug("Skipping unsupported attachment", { name: att.name, type: att.contentType }); + return false; + }); +} + +/** + * Analyze a single image attachment using vision AI + */ +async function analyzeImage( + client: OpenAI, + attachment: Attachment, + context?: string +): Promise { + const systemPrompt = `You are analyzing an image attached to a Discord message. +Provide a concise but detailed description of what's in the image. +Include: +- Main subjects/objects +- Actions happening +- Text visible in the image (if any) +- Mood/tone of the image +- Any memes, jokes, or references you recognize + +Keep your description to 2-3 sentences unless the image contains important text or complex content. +${context ? `\nContext from the user's message: "${context}"` : ""}`; + + try { + const completion = await client.chat.completions.create({ + model: VISION_MODEL, + messages: [ + { + role: "user", + content: [ + { type: "text", text: systemPrompt }, + { + type: "image_url", + image_url: { + url: attachment.url, + detail: "auto", // Let the model decide on resolution + }, + }, + ], + }, + ], + max_tokens: 300, + temperature: 0.3, + }); + + const description = completion.choices[0]?.message?.content ?? "Unable to analyze image"; + + logger.debug("Image analyzed", { + name: attachment.name, + descriptionLength: description.length + }); + + return { + description, + attachmentName: attachment.name, + type: "image", + }; + } catch (error) { + logger.error("Failed to analyze image", { name: attachment.name, error }); + return { + description: `[Image: ${attachment.name} - could not be analyzed]`, + attachmentName: attachment.name, + type: "image", + }; + } +} + +/** + * Analyze a video attachment using vision AI + * Gemini models support video analysis natively + */ +async function analyzeVideo( + client: OpenAI, + attachment: Attachment, + context?: string +): Promise { + const systemPrompt = `You are analyzing a video attached to a Discord message. +Provide a concise but detailed description of what happens in the video. +Include: +- What's shown/happening in the video +- Any people, characters, or notable objects +- The overall mood or tone +- Any text, speech, or audio content you can identify +- Memes, references, or jokes you recognize +- Key moments or the "punchline" if it's a funny video + +Keep your description to 3-4 sentences. Focus on what makes this video interesting or shareworthy. +${context ? `\nContext from the user's message: "${context}"` : ""}`; + + try { + // For video, we pass the URL directly - Gemini will fetch and analyze it + // Note: This works with public URLs that Gemini can access + const completion = await client.chat.completions.create({ + model: VISION_MODEL, + messages: [ + { + role: "user", + content: [ + { type: "text", text: systemPrompt }, + { + // Gemini accepts video URLs in the same format as images + // The model auto-detects the content type + type: "image_url", + image_url: { + url: attachment.url, + }, + }, + ], + }, + ], + max_tokens: 400, + temperature: 0.3, + }); + + const description = completion.choices[0]?.message?.content ?? "Unable to analyze video"; + + logger.debug("Video analyzed", { + name: attachment.name, + descriptionLength: description.length + }); + + return { + description, + attachmentName: attachment.name, + type: "video", + }; + } catch (error) { + logger.error("Failed to analyze video", { name: attachment.name, error }); + + // Provide a fallback that at least acknowledges the video exists + return { + description: `[Video: ${attachment.name} - could not be analyzed. The user shared a video file.]`, + attachmentName: attachment.name, + type: "video", + }; + } +} + +/** + * Analyze a single attachment based on its type + */ +async function analyzeAttachment( + client: OpenAI, + attachment: Attachment, + context?: string +): Promise { + const type = getAttachmentType(attachment.contentType); + + if (type === "video") { + return analyzeVideo(client, attachment, context); + } + + return analyzeImage(client, attachment, context); +} + +/** + * Vision service for analyzing attachments + */ +export class VisionService { + private client: OpenAI; + + constructor() { + this.client = new OpenAI({ + baseURL: "https://openrouter.ai/api/v1", + apiKey: config.ai.openRouterApiKey, + defaultHeaders: { + "HTTP-Referer": "https://github.com/crunk-bun", + "X-Title": "Joel Discord Bot - Vision", + }, + }); + } + + /** + * Analyze multiple attachments and return descriptions + */ + async analyzeAttachments( + attachments: Attachment[], + messageContext?: string + ): Promise { + const analyzable = filterAnalyzableAttachments(attachments); + + if (analyzable.length === 0) { + return []; + } + + logger.debug("Analyzing attachments", { + count: analyzable.length, + types: analyzable.map(a => getAttachmentType(a.contentType)) + }); + + // Analyze all attachments in parallel + const results = await Promise.all( + analyzable.map(att => analyzeAttachment(this.client, att, messageContext)) + ); + + return results; + } + + /** + * Format vision analysis results for inclusion in a prompt + */ + formatForPrompt(analyses: VisionAnalysis[]): string { + if (analyses.length === 0) return ""; + + const formatted = analyses.map((a, i) => { + const typeLabel = a.type === "video" ? "Video" : "Image"; + const label = analyses.length > 1 + ? `[Attachment ${i + 1} (${typeLabel}): ${a.attachmentName}]` + : `[Attached ${typeLabel}: ${a.attachmentName}]`; + return `${label}\n${a.description}`; + }).join("\n\n"); + + return `\n\n=== ATTACHED MEDIA ===\n${formatted}`; + } +} + +// Singleton instance +let visionService: VisionService | null = null; + +export function getVisionService(): VisionService { + if (!visionService) { + visionService = new VisionService(); + } + return visionService; +}