Publishing and receiving audio

import {
  VoiceMediaSession,
  ffmpegResource,
} from '@cloak-software/bot-sdk';

// 1. Control plane. Join DEAFENED: the SFU then forwards this bot nothing at
//    all. Crypto and transport both say no, which is the right shape for a
//    publisher.
const conn = await client.joinVoice(serverId, voiceChannelId, { deafened: true });

// 2. Key lane. A bot never derives a voice key. A human member seals one to it.
const keys = client.voiceKeys;
const voiceKey = await keys.acquireVoiceKey(serverId);

// 3. The scope is not this bot's choice. Publish-only means participant scope.
if (voiceKey.scope !== 'participant') {
  throw new Error('this server granted voice_listen; this bot is publish-only');
}

// 4. Media. attach() installs the key, reports the connection up and waits
//    for the microphone admission, and only then publishes. That order is
//    load-bearing.
const session = await VoiceMediaSession.attach({
  url: conn.url,
  token: conn.token,      // a live credential, never log it
  botId: client.user!.id, // its media identity is derived from this
  serverId,
  voiceKey,               // for the scope and the length check
  keys,                   // where frame keys actually come from
  deafened: conn.selfDeafened,
  connectionReporter: client.voiceConnectionReporter(), // admits the microphone; see below
  // admissionTimeoutMs: 5000,                          // how long to wait for it
});

// 5. Resolves at the end of the track, and REJECTS if the decoder exits non-zero.
await session.player.play(ffmpegResource('./track.mp3'));

await session.detach();
await client.leaveVoice();

VoiceMediaSession.attach(options: VoiceMediaOptions): Promise<VoiceMediaSession>

await session.player.play(resource);

session.player.on('error', (e) => console.error('playback failed', e));

interface AudioResource {
  pcm: AsyncIterable<Int16Array>;
  sampleRate?: number;  // default 48000
  channels?: number;    // default 2
  close?(): void;       // called once when the pump ends, for any reason
}

pcmChunks(bytes: AsyncIterable<Uint8Array>, channels?: number): AsyncIterable<Int16Array>

await session.player.play({
  pcm: pcmChunks(myByteStream, 2),
  sampleRate: 48000,
  channels: 2,
});

ffmpegResource(input: string, opts?: FfmpegResourceOptions): AudioResource

import { ytdlpResource, resolveMediaUrl } from '@cloak-software/bot-sdk';

// async, unlike ffmpegResource: resolving is a round trip through yt-dlp
await session.player.play(await ytdlpResource('https://www.youtube.com/watch?v=...'));

// or resolve only, to log, cache, or inspect the url yourself
const direct = await resolveMediaUrl(pageUrl);

session.player.stop();  // end the current resource; the in-flight frame is still awaited
await session.detach(); // stop the pump, unpublish, disconnect
await client.leaveVoice();

import { VoiceMediaSession, pcmFrameBytes } from '@cloak-software/bot-sdk';

const conn = await client.joinVoice(serverId, voiceChannelId, { deafened: false });
const voiceKey = await client.voiceKeys.acquireVoiceKey(serverId);
if (voiceKey.scope !== 'room') throw new Error('this bot needs voice_listen to hear anyone');

const session = await VoiceMediaSession.attach({
  url: conn.url,
  token: conn.token,
  botId: client.user!.id,
  serverId,
  voiceKey,
  keys: client.voiceKeys,
  deafened: conn.selfDeafened,
  connectionReporter: client.voiceConnectionReporter(),
  receiveSampleRate: 16000, // the engine resamples; default 48000
  receiveChannels: 1,       // default 1
});

session.on('audio', async ({ identity, frames }) => {
  for await (const frame of frames) {
    // frame.data is interleaved signed 16-bit PCM, frame.samplesPerChannel * frame.channels long
    handle(identity, pcmFrameBytes(frame));
  }
  // A normal return: the track was unsubscribed, or the session detached.
});

for await (const { identity, frame } of session.audio()) {
  meter(identity, frame.data);
}
// Ends with a normal return on detach(). A `break` releases every stream this loop held open.

await session.setSubscribed(identity, false); // ends that identity's iterable now, with a normal return
await session.setSubscribed(identity, true);  // a fresh `audio` event, with a fresh iterable

import { spawn } from 'node:child_process';
import { pcmFrameBytes } from '@cloak-software/bot-sdk';

session.on('audio', async ({ identity, frames }) => {
  const ffmpeg = spawn('ffmpeg', [
    '-f', 's16le', '-ar', '16000', '-ac', '1', // must match receiveSampleRate / receiveChannels
    '-i', 'pipe:0',
    `${identity}.ogg`,
  ], { stdio: ['pipe', 'ignore', 'inherit'] });

  for await (const frame of frames) {
    if (!ffmpeg.stdin.write(pcmFrameBytes(frame))) {
      await new Promise((resolve) => ffmpeg.stdin.once('drain', resolve));
    }
  }
  ffmpeg.stdin.end();
});

import {
  Client,
  VoiceMediaSession,
  ffmpegResource,
  ytdlpResource,
  probeVoiceRuntime,
} from '@cloak-software/bot-sdk';
import type { AudioResource, VoiceConnection } from '@cloak-software/bot-sdk';

const token = process.env.CLOAK_TOKEN;
if (!token) throw new Error('set CLOAK_TOKEN');

const client = new Client({
  token,
  // Not optional. See /concepts/keys-and-keystore.
  keystorePath: './dj-bot.cloak-keystore.json',
  requiredPermissions: [
    'view_text',
    'message_send',
    'commands',
    'voice_connect',
    'voice_speak',
  ],
});

// userId -> channelId. The roster is keyed by channel, so keep the reverse.
const whereTheyAre = new Map<string, string>();
client.on('voiceJoin', (p) => whereTheyAre.set(p.userId, p.channelId));
client.on('voiceLeave', ({ userId }) => whereTheyAre.delete(userId));

let conn: VoiceConnection | undefined;
let session: VoiceMediaSession | undefined;

function resource(source: string): AudioResource | Promise<AudioResource> {
  // Any http(s) url goes through yt-dlp; local paths skip it.
  return /^https?:\/\//i.test(source) ? ytdlpResource(source) : ffmpegResource(source);
}

async function leave(): Promise<void> {
  try {
    await session?.detach();
  } finally {
    session = undefined;
    if (conn) {
      await client.leaveVoice().catch(() => undefined);
      conn = undefined;
    }
  }
}

client.commands.register(
  {
    name: 'play',
    description: 'play something in the voice channel you are in',
    options: [
      {
        name: 'source',
        type: 'string',
        description: 'a file path or a URL',
        required: true,
      },
    ],
  },
  async (ctx) => {
    // Plain answers use channel.send(), not reply(): a true reply fires a
    // mute-piercing "replied to you" notification.
    const say = (text: string) => ctx.message.channel.send(text);

    // Voice is server-scoped, and a direct message has a null serverId.
    const serverId = ctx.message.serverId;
    if (ctx.message.isDM || !serverId) {
      await say('run this in a server, not a DM');
      return;
    }

    const channelId = whereTheyAre.get(ctx.message.authorId);
    if (!channelId) {
      await say('join a voice channel first');
      return;
    }

    // The engine is an optional dependency. Probe rather than letting an
    // import throw halfway through a join.
    const probe = await probeVoiceRuntime();
    if (!probe.available) {
      await say(`voice engine unavailable: ${probe.remedy ?? probe.error}`);
      return;
    }

    await leave();
    conn = await client.joinVoice(serverId, channelId, { deafened: true });

    const voiceKey = await client.voiceKeys.acquireVoiceKey(serverId);
    if (voiceKey.scope !== 'participant') {
      await leave();
      await say('this server granted me voice_listen and I am deliberately publish-only');
      return;
    }

    session = await VoiceMediaSession.attach({
      url: conn.url,
      token: conn.token,
      botId: client.user!.id,
      serverId,
      voiceKey,
      keys: client.voiceKeys,
      deafened: conn.selfDeafened,
      // Not optional in practice: this is what admits the microphone.
      connectionReporter: client.voiceConnectionReporter(),
    });

    await say('playing, and I cannot hear any of you');
    try {
      await session.player.play(await resource(ctx.getString('source')!));
    } catch (e) {
      // Report it in channel. A decoder failure is the likeliest thing to go
      // wrong, and CloakEnvironmentError carries a remedy written for it.
      const err = e as Error & { remedy?: string };
      await leave();
      await say(`could not play that: ${err.message}${err.remedy ? `\n\n${err.remedy}` : ''}`);
      return;
    }
    await say('finished');
  },
);

client.commands.register(
  { name: 'stop', description: 'stop playing and leave voice' },
  async (ctx) => {
    await leave();
    await ctx.message.channel.send('stopped');
  },
);

// A scope flip re-pins at the SAME version, so a version-only handler misses it.
client.on('voiceKeyUpdate', ({ scope }) => {
  if (session && scope !== 'participant') void leave();
});

client.on('error', (e) => console.error('sdk error', e.message));

process.on('SIGINT', () => {
  void leave().finally(() => process.exit(0));
});

await client.login();