Lip sync a video

import { readFile } from "node:fs/promises";
import { basename } from "node:path";

const VIDEO = "presenter.mp4";
const AUDIO = "spanish-voiceover.mp3";
// Length of the video in seconds. Lip sync is billed per second, and the
// API does not probe Library assets, so the request has to state it.
const VIDEO_SECONDS = 12;


// Files live inside a workspace, so the upload needs one. An account
// always has at least one, and the first is the default.
async function firstWorkspaceId() {
  const response = await fetch(`${API}/workspaces`, { headers });

  if (!response.ok) {
    throw new Error("Could not list workspaces: " + response.status);
  }

  const body = await response.json();
  return body.data[0].id;
}


// Push the raw bytes into the media library. The response carries the
// file_hash the AI endpoints take as input.
async function upload(path, workspaceId) {
  const bytes = await readFile(path);

  const query = new URLSearchParams({
    file_name: basename(path),
    mime_type: path.endsWith(".mp3") ? "audio/mpeg" : "video/mp4",
  });

  const url = `${API}/workspaces/${workspaceId}/library?${query}`;

  const response = await fetch(url, {
    method: "POST",
    headers: {
      ...headers,
      "Content-Type": "application/octet-stream",
    },
    body: bytes,
  });

  if (!response.ok) {
    throw new Error("Upload failed: " + response.status);
  }

  const body = await response.json();
  return body.data;
}


// Start the lip-sync job. This returns straight away with a job id.
async function startLipSync(videoHash, audioHash, workspaceId) {
  const response = await fetch(`${API}/ai/lipsync`, {
    method: "POST",
    headers: {
      ...headers,
      "Content-Type": "application/json",
    },
    // To skip an upload, drop its hash and send a public https URL as
    // video_file_url or audio_file_url at the top level, next to params.
    body: JSON.stringify({
      workspace_id: workspaceId,
      duration: VIDEO_SECONDS,
      params: {
        video_file_hash: videoHash,
        audio_file_hash: audioHash,
      },
    }),
  });

  if (!response.ok) {
    throw new Error("Could not start the job: " + response.status);
  }

  const body = await response.json();
  return body.data.job_id;
}


const workspaceId = await firstWorkspaceId();

// Lip sync needs both sides: the footage and the audio to match it to.
const video = await upload(VIDEO, workspaceId);
const audio = await upload(AUDIO, workspaceId);

const jobId = await startLipSync(video.file_hash, audio.file_hash, workspaceId);
const job = await waitForJob(jobId);

// The signed URL is the result. Fetch it, pipe it to your storage,
// or hand it to the browser.
console.log(job.output.url);