Face.txt

1. src/webgl/shader-programs/faceMaskVertexShader.ts

import { glsl } from '../utils';

export const faceMaskVertexShader = glsl`#version 300 es
in vec2 position;
in vec2 attrUV;
out vec2 vUV;

void main() {
  vUV = attrUV;
  gl_Position = vec4(position, 0.0, 1.0);
}
`;

2. src/webgl/shader-programs/faceMaskFragmentShader.ts

import { glsl } from '../utils';

export const faceMaskFragmentShader = glsl`#version 300 es
precision mediump float;

in vec2 vUV;
uniform sampler2D u_frame;
out vec4 fragColor;

void main() {
  // just output the video
  fragColor = texture(u_frame, vUV);
}
`;

3. src/webgl/shader-programs/faceOverlayVertexShader.ts

import { glsl } from '../utils';

export const overlayVertexShader = glsl`#version 300 es
in vec2 position;
void main() {
  gl_Position = vec4(position, 0.0, 1.0);
}
`;

4. src/webgl/shader-programs/faceOverlayFragmentShader.ts

import { glsl } from '../utils';

export const overlayFragmentShader = glsl`#version 300 es
precision mediump float;

// Hard-coded purple mask at 40% opacity
out vec4 fragColor;
void main() {
  fragColor = vec4(0.6, 0.0, 0.8, 0.4);
}
`;


---

5. src/webgl/setupWebGLFaceMask.ts

import {
  createProgram,
  createShader,
  createVertexBuffer,
  initTexture,
} from './utils';
import { faceMaskVertexShader } from './shader-programs/faceMaskVertexShader';
import { faceMaskFragmentShader } from './shader-programs/faceMaskFragmentShader';
import { overlayVertexShader }     from './shader-programs/faceOverlayVertexShader';
import { overlayFragmentShader }   from './shader-programs/faceOverlayFragmentShader';

export function setupWebGLFaceMask(canvas: OffscreenCanvas) {
  const gl = canvas.getContext('webgl2')!;
  if (!gl) throw new Error('WebGL2 required');

  // ----- compile programs -----
  const vsFrame  = createShader(gl, gl.VERTEX_SHADER,   faceMaskVertexShader);
  const fsFrame  = createShader(gl, gl.FRAGMENT_SHADER, faceMaskFragmentShader);
  const progFrame= createProgram(gl, vsFrame, fsFrame);

  const vsOvl    = createShader(gl, gl.VERTEX_SHADER,   overlayVertexShader);
  const fsOvl    = createShader(gl, gl.FRAGMENT_SHADER, overlayFragmentShader);
  const progOvl  = createProgram(gl, vsOvl, fsOvl);

  // ----- fullscreen quad (two triangles) -----
  //   positions in clip space + UVs
  const quadData = new Float32Array([
    //  position   UV
    -1, -1,      0, 0,
     1, -1,      1, 0,
     1,  1,      1, 1,
    -1, -1,      0, 0,
     1,  1,      1, 1,
    -1,  1,      0, 1,
  ]);
  const quadVBO = createVertexBuffer(gl);
  gl.bindBuffer(gl.ARRAY_BUFFER, quadVBO);
  gl.bufferData(gl.ARRAY_BUFFER, quadData, gl.STATIC_DRAW);

  // attribute offsets
  const posLocFrame = gl.getAttribLocation(progFrame, 'position');
  const uvLocFrame  = gl.getAttribLocation(progFrame, 'attrUV');

  // overlay uses only position
  const posLocOvl   = gl.getAttribLocation(progOvl, 'position');

  // video texture
  const frameTex = initTexture(gl, 0);

  // dynamic VBO for the face polygon
  const faceVBO = gl.createBuffer()!;

  gl.enable(gl.BLEND);
  gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);

  function updateFaceVertices(coords: Float32Array) {
    // coords: [x0,y0, x1,y1, …] in clip space
    gl.bindBuffer(gl.ARRAY_BUFFER, faceVBO);
    gl.bufferData(gl.ARRAY_BUFFER, coords, gl.DYNAMIC_DRAW);
  }

  function renderFrame(frame: VideoFrame) {
    const w = frame.displayWidth, h = frame.displayHeight;
    gl.viewport(0, 0, w, h);

    // upload video frame
    gl.activeTexture(gl.TEXTURE0);
    gl.bindTexture(gl.TEXTURE_2D, frameTex);
    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA,
                  gl.UNSIGNED_BYTE, frame);

    // draw the video fullscreen
    gl.useProgram(progFrame);
    gl.bindBuffer(gl.ARRAY_BUFFER, quadVBO);
    gl.enableVertexAttribArray(posLocFrame);
    gl.vertexAttribPointer(posLocFrame, 2, gl.FLOAT, false, 16, 0);
    gl.enableVertexAttribArray(uvLocFrame);
    gl.vertexAttribPointer(uvLocFrame, 2, gl.FLOAT, false, 16, 8);
    gl.drawArrays(gl.TRIANGLES, 0, 6);
  }

  function renderOverlay(vertexCount: number) {
    gl.useProgram(progOvl);
    gl.bindBuffer(gl.ARRAY_BUFFER, faceVBO);
    gl.enableVertexAttribArray(posLocOvl);
    gl.vertexAttribPointer(posLocOvl, 2, gl.FLOAT, false, 8, 0);
    gl.drawArrays(gl.TRIANGLE_FAN, 0, vertexCount);
  }

  function cleanup() {
    gl.deleteProgram(progFrame);
    gl.deleteProgram(progOvl);
    gl.deleteBuffer(quadVBO);
    gl.deleteBuffer(faceVBO);
    gl.deleteTexture(frameTex);
  }

  return { renderFrame, updateFaceVertices, renderOverlay, cleanup };
}


---

6. src/transformers/FaceMaskProcessor.ts

import * as vision from '@mediapipe/tasks-vision';
import { dependencies } from '../../package.json';
import VideoTransformer from './VideoTransformer';
import { VideoTransformerInitOptions } from './types';
import { setupWebGLFaceMask } from '../webgl/setupWebGLFaceMask';

export default class FaceMaskProcessor extends VideoTransformer<{}> {
  private faceLandmarker?: vision.FaceLandmarker;
  private glPipeline?: ReturnType<typeof setupWebGLFaceMask>;

  async init({ outputCanvas, inputElement }: VideoTransformerInitOptions) {
    if (!(inputElement instanceof HTMLVideoElement)) {
      throw new TypeError('Expected HTMLVideoElement');
    }

    // 1) create FaceLandmarker
    const fs = await vision.FilesetResolver.forVisionTasks(
      `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`
    );
    this.faceLandmarker = await vision.FaceLandmarker.createFromOptions(fs, {
      baseOptions: {
        modelAssetPath:
          'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task',
        delegate: 'GPU',
      },
      runningMode: 'VIDEO',
      outputFaceBlendshapes: false,
      outputFaceLandmarks: true,
      numFaces: 1,
    });

    // 2) set up WebGL pipeline
    this.transformer = new TransformStream({ transform: (f,c)=>this.transform(f,c) });
    this.canvas = outputCanvas!;
    this.glPipeline = setupWebGLFaceMask(this.canvas);
    this.inputVideo = inputElement;
    this.isDisabled = false;
  }

  async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {
    if (!this.glPipeline || !this.faceLandmarker) {
      controller.enqueue(frame);
      return;
    }

    this.canvas!.width  = frame.displayWidth;
    this.canvas!.height = frame.displayHeight;

    // 3) run face detection on this video frame
    const now = performance.now();
    const result = this.faceLandmarker.detectForVideo(this.inputVideo!, now);

    // 4) convert landmarks to clip-space XY pairs
    if (result.faceLandmarks?.length) {
      const lm = result.faceLandmarks[0]; // single face
      const coords = new Float32Array(lm.length * 2);
      for (let i = 0; i < lm.length; i++) {
        // lm[i].x/y are normalized [0,1], origin top-left
        const x = lm[i].x * 2 - 1;
        const y = 1 - 2 * lm[i].y;
        coords[2*i  ] = x;
        coords[2*i+1] = y;
      }
      this.glPipeline.updateFaceVertices(coords);
    }

    // 5) draw video + overlay polygon
    this.glPipeline.renderFrame(frame);
    if (result.faceLandmarks?.length) {
      this.glPipeline.renderOverlay(result.faceLandmarks[0].length);
    }

    // 6) output new frame
    const out = new VideoFrame(this.canvas!, { timestamp: frame.timestamp });
    controller.enqueue(out);
    frame.close();
  }

  async destroy() {
    await super.destroy();
    this.faceLandmarker?.close();
    this.glPipeline?.cleanup();
  }
}




import { FaceLandmarker } from '@mediapipe/tasks-vision';

// …

async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {
  if (!this.glPipeline || !this.faceLandmarker) {
    controller.enqueue(frame);
    return;
  }

  // 1) Resize your offscreen
  this.canvas!.width  = frame.displayWidth;
  this.canvas!.height = frame.displayHeight;

  // 2) Run MediaPipe
  const nowMs = performance.now();
  const results = this.faceLandmarker.detectForVideo(this.inputVideo!, nowMs);

  // 3) If we got a face, build a fan of the face‐oval
  const ovalIdx = FaceLandmarker.FACE_LANDMARKS_FACE_OVAL; 
  if (results.faceLandmarks?.length) {
    const lm = results.faceLandmarks[0];

    // compute polygon center (average of oval points)
    let cx = 0, cy = 0;
    for (const i of ovalIdx) {
      cx += lm[i].x;
      cy += lm[i].y;
    }
    cx /= ovalIdx.length;
    cy /= ovalIdx.length;

    // build [ center, pt0, pt1, …, ptN−1, pt0 ] in clip-space
    const verts = new Float32Array((ovalIdx.length + 2) * 2);
    // center
    verts[0] = cx * 2 - 1;
    verts[1] = 1 - 2 * cy;
    // boundary
    for (let j = 0; j < ovalIdx.length; j++) {
      const p = lm[ ovalIdx[j] ];
      verts[2 * (j + 1)    ] = p.x * 2 - 1;
      verts[2 * (j + 1) + 1] = 1 - 2 * p.y;
    }
    // close loop by repeating pt0
    verts[2*(ovalIdx.length+1)    ] = verts[2*1    ];
    verts[2*(ovalIdx.length+1) + 1] = verts[2*1 + 1];

    // upload to GL and draw fan
    this.glPipeline.updateFaceVertices(verts);
    this.glPipeline.renderOverlay(ovalIdx.length + 2);
  }

  // 4) Draw the video itself
  this.glPipeline.renderFrame(frame);

  // 5) Emit new VideoFrame
  const out = new VideoFrame(this.canvas!, { timestamp: frame.timestamp });
  controller.enqueue(out);
  frame.close();
}



const FACE_OVAL_INDICES = [
  10, 338, 297, 332, 284, 251, 389, 356,
  454, 323, 361, 288, 397, 365, 379, 378,
  400, 377, 152, 148, 176, 149, 150, 136,
  172, 58,  132, 93,  234, 127, 162, 21,
  54,  103, 67,  109
];







------



function renderLandmarks(landmarkSets: Array<Array<{ x: number; y: number }>>) {
  if (landmarkSets.length === 0) return;

  gl.useProgram(landmarkProgram);
  gl.bindTexture(gl.TEXTURE_2D, null);

  const facesDetected = landmarkSets.length; // <--- Add here

  for (const landmarks of landmarkSets) {
    const data = new Float32Array(landmarks.length * 2);
    landmarks.forEach((pt, i) => {
      data[i * 2 + 0] = pt.x;
      data[i * 2 + 1] = pt.y;
    });

    gl.bindBuffer(gl.ARRAY_BUFFER, landmarkVBO);
    gl.bufferData(gl.ARRAY_BUFFER, data, gl.DYNAMIC_DRAW);
    gl.enableVertexAttribArray(uvLoc);
    gl.vertexAttribPointer(uvLoc, 2, gl.FLOAT, false, 0, 0);

    connectorInfo.forEach(({ ebo, count, color }) => {
      gl.uniform4fv(colorLoc, color);
      gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, ebo);
      gl.drawElements(gl.LINES, count, gl.UNSIGNED_SHORT, 0);
    });

    gl.uniform4fv(colorLoc, [1, 1, 1, 1]);
    gl.drawArrays(gl.POINTS, 0, landmarks.length);
  }

  // ✅ After all faces drawn, overlay alert if needed
  if (facesDetected > 2) {
    const overlayVertices = new Float32Array([
      -1.0, 0.9, // left-top
      -1.0, 1.0, // left-very-top
       1.0, 0.9, // right-top
       1.0, 1.0  // right-very-top
    ]);

    const overlayBuffer = gl.createBuffer()!;
    gl.bindBuffer(gl.ARRAY_BUFFER, overlayBuffer);
    gl.bufferData(gl.ARRAY_BUFFER, overlayVertices, gl.STATIC_DRAW);

    gl.enableVertexAttribArray(uvLoc);
    gl.vertexAttribPointer(uvLoc, 2, gl.FLOAT, false, 0, 0);

    gl.uniform4fv(colorLoc, [1.0, 0.0, 0.0, 0.7]); // semi-transparent red
    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);

    gl.deleteBuffer(overlayBuffer);
  }
}