How to perform image processing and pixel manipulation in Canvas? Please explain the relevant methods and application scenarios in detail.

Image Processing Methods in Canvas

1. Drawing Images

Canvas provides the drawImage() method to draw images, which has three different overload forms:

javascript
// Basic form: draw the entire image
ctx.drawImage(image, dx, dy);

// Scaling form: draw and scale the image
ctx.drawImage(image, dx, dy, dWidth, dHeight);

// Cropping form: crop and scale the image
ctx.drawImage(image, sx, sy, sWidth, sHeight, dx, dy, dWidth, dHeight);

• image: The image object to draw (HTMLImageElement, HTMLCanvasElement, HTMLVideoElement, etc.)
• dx, dy: The position of the image on the target Canvas
• dWidth, dHeight: The width and height of the image on the target Canvas (scaling)
• sx, sy: The starting position of the cropped area in the source image
• sWidth, sHeight: The width and height of the cropped area in the source image

2. Image Transformation

You can use Canvas transformation methods (such as translate, rotate, scale, etc.) to transform images:

javascript
// Rotate image
ctx.save();
ctx.translate(x, y);
ctx.rotate(angle);
ctx.drawImage(image, -image.width/2, -image.height/2);
ctx.restore();

3. Image Composition

Canvas provides the globalCompositeOperation property to control how images are composed:

javascript
ctx.globalCompositeOperation = "source-over"; // Default: new image covers old image
ctx.globalCompositeOperation = "destination-over"; // Old image covers new image
ctx.globalCompositeOperation = "source-in"; // Only show overlapping part of new and old images
ctx.globalCompositeOperation = "source-out"; // Only show non-overlapping part of new image
ctx.globalCompositeOperation = "destination-in"; // Only show overlapping part of old and new images
ctx.globalCompositeOperation = "destination-out"; // Only show non-overlapping part of old image
ctx.globalCompositeOperation = "lighter"; // New image overlays old image
ctx.globalCompositeOperation = "copy"; // Only show new image, ignore old image
ctx.globalCompositeOperation = "xor"; // Only show non-overlapping parts of new and old images

Pixel Manipulation in Canvas

1. Getting Pixel Data

Use the getImageData() method to get pixel data for a specified area in Canvas:

javascript
const imageData = ctx.getImageData(x, y, width, height);
const data = imageData.data;
// data is a Uint8ClampedArray containing RGBA values for each pixel
// Format: [R1, G1, B1, A1, R2, G2, B2, A2, ...]

2. Setting Pixel Data

Use the putImageData() method to draw pixel data onto Canvas:

javascript
ctx.putImageData(imageData, x, y);
// Or specify offset
ctx.putImageData(imageData, dx, dy, dirtyX, dirtyY, dirtyWidth, dirtyHeight);

3. Creating New Pixel Data

Use the createImageData() method to create a new blank ImageData object:

javascript
// Create ImageData with specified size
const imageData = ctx.createImageData(width, height);

// Create a new ImageData from existing ImageData
const newImageData = ctx.createImageData(imageData);

Common Image Processing Operations

1. Image Filters

By manipulating pixel data, various image filter effects can be implemented:

Grayscale Filter

javascript
const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
const data = imageData.data;

for (let i = 0; i < data.length; i += 4) {
  const r = data[i];
  const g = data[i + 1];
  const b = data[i + 2];
  
  // Calculate grayscale value (using luminance formula)
  const gray = 0.299 * r + 0.587 * g + 0.114 * b;
  
  // Set pixel to grayscale value
  data[i] = gray;     // R
  data[i + 1] = gray; // G
  data[i + 2] = gray; // B
  // A channel remains unchanged
}

ctx.putImageData(imageData, 0, 0);

Invert Filter

javascript
const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
const data = imageData.data;

for (let i = 0; i < data.length; i += 4) {
  data[i] = 255 - data[i];     // R
  data[i + 1] = 255 - data[i + 1]; // G
  data[i + 2] = 255 - data[i + 2]; // B
  // A channel remains unchanged
}

ctx.putImageData(imageData, 0, 0);

Blur Filter

Blur filters are usually implemented using convolution kernels. Here's a simple mean blur as an example:

javascript
function blurImage(ctx, canvas, radius) {
  const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
  const data = imageData.data;
  const tempData = new Uint8ClampedArray(data);
  
  const size = radius * 2 + 1;
  const offset = canvas.width * 4;
  
  for (let y = radius; y < canvas.height - radius; y++) {
    for (let x = radius; x < canvas.width - radius; x++) {
      let r = 0, g = 0, b = 0, a = 0;
      
      for (let dy = -radius; dy <= radius; dy++) {
        for (let dx = -radius; dx <= radius; dx++) {
          const i = ((y + dy) * canvas.width + (x + dx)) * 4;
          r += tempData[i];
          g += tempData[i + 1];
          b += tempData[i + 2];
          a += tempData[i + 3];
        }
      }
      
      const i = (y * canvas.width + x) * 4;
      data[i] = r / (size * size);
      data[i + 1] = g / (size * size);
      data[i + 2] = b / (size * size);
      data[i + 3] = a / (size * size);
    }
  }
  
  ctx.putImageData(imageData, 0, 0);
}

2. Image Scaling

In addition to using the drawImage() method for scaling, you can also implement more precise scaling through pixel manipulation:

javascript
function resizeImage(ctx, canvas, newWidth, newHeight) {
  const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
  const newImageData = ctx.createImageData(newWidth, newHeight);
  
  const scaleX = canvas.width / newWidth;
  const scaleY = canvas.height / newHeight;
  
  for (let y = 0; y < newHeight; y++) {
    for (let x = 0; x < newWidth; x++) {
      const srcX = Math.floor(x * scaleX);
      const srcY = Math.floor(y * scaleY);
      const srcIndex = (srcY * canvas.width + srcX) * 4;
      const dstIndex = (y * newWidth + x) * 4;
      
      newImageData.data[dstIndex] = imageData.data[srcIndex];
      newImageData.data[dstIndex + 1] = imageData.data[srcIndex + 1];
      newImageData.data[dstIndex + 2] = imageData.data[srcIndex + 2];
      newImageData.data[dstIndex + 3] = imageData.data[srcIndex + 3];
    }
  }
  
  // Clear canvas and draw scaled image
  ctx.clearRect(0, 0, canvas.width, canvas.height);
  ctx.putImageData(newImageData, 0, 0);
}

3. Image Clipping

Use the clip() method to implement image clipping:

javascript
// Create clipping path
ctx.beginPath();
ctx.arc(canvas.width/2, canvas.height/2, 100, 0, Math.PI * 2);
ctx.clip();

// Draw image, only show part inside clipping path
ctx.drawImage(image, 0, 0, canvas.width, canvas.height);

Performance Considerations for Pixel Manipulation

1. Performance Overhead of Pixel Manipulation: Directly manipulating pixel data is a CPU-intensive operation and may cause performance issues for large images.

2. Optimization Strategies:

   • Use the data array of ImageData for direct manipulation, avoiding frequent method calls
   • For complex image processing, consider using Web Workers to process in background threads
   • Use Uint32Array view to access pixel data, which can operate on a 32-bit value of a pixel at once
   • For frequent image processing, consider using off-screen Canvas

3. Example: Optimizing Pixel Manipulation with Uint32Array

javascript
const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
const data = imageData.data;
const uint32Data = new Uint32Array(data.buffer);

for (let i = 0; i < uint32Data.length; i++) {
  // Directly operate on 32-bit pixel value (format: 0xAABBGGRR)
  const pixel = uint32Data[i];
  // Process pixel...
  uint32Data[i] = processedPixel;
}

ctx.putImageData(imageData, 0, 0);

Application Scenarios of Image Processing

1. Image Editors: Implement basic image editing functions such as cropping, adjusting brightness/contrast, applying filters, etc.
2. Real-time Video Processing: Capture camera video and process it in real-time, such as face detection, filter effects, etc.
3. Game Development: Implement special effects, sprite animations, particle effects, etc. in games.
4. Data Visualization: Convert data into images, such as heat maps, spectrograms, etc.
5. CAPTCHA Generation: Generate CAPTCHA images with interference lines, noise, etc.
6. Image Compression: Compress images by reducing color depth, applying filters, etc.
7. Watermark Addition: Add text or image watermarks to images.

Image Processing Example

javascript
const canvas = document.getElementById('myCanvas');
const ctx = canvas.getContext('2d');

// Load image
const image = new Image();
image.crossOrigin = 'Anonymous'; // Allow cross-origin loading
image.src = 'https://example.com/image.jpg';

image.onload = function() {
  // Draw original image
  ctx.drawImage(image, 0, 0, 200, 200);
  
  // Apply grayscale filter
  applyGrayscaleFilter(ctx, 0, 0, 200, 200);
  
  // Draw to right side
  ctx.drawImage(canvas, 0, 0, 200, 200, 250, 0, 200, 200);
  
  // Apply blur filter
  applyBlurFilter(ctx, 250, 0, 200, 200, 5);
  
  // Draw to bottom
  ctx.drawImage(canvas, 250, 0, 200, 200, 0, 250, 200, 200);
  
  // Apply invert filter
  applyInvertFilter(ctx, 0, 250, 200, 200);
};

function applyGrayscaleFilter(ctx, x, y, width, height) {
  const imageData = ctx.getImageData(x, y, width, height);
  const data = imageData.data;
  
  for (let i = 0; i < data.length; i += 4) {
    const r = data[i];
    const g = data[i + 1];
    const b = data[i + 2];
    const gray = 0.299 * r + 0.587 * g + 0.114 * b;
    
    data[i] = gray;
    data[i + 1] = gray;
    data[i + 2] = gray;
  }
  
  ctx.putImageData(imageData, x, y);
}

function applyBlurFilter(ctx, x, y, width, height, radius) {
  // Implement blur filter...
}

function applyInvertFilter(ctx, x, y, width, height) {
  const imageData = ctx.getImageData(x, y, width, height);
  const data = imageData.data;
  
  for (let i = 0; i < data.length; i += 4) {
    data[i] = 255 - data[i];
    data[i + 1] = 255 - data[i + 1];
    data[i + 2] = 255 - data[i + 2];
  }
  
  ctx.putImageData(imageData, x, y);
}

Notes

1. Cross-domain Restrictions: When using the getImageData() method to get image data loaded from other domains, it is subject to the same-origin policy. Ensure the image server sets the correct CORS headers, or use the crossOrigin attribute.
2. Image Loading: Before drawing an image, ensure the image has been fully loaded. You can use the onload event to listen for image loading completion.
3. Canvas Size Limitations: Different browsers have different limitations on Canvas size, and an overly large Canvas may cause memory issues.
4. Performance Monitoring: For complex image processing, it is recommended to use the browser's performance analysis tools to monitor and optimize performance.