ExternalTextureFlutterActivity.java

Go to the documentation of this file.

// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
 
package dev.flutter.scenarios;
 
import static io.flutter.Build.API_LEVELS;
 
import android.content.res.AssetFileDescriptor;
import android.graphics.Canvas;
import android.graphics.ImageFormat;
import android.graphics.LinearGradient;
import android.graphics.Paint;
import android.graphics.Shader.TileMode;
import android.hardware.HardwareBuffer;
import android.media.Image;
import android.media.ImageReader;
import android.media.ImageWriter;
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.os.Build.VERSION;
import android.os.Bundle;
import android.os.Handler;
import android.os.HandlerThread;
import android.util.Log;
import android.view.Gravity;
import android.view.Surface;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
import android.view.ViewGroup;
import android.widget.FrameLayout;
import android.widget.FrameLayout.LayoutParams;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.core.util.Supplier;
import io.flutter.view.TextureRegistry;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.CountDownLatch;
 
public class ExternalTextureFlutterActivity extends TestActivity {
  static final String TAG = "Scenarios";
  private static final int SURFACE_WIDTH = 192;
  private static final int SURFACE_HEIGHT = 256;
 
  private SurfaceRenderer flutterRenderer;
 
  // Latch used to ensure both SurfaceRenderers produce a frame before taking a screenshot.
  private final CountDownLatch firstFrameLatch = new CountDownLatch(1);
 
  private long textureId = 0;
  private TextureRegistry.SurfaceProducer surfaceProducer;
 
  @Override
  protected void onCreate(@Nullable Bundle savedInstanceState) {
    super.onCreate(savedInstanceState);
 
    String surfaceRenderer = getIntent().getStringExtra("surface_renderer");
    assert surfaceRenderer != null;
    flutterRenderer = selectSurfaceRenderer(surfaceRenderer);
 
    // Create and place a SurfaceView above the Flutter content.
    SurfaceView surfaceView = new SurfaceView(getContext());
    surfaceView.setZOrderMediaOverlay(true);
    surfaceView.setMinimumWidth(SURFACE_WIDTH);
    surfaceView.setMinimumHeight(SURFACE_HEIGHT);
 
    FrameLayout frameLayout = new FrameLayout(getContext());
    frameLayout.addView(
        surfaceView,
        new LayoutParams(
            ViewGroup.LayoutParams.WRAP_CONTENT,
            ViewGroup.LayoutParams.WRAP_CONTENT,
            Gravity.BOTTOM | Gravity.CENTER_HORIZONTAL));
 
    addContentView(
        frameLayout,
        new ViewGroup.LayoutParams(
            ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT));
 
    SurfaceHolder surfaceHolder = surfaceView.getHolder();
    surfaceHolder.setFixedSize(SURFACE_WIDTH, SURFACE_HEIGHT);
  }
 
  @Override
  public void waitUntilFlutterRendered() {
    super.waitUntilFlutterRendered();
 
    try {
      firstFrameLatch.await();
    } catch (InterruptedException e) {
      throw new RuntimeException(e);
    }
  }
 
  private SurfaceRenderer selectSurfaceRenderer(String surfaceRenderer) {
    switch (surfaceRenderer) {
      case "image":
        if (VERSION.SDK_INT >= API_LEVELS.API_23) {
          // CanvasSurfaceRenderer doesn't work correctly when used with ImageSurfaceRenderer.
          // Use MediaSurfaceRenderer for now.
          return new ImageSurfaceRenderer(selectSurfaceRenderer("media"));
        } else {
          throw new RuntimeException("ImageSurfaceRenderer not supported");
        }
      case "media":
        return new MediaSurfaceRenderer(this::createMediaExtractor);
      case "canvas":
      default:
        return new CanvasSurfaceRenderer();
    }
  }
 
  private MediaExtractor createMediaExtractor() {
    // Sample Video generated with FFMPEG.
    // ffmpeg -loop 1 -i ~/engine/src/flutter/lib/ui/fixtures/DashInNooglerHat.jpg -c:v libx264
    // -profile:v main -level:v 5.2 -t 1 -r 1 -vf scale=192:256 -b:v 1M sample.mp4
    try {
      MediaExtractor extractor = new MediaExtractor();
      try (AssetFileDescriptor afd = getAssets().openFd("sample.mp4")) {
        extractor.setDataSource(afd.getFileDescriptor(), afd.getStartOffset(), afd.getLength());
      }
      return extractor;
    } catch (IOException e) {
      e.printStackTrace();
      throw new RuntimeException(e);
    }
  }
 
  @Override
  public void onPause() {
    flutterRenderer.destroy();
    surfaceProducer.release();
    super.onPause();
  }
 
  @Override
  public void onFlutterUiDisplayed() {
    surfaceProducer =
        Objects.requireNonNull(getFlutterEngine()).getRenderer().createSurfaceProducer();
    surfaceProducer.setSize(SURFACE_WIDTH, SURFACE_HEIGHT);
    flutterRenderer.attach(surfaceProducer.getSurface(), firstFrameLatch);
    flutterRenderer.repaint();
    textureId = surfaceProducer.id();
 
    super.onFlutterUiDisplayed();
  }
 
  @Override
  protected void getScenarioParams(@NonNull Map<String, Object> args) {
    super.getScenarioParams(args);
    args.put("texture_id", textureId);
    args.put("texture_width", SURFACE_WIDTH);
    args.put("texture_height", SURFACE_HEIGHT);
  }
 
  private interface SurfaceRenderer {
    void attach(Surface surface, CountDownLatch onFirstFrame);
 
    void repaint();
 
    void destroy();
  }
 
  /** Paints a simple gradient onto the attached Surface. */
  private static class CanvasSurfaceRenderer implements SurfaceRenderer {
    private Surface surface;
    private CountDownLatch onFirstFrame;
 
    protected CanvasSurfaceRenderer() {}
 
    @Override
    public void attach(Surface surface, CountDownLatch onFirstFrame) {
      this.surface = surface;
      this.onFirstFrame = onFirstFrame;
    }
 
    @Override
    public void repaint() {
      Canvas canvas =
          VERSION.SDK_INT >= API_LEVELS.API_23
              ? surface.lockHardwareCanvas()
              : surface.lockCanvas(null);
      Paint paint = new Paint();
      paint.setShader(
          new LinearGradient(
              0.0f,
              0.0f,
              canvas.getWidth(),
              canvas.getHeight(),
              new int[] {
                // Cyan (#00FFFF)
                0xFF00FFFF,
                // Magenta (#FF00FF)
                0xFFFF00FF,
                // Yellow (#FFFF00)
                0xFFFFFF00,
              },
              null,
              TileMode.REPEAT));
      canvas.drawPaint(paint);
      surface.unlockCanvasAndPost(canvas);
 
      if (onFirstFrame != null) {
        onFirstFrame.countDown();
        onFirstFrame = null;
      }
    }
 
    @Override
    public void destroy() {}
  }
 
  /** Decodes a sample video into the attached Surface. */
  private static class MediaSurfaceRenderer implements SurfaceRenderer {
    private final Supplier<MediaExtractor> extractorSupplier;
    private CountDownLatch onFirstFrame;
 
    private Surface surface;
    private MediaExtractor extractor;
    private MediaFormat format;
    private Thread decodeThread;
 
    protected MediaSurfaceRenderer(Supplier<MediaExtractor> extractorSupplier) {
      this.extractorSupplier = extractorSupplier;
    }
 
    @Override
    public void attach(Surface surface, CountDownLatch onFirstFrame) {
      this.surface = surface;
      this.onFirstFrame = onFirstFrame;
 
      extractor = extractorSupplier.get();
      format = extractor.getTrackFormat(0);
 
      decodeThread = new Thread(this::decodeThreadMain);
      decodeThread.start();
    }
 
    private void decodeThreadMain() {
      try {
        MediaCodec codec =
            MediaCodec.createDecoderByType(
                Objects.requireNonNull(format.getString(MediaFormat.KEY_MIME)));
        codec.configure(format, surface, null, 0);
        codec.start();
 
        // Track 0 is always the video track, since the sample video doesn't contain audio.
        extractor.selectTrack(0);
 
        MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
        boolean seenEOS = false;
        long startTimeNs = System.nanoTime();
        int frameCount = 0;
 
        while (true) {
          // Move samples (video frames) from the extractor into the decoder, as long as we haven't
          // consumed all samples.
          if (!seenEOS) {
            int inputBufferIndex = codec.dequeueInputBuffer(-1);
            ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferIndex);
            assert inputBuffer != null;
            int sampleSize = extractor.readSampleData(inputBuffer, 0);
            if (sampleSize >= 0) {
              long presentationTimeUs = extractor.getSampleTime();
              codec.queueInputBuffer(inputBufferIndex, 0, sampleSize, presentationTimeUs, 0);
              extractor.advance();
            } else {
              codec.queueInputBuffer(
                  inputBufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
              seenEOS = true;
            }
          }
 
          // Then consume decoded video frames from the decoder. These frames are automatically
          // pushed to the attached Surface, so this schedules them for present.
          int outputBufferIndex = codec.dequeueOutputBuffer(bufferInfo, 10000);
          boolean lastBuffer = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
          if (outputBufferIndex >= 0) {
            if (bufferInfo.size > 0) {
              if (onFirstFrame != null) {
                onFirstFrame.countDown();
                onFirstFrame = null;
              }
              Log.w(TAG, "Presenting frame " + frameCount);
              frameCount++;
 
              codec.releaseOutputBuffer(
                  outputBufferIndex, startTimeNs + (bufferInfo.presentationTimeUs * 1000));
            }
          }
 
          // Exit the loop if there are no more frames available.
          if (lastBuffer) {
            break;
          }
        }
 
        codec.stop();
        codec.release();
        extractor.release();
      } catch (IOException e) {
        e.printStackTrace();
        throw new RuntimeException(e);
      }
    }
 
    @Override
    public void repaint() {}
 
    @Override
    public void destroy() {
      try {
        decodeThread.join();
      } catch (InterruptedException e) {
        e.printStackTrace();
        throw new RuntimeException(e);
      }
    }
  }
 
  /**
   * Takes frames from the inner SurfaceRenderer and feeds it through an ImageReader and ImageWriter
   * pair.
   */
  @RequiresApi(API_LEVELS.API_23)
  private static class ImageSurfaceRenderer implements SurfaceRenderer {
    private final SurfaceRenderer inner;
    private CountDownLatch onFirstFrame;
    private ImageReader reader;
    private ImageWriter writer;
 
    private Handler handler;
    private HandlerThread handlerThread;
 
    private boolean canReadImage = true;
    private boolean canWriteImage = true;
 
    protected ImageSurfaceRenderer(SurfaceRenderer inner) {
      this.inner = inner;
    }
 
    @Override
    public void attach(Surface surface, CountDownLatch onFirstFrame) {
      this.onFirstFrame = onFirstFrame;
      if (VERSION.SDK_INT >= API_LEVELS.API_29) {
        // On Android Q+, use PRIVATE image format.
        // Also let the frame producer know the images will
        // be sampled from by the GPU.
        writer = ImageWriter.newInstance(surface, 3, ImageFormat.PRIVATE);
        reader =
            ImageReader.newInstance(
                SURFACE_WIDTH,
                SURFACE_HEIGHT,
                ImageFormat.PRIVATE,
                2,
                HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE);
      } else {
        // Before Android Q, this will change the format of the surface to match the images.
        writer = ImageWriter.newInstance(surface, 3);
        reader = ImageReader.newInstance(SURFACE_WIDTH, SURFACE_HEIGHT, writer.getFormat(), 2);
      }
      inner.attach(reader.getSurface(), null);
 
      handlerThread = new HandlerThread("image reader/writer thread");
      handlerThread.start();
 
      handler = new Handler(handlerThread.getLooper());
      reader.setOnImageAvailableListener(this::onImageAvailable, handler);
      writer.setOnImageReleasedListener(this::onImageReleased, handler);
    }
 
    private void onImageAvailable(ImageReader reader) {
      Log.v(TAG, "Image available");
 
      if (!canWriteImage) {
        // If the ImageWriter hasn't released the latest image, don't attempt to enqueue another
        // image.
        // Otherwise the reader writer pair locks up if the writer runs behind, as the reader runs
        // out of images and the writer has no more space for images.
        canReadImage = true;
        return;
      }
 
      canReadImage = false;
      Image image = reader.acquireLatestImage();
      try {
        canWriteImage = false;
        writer.queueInputImage(image);
      } catch (IllegalStateException e) {
        // If the output surface disconnects, this method will be interrupted with an
        // IllegalStateException.
        // Simply log and return.
        Log.i(TAG, "Surface disconnected from ImageWriter", e);
        image.close();
      }
 
      Log.v(TAG, "Output image");
 
      if (onFirstFrame != null) {
        onFirstFrame.countDown();
        onFirstFrame = null;
      }
    }
 
    private void tryAcquireImage() {
      if (canReadImage) {
        onImageAvailable(reader);
      }
    }
 
    private void onImageReleased(ImageWriter imageWriter) {
      Log.v(TAG, "Image released");
 
      if (!canWriteImage) {
        canWriteImage = true;
        if (canReadImage) {
          // Try acquire the image in a handler message, as we may have another call to
          // onImageAvailable in the thread's message queue.
          handler.post(this::tryAcquireImage);
        }
      }
    }
 
    @Override
    public void repaint() {
      inner.repaint();
    }
 
    @Override
    public void destroy() {
      Log.i(TAG, "Destroying ImageSurfaceRenderer");
      inner.destroy();
      handler.post(this::destroyReaderWriter);
    }
 
    private void destroyReaderWriter() {
      writer.close();
      Log.i(TAG, "ImageWriter destroyed");
      reader.close();
      Log.i(TAG, "ImageReader destroyed");
      handlerThread.quitSafely();
    }
  }
}