Flutter Engine
ios_external_texture_gl.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #import "flutter/shell/platform/darwin/ios/ios_external_texture_gl.h"
6 
7 #import <OpenGLES/EAGL.h>
8 #import <OpenGLES/ES2/gl.h>
9 #import <OpenGLES/ES2/glext.h>
10 
11 #import "flutter/shell/platform/darwin/ios/framework/Source/vsync_waiter_ios.h"
12 #include "third_party/skia/include/core/SkSurface.h"
13 #include "third_party/skia/include/core/SkYUVAIndex.h"
14 #include "third_party/skia/include/gpu/GrBackendSurface.h"
15 #include "third_party/skia/include/gpu/GrDirectContext.h"
16 #include "third_party/skia/src/gpu/gl/GrGLDefines.h"
17 
18 namespace flutter {
19 
21  NSObject<FlutterTexture>* externalTexture)
22  : Texture(textureId),
23  external_texture_(fml::scoped_nsobject<NSObject<FlutterTexture>>([externalTexture retain])) {
24  FML_DCHECK(external_texture_);
25 }
26 
28 
29 void IOSExternalTextureGL::EnsureTextureCacheExists() {
30  if (!cache_ref_) {
31  CVOpenGLESTextureCacheRef cache;
32  CVReturn err = CVOpenGLESTextureCacheCreate(kCFAllocatorDefault, NULL,
33  [EAGLContext currentContext], NULL, &cache);
34  if (err == noErr) {
35  cache_ref_.Reset(cache);
36  } else {
37  FML_LOG(WARNING) << "Failed to create GLES texture cache: " << err;
38  return;
39  }
40  }
41 }
42 
43 void IOSExternalTextureGL::CreateTextureFromPixelBuffer() {
44  if (buffer_ref_ == nullptr) {
45  return;
46  }
47  if (pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
48  pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
49  CreateYUVTexturesFromPixelBuffer();
50  } else {
51  CreateRGBATextureFromPixelBuffer();
52  }
53 }
54 
55 void IOSExternalTextureGL::CreateRGBATextureFromPixelBuffer() {
56  CVOpenGLESTextureRef texture;
57  CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(
58  kCFAllocatorDefault, cache_ref_, buffer_ref_, /*textureAttributes=*/nullptr, GL_TEXTURE_2D,
59  GL_RGBA, static_cast<int>(CVPixelBufferGetWidth(buffer_ref_)),
60  static_cast<int>(CVPixelBufferGetHeight(buffer_ref_)), GL_BGRA, GL_UNSIGNED_BYTE, 0,
61  &texture);
62  if (err != noErr) {
63  FML_LOG(WARNING) << "Could not create texture from pixel buffer: " << err;
64  } else {
65  texture_ref_.Reset(texture);
66  }
67 }
68 
69 void IOSExternalTextureGL::CreateYUVTexturesFromPixelBuffer() {
70  size_t width = CVPixelBufferGetWidth(buffer_ref_);
71  size_t height = CVPixelBufferGetHeight(buffer_ref_);
72  {
73  CVOpenGLESTextureRef yTexture;
74  CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(
75  kCFAllocatorDefault, cache_ref_, buffer_ref_, /*textureAttributes=*/nullptr, GL_TEXTURE_2D,
76  GL_LUMINANCE, width, height, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0, &yTexture);
77  if (err != noErr) {
78  FML_DCHECK(yTexture) << "Could not create texture from pixel buffer: " << err;
79  } else {
80  y_texture_ref_.Reset(yTexture);
81  }
82  }
83 
84  {
85  CVOpenGLESTextureRef uvTexture;
86  CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(
87  kCFAllocatorDefault, cache_ref_, buffer_ref_, /*textureAttributes=*/nullptr, GL_TEXTURE_2D,
88  GL_LUMINANCE_ALPHA, width / 2, height / 2, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, 1,
89  &uvTexture);
90  if (err != noErr) {
91  FML_DCHECK(uvTexture) << "Could not create texture from pixel buffer: " << err;
92  } else {
93  uv_texture_ref_.Reset(uvTexture);
94  }
95  }
96 }
97 
98 sk_sp<SkImage> IOSExternalTextureGL::CreateImageFromRGBATexture(GrDirectContext* context,
99  const SkRect& bounds) {
100  GrGLTextureInfo textureInfo = {CVOpenGLESTextureGetTarget(texture_ref_),
101  CVOpenGLESTextureGetName(texture_ref_), GL_RGBA8_OES};
102  GrBackendTexture backendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, textureInfo);
103  sk_sp<SkImage> image = SkImage::MakeFromTexture(context, backendTexture, kTopLeft_GrSurfaceOrigin,
104  kRGBA_8888_SkColorType, kPremul_SkAlphaType,
105  /*imageColorSpace=*/nullptr);
106  return image;
107 }
108 
109 sk_sp<SkImage> IOSExternalTextureGL::CreateImageFromYUVTextures(GrDirectContext* context,
110  const SkRect& bounds) {
111  GrGLTextureInfo yTextureInfo = {CVOpenGLESTextureGetTarget(y_texture_ref_),
112  CVOpenGLESTextureGetName(y_texture_ref_), GR_GL_LUMINANCE8};
113  GrBackendTexture yBackendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, yTextureInfo);
114  GrGLTextureInfo uvTextureInfo = {CVOpenGLESTextureGetTarget(uv_texture_ref_),
115  CVOpenGLESTextureGetName(uv_texture_ref_), GR_GL_RGBA8};
116  GrBackendTexture uvBackendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo,
117  uvTextureInfo);
118  GrBackendTexture nv12TextureHandles[] = {yBackendTexture, uvBackendTexture};
119  SkYUVAIndex yuvaIndices[4] = {
120  SkYUVAIndex{0, SkColorChannel::kR}, // Read Y data from the red channel of the first texture
121  SkYUVAIndex{1, SkColorChannel::kR}, // Read U data from the red channel of the second texture
122  SkYUVAIndex{
123  1, SkColorChannel::kA}, // Read V data from the alpha channel of the second texture,
124  // normal NV12 data V should be taken from the green channel, but
125  // currently only the uv texture created by GL_LUMINANCE_ALPHA
126  // can be used, so the V value is taken from the alpha channel
127  SkYUVAIndex{-1, SkColorChannel::kA}}; //-1 means to omit the alpha data of YUVA
128  SkISize size{yBackendTexture.width(), yBackendTexture.height()};
129  sk_sp<SkImage> image = SkImage::MakeFromYUVATextures(
130  context, kRec601_SkYUVColorSpace, nv12TextureHandles, yuvaIndices, size,
131  kTopLeft_GrSurfaceOrigin, /*imageColorSpace=*/nullptr);
132  return image;
133 }
134 
135 bool IOSExternalTextureGL::IsTexturesAvailable() const {
136  return ((pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
137  pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) &&
138  (y_texture_ref_ && uv_texture_ref_)) ||
139  (pixel_format_ == kCVPixelFormatType_32BGRA && texture_ref_);
140 }
141 
142 bool IOSExternalTextureGL::NeedUpdateTexture(bool freeze) {
143  // Update texture if `texture_ref_` is reset to `nullptr` when GrContext
144  // is destroyed or new frame is ready.
145  return (!freeze && new_frame_ready_) || !IsTexturesAvailable();
146 }
147 
148 void IOSExternalTextureGL::Paint(SkCanvas& canvas,
149  const SkRect& bounds,
150  bool freeze,
151  GrDirectContext* context,
152  SkFilterQuality filter_quality) {
153  EnsureTextureCacheExists();
154  if (NeedUpdateTexture(freeze)) {
155  auto pixelBuffer = [external_texture_.get() copyPixelBuffer];
156  if (pixelBuffer) {
157  buffer_ref_.Reset(pixelBuffer);
158  pixel_format_ = CVPixelBufferGetPixelFormatType(buffer_ref_);
159  }
160  CreateTextureFromPixelBuffer();
161  new_frame_ready_ = false;
162  }
163  if (!IsTexturesAvailable()) {
164  return;
165  }
166 
167  sk_sp<SkImage> image = nullptr;
168  if (pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
169  pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
170  image = CreateImageFromYUVTextures(context, bounds);
171  } else {
172  image = CreateImageFromRGBATexture(context, bounds);
173  }
174 
175  FML_DCHECK(image) << "Failed to create SkImage from Texture.";
176  if (image) {
177  SkPaint paint;
178  paint.setFilterQuality(filter_quality);
179  canvas.drawImage(image, bounds.x(), bounds.y(), &paint);
180  }
181 }
182 
183 void IOSExternalTextureGL::OnGrContextCreated() {
184  // Re-create texture from pixel buffer that was saved before
185  // OnGrContextDestroyed gets called.
186  // https://github.com/flutter/flutter/issues/30491
187  EnsureTextureCacheExists();
188  CreateTextureFromPixelBuffer();
189 }
190 
191 void IOSExternalTextureGL::OnGrContextDestroyed() {
192  texture_ref_.Reset(nullptr);
193  cache_ref_.Reset(nullptr);
194 }
195 
196 void IOSExternalTextureGL::MarkNewFrameAvailable() {
197  new_frame_ready_ = true;
198 }
199 
200 void IOSExternalTextureGL::OnTextureUnregistered() {
201  if ([external_texture_ respondsToSelector:@selector(onTextureUnregistered:)]) {
202  [external_texture_ onTextureUnregistered:external_texture_];
203  }
204 }
205 
206 } // namespace flutter
#define FML_DCHECK(condition)
Definition: logging.h:86
constexpr std::size_t size(T(&array)[N])
Definition: size.h:13
#define FML_LOG(severity)
Definition: logging.h:65
void Reset(T instance=nullptr)
Definition: cf_utils.h:43
Definition: ascii_trie.cc:9
int32_t height
int32_t width
IOSExternalTextureGL(int64_t textureId, NSObject< FlutterTexture > *externalTexture)