Flutter Engine
ios_external_texture_metal.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #import "flutter/shell/platform/darwin/ios/ios_external_texture_metal.h"
6 
7 #include "flutter/fml/logging.h"
8 #include "third_party/skia/include/core/SkYUVAIndex.h"
9 #include "third_party/skia/include/gpu/GrBackendSurface.h"
10 #include "third_party/skia/include/gpu/GrDirectContext.h"
11 #include "third_party/skia/include/gpu/mtl/GrMtlTypes.h"
12 
13 namespace flutter {
14 
16  int64_t texture_id,
18  fml::scoped_nsobject<NSObject<FlutterTexture>> external_texture)
19  : Texture(texture_id),
20  texture_cache_(std::move(texture_cache)),
21  external_texture_(std::move(external_texture)) {
22  FML_DCHECK(texture_cache_);
23  FML_DCHECK(external_texture_);
24 }
25 
27 
28 void IOSExternalTextureMetal::Paint(SkCanvas& canvas,
29  const SkRect& bounds,
30  bool freeze,
31  GrDirectContext* context,
32  SkFilterQuality filter_quality) {
33  const bool needs_updated_texture = (!freeze && texture_frame_available_) || !external_image_;
34 
35  if (needs_updated_texture) {
36  auto pixel_buffer = fml::CFRef<CVPixelBufferRef>([external_texture_ copyPixelBuffer]);
37  if (!pixel_buffer) {
38  pixel_buffer = std::move(last_pixel_buffer_);
39  } else {
40  pixel_format_ = CVPixelBufferGetPixelFormatType(pixel_buffer);
41  }
42 
43  // If the application told us there was a texture frame available but did not provide one when
44  // asked for it, reuse the previous texture but make sure to ask again the next time around.
45  if (auto wrapped_texture = WrapExternalPixelBuffer(pixel_buffer, context)) {
46  external_image_ = wrapped_texture;
47  texture_frame_available_ = false;
48  last_pixel_buffer_ = std::move(pixel_buffer);
49  }
50  }
51 
52  if (external_image_) {
53  SkPaint paint;
54  paint.setFilterQuality(filter_quality);
55  canvas.drawImageRect(external_image_, // image
56  external_image_->bounds(), // source rect
57  bounds, // destination rect
58  &paint, // paint
59  SkCanvas::SrcRectConstraint::kFast_SrcRectConstraint // constraint
60  );
61  }
62 }
63 
64 sk_sp<SkImage> IOSExternalTextureMetal::WrapExternalPixelBuffer(
65  fml::CFRef<CVPixelBufferRef> pixel_buffer,
66  GrDirectContext* context) const {
67  if (!pixel_buffer) {
68  return nullptr;
69  }
70 
71  sk_sp<SkImage> image = nullptr;
72  if (pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
73  pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
74  image = WrapNV12ExternalPixelBuffer(pixel_buffer, context);
75  } else {
76  image = WrapRGBAExternalPixelBuffer(pixel_buffer, context);
77  }
78 
79  if (!image) {
80  FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image.";
81  }
82 
83  return image;
84 }
85 
86 sk_sp<SkImage> IOSExternalTextureMetal::WrapNV12ExternalPixelBuffer(
87  fml::CFRef<CVPixelBufferRef> pixel_buffer,
88  GrDirectContext* context) const {
89  auto texture_size =
90  SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer));
91  CVMetalTextureRef y_metal_texture_raw = nullptr;
92  {
93  auto cv_return =
94  CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
95  /*textureCache=*/texture_cache_,
96  /*sourceImage=*/pixel_buffer,
97  /*textureAttributes=*/nullptr,
98  /*pixelFormat=*/MTLPixelFormatR8Unorm,
99  /*width=*/texture_size.width(),
100  /*height=*/texture_size.height(),
101  /*planeIndex=*/0u,
102  /*texture=*/&y_metal_texture_raw);
103 
104  if (cv_return != kCVReturnSuccess) {
105  FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
106  return nullptr;
107  }
108  }
109 
110  CVMetalTextureRef uv_metal_texture_raw = nullptr;
111  {
112  auto cv_return =
113  CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
114  /*textureCache=*/texture_cache_,
115  /*sourceImage=*/pixel_buffer,
116  /*textureAttributes=*/nullptr,
117  /*pixelFormat=*/MTLPixelFormatRG8Unorm,
118  /*width=*/texture_size.width() / 2,
119  /*height=*/texture_size.height() / 2,
120  /*planeIndex=*/1u,
121  /*texture=*/&uv_metal_texture_raw);
122 
123  if (cv_return != kCVReturnSuccess) {
124  FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
125  return nullptr;
126  }
127  }
128 
129  fml::CFRef<CVMetalTextureRef> y_metal_texture(y_metal_texture_raw);
130 
131  GrMtlTextureInfo y_skia_texture_info;
132  y_skia_texture_info.fTexture = sk_cf_obj<const void*>{
133  [reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(y_metal_texture)) retain]};
134 
135  GrBackendTexture y_skia_backend_texture(/*width=*/texture_size.width(),
136  /*height=*/texture_size.height(),
137  /*mipMapped=*/GrMipMapped ::kNo,
138  /*textureInfo=*/y_skia_texture_info);
139 
140  fml::CFRef<CVMetalTextureRef> uv_metal_texture(uv_metal_texture_raw);
141 
142  GrMtlTextureInfo uv_skia_texture_info;
143  uv_skia_texture_info.fTexture = sk_cf_obj<const void*>{
144  [reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(uv_metal_texture)) retain]};
145 
146  GrBackendTexture uv_skia_backend_texture(/*width=*/texture_size.width(),
147  /*height=*/texture_size.height(),
148  /*mipMapped=*/GrMipMapped ::kNo,
149  /*textureInfo=*/uv_skia_texture_info);
150  GrBackendTexture nv12TextureHandles[] = {y_skia_backend_texture, uv_skia_backend_texture};
151  SkYUVAIndex yuvaIndices[4] = {
152  SkYUVAIndex{0, SkColorChannel::kR}, // Read Y data from the red channel of the first texture
153  SkYUVAIndex{1, SkColorChannel::kR}, // Read U data from the red channel of the second texture
154  SkYUVAIndex{1,
155  SkColorChannel::kG}, // Read V data from the green channel of the second texture
156  SkYUVAIndex{-1, SkColorChannel::kA}}; //-1 means to omit the alpha data of YUVA
157 
158  struct ImageCaptures {
162  };
163 
164  auto captures = std::make_unique<ImageCaptures>();
165  captures->buffer = std::move(pixel_buffer);
166  captures->y_texture = std::move(y_metal_texture);
167  captures->uv_texture = std::move(uv_metal_texture);
168 
169  SkImage::TextureReleaseProc release_proc = [](SkImage::ReleaseContext release_context) {
170  auto captures = reinterpret_cast<ImageCaptures*>(release_context);
171  delete captures;
172  };
173  sk_sp<SkImage> image = SkImage::MakeFromYUVATextures(
174  context, kRec601_SkYUVColorSpace, nv12TextureHandles, yuvaIndices, texture_size,
175  kTopLeft_GrSurfaceOrigin, /*imageColorSpace=*/nullptr, release_proc, captures.release());
176  return image;
177 }
178 
179 sk_sp<SkImage> IOSExternalTextureMetal::WrapRGBAExternalPixelBuffer(
180  fml::CFRef<CVPixelBufferRef> pixel_buffer,
181  GrDirectContext* context) const {
182  auto texture_size =
183  SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer));
184  CVMetalTextureRef metal_texture_raw = nullptr;
185  auto cv_return =
186  CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
187  /*textureCache=*/texture_cache_,
188  /*sourceImage=*/pixel_buffer,
189  /*textureAttributes=*/nullptr,
190  /*pixelFormat=*/MTLPixelFormatBGRA8Unorm,
191  /*width=*/texture_size.width(),
192  /*height=*/texture_size.height(),
193  /*planeIndex=*/0u,
194  /*texture=*/&metal_texture_raw);
195 
196  if (cv_return != kCVReturnSuccess) {
197  FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
198  return nullptr;
199  }
200 
201  fml::CFRef<CVMetalTextureRef> metal_texture(metal_texture_raw);
202 
203  GrMtlTextureInfo skia_texture_info;
204  skia_texture_info.fTexture = sk_cf_obj<const void*>{
205  [reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(metal_texture)) retain]};
206 
207  GrBackendTexture skia_backend_texture(/*width=*/texture_size.width(),
208  /*height=*/texture_size.height(),
209  /*mipMapped=*/GrMipMapped ::kNo,
210  /*textureInfo=*/skia_texture_info);
211 
212  struct ImageCaptures {
215  };
216 
217  auto captures = std::make_unique<ImageCaptures>();
218  captures->buffer = std::move(pixel_buffer);
219  captures->texture = std::move(metal_texture);
220 
221  SkImage::TextureReleaseProc release_proc = [](SkImage::ReleaseContext release_context) {
222  auto captures = reinterpret_cast<ImageCaptures*>(release_context);
223  delete captures;
224  };
225 
226  auto image =
227  SkImage::MakeFromTexture(context, skia_backend_texture, kTopLeft_GrSurfaceOrigin,
228  kBGRA_8888_SkColorType, kPremul_SkAlphaType,
229  /*imageColorSpace=*/nullptr, release_proc, captures.release()
230 
231  );
232  return image;
233 }
234 
235 void IOSExternalTextureMetal::OnGrContextCreated() {
236  // External images in this backend have no thread affinity and are not tied to the context in any
237  // way. Instead, they are tied to the Metal device which is associated with the cache already and
238  // is consistent throughout the shell run.
239 }
240 
241 void IOSExternalTextureMetal::OnGrContextDestroyed() {
242  // The image must be reset because it is tied to the onscreen context. But the pixel buffer that
243  // created the image is still around. In case of context reacquisition, that last pixel
244  // buffer will be used to materialize the image in case the application fails to provide a new
245  // one.
246  external_image_.reset();
247  CVMetalTextureCacheFlush(texture_cache_, // cache
248  0 // options (must be zero)
249  );
250 }
251 
252 void IOSExternalTextureMetal::MarkNewFrameAvailable() {
253  texture_frame_available_ = true;
254 }
255 
256 void IOSExternalTextureMetal::OnTextureUnregistered() {
257  if ([external_texture_ respondsToSelector:@selector(onTextureUnregistered:)]) {
258  [external_texture_ onTextureUnregistered:external_texture_];
259  }
260 }
261 
262 } // namespace flutter
#define FML_DCHECK(condition)
Definition: logging.h:86
Definition: ref_ptr.h:252
IOSExternalTextureMetal(int64_t texture_id, fml::CFRef< CVMetalTextureCacheRef > texture_cache, fml::scoped_nsobject< NSObject< FlutterTexture >> external_texture)
#define FML_DLOG(severity)
Definition: logging.h:85