Flutter Engine
image_decoder.cc
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "flutter/lib/ui/painting/image_decoder.h"
6 
7 #include <algorithm>
8 
9 #include "flutter/fml/make_copyable.h"
10 #include "third_party/skia/include/codec/SkCodec.h"
11 
12 namespace flutter {
13 
15  TaskRunners runners,
16  std::shared_ptr<fml::ConcurrentTaskRunner> concurrent_task_runner,
17  fml::WeakPtr<IOManager> io_manager)
18  : runners_(std::move(runners)),
19  concurrent_task_runner_(std::move(concurrent_task_runner)),
20  io_manager_(std::move(io_manager)),
21  weak_factory_(this) {
22  FML_DCHECK(runners_.IsValid());
24  << "The image decoder must be created & collected on the UI thread.";
25 }
26 
27 ImageDecoder::~ImageDecoder() = default;
28 
29 static sk_sp<SkImage> ResizeRasterImage(sk_sp<SkImage> image,
30  const SkISize& resized_dimensions,
31  const fml::tracing::TraceFlow& flow) {
32  FML_DCHECK(!image->isTextureBacked());
33 
34  TRACE_EVENT0("flutter", __FUNCTION__);
35  flow.Step(__FUNCTION__);
36 
37  if (resized_dimensions.isEmpty()) {
38  FML_LOG(ERROR) << "Could not resize to empty dimensions.";
39  return nullptr;
40  }
41 
42  if (image->dimensions() == resized_dimensions) {
43  return image->makeRasterImage();
44  }
45 
46  const auto scaled_image_info =
47  image->imageInfo().makeDimensions(resized_dimensions);
48 
49  SkBitmap scaled_bitmap;
50  if (!scaled_bitmap.tryAllocPixels(scaled_image_info)) {
51  FML_LOG(ERROR) << "Failed to allocate memory for bitmap of size "
52  << scaled_image_info.computeMinByteSize() << "B";
53  return nullptr;
54  }
55 
56  if (!image->scalePixels(
57  scaled_bitmap.pixmap(),
58  SkSamplingOptions(SkFilterMode::kLinear, SkMipmapMode::kNone),
59  SkImage::kDisallow_CachingHint)) {
60  FML_LOG(ERROR) << "Could not scale pixels";
61  return nullptr;
62  }
63 
64  // Marking this as immutable makes the MakeFromBitmap call share the pixels
65  // instead of copying.
66  scaled_bitmap.setImmutable();
67 
68  auto scaled_image = SkImage::MakeFromBitmap(scaled_bitmap);
69  if (!scaled_image) {
70  FML_LOG(ERROR) << "Could not create a scaled image from a scaled bitmap.";
71  return nullptr;
72  }
73 
74  return scaled_image;
75 }
76 
77 static sk_sp<SkImage> ImageFromDecompressedData(
78  ImageDescriptor* descriptor,
79  uint32_t target_width,
80  uint32_t target_height,
81  const fml::tracing::TraceFlow& flow) {
82  TRACE_EVENT0("flutter", __FUNCTION__);
83  flow.Step(__FUNCTION__);
84  auto image = SkImage::MakeRasterData(
85  descriptor->image_info(), descriptor->data(), descriptor->row_bytes());
86 
87  if (!image) {
88  FML_LOG(ERROR) << "Could not create image from decompressed bytes.";
89  return nullptr;
90  }
91 
92  if (!target_width && !target_height) {
93  // No resizing requested. Just rasterize the image.
94  return image->makeRasterImage();
95  }
96 
97  return ResizeRasterImage(std::move(image),
98  SkISize::Make(target_width, target_height), flow);
99 }
100 
101 sk_sp<SkImage> ImageFromCompressedData(ImageDescriptor* descriptor,
102  uint32_t target_width,
103  uint32_t target_height,
104  const fml::tracing::TraceFlow& flow) {
105  TRACE_EVENT0("flutter", __FUNCTION__);
106  flow.Step(__FUNCTION__);
107 
108  if (!descriptor->should_resize(target_width, target_height)) {
109  // No resizing requested. Just decode & rasterize the image.
110  sk_sp<SkImage> image = descriptor->image();
111  return image ? image->makeRasterImage() : nullptr;
112  }
113 
114  const SkISize source_dimensions = descriptor->image_info().dimensions();
115  const SkISize resized_dimensions = {static_cast<int32_t>(target_width),
116  static_cast<int32_t>(target_height)};
117 
118  auto decode_dimensions = descriptor->get_scaled_dimensions(
119  std::max(static_cast<double>(resized_dimensions.width()) /
120  source_dimensions.width(),
121  static_cast<double>(resized_dimensions.height()) /
122  source_dimensions.height()));
123 
124  // If the codec supports efficient sub-pixel decoding, decoded at a resolution
125  // close to the target resolution before resizing.
126  if (decode_dimensions != source_dimensions) {
127  auto scaled_image_info =
128  descriptor->image_info().makeDimensions(decode_dimensions);
129 
130  SkBitmap scaled_bitmap;
131  if (!scaled_bitmap.tryAllocPixels(scaled_image_info)) {
132  FML_LOG(ERROR) << "Failed to allocate memory for bitmap of size "
133  << scaled_image_info.computeMinByteSize() << "B";
134  return nullptr;
135  }
136 
137  const auto& pixmap = scaled_bitmap.pixmap();
138  if (descriptor->get_pixels(pixmap)) {
139  // Marking this as immutable makes the MakeFromBitmap call share
140  // the pixels instead of copying.
141  scaled_bitmap.setImmutable();
142 
143  auto decoded_image = SkImage::MakeFromBitmap(scaled_bitmap);
144  FML_DCHECK(decoded_image);
145  if (!decoded_image) {
146  FML_LOG(ERROR)
147  << "Could not create a scaled image from a scaled bitmap.";
148  return nullptr;
149  }
150  return ResizeRasterImage(std::move(decoded_image), resized_dimensions,
151  flow);
152  }
153  }
154 
155  auto image = descriptor->image();
156  if (!image) {
157  return nullptr;
158  }
159 
160  return ResizeRasterImage(std::move(image), resized_dimensions, flow);
161 }
162 
164  sk_sp<SkImage> image,
165  fml::WeakPtr<IOManager> io_manager,
166  const fml::tracing::TraceFlow& flow) {
167  TRACE_EVENT0("flutter", __FUNCTION__);
168  flow.Step(__FUNCTION__);
169 
170  // Should not already be a texture image because that is the entire point of
171  // the this method.
172  FML_DCHECK(!image->isTextureBacked());
173 
174  if (!io_manager->GetResourceContext() || !io_manager->GetSkiaUnrefQueue()) {
175  FML_LOG(ERROR)
176  << "Could not acquire context of release queue for texture upload.";
177  return {};
178  }
179 
180  SkPixmap pixmap;
181  if (!image->peekPixels(&pixmap)) {
182  FML_LOG(ERROR) << "Could not peek pixels of image for texture upload.";
183  return {};
184  }
185 
187  io_manager->GetIsGpuDisabledSyncSwitch()->Execute(
189  .SetIfTrue([&result, &pixmap, &image] {
190  SkSafeRef(image.get());
191  sk_sp<SkImage> texture_image = SkImage::MakeFromRaster(
192  pixmap,
193  [](const void* pixels, SkImage::ReleaseContext context) {
194  SkSafeUnref(static_cast<SkImage*>(context));
195  },
196  image.get());
197  result = {std::move(texture_image), nullptr};
198  })
199  .SetIfFalse([&result, context = io_manager->GetResourceContext(),
200  &pixmap, queue = io_manager->GetSkiaUnrefQueue()] {
201  TRACE_EVENT0("flutter", "MakeCrossContextImageFromPixmap");
202  sk_sp<SkImage> texture_image = SkImage::MakeCrossContextFromPixmap(
203  context.get(), // context
204  pixmap, // pixmap
205  true, // buildMips,
206  true // limitToMaxTextureSize
207  );
208  if (!texture_image) {
209  FML_LOG(ERROR) << "Could not make x-context image.";
210  result = {};
211  } else {
212  result = {std::move(texture_image), queue};
213  }
214  }));
215 
216  return result;
217 }
218 
220  uint32_t target_width,
221  uint32_t target_height,
222  const ImageResult& callback) {
223  TRACE_EVENT0("flutter", __FUNCTION__);
224  fml::tracing::TraceFlow flow(__FUNCTION__);
225 
226  // ImageDescriptors have Dart peers that must be collected on the UI thread.
227  // However, closures in MakeCopyable below capture the descriptor. The
228  // captures of copyable closures may be collected on any of the thread
229  // participating in task execution.
230  //
231  // To avoid this issue, we resort to manually reference counting the
232  // descriptor. Since all task flows invoke the `result` callback, the raw
233  // descriptor is retained in the beginning and released in the `result`
234  // callback.
235  //
236  // `ImageDecoder::Decode` itself is invoked on the UI thread, so the
237  // collection of the smart pointer from which we obtained the raw descriptor
238  // is fine in this scope.
239  auto raw_descriptor = descriptor_ref_ptr.get();
240  raw_descriptor->AddRef();
241 
242  FML_DCHECK(callback);
244 
245  // Always service the callback (and cleanup the descriptor) on the UI thread.
246  auto result =
247  [callback, raw_descriptor, ui_runner = runners_.GetUITaskRunner()](
249  ui_runner->PostTask(fml::MakeCopyable(
250  [callback, raw_descriptor, image = std::move(image),
251  flow = std::move(flow)]() mutable {
252  // We are going to terminate the trace flow here. Flows cannot
253  // terminate without a base trace. Add one explicitly.
254  TRACE_EVENT0("flutter", "ImageDecodeCallback");
255  flow.End();
256  callback(std::move(image));
257  raw_descriptor->Release();
258  }));
259  };
260 
261  if (!raw_descriptor->data() || raw_descriptor->data()->size() == 0) {
262  result({}, std::move(flow));
263  return;
264  }
265 
266  concurrent_task_runner_->PostTask(
267  fml::MakeCopyable([raw_descriptor, //
268  io_manager = io_manager_, //
269  io_runner = runners_.GetIOTaskRunner(), //
270  result, //
271  target_width = target_width, //
272  target_height = target_height, //
273  flow = std::move(flow) //
274  ]() mutable {
275  // Step 1: Decompress the image.
276  // On Worker.
277 
278  auto decompressed = raw_descriptor->is_compressed()
279  ? ImageFromCompressedData(raw_descriptor, //
280  target_width, //
281  target_height, //
282  flow)
283  : ImageFromDecompressedData(raw_descriptor, //
284  target_width, //
285  target_height, //
286  flow);
287 
288  if (!decompressed) {
289  FML_DLOG(ERROR) << "Could not decompress image.";
290  result({}, std::move(flow));
291  return;
292  }
293 
294  // Step 2: Update the image to the GPU.
295  // On IO Thread.
296 
297  io_runner->PostTask(fml::MakeCopyable([io_manager, decompressed, result,
298  flow =
299  std::move(flow)]() mutable {
300  if (!io_manager) {
301  FML_DLOG(ERROR) << "Could not acquire IO manager.";
302  result({}, std::move(flow));
303  return;
304  }
305 
306  // If the IO manager does not have a resource context, the caller
307  // might not have set one or a software backend could be in use.
308  // Either way, just return the image as-is.
309  if (!io_manager->GetResourceContext()) {
310  result({std::move(decompressed), io_manager->GetSkiaUnrefQueue()},
311  std::move(flow));
312  return;
313  }
314 
315  auto uploaded =
316  UploadRasterImage(std::move(decompressed), io_manager, flow);
317 
318  if (!uploaded.skia_object()) {
319  FML_DLOG(ERROR) << "Could not upload image to the GPU.";
320  result({}, std::move(flow));
321  return;
322  }
323 
324  // Finally, all done.
325  result(std::move(uploaded), std::move(flow));
326  }));
327  }));
328 }
329 
331  return weak_factory_.GetWeakPtr();
332 }
333 
334 } // namespace flutter
fml::WeakPtr< IOManager > io_manager_
static SkiaGPUObject< SkImage > UploadRasterImage(sk_sp< SkImage > image, fml::WeakPtr< IOManager > io_manager, const fml::tracing::TraceFlow &flow)
bool IsValid() const
Definition: task_runners.cc:46
#define TRACE_EVENT0(category_group, name)
Definition: trace_event.h:90
bool should_resize(int target_width, int target_height) const
Whether the given target_width or target_height differ from width() and height() respectively.
sk_sp< SkImage > image() const
#define FML_DCHECK(condition)
Definition: logging.h:86
Definition: ref_ptr.h:255
GAsyncResult * result
#define FML_LOG(severity)
Definition: logging.h:65
static sk_sp< SkImage > ImageFromDecompressedData(ImageDescriptor *descriptor, uint32_t target_width, uint32_t target_height, const fml::tracing::TraceFlow &flow)
SkISize get_scaled_dimensions(float scale)
Gets the scaled dimensions of this image, if backed by an ImageGenerator that can perform efficient s...
void Decode(fml::RefPtr< ImageDescriptor > descriptor, uint32_t target_width, uint32_t target_height, const ImageResult &result)
FlKeyEvent FlKeyResponderAsyncCallback callback
std::function< void(SkiaGPUObject< SkImage >)> ImageResult
Definition: image_decoder.h:41
Creates an image descriptor for encoded or decoded image data, describing the width, height, and bytes per pixel for that image. This class will hold a reference on the underlying image data, and in the case of compressed data, an ImageGenerator for the data. The Codec initialization actually happens in initEncoded, making initstantiateCodec a lightweight operation.
internal::CopyableLambda< T > MakeCopyable(T lambda)
Definition: make_copyable.h:57
void End(const char *label=nullptr)
Definition: trace_event.h:377
void Step(const char *label=nullptr) const
Definition: trace_event.h:373
int row_bytes() const
The byte length of the first row of the image. Defaults to width() * 4.
sk_sp< SkImage > ImageFromCompressedData(ImageDescriptor *descriptor, uint32_t target_width, uint32_t target_height, const fml::tracing::TraceFlow &flow)
static sk_sp< SkImage > ResizeRasterImage(sk_sp< SkImage > image, const SkISize &resized_dimensions, const fml::tracing::TraceFlow &flow)
T * get() const
Definition: ref_ptr.h:115
ImageDecoder(TaskRunners runners, std::shared_ptr< fml::ConcurrentTaskRunner > concurrent_task_runner, fml::WeakPtr< IOManager > io_manager)
sk_sp< SkData > data() const
The underlying buffer for this image.
fml::RefPtr< fml::TaskRunner > GetIOTaskRunner() const
Definition: task_runners.cc:38
fml::RefPtr< fml::TaskRunner > GetUITaskRunner() const
Definition: task_runners.cc:34
const SkImageInfo & image_info() const
The orientation corrected image info for this image.
virtual bool RunsTasksOnCurrentThread()
Definition: task_runner.cc:43
#define FML_DLOG(severity)
Definition: logging.h:85
fml::WeakPtr< ImageDecoder > GetWeakPtr() const
bool get_pixels(const SkPixmap &pixmap) const
Gets pixels for this image transformed based on the EXIF orientation tag, if applicable.
Represents the 2 code paths available when calling |SyncSwitch::Execute|.
Definition: sync_switch.h:24