Flutter Engine
rasterizer.h
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #ifndef SHELL_COMMON_RASTERIZER_H_
6 #define SHELL_COMMON_RASTERIZER_H_
7 
8 #include <memory>
9 #include <optional>
10 
11 #include "flutter/common/settings.h"
12 #include "flutter/common/task_runners.h"
13 #include "flutter/flow/compositor_context.h"
14 #include "flutter/flow/layers/layer_tree.h"
15 #include "flutter/flow/surface.h"
16 #include "flutter/fml/closure.h"
17 #include "flutter/fml/memory/weak_ptr.h"
18 #include "flutter/fml/raster_thread_merger.h"
19 #include "flutter/fml/synchronization/sync_switch.h"
20 #include "flutter/fml/synchronization/waitable_event.h"
21 #include "flutter/fml/time/time_delta.h"
22 #include "flutter/fml/time/time_point.h"
23 #include "flutter/lib/ui/snapshot_delegate.h"
24 #include "flutter/shell/common/pipeline.h"
25 
26 namespace flutter {
27 
28 //------------------------------------------------------------------------------
29 /// The rasterizer is a component owned by the shell that resides on the GPU
30 /// task runner. Each shell owns exactly one instance of a rasterizer. The
31 /// rasterizer may only be created, used and collected on the GPU task runner.
32 ///
33 /// The rasterizer owns the instance of the currently active on-screen render
34 /// surface. On this surface, it renders the contents of layer trees submitted
35 /// to it by the `Engine` (which lives on the UI task runner).
36 ///
37 /// The primary components owned by the rasterizer are the compositor context
38 /// and the on-screen render surface. The compositor context has all the GPU
39 /// state necessary to render frames to the render surface.
40 ///
41 class Rasterizer final : public SnapshotDelegate {
42  public:
43  //----------------------------------------------------------------------------
44  /// @brief Used to forward events from the rasterizer to interested
45  /// subsystems. Currently, the shell sets itself up as the
46  /// rasterizer delegate to listen for frame rasterization events.
47  /// It can then forward these events to the engine.
48  ///
49  /// Like all rasterizer operation, the rasterizer delegate call
50  /// are made on the GPU task runner. Any delegate must ensure that
51  /// they can handle the threading implications.
52  ///
53  class Delegate {
54  public:
55  //--------------------------------------------------------------------------
56  /// @brief Notifies the delegate that a frame has been rendered. The
57  /// rasterizer collects profiling information for each part of
58  /// the frame workload. This profiling information is made
59  /// available to the delegate for forwarding to subsystems
60  /// interested in collecting such profiles. Currently, the shell
61  /// (the delegate) forwards this to the engine where Dart code
62  /// can react to this information.
63  ///
64  /// @see `FrameTiming`
65  ///
66  /// @param[in] frame_timing Instrumentation information for each phase of
67  /// the frame workload.
68  ///
69  virtual void OnFrameRasterized(const FrameTiming& frame_timing) = 0;
70 
71  /// Time limit for a smooth frame.
72  ///
73  /// See: `DisplayManager::GetMainDisplayRefreshRate`.
74  virtual fml::Milliseconds GetFrameBudget() = 0;
75 
76  /// Target time for the latest frame. See also `Shell::OnAnimatorBeginFrame`
77  /// for when this time gets updated.
78  virtual fml::TimePoint GetLatestFrameTargetTime() const = 0;
79 
80  /// Task runners used by the shell.
81  virtual const TaskRunners& GetTaskRunners() const = 0;
82 
83  /// Accessor for the shell's GPU sync switch, which determines whether GPU
84  /// operations are allowed on the current thread.
85  ///
86  /// For example, on some platforms when the application is backgrounded it
87  /// is critical that GPU operations are not processed.
88  virtual std::shared_ptr<fml::SyncSwitch> GetIsGpuDisabledSyncSwitch()
89  const = 0;
90  };
91 
92  //----------------------------------------------------------------------------
93  /// @brief Creates a new instance of a rasterizer. Rasterizers may only
94  /// be created on the GPU task runner. Rasterizers are currently
95  /// only created by the shell (which also sets itself up as the
96  /// rasterizer delegate).
97  ///
98  /// @param[in] delegate The rasterizer delegate.
99  ///
100  Rasterizer(Delegate& delegate);
101 
102 #if defined(LEGACY_FUCHSIA_EMBEDDER)
103  //----------------------------------------------------------------------------
104  /// @brief Creates a new instance of a rasterizer. Rasterizers may only
105  /// be created on the GPU task runner. Rasterizers are currently
106  /// only created by the shell (which also sets itself up as the
107  /// rasterizer delegate).
108  ///
109  /// @param[in] delegate The rasterizer delegate.
110  /// @param[in] compositor_context The compositor context used to hold all
111  /// the GPU state used by the rasterizer.
112  ///
113  Rasterizer(Delegate& delegate,
114  std::unique_ptr<flutter::CompositorContext> compositor_context);
115 #endif
116 
117  //----------------------------------------------------------------------------
118  /// @brief Destroys the rasterizer. This must happen on the GPU task
119  /// runner. All GPU resources are collected before this call
120  /// returns. Any context setup by the embedder to hold these
121  /// resources can be immediately collected as well.
122  ///
123  ~Rasterizer();
124 
125  //----------------------------------------------------------------------------
126  /// @brief Rasterizers may be created well before an on-screen surface is
127  /// available for rendering. Shells usually create a rasterizer in
128  /// their constructors. Once an on-screen surface is available
129  /// however, one may be provided to the rasterizer using this
130  /// call. No rendering may occur before this call. The surface is
131  /// held till the balancing call to `Rasterizer::Teardown` is
132  /// made. Calling a setup before tearing down the previous surface
133  /// (if this is not the first time the surface has been setup) is
134  /// user error.
135  ///
136  /// @see `Rasterizer::Teardown`
137  ///
138  /// @param[in] surface The on-screen render surface.
139  ///
140  void Setup(std::unique_ptr<Surface> surface);
141 
142  //----------------------------------------------------------------------------
143  /// @brief Releases the previously setup on-screen render surface and
144  /// collects associated resources. No more rendering may occur
145  /// till the next call to `Rasterizer::Setup` with a new render
146  /// surface. Calling a teardown without a setup is user error.
147  ///
148  void Teardown();
149 
150  //----------------------------------------------------------------------------
151  /// @brief Notifies the rasterizer that there is a low memory situation
152  /// and it must purge as many unnecessary resources as possible.
153  /// Currently, the Skia context associated with onscreen rendering
154  /// is told to free GPU resources.
155  ///
156  void NotifyLowMemoryWarning() const;
157 
158  //----------------------------------------------------------------------------
159  /// @brief Gets a weak pointer to the rasterizer. The rasterizer may only
160  /// be accessed on the GPU task runner.
161  ///
162  /// @return The weak pointer to the rasterizer.
163  ///
165 
167 
168  //----------------------------------------------------------------------------
169  /// @brief Sometimes, it may be necessary to render the same frame again
170  /// without having to wait for the framework to build a whole new
171  /// layer tree describing the same contents. One such case is when
172  /// external textures (video or camera streams for example) are
173  /// updated in an otherwise static layer tree. To support this use
174  /// case, the rasterizer holds onto the last rendered layer tree.
175  ///
176  /// @bug https://github.com/flutter/flutter/issues/33939
177  ///
178  /// @return A pointer to the last layer or `nullptr` if this rasterizer
179  /// has never rendered a frame.
180  ///
182 
183  //----------------------------------------------------------------------------
184  /// @brief Draws a last layer tree to the render surface. This may seem
185  /// entirely redundant at first glance. After all, on surface loss
186  /// and re-acquisition, the framework generates a new layer tree.
187  /// Otherwise, why render the same contents to the screen again?
188  /// This is used as an optimization in cases where there are
189  /// external textures (video or camera streams for example) in
190  /// referenced in the layer tree. These textures may be updated at
191  /// a cadence different from that of the Flutter application.
192  /// Flutter can re-render the layer tree with just the updated
193  /// textures instead of waiting for the framework to do the work
194  /// to generate the layer tree describing the same contents.
195  ///
196  void DrawLastLayerTree();
197 
198  //----------------------------------------------------------------------------
199  /// @brief Gets the registry of external textures currently in use by the
200  /// rasterizer. These textures may be updated at a cadence
201  /// different from that of the Flutter application. When an
202  /// external texture is referenced in the Flutter layer tree, that
203  /// texture is composited within the Flutter layer tree.
204  ///
205  /// @return A pointer to the external texture registry.
206  ///
208 
209  using LayerTreeDiscardCallback = std::function<bool(flutter::LayerTree&)>;
210 
211  //----------------------------------------------------------------------------
212  /// @brief Takes the next item from the layer tree pipeline and executes
213  /// the raster thread frame workload for that pipeline item to
214  /// render a frame on the on-screen surface.
215  ///
216  /// Why does the draw call take a layer tree pipeline and not the
217  /// layer tree directly?
218  ///
219  /// The pipeline is the way book-keeping of frame workloads
220  /// distributed across the multiple threads is managed. The
221  /// rasterizer deals with the pipelines directly (instead of layer
222  /// trees which is what it actually renders) because the pipeline
223  /// consumer's workload must be accounted for within the pipeline
224  /// itself. If the rasterizer took the layer tree directly, it
225  /// would have to be taken out of the pipeline. That would signal
226  /// the end of the frame workload and the pipeline would be ready
227  /// for new frames. But the last frame has not been rendered by
228  /// the frame yet! On the other hand, the pipeline must own the
229  /// layer tree it renders because it keeps a reference to the last
230  /// layer tree around till a new frame is rendered. So a simple
231  /// reference wont work either. The `Rasterizer::DoDraw` method
232  /// actually performs the GPU operations within the layer tree
233  /// pipeline.
234  ///
235  /// @see `Rasterizer::DoDraw`
236  ///
237  /// @param[in] pipeline The layer tree pipeline to take the next layer tree
238  /// to render from.
239  /// @param[in] discardCallback if specified and returns true, the layer tree
240  /// is discarded instead of being rendered
241  ///
243  LayerTreeDiscardCallback discardCallback = NoDiscard);
244 
245  //----------------------------------------------------------------------------
246  /// @brief The type of the screenshot to obtain of the previously
247  /// rendered layer tree.
248  ///
249  enum class ScreenshotType {
250  //--------------------------------------------------------------------------
251  /// A format used to denote a Skia picture. A Skia picture is a serialized
252  /// representation of an `SkPicture` that can be used to introspect the
253  /// series of commands used to draw that picture.
254  ///
255  /// Skia pictures are typically stored as files with the .skp extension on
256  /// disk. These files may be viewed in an interactive debugger available at
257  /// https://debugger.skia.org/
258  ///
259  SkiaPicture,
260 
261  //--------------------------------------------------------------------------
262  /// A format used to denote uncompressed image data. This format
263  /// is 32 bits per pixel, 8 bits per component and
264  /// denoted by the `kN32_SkColorType ` Skia color type.
265  ///
266  UncompressedImage,
267 
268  //--------------------------------------------------------------------------
269  /// A format used to denote compressed image data. The PNG compressed
270  /// container is used.
271  ///
272  CompressedImage,
273  };
274 
275  //----------------------------------------------------------------------------
276  /// @brief A POD type used to return the screenshot data along with the
277  /// size of the frame.
278  ///
279  struct Screenshot {
280  //--------------------------------------------------------------------------
281  /// The data used to describe the screenshot. The data format depends on the
282  /// type of screenshot taken and any further encoding done to the same.
283  ///
284  /// @see `ScreenshotType`
285  ///
286  sk_sp<SkData> data;
287 
288  //--------------------------------------------------------------------------
289  /// The size of the screenshot in texels.
290  ///
291  SkISize frame_size = SkISize::MakeEmpty();
292 
293  //--------------------------------------------------------------------------
294  /// @brief Creates an empty screenshot
295  ///
296  Screenshot();
297 
298  //--------------------------------------------------------------------------
299  /// @brief Creates a screenshot with the specified data and size.
300  ///
301  /// @param[in] p_data The screenshot data
302  /// @param[in] p_size The screenshot size.
303  ///
304  Screenshot(sk_sp<SkData> p_data, SkISize p_size);
305 
306  //--------------------------------------------------------------------------
307  /// @brief The copy constructor for a screenshot.
308  ///
309  /// @param[in] other The screenshot to copy from.
310  ///
311  Screenshot(const Screenshot& other);
312 
313  //--------------------------------------------------------------------------
314  /// @brief Destroys the screenshot object and releases underlying data.
315  ///
316  ~Screenshot();
317  };
318 
319  //----------------------------------------------------------------------------
320  /// @brief Screenshots the last layer tree to one of the supported
321  /// screenshot types and optionally Base 64 encodes that data for
322  /// easier transmission and packaging (usually over the service
323  /// protocol for instrumentation tools running on the host).
324  ///
325  /// @param[in] type The type of the screenshot to gather.
326  /// @param[in] base64_encode Whether Base 64 encoding must be applied to the
327  /// data after a screenshot has been captured.
328  ///
329  /// @return A non-empty screenshot if one could be captured. A screenshot
330  /// capture may fail if there were no layer trees previously
331  /// rendered by this rasterizer, or, due to an unspecified
332  /// internal error. Internal error will be logged to the console.
333  ///
334  Screenshot ScreenshotLastLayerTree(ScreenshotType type, bool base64_encode);
335 
336  //----------------------------------------------------------------------------
337  /// @brief Sets a callback that will be executed when the next layer tree
338  /// in rendered to the on-screen surface. This is used by
339  /// embedders to listen for one time operations like listening for
340  /// when the first frame is rendered so that they may hide splash
341  /// screens.
342  ///
343  /// The callback is only executed once and dropped on the GPU
344  /// thread when executed (lambda captures must be able to deal
345  /// with the threading repercussions of this behavior).
346  ///
347  /// @param[in] callback The callback to execute when the next layer tree is
348  /// rendered on-screen.
349  ///
350  void SetNextFrameCallback(const fml::closure& callback);
351 
352  //----------------------------------------------------------------------------
353  /// @brief Returns a pointer to the compositor context used by this
354  /// rasterizer. This pointer will never be `nullptr`.
355  ///
356  /// @return The compositor context used by this rasterizer.
357  ///
359  return compositor_context_.get();
360  }
361 
362  //----------------------------------------------------------------------------
363  /// @brief Skia has no notion of time. To work around the performance
364  /// implications of this, it may cache GPU resources to reference
365  /// them from one frame to the next. Using this call, embedders
366  /// may set the maximum bytes cached by Skia in its caches
367  /// dedicated to on-screen rendering.
368  ///
369  /// @attention This cache setting will be invalidated when the surface is
370  /// torn down via `Rasterizer::Teardown`. This call must be made
371  /// again with new limits after surface re-acquisition.
372  ///
373  /// @attention This cache does not describe the entirety of GPU resources
374  /// that may be cached. The `RasterCache` also holds very large
375  /// GPU resources.
376  ///
377  /// @see `RasterCache`
378  ///
379  /// @param[in] max_bytes The maximum byte size of resource that may be
380  /// cached for GPU rendering.
381  /// @param[in] from_user Whether this request was from user code, e.g. via
382  /// the flutter/skia message channel, in which case
383  /// it should not be overridden by the platform.
384  ///
385  void SetResourceCacheMaxBytes(size_t max_bytes, bool from_user);
386 
387  //----------------------------------------------------------------------------
388  /// @brief The current value of Skia's resource cache size, if a surface
389  /// is present.
390  ///
391  /// @attention This cache does not describe the entirety of GPU resources
392  /// that may be cached. The `RasterCache` also holds very large
393  /// GPU resources.
394  ///
395  /// @see `RasterCache`
396  ///
397  /// @return The size of Skia's resource cache, if available.
398  ///
399  std::optional<size_t> GetResourceCacheMaxBytes() const;
400 
401  //----------------------------------------------------------------------------
402  /// @brief Enables the thread merger if the external view embedder
403  /// supports dynamic thread merging.
404  ///
405  /// @attention This method is thread-safe. When the thread merger is enabled,
406  /// the raster task queue can run in the platform thread at any
407  /// time.
408  ///
409  /// @see `ExternalViewEmbedder`
410  ///
412 
413  //----------------------------------------------------------------------------
414  /// @brief Disables the thread merger if the external view embedder
415  /// supports dynamic thread merging.
416  ///
417  /// @attention This method is thread-safe. When the thread merger is
418  /// disabled, the raster task queue will continue to run in the
419  /// same thread until |EnableThreadMergerIfNeeded| is called.
420  ///
421  /// @see `ExternalViewEmbedder`
422  ///
424 
425  private:
427  std::unique_ptr<Surface> surface_;
428  std::unique_ptr<flutter::CompositorContext> compositor_context_;
429  // This is the last successfully rasterized layer tree.
430  std::unique_ptr<flutter::LayerTree> last_layer_tree_;
431  // Set when we need attempt to rasterize the layer tree again. This layer_tree
432  // has not successfully rasterized. This can happen due to the change in the
433  // thread configuration. This will be inserted to the front of the pipeline.
434  std::unique_ptr<flutter::LayerTree> resubmitted_layer_tree_;
435  fml::closure next_frame_callback_;
436  bool user_override_resource_cache_bytes_;
437  std::optional<size_t> max_cache_bytes_;
438  fml::RefPtr<fml::RasterThreadMerger> raster_thread_merger_;
440 
441  // |SnapshotDelegate|
442  sk_sp<SkImage> MakeRasterSnapshot(sk_sp<SkPicture> picture,
443  SkISize picture_size) override;
444 
445  // |SnapshotDelegate|
446  sk_sp<SkImage> ConvertToRasterImage(sk_sp<SkImage> image) override;
447 
448  sk_sp<SkData> ScreenshotLayerTreeAsImage(
449  flutter::LayerTree* tree,
450  flutter::CompositorContext& compositor_context,
451  GrDirectContext* surface_context,
452  bool compressed);
453 
454  sk_sp<SkImage> DoMakeRasterSnapshot(
455  SkISize size,
456  std::function<void(SkCanvas*)> draw_callback);
457 
458  RasterStatus DoDraw(std::unique_ptr<flutter::LayerTree> layer_tree);
459 
460  RasterStatus DrawToSurface(flutter::LayerTree& layer_tree);
461 
462  void FireNextFrameCallbackIfPresent();
463 
464  static bool NoDiscard(const flutter::LayerTree& layer_tree) { return false; }
465 
467 };
468 
469 } // namespace flutter
470 
471 #endif // SHELL_COMMON_RASTERIZER_H_
std::optional< size_t > GetResourceCacheMaxBytes() const
The current value of Skia&#39;s resource cache size, if a surface is present.
Definition: rasterizer.cc:686
void Setup(std::unique_ptr< Surface > surface)
Rasterizers may be created well before an on-screen surface is available for rendering. Shells usually create a rasterizer in their constructors. Once an on-screen surface is available however, one may be provided to the rasterizer using this call. No rendering may occur before this call. The surface is held till the balancing call to Rasterizer::Teardown is made. Calling a setup before tearing down the previous surface (if this is not the first time the surface has been setup) is user error.
Definition: rasterizer.cc:72
virtual std::shared_ptr< fml::SyncSwitch > GetIsGpuDisabledSyncSwitch() const =0
fml::TaskRunnerAffineWeakPtr< Rasterizer > GetWeakPtr() const
Gets a weak pointer to the rasterizer. The rasterizer may only be accessed on the GPU task runner...
Definition: rasterizer.cc:63
Dart_NativeFunction function
Definition: fuchsia.cc:51
constexpr std::size_t size(T(&array)[N])
Definition: size.h:13
void Teardown()
Releases the previously setup on-screen render surface and collects associated resources. No more rendering may occur till the next call to Rasterizer::Setup with a new render surface. Calling a teardown without a setup is user error.
Definition: rasterizer.cc:99
void DrawLastLayerTree()
Draws a last layer tree to the render surface. This may seem entirely redundant at first glance...
Definition: rasterizer.cc:147
Screenshot ScreenshotLastLayerTree(ScreenshotType type, bool base64_encode)
Screenshots the last layer tree to one of the supported screenshot types and optionally Base 64 encod...
Definition: rasterizer.cc:607
virtual const TaskRunners & GetTaskRunners() const =0
Task runners used by the shell.
virtual fml::TimePoint GetLatestFrameTargetTime() const =0
void DisableThreadMergerIfNeeded()
Disables the thread merger if the external view embedder supports dynamic thread merging.
Definition: rasterizer.cc:118
MockDelegate delegate_
std::function< void()> closure
Definition: closure.h:14
void EnableThreadMergerIfNeeded()
Enables the thread merger if the external view embedder supports dynamic thread merging.
Definition: rasterizer.cc:112
A POD type used to return the screenshot data along with the size of the frame.
Definition: rasterizer.h:279
ScreenshotType
The type of the screenshot to obtain of the previously rendered layer tree.
Definition: rasterizer.h:249
void SetNextFrameCallback(const fml::closure &callback)
Sets a callback that will be executed when the next layer tree in rendered to the on-screen surface...
Definition: rasterizer.cc:650
void Draw(fml::RefPtr< Pipeline< flutter::LayerTree >> pipeline, LayerTreeDiscardCallback discardCallback=NoDiscard)
Takes the next item from the layer tree pipeline and executes the raster thread frame workload for th...
Definition: rasterizer.cc:154
Used to forward events from the rasterizer to interested subsystems. Currently, the shell sets itself...
Definition: rasterizer.h:53
std::function< bool(flutter::LayerTree &)> LayerTreeDiscardCallback
Definition: rasterizer.h:209
~Rasterizer()
Destroys the rasterizer. This must happen on the GPU task runner. All GPU resources are collected bef...
flutter::TextureRegistry * GetTextureRegistry()
Gets the registry of external textures currently in use by the rasterizer. These textures may be upda...
Definition: rasterizer.cc:139
fml::TaskRunnerAffineWeakPtr< SnapshotDelegate > GetSnapshotDelegate() const
Definition: rasterizer.cc:67
virtual fml::Milliseconds GetFrameBudget()=0
Rasterizer(Delegate &delegate)
Creates a new instance of a rasterizer. Rasterizers may only be created on the GPU task runner...
Definition: rasterizer.cc:39
#define FML_DISALLOW_COPY_AND_ASSIGN(TypeName)
Definition: macros.h:27
flutter::CompositorContext * compositor_context()
Returns a pointer to the compositor context used by this rasterizer. This pointer will never be nullp...
Definition: rasterizer.h:358
flutter::LayerTree * GetLastLayerTree()
Sometimes, it may be necessary to render the same frame again without having to wait for the framewor...
Definition: rasterizer.cc:143
void SetResourceCacheMaxBytes(size_t max_bytes, bool from_user)
Skia has no notion of time. To work around the performance implications of this, it may cache GPU res...
Definition: rasterizer.cc:664
virtual void OnFrameRasterized(const FrameTiming &frame_timing)=0
Notifies the delegate that a frame has been rendered. The rasterizer collects profiling information f...
std::chrono::duration< double, std::milli > Milliseconds
Definition: time_delta.h:18
void NotifyLowMemoryWarning() const
Notifies the rasterizer that there is a low memory situation and it must purge as many unnecessary re...
Definition: rasterizer.cc:124