Flutter Engine
persistent_cache.cc
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "flutter/shell/common/persistent_cache.h"
6 
7 #include <future>
8 #include <memory>
9 #include <string>
10 #include <string_view>
11 
12 #include "flutter/fml/base32.h"
13 #include "flutter/fml/file.h"
14 #include "flutter/fml/logging.h"
15 #include "flutter/fml/make_copyable.h"
16 #include "flutter/fml/mapping.h"
17 #include "flutter/fml/paths.h"
18 #include "flutter/fml/trace_event.h"
19 #include "flutter/shell/version/version.h"
20 #include "rapidjson/document.h"
21 #include "third_party/skia/include/utils/SkBase64.h"
22 
23 namespace flutter {
24 
25 std::string PersistentCache::cache_base_path_;
26 
27 std::shared_ptr<AssetManager> PersistentCache::asset_manager_;
28 
29 std::mutex PersistentCache::instance_mutex_;
30 std::unique_ptr<PersistentCache> PersistentCache::gPersistentCache;
31 
32 std::string PersistentCache::SkKeyToFilePath(const SkData& data) {
33  if (data.data() == nullptr || data.size() == 0) {
34  return "";
35  }
36 
37  std::string_view view(reinterpret_cast<const char*>(data.data()),
38  data.size());
39 
40  auto encode_result = fml::Base32Encode(view);
41 
42  if (!encode_result.first) {
43  return "";
44  }
45 
46  return encode_result.second;
47 }
48 
49 bool PersistentCache::gIsReadOnly = false;
50 
51 std::atomic<bool> PersistentCache::cache_sksl_ = false;
52 std::atomic<bool> PersistentCache::strategy_set_ = false;
53 
55  if (strategy_set_ && value != cache_sksl_) {
56  FML_LOG(ERROR) << "Cache SkSL can only be set before the "
57  "GrContextOptions::fShaderCacheStrategy is set.";
58  return;
59  }
60  cache_sksl_ = value;
61 }
62 
64  std::scoped_lock lock(instance_mutex_);
65  if (gPersistentCache == nullptr) {
66  gPersistentCache.reset(new PersistentCache(gIsReadOnly));
67  }
68  return gPersistentCache.get();
69 }
70 
72  std::scoped_lock lock(instance_mutex_);
73  gPersistentCache.reset(new PersistentCache(gIsReadOnly));
74  strategy_set_ = false;
75 }
76 
78  cache_base_path_ = path;
79 }
80 
82  // Make sure that this is called after the worker task runner setup so all the
83  // file system modifications would happen on that single thread to avoid
84  // racing.
85  FML_CHECK(GetWorkerTaskRunner());
86 
87  std::promise<bool> removed;
88  GetWorkerTaskRunner()->PostTask([&removed,
89  cache_directory = cache_directory_]() {
90  if (cache_directory->is_valid()) {
91  // Only remove files but not directories.
92  FML_LOG(INFO) << "Purge persistent cache.";
93  fml::FileVisitor delete_file = [](const fml::UniqueFD& directory,
94  const std::string& filename) {
95  // Do not delete directories. Return true to continue with other files.
96  if (fml::IsDirectory(directory, filename.c_str())) {
97  return true;
98  }
99  return fml::UnlinkFile(directory, filename.c_str());
100  };
101  removed.set_value(VisitFilesRecursively(*cache_directory, delete_file));
102  } else {
103  removed.set_value(false);
104  }
105  });
106  return removed.get_future().get();
107 }
108 
109 namespace {
110 
111 constexpr char kEngineComponent[] = "flutter_engine";
112 
113 static void FreeOldCacheDirectory(const fml::UniqueFD& cache_base_dir) {
114  fml::UniqueFD engine_dir =
115  fml::OpenDirectoryReadOnly(cache_base_dir, kEngineComponent);
116  if (!engine_dir.is_valid()) {
117  return;
118  }
119  fml::VisitFiles(engine_dir, [](const fml::UniqueFD& directory,
120  const std::string& filename) {
121  if (filename != GetFlutterEngineVersion()) {
122  auto dir = fml::OpenDirectory(directory, filename.c_str(), false,
124  if (dir.is_valid()) {
125  fml::RemoveDirectoryRecursively(directory, filename.c_str());
126  }
127  }
128  return true;
129  });
130 }
131 
132 static std::shared_ptr<fml::UniqueFD> MakeCacheDirectory(
133  const std::string& global_cache_base_path,
134  bool read_only,
135  bool cache_sksl) {
136  fml::UniqueFD cache_base_dir;
137  if (global_cache_base_path.length()) {
138  cache_base_dir = fml::OpenDirectory(global_cache_base_path.c_str(), false,
140  } else {
141  cache_base_dir = fml::paths::GetCachesDirectory();
142  }
143 
144  if (cache_base_dir.is_valid()) {
145  FreeOldCacheDirectory(cache_base_dir);
146  std::vector<std::string> components = {
147  kEngineComponent, GetFlutterEngineVersion(), "skia", GetSkiaVersion()};
148  if (cache_sksl) {
149  components.push_back(PersistentCache::kSkSLSubdirName);
150  }
151  return std::make_shared<fml::UniqueFD>(
152  CreateDirectory(cache_base_dir, components,
153  read_only ? fml::FilePermission::kRead
155  } else {
156  return std::make_shared<fml::UniqueFD>();
157  }
158 }
159 } // namespace
160 
161 sk_sp<SkData> ParseBase32(const std::string& input) {
162  std::pair<bool, std::string> decode_result = fml::Base32Decode(input);
163  if (!decode_result.first) {
164  FML_LOG(ERROR) << "Base32 can't decode: " << input;
165  return nullptr;
166  }
167  const std::string& data_string = decode_result.second;
168  return SkData::MakeWithCopy(data_string.data(), data_string.length());
169 }
170 
171 sk_sp<SkData> ParseBase64(const std::string& input) {
172  SkBase64 decoder;
173  auto error = decoder.decode(input.c_str(), input.length());
175  FML_LOG(ERROR) << "Base64 decode error: " << error;
176  FML_LOG(ERROR) << "Base64 can't decode: " << input;
177  return nullptr;
178  }
179  return SkData::MakeWithCopy(decoder.getData(), decoder.getDataSize());
180 }
181 
182 std::vector<PersistentCache::SkSLCache> PersistentCache::LoadSkSLs() {
183  TRACE_EVENT0("flutter", "PersistentCache::LoadSkSLs");
184  std::vector<PersistentCache::SkSLCache> result;
185  fml::FileVisitor visitor = [&result](const fml::UniqueFD& directory,
186  const std::string& filename) {
187  sk_sp<SkData> key = ParseBase32(filename);
188  sk_sp<SkData> data = LoadFile(directory, filename);
189  if (key != nullptr && data != nullptr) {
190  result.push_back({key, data});
191  } else {
192  FML_LOG(ERROR) << "Failed to load: " << filename;
193  }
194  return true;
195  };
196 
197  // Only visit sksl_cache_directory_ if this persistent cache is valid.
198  // However, we'd like to continue visit the asset dir even if this persistent
199  // cache is invalid.
200  if (IsValid()) {
201  // In case `rewinddir` doesn't work reliably, load SkSLs from a freshly
202  // opened directory (https://github.com/flutter/flutter/issues/65258).
203  fml::UniqueFD fresh_dir =
204  fml::OpenDirectoryReadOnly(*cache_directory_, kSkSLSubdirName);
205  if (fresh_dir.is_valid()) {
206  fml::VisitFiles(fresh_dir, visitor);
207  }
208  }
209 
210  std::unique_ptr<fml::Mapping> mapping = nullptr;
211  if (asset_manager_ != nullptr) {
212  mapping = asset_manager_->GetAsMapping(kAssetFileName);
213  }
214  if (mapping == nullptr) {
215  FML_LOG(INFO) << "No sksl asset found.";
216  } else {
217  FML_LOG(INFO) << "Found sksl asset. Loading SkSLs from it...";
218  rapidjson::Document json_doc;
219  rapidjson::ParseResult parse_result =
220  json_doc.Parse(reinterpret_cast<const char*>(mapping->GetMapping()),
221  mapping->GetSize());
222  if (parse_result != rapidjson::ParseErrorCode::kParseErrorNone) {
223  FML_LOG(ERROR) << "Failed to parse json file: " << kAssetFileName;
224  } else {
225  for (auto& item : json_doc["data"].GetObject()) {
226  sk_sp<SkData> key = ParseBase32(item.name.GetString());
227  sk_sp<SkData> sksl = ParseBase64(item.value.GetString());
228  if (key != nullptr && sksl != nullptr) {
229  result.push_back({key, sksl});
230  } else {
231  FML_LOG(ERROR) << "Failed to load: " << item.name.GetString();
232  }
233  }
234  }
235  }
236 
237  return result;
238 }
239 
240 PersistentCache::PersistentCache(bool read_only)
241  : is_read_only_(read_only),
242  cache_directory_(MakeCacheDirectory(cache_base_path_, read_only, false)),
243  sksl_cache_directory_(
244  MakeCacheDirectory(cache_base_path_, read_only, true)) {
245  if (!IsValid()) {
246  FML_LOG(WARNING) << "Could not acquire the persistent cache directory. "
247  "Caching of GPU resources on disk is disabled.";
248  }
249 }
250 
252 
253 bool PersistentCache::IsValid() const {
254  return cache_directory_ && cache_directory_->is_valid();
255 }
256 
257 sk_sp<SkData> PersistentCache::LoadFile(const fml::UniqueFD& dir,
258  const std::string& file_name) {
259  auto file = fml::OpenFileReadOnly(dir, file_name.c_str());
260  if (!file.is_valid()) {
261  return nullptr;
262  }
263  auto mapping = std::make_unique<fml::FileMapping>(file);
264  if (mapping->GetSize() == 0) {
265  return nullptr;
266  }
267  return SkData::MakeWithCopy(mapping->GetMapping(), mapping->GetSize());
268 }
269 
270 // |GrContextOptions::PersistentCache|
271 sk_sp<SkData> PersistentCache::load(const SkData& key) {
272  TRACE_EVENT0("flutter", "PersistentCacheLoad");
273  if (!IsValid()) {
274  return nullptr;
275  }
276  auto file_name = SkKeyToFilePath(key);
277  if (file_name.size() == 0) {
278  return nullptr;
279  }
280  auto result = PersistentCache::LoadFile(*cache_directory_, file_name);
281  if (result != nullptr) {
282  TRACE_EVENT0("flutter", "PersistentCacheLoadHit");
283  }
284  return result;
285 }
286 
288  std::shared_ptr<fml::UniqueFD> cache_directory,
289  std::string key,
290  std::unique_ptr<fml::Mapping> value) {
291  auto task = fml::MakeCopyable([cache_directory, //
292  file_name = std::move(key), //
293  mapping = std::move(value) //
294  ]() mutable {
295  TRACE_EVENT0("flutter", "PersistentCacheStore");
296  if (!fml::WriteAtomically(*cache_directory, //
297  file_name.c_str(), //
298  *mapping) //
299  ) {
300  FML_LOG(WARNING) << "Could not write cache contents to persistent store.";
301  }
302  });
303 
304  if (!worker) {
305  FML_LOG(WARNING)
306  << "The persistent cache has no available workers. Performing the task "
307  "on the current thread. This slow operation is going to occur on a "
308  "frame workload.";
309  task();
310  } else {
311  worker->PostTask(std::move(task));
312  }
313 }
314 
315 // |GrContextOptions::PersistentCache|
316 void PersistentCache::store(const SkData& key, const SkData& data) {
317  stored_new_shaders_ = true;
318 
319  if (is_read_only_) {
320  return;
321  }
322 
323  if (!IsValid()) {
324  return;
325  }
326 
327  auto file_name = SkKeyToFilePath(key);
328 
329  if (file_name.size() == 0) {
330  return;
331  }
332 
333  auto mapping = std::make_unique<fml::DataMapping>(
334  std::vector<uint8_t>{data.bytes(), data.bytes() + data.size()});
335 
336  if (mapping == nullptr || mapping->GetSize() == 0) {
337  return;
338  }
339 
340  PersistentCacheStore(GetWorkerTaskRunner(),
341  cache_sksl_ ? sksl_cache_directory_ : cache_directory_,
342  std::move(file_name), std::move(mapping));
343 }
344 
345 void PersistentCache::DumpSkp(const SkData& data) {
346  if (is_read_only_ || !IsValid()) {
347  FML_LOG(ERROR) << "Could not dump SKP from read-only or invalid persistent "
348  "cache.";
349  return;
350  }
351 
352  std::stringstream name_stream;
354  name_stream << "shader_dump_" << std::to_string(ticks) << ".skp";
355  std::string file_name = name_stream.str();
356  FML_LOG(INFO) << "Dumping " << file_name;
357  auto mapping = std::make_unique<fml::DataMapping>(
358  std::vector<uint8_t>{data.bytes(), data.bytes() + data.size()});
359  PersistentCacheStore(GetWorkerTaskRunner(), cache_directory_,
360  std::move(file_name), std::move(mapping));
361 }
362 
364  fml::RefPtr<fml::TaskRunner> task_runner) {
365  std::scoped_lock lock(worker_task_runners_mutex_);
366  worker_task_runners_.insert(task_runner);
367 }
368 
370  fml::RefPtr<fml::TaskRunner> task_runner) {
371  std::scoped_lock lock(worker_task_runners_mutex_);
372  auto found = worker_task_runners_.find(task_runner);
373  if (found != worker_task_runners_.end()) {
374  worker_task_runners_.erase(found);
375  }
376 }
377 
378 fml::RefPtr<fml::TaskRunner> PersistentCache::GetWorkerTaskRunner() const {
380 
381  std::scoped_lock lock(worker_task_runners_mutex_);
382  if (!worker_task_runners_.empty()) {
383  worker = *worker_task_runners_.begin();
384  }
385 
386  return worker;
387 }
388 
389 void PersistentCache::SetAssetManager(std::shared_ptr<AssetManager> value) {
390  TRACE_EVENT_INSTANT0("flutter", "PersistentCache::SetAssetManager");
391  asset_manager_ = value;
392 }
393 
394 } // namespace flutter
const char * GetFlutterEngineVersion()
Definition: version.cc:11
DEF_SWITCHES_START snapshot asset path
Definition: switches.h:32
sk_sp< SkData > ParseBase64(const std::string &input)
#define TRACE_EVENT0(category_group, name)
Definition: trace_event.h:75
void DumpSkp(const SkData &data)
#define TRACE_EVENT_INSTANT0(category_group, name)
Definition: trace_event.h:104
std::function< bool(const fml::UniqueFD &directory, const std::string &filename)> FileVisitor
Definition: file.h:98
FlMethodResponse GError ** error
fml::UniqueFD GetCachesDirectory()
virtual void PostTask(const fml::closure &task)
Definition: task_runner.cc:24
TimeDelta ToEpochDelta() const
Definition: time_point.h:40
sk_sp< SkData > ParseBase32(const std::string &input)
#define FML_LOG(severity)
Definition: logging.h:65
static void SetCacheSkSL(bool value)
static constexpr char kAssetFileName[]
void AddWorkerTaskRunner(fml::RefPtr< fml::TaskRunner > task_runner)
bool is_valid() const
Definition: unique_object.h:89
fml::UniqueFD OpenDirectory(const char *path, bool create_if_necessary, FilePermission permission)
Definition: file_posix.cc:94
static constexpr char kSkSLSubdirName[]
internal::CopyableLambda< T > MakeCopyable(T lambda)
Definition: make_copyable.h:57
bool VisitFiles(const fml::UniqueFD &directory, const FileVisitor &visitor)
Definition: file_posix.cc:233
static void SetAssetManager(std::shared_ptr< AssetManager > value)
uint8_t value
static void PersistentCacheStore(fml::RefPtr< fml::TaskRunner > worker, std::shared_ptr< fml::UniqueFD > cache_directory, std::string key, std::unique_ptr< fml::Mapping > value)
static void SetCacheDirectoryPath(std::string path)
void RemoveWorkerTaskRunner(fml::RefPtr< fml::TaskRunner > task_runner)
std::vector< SkSLCache > LoadSkSLs()
Load all the SkSL shader caches in the right directory.
bool IsDirectory(const fml::UniqueFD &directory)
Definition: file_posix.cc:124
std::pair< bool, std::string > Base32Encode(std::string_view input)
Definition: base32.cc:14
std::pair< bool, std::string > Base32Decode(const std::string &input)
Definition: base32.cc:54
static fml::UniqueFD CreateDirectory(const fml::UniqueFD &base_directory, const std::vector< std::string > &components, FilePermission permission, size_t index)
Definition: file.cc:12
constexpr int64_t ToNanoseconds() const
Definition: time_delta.h:61
sk_sp< SkData > load(const SkData &key) override
static std::string SkKeyToFilePath(const SkData &data)
const char * GetSkiaVersion()
Definition: version.cc:15
fml::UniqueFD OpenDirectoryReadOnly(const fml::UniqueFD &base_directory, const char *path)
Definition: file.cc:97
bool VisitFilesRecursively(const fml::UniqueFD &directory, const FileVisitor &visitor)
Definition: file.cc:71
bool UnlinkFile(const char *path)
Definition: file_posix.cc:168
static void ResetCacheForProcess()
static PersistentCache * GetCacheForProcess()
#define FML_CHECK(condition)
Definition: logging.h:68
bool WriteAtomically(const fml::UniqueFD &base_directory, const char *file_name, const Mapping &mapping)
Definition: file_posix.cc:188
bool RemoveDirectoryRecursively(const fml::UniqueFD &parent, const char *directory_name)
Definition: file.cc:120
static TimePoint Now()
Definition: time_point.cc:26
fml::UniqueFD OpenFileReadOnly(const fml::UniqueFD &base_directory, const char *path)
Definition: file.cc:92