pointer_delegate.cc

Go to the documentation of this file.

// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
 
#include "pointer_delegate.h"
 
#include <lib/trace/event.h>
#include <zircon/status.h>
#include <zircon/types.h>
 
#include <limits>
 
#include "flutter/fml/logging.h"
#include "flutter/fml/trace_event.h"
 
// TODO(fxbug.dev/87076): Add MouseSource tests.
 
namespace fuchsia::ui::pointer {
// For using TouchInteractionId as a map key.
bool operator==(const fuchsia::ui::pointer::TouchInteractionId& a,
                const fuchsia::ui::pointer::TouchInteractionId& b) {
  return a.device_id == b.device_id && a.pointer_id == b.pointer_id &&
         a.interaction_id == b.interaction_id;
}
}  // namespace fuchsia::ui::pointer
 
namespace flutter_runner {
 
using fup_EventPhase = fuchsia::ui::pointer::EventPhase;
using fup_MouseDeviceInfo = fuchsia::ui::pointer::MouseDeviceInfo;
using fup_MouseEvent = fuchsia::ui::pointer::MouseEvent;
using fup_TouchEvent = fuchsia::ui::pointer::TouchEvent;
using fup_TouchIxnStatus = fuchsia::ui::pointer::TouchInteractionStatus;
using fup_TouchResponse = fuchsia::ui::pointer::TouchResponse;
using fup_TouchResponseType = fuchsia::ui::pointer::TouchResponseType;
using fup_ViewParameters = fuchsia::ui::pointer::ViewParameters;
 
namespace {
void IssueTouchTraceEvent(const fup_TouchEvent& event) {
  FML_DCHECK(event.has_trace_flow_id()) << "API guarantee";
  TRACE_FLOW_END("input", "dispatch_event_to_client", event.trace_flow_id());
}
 
void IssueMouseTraceEvent(const fup_MouseEvent& event) {
  FML_DCHECK(event.has_trace_flow_id()) << "API guarantee";
  TRACE_FLOW_END("input", "dispatch_event_to_client", event.trace_flow_id());
}
 
bool HasValidatedTouchSample(const fup_TouchEvent& event) {
  if (!event.has_pointer_sample()) {
    return false;
  }
  FML_DCHECK(event.pointer_sample().has_interaction()) << "API guarantee";
  FML_DCHECK(event.pointer_sample().has_phase()) << "API guarantee";
  FML_DCHECK(event.pointer_sample().has_position_in_viewport())
      << "API guarantee";
  return true;
}
 
bool HasValidatedMouseSample(const fup_MouseEvent& event) {
  if (!event.has_pointer_sample()) {
    return false;
  }
  const auto& sample = event.pointer_sample();
  FML_DCHECK(sample.has_device_id()) << "API guarantee";
  FML_DCHECK(sample.has_position_in_viewport()) << "API guarantee";
  FML_DCHECK(!sample.has_pressed_buttons() || !sample.pressed_buttons().empty())
      << "API guarantee";
 
  return true;
}
 
std::array<float, 2> ViewportToViewCoordinates(
    std::array<float, 2> viewport_coordinates,
    const std::array<float, 9>& viewport_to_view_transform) {
  // The transform matrix is a FIDL array with matrix data in column-major
  // order. For a matrix with data [a b c d e f g h i], and with the viewport
  // coordinates expressed as homogeneous coordinates, the logical view
  // coordinates are obtained with the following formula:
  //   |a d g|   |x|   |x'|
  //   |b e h| * |y| = |y'|
  //   |c f i|   |1|   |w'|
  // which we then normalize based on the w component:
  //   if z' not zero: (x'/w', y'/w')
  //   else (x', y')
  const auto& M = viewport_to_view_transform;
  const float x = viewport_coordinates[0];
  const float y = viewport_coordinates[1];
  const float xp = M[0] * x + M[3] * y + M[6];
  const float yp = M[1] * x + M[4] * y + M[7];
  const float wp = M[2] * x + M[5] * y + M[8];
  if (wp != 0) {
    return {xp / wp, yp / wp};
  } else {
    return {xp, yp};
  }
}
 
flutter::PointerData::Change GetChangeFromTouchEventPhase(
    fup_EventPhase phase) {
  switch (phase) {
    case fup_EventPhase::ADD:
      return flutter::PointerData::Change::kAdd;
    case fup_EventPhase::CHANGE:
      return flutter::PointerData::Change::kMove;
    case fup_EventPhase::REMOVE:
      return flutter::PointerData::Change::kRemove;
    case fup_EventPhase::CANCEL:
      return flutter::PointerData::Change::kCancel;
    default:
      return flutter::PointerData::Change::kCancel;
  }
}
 
std::array<float, 2> ClampToViewSpace(const float x,
                                      const float y,
                                      const fup_ViewParameters& p) {
  const float min_x = p.view.min[0];
  const float min_y = p.view.min[1];
  const float max_x = p.view.max[0];
  const float max_y = p.view.max[1];
  if (min_x <= x && x < max_x && min_y <= y && y < max_y) {
    return {x, y};  // No clamping to perform.
  }
 
  // View boundary is [min_x, max_x) x [min_y, max_y). Note that min is
  // inclusive, but max is exclusive - so we subtract epsilon.
  const float max_x_inclusive = max_x - std::numeric_limits<float>::epsilon();
  const float max_y_inclusive = max_y - std::numeric_limits<float>::epsilon();
  const float& clamped_x = std::clamp(x, min_x, max_x_inclusive);
  const float& clamped_y = std::clamp(y, min_y, max_y_inclusive);
  FML_LOG(INFO) << "Clamped (" << x << ", " << y << ") to (" << clamped_x
                << ", " << clamped_y << ").";
  return {clamped_x, clamped_y};
}
 
flutter::PointerData::Change ComputePhase(
    bool any_button_down,
    std::unordered_set<uint32_t>& mouse_down,
    uint32_t id) {
  if (!mouse_down.count(id) && !any_button_down) {
    return flutter::PointerData::Change::kHover;
  } else if (!mouse_down.count(id) && any_button_down) {
    mouse_down.insert(id);
    return flutter::PointerData::Change::kDown;
  } else if (mouse_down.count(id) && any_button_down) {
    return flutter::PointerData::Change::kMove;
  } else if (mouse_down.count(id) && !any_button_down) {
    mouse_down.erase(id);
    return flutter::PointerData::Change::kUp;
  }
 
  FML_UNREACHABLE();
  return flutter::PointerData::Change::kCancel;
}
 
// Flutter's PointerData.device field is 64 bits and is expected to be unique
// for each pointer. We pack Fuchsia's device ID (hi) and pointer ID (lo) into
// 64 bits to retain uniqueness across multiple touch devices.
uint64_t PackFuchsiaDeviceIdAndPointerId(uint32_t fuchsia_device_id,
                                         uint32_t fuchsia_pointer_id) {
  return (((uint64_t)fuchsia_device_id) << 32) | fuchsia_pointer_id;
}
 
// It returns a "draft" because the coordinates are logical. Later, view pixel
// ratio is applied to obtain physical coordinates.
//
// The flutter pointerdata state machine has extra phases, which this function
// synthesizes on the fly. Hence the return data is a flutter pointerdata, and
// optionally a second one.
// For example: <ADD, DOWN>, <MOVE, nullopt>, <UP, REMOVE>.
// TODO(fxbug.dev/87074): Let PointerDataPacketConverter synthesize events.
//
// Flutter gestures expect a gesture to start within the logical view space, and
// is not tolerant of floating point drift. This function coerces just the DOWN
// event's coordinate to start within the logical view.
std::pair<flutter::PointerData, std::optional<flutter::PointerData>>
CreateTouchDraft(const fup_TouchEvent& event,
                 const fup_ViewParameters& view_parameters) {
  FML_DCHECK(HasValidatedTouchSample(event)) << "precondition";
  const auto& sample = event.pointer_sample();
  const auto& ixn = sample.interaction();
 
  flutter::PointerData ptr;
  ptr.Clear();
  ptr.time_stamp = event.timestamp() / 1000;  // in microseconds
  ptr.change = GetChangeFromTouchEventPhase(sample.phase());
  ptr.kind = flutter::PointerData::DeviceKind::kTouch;
  // Load Fuchsia's pointer ID onto Flutter's |device| field, and not the
  // |pointer_identifier| field. The latter is written by
  // PointerDataPacketConverter, to track individual gesture interactions.
  ptr.device = PackFuchsiaDeviceIdAndPointerId(ixn.device_id, ixn.pointer_id);
  // View parameters can change mid-interaction; apply transform on the fly.
  auto logical =
      ViewportToViewCoordinates(sample.position_in_viewport(),
                                view_parameters.viewport_to_view_transform);
  ptr.physical_x = logical[0];  // Not yet physical; adjusted in PlatformView.
  ptr.physical_y = logical[1];  // Not yet physical; adjusted in PlatformView.
 
  // Match Flutter pointer's state machine with synthesized events.
  if (ptr.change == flutter::PointerData::Change::kAdd) {
    flutter::PointerData down;
    memcpy(&down, &ptr, sizeof(flutter::PointerData));
    down.change = flutter::PointerData::Change::kDown;
    {  // Ensure gesture recognition: DOWN starts in the logical view space.
      auto [x, y] =
          ClampToViewSpace(down.physical_x, down.physical_y, view_parameters);
      down.physical_x = x;
      down.physical_y = y;
    }
    return {std::move(ptr), std::move(down)};
  } else if (ptr.change == flutter::PointerData::Change::kRemove) {
    flutter::PointerData up;
    memcpy(&up, &ptr, sizeof(flutter::PointerData));
    up.change = flutter::PointerData::Change::kUp;
    return {std::move(up), std::move(ptr)};
  } else {
    return {std::move(ptr), std::nullopt};
  }
}
 
// It returns a "draft" because the coordinates are logical. Later, view pixel
// ratio is applied to obtain physical coordinates.
//
// Phase data is computed before this call; it involves state tracking based on
// button-down state.
//
// Button data, if available, gets packed into the |buttons| field, in flutter
// button order (kMousePrimaryButton, etc). The device-assigned button IDs are
// provided in priority order in MouseEvent.device_info (at the start of channel
// connection), and maps from device button ID (given in fup_MouseEvent) to
// flutter button ID (flutter::PointerData).
//
// Scroll data, if available, gets packed into the |scroll_delta_x| or
// |scroll_delta_y| fields, and the |signal_kind| field is set to kScroll.
// The PointerDataPacketConverter reads this field to synthesize events to match
// Flutter's expected pointer stream.
// TODO(fxbug.dev/87073): PointerDataPacketConverter should synthesize a
// discrete scroll event on kDown or kUp, to match engine expectations.
//
// Flutter gestures expect a gesture to start within the logical view space, and
// is not tolerant of floating point drift. This function coerces just the DOWN
// event's coordinate to start within the logical view.
flutter::PointerData CreateMouseDraft(const fup_MouseEvent& event,
                                      const flutter::PointerData::Change phase,
                                      const fup_ViewParameters& view_parameters,
                                      const fup_MouseDeviceInfo& device_info) {
  FML_DCHECK(HasValidatedMouseSample(event)) << "precondition";
  const auto& sample = event.pointer_sample();
 
  flutter::PointerData ptr;
  ptr.Clear();
  ptr.time_stamp = event.timestamp() / 1000;  // in microseconds
  ptr.change = phase;
  ptr.kind = flutter::PointerData::DeviceKind::kMouse;
  ptr.device = sample.device_id();
 
  // View parameters can change mid-interaction; apply transform on the fly.
  auto logical =
      ViewportToViewCoordinates(sample.position_in_viewport(),
                                view_parameters.viewport_to_view_transform);
  ptr.physical_x = logical[0];  // Not yet physical; adjusted in PlatformView.
  ptr.physical_y = logical[1];  // Not yet physical; adjusted in PlatformView.
 
  // Ensure gesture recognition: DOWN starts in the logical view space.
  if (ptr.change == flutter::PointerData::Change::kDown) {
    auto [x, y] =
        ClampToViewSpace(ptr.physical_x, ptr.physical_y, view_parameters);
    ptr.physical_x = x;
    ptr.physical_y = y;
  }
 
  if (sample.has_pressed_buttons()) {
    int64_t flutter_buttons = 0;
    const auto& pressed = sample.pressed_buttons();
    for (size_t idx = 0; idx < pressed.size(); ++idx) {
      const uint8_t button_id = pressed[idx];
      FML_DCHECK(device_info.has_buttons()) << "API guarantee";
      // Priority 0 maps to kPrimaryButton, and so on.
      for (uint8_t prio = 0; prio < device_info.buttons().size(); ++prio) {
        if (button_id == device_info.buttons()[prio]) {
          flutter_buttons |= (1 << prio);
        }
      }
    }
    FML_DCHECK(flutter_buttons != 0);
    ptr.buttons = flutter_buttons;
  }
 
  // Fuchsia previously only provided scroll data in "ticks", not physical
  // pixels. On legacy platforms, since Flutter expects scroll data in physical
  // pixels, to compensate for lack of guidance, we make up a "reasonable
  // amount".
  // TODO(fxbug.dev/103443): Remove the tick based scrolling after the
  // transition.
  const int kScrollOffsetMultiplier = 20;
 
  double dy = 0;
  double dx = 0;
  bool is_scroll = false;
 
  if (sample.has_scroll_v_physical_pixel()) {
    dy = -sample.scroll_v_physical_pixel();
    is_scroll = true;
  } else if (sample.has_scroll_v()) {
    dy = -sample.scroll_v() *
         kScrollOffsetMultiplier;  // logical amount, not yet physical; adjusted
                                   // in Platform View.
    is_scroll = true;
  }
 
  if (sample.has_scroll_h_physical_pixel()) {
    dx = sample.scroll_h_physical_pixel();
    is_scroll = true;
  } else if (sample.has_scroll_h()) {
    dx = sample.scroll_h() * kScrollOffsetMultiplier;  // logical amount
    is_scroll = true;
  }
 
  if (is_scroll) {
    ptr.signal_kind = flutter::PointerData::SignalKind::kScroll;
    ptr.scroll_delta_y = dy;
    ptr.scroll_delta_x = dx;
  }
 
  return ptr;
}
 
// Helper to insert one or two events into a vector buffer.
void InsertIntoBuffer(
    std::pair<flutter::PointerData, std::optional<flutter::PointerData>> events,
    std::vector<flutter::PointerData>* buffer) {
  FML_DCHECK(buffer);
  buffer->emplace_back(std::move(events.first));
  if (events.second.has_value()) {
    buffer->emplace_back(std::move(events.second.value()));
  }
}
}  // namespace
 
PointerDelegate::PointerDelegate(
    fuchsia::ui::pointer::TouchSourceHandle touch_source,
    fuchsia::ui::pointer::MouseSourceHandle mouse_source)
    : touch_source_(touch_source.Bind()), mouse_source_(mouse_source.Bind()) {
  if (touch_source_) {
    touch_source_.set_error_handler([](zx_status_t status) {
      FML_LOG(ERROR) << "TouchSource channel error: << "
                     << zx_status_get_string(status);
    });
  }
  if (mouse_source_) {
    mouse_source_.set_error_handler([](zx_status_t status) {
      FML_LOG(ERROR) << "MouseSource channel error: << "
                     << zx_status_get_string(status);
    });
  }
}
// Core logic of this class.
// Aim to keep state management in this function.
void PointerDelegate::WatchLoop(
    std::function<void(std::vector<flutter::PointerData>)> callback) {
  FML_LOG(INFO) << "Flutter - PointerDelegate started.";
  if (touch_responder_) {
    FML_LOG(ERROR) << "PointerDelegate::WatchLoop() must be called once.";
    return;
  }
 
  touch_responder_ = [this, callback](std::vector<fup_TouchEvent> events) {
    TRACE_EVENT0("flutter", "PointerDelegate::TouchHandler");
    FML_DCHECK(touch_responses_.empty()) << "precondition";
    std::vector<flutter::PointerData> to_client;
    for (const fup_TouchEvent& event : events) {
      IssueTouchTraceEvent(event);
      fup_TouchResponse
          response;  // Response per event, matched on event's index.
      if (event.has_view_parameters()) {
        touch_view_parameters_ = std::move(event.view_parameters());
      }
      if (HasValidatedTouchSample(event)) {
        const auto& sample = event.pointer_sample();
        const auto& ixn = sample.interaction();
        if (sample.phase() == fup_EventPhase::ADD &&
            !event.has_interaction_result()) {
          touch_buffer_.emplace(ixn, std::vector<flutter::PointerData>());
        }
 
        FML_DCHECK(touch_view_parameters_.has_value()) << "API guarantee";
        auto events = CreateTouchDraft(event, touch_view_parameters_.value());
        if (touch_buffer_.count(ixn) > 0) {
          InsertIntoBuffer(std::move(events), &touch_buffer_[ixn]);
        } else {
          InsertIntoBuffer(std::move(events), &to_client);
        }
        // For this simple client, always claim we want the gesture.
        response.set_response_type(fup_TouchResponseType::YES);
      }
      if (event.has_interaction_result()) {
        const auto& result = event.interaction_result();
        const auto& ixn = result.interaction;
        if (result.status == fup_TouchIxnStatus::GRANTED &&
            touch_buffer_.count(ixn) > 0) {
          FML_DCHECK(to_client.empty()) << "invariant";
          to_client.insert(to_client.end(), touch_buffer_[ixn].begin(),
                           touch_buffer_[ixn].end());
        }
        touch_buffer_.erase(ixn);  // Result seen, delete the buffer.
      }
      touch_responses_.push_back(std::move(response));
    }
    callback(std::move(to_client));  // Notify client of touch events, if any.
 
    touch_source_->Watch(std::move(touch_responses_),
                         /*copy*/ touch_responder_);
    touch_responses_.clear();
  };
 
  mouse_responder_ = [this, callback](std::vector<fup_MouseEvent> events) {
    TRACE_EVENT0("flutter", "PointerDelegate::MouseHandler");
    std::vector<flutter::PointerData> to_client;
    for (fup_MouseEvent& event : events) {
      IssueMouseTraceEvent(event);
      if (event.has_device_info()) {
        const auto& id = event.device_info().id();
        mouse_device_info_[id] = std::move(*event.mutable_device_info());
      }
      if (event.has_view_parameters()) {
        mouse_view_parameters_ = std::move(event.view_parameters());
      }
      if (HasValidatedMouseSample(event)) {
        const auto& sample = event.pointer_sample();
        const auto& id = sample.device_id();
        const bool any_button_down = sample.has_pressed_buttons();
        FML_DCHECK(mouse_view_parameters_.has_value()) << "API guarantee";
        FML_DCHECK(mouse_device_info_.count(id) > 0) << "API guarantee";
 
        const auto phase = ComputePhase(any_button_down, mouse_down_, id);
        flutter::PointerData data =
            CreateMouseDraft(event, phase, mouse_view_parameters_.value(),
                             mouse_device_info_[id]);
        to_client.emplace_back(std::move(data));
      }
    }
    callback(std::move(to_client));
    mouse_source_->Watch(/*copy*/ mouse_responder_);
  };
 
  // Start watching both channels.
  touch_source_->Watch(std::move(touch_responses_), /*copy*/ touch_responder_);
  touch_responses_.clear();
  if (mouse_source_) {
    mouse_source_->Watch(/*copy*/ mouse_responder_);
  }
}
 
}  // namespace flutter_runner