Merge pull request #6267 from german77/gestureRewrite

hid: Improve hardware accuracy of gestures

src/core/hle/service/hid/controllers/gesture.cpp

@@ -1,10 +1,9 @@
-// Copyright 2018 yuzu emulator team
+// Copyright 2021 yuzu Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
-#include <cstring>
-#include "common/common_types.h"
 #include "common/logging/log.h"
+#include "common/math_util.h"
 #include "common/settings.h"
 #include "core/core_timing.h"
 #include "core/frontend/emu_window.h"
@@ -12,10 +11,19 @@
 
 namespace Service::HID {
 constexpr std::size_t SHARED_MEMORY_OFFSET = 0x3BA00;
-constexpr f32 angle_threshold = 0.08f;
-constexpr f32 pinch_threshold = 100.0f;
 
-Controller_Gesture::Controller_Gesture(Core::System& system_) : ControllerBase{system_} {}
+// HW is around 700, value is set to 400 to make it easier to trigger with mouse
+constexpr f32 swipe_threshold = 400.0f; // Threshold in pixels/s
+constexpr f32 angle_threshold = 0.015f; // Threshold in radians
+constexpr f32 pinch_threshold = 0.5f;   // Threshold in pixels
+constexpr f32 press_delay = 0.5f;       // Time in seconds
+constexpr f32 double_tap_delay = 0.35f; // Time in seconds
+
+constexpr f32 Square(s32 num) {
+    return static_cast<f32>(num * num);
+}
+
+Controller_Gesture::Controller_Gesture(Core::System& system) : ControllerBase(system) {}
 Controller_Gesture::~Controller_Gesture() = default;
 
 void Controller_Gesture::OnInit() {
@@ -24,6 +32,8 @@ void Controller_Gesture::OnInit() {
         keyboard_finger_id[id] = MAX_POINTS;
         udp_finger_id[id] = MAX_POINTS;
     }
+    shared_memory.header.entry_count = 0;
+    force_update = true;
 }
 
 void Controller_Gesture::OnRelease() {}
@@ -38,17 +48,23 @@ void Controller_Gesture::OnUpdate(const Core::Timing::CoreTiming& core_timing, u
         shared_memory.header.last_entry_index = 0;
         return;
     }
-    shared_memory.header.entry_count = 16;
 
-    const auto& last_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index];
+    ReadTouchInput();
-    shared_memory.header.last_entry_index = (shared_memory.header.last_entry_index + 1) % 17;
-    auto& cur_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index];
 
-    cur_entry.sampling_number = last_entry.sampling_number + 1;
+    GestureProperties gesture = GetGestureProperties();
-    cur_entry.sampling_number2 = cur_entry.sampling_number;
+    f32 time_difference = static_cast<f32>(shared_memory.header.timestamp - last_update_timestamp) /
+                          (1000 * 1000 * 1000);
 
-    // TODO(german77): Implement all gesture types
+    // Only update if necesary
+    if (!ShouldUpdateGesture(gesture, time_difference)) {
+        return;
+    }
 
+    last_update_timestamp = shared_memory.header.timestamp;
+    UpdateGestureSharedMemory(data, size, gesture, time_difference);
+}
+
+void Controller_Gesture::ReadTouchInput() {
     const Input::TouchStatus& mouse_status = touch_mouse_device->GetStatus();
     const Input::TouchStatus& udp_status = touch_udp_device->GetStatus();
     for (std::size_t id = 0; id < mouse_status.size(); ++id) {
@@ -63,50 +79,71 @@ void Controller_Gesture::OnUpdate(const Core::Timing::CoreTiming& core_timing, u
                 UpdateTouchInputEvent(keyboard_status[id], keyboard_finger_id[id]);
         }
     }
+}
 
+bool Controller_Gesture::ShouldUpdateGesture(const GestureProperties& gesture,
+                                             f32 time_difference) {
+    const auto& last_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index];
+    if (force_update) {
+        force_update = false;
+        return true;
+    }
+
+    // Update if coordinates change
+    for (size_t id = 0; id < MAX_POINTS; id++) {
+        if (gesture.points[id].x != last_gesture.points[id].x ||
+            gesture.points[id].y != last_gesture.points[id].y) {
+            return true;
+        }
+    }
+
+    // Update on press and hold event after 0.5 seconds
+    if (last_entry.type == TouchType::Touch && last_entry.point_count == 1 &&
+        time_difference > press_delay) {
+        return enable_press_and_tap;
+    }
+
+    return false;
+}
+
+void Controller_Gesture::UpdateGestureSharedMemory(u8* data, std::size_t size,
+                                                   GestureProperties& gesture,
+                                                   f32 time_difference) {
     TouchType type = TouchType::Idle;
     Attribute attributes{};
-    GestureProperties gesture = GetGestureProperties();
+
-    if (last_gesture.active_points != gesture.active_points) {
+    const auto& last_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index];
-        ++last_gesture.detection_count;
+    shared_memory.header.last_entry_index = (shared_memory.header.last_entry_index + 1) % 17;
-    }
+    auto& cur_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index];
-    if (gesture.active_points > 0) {
+
-        if (last_gesture.active_points == 0) {
+    if (shared_memory.header.entry_count < 16) {
-            attributes.is_new_touch.Assign(true);
+        shared_memory.header.entry_count++;
-            last_gesture.average_distance = gesture.average_distance;
-            last_gesture.angle = gesture.angle;
     }
 
-        type = TouchType::Touch;
+    cur_entry.sampling_number = last_entry.sampling_number + 1;
-        if (gesture.mid_point.x != last_entry.x || gesture.mid_point.y != last_entry.y) {
+    cur_entry.sampling_number2 = cur_entry.sampling_number;
-            type = TouchType::Pan;
-        }
-        if (std::abs(gesture.average_distance - last_gesture.average_distance) > pinch_threshold) {
-            type = TouchType::Pinch;
-        }
-        if (std::abs(gesture.angle - last_gesture.angle) > angle_threshold) {
-            type = TouchType::Rotate;
-        }
 
-        cur_entry.delta_x = gesture.mid_point.x - last_entry.x;
+    // Reset values to default
-        cur_entry.delta_y = gesture.mid_point.y - last_entry.y;
-        // TODO: Find how velocities are calculated
-        cur_entry.vel_x = static_cast<float>(cur_entry.delta_x) * 150.1f;
-        cur_entry.vel_y = static_cast<float>(cur_entry.delta_y) * 150.1f;
-
-        // Slowdown the rate of change for less flapping
-        last_gesture.average_distance =
-            (last_gesture.average_distance * 0.9f) + (gesture.average_distance * 0.1f);
-        last_gesture.angle = (last_gesture.angle * 0.9f) + (gesture.angle * 0.1f);
-
-    } else {
     cur_entry.delta_x = 0;
     cur_entry.delta_y = 0;
     cur_entry.vel_x = 0;
     cur_entry.vel_y = 0;
+    cur_entry.direction = Direction::None;
+    cur_entry.rotation_angle = 0;
+    cur_entry.scale = 0;
+
+    if (gesture.active_points > 0) {
+        if (last_gesture.active_points == 0) {
+            NewGesture(gesture, type, attributes);
+        } else {
+            UpdateExistingGesture(gesture, type, time_difference);
         }
-    last_gesture.active_points = gesture.active_points;
+    } else {
-    cur_entry.detection_count = last_gesture.detection_count;
+        EndGesture(gesture, last_gesture, type, attributes, time_difference);
+    }
+
+    // Apply attributes
+    cur_entry.detection_count = gesture.detection_count;
     cur_entry.type = type;
     cur_entry.attributes = attributes;
     cur_entry.x = gesture.mid_point.x;
@@ -116,12 +153,190 @@ void Controller_Gesture::OnUpdate(const Core::Timing::CoreTiming& core_timing, u
         cur_entry.points[id].x = gesture.points[id].x;
         cur_entry.points[id].y = gesture.points[id].y;
     }
-    cur_entry.rotation_angle = 0;
+    last_gesture = gesture;
-    cur_entry.scale = 0;
 
     std::memcpy(data + SHARED_MEMORY_OFFSET, &shared_memory, sizeof(SharedMemory));
 }
 
+void Controller_Gesture::NewGesture(GestureProperties& gesture, TouchType& type,
+                                    Attribute& attributes) {
+    const auto& last_entry =
+        shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17];
+    gesture.detection_count++;
+    type = TouchType::Touch;
+
+    // New touch after cancel is not considered new
+    if (last_entry.type != TouchType::Cancel) {
+        attributes.is_new_touch.Assign(1);
+        enable_press_and_tap = true;
+    }
+}
+
+void Controller_Gesture::UpdateExistingGesture(GestureProperties& gesture, TouchType& type,
+                                               f32 time_difference) {
+    const auto& last_entry =
+        shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17];
+
+    // Promote to pan type if touch moved
+    for (size_t id = 0; id < MAX_POINTS; id++) {
+        if (gesture.points[id].x != last_gesture.points[id].x ||
+            gesture.points[id].y != last_gesture.points[id].y) {
+            type = TouchType::Pan;
+            break;
+        }
+    }
+
+    // Number of fingers changed cancel the last event and clear data
+    if (gesture.active_points != last_gesture.active_points) {
+        type = TouchType::Cancel;
+        enable_press_and_tap = false;
+        gesture.active_points = 0;
+        gesture.mid_point = {};
+        for (size_t id = 0; id < MAX_POINTS; id++) {
+            gesture.points[id].x = 0;
+            gesture.points[id].y = 0;
+        }
+        return;
+    }
+
+    // Calculate extra parameters of panning
+    if (type == TouchType::Pan) {
+        UpdatePanEvent(gesture, last_gesture, type, time_difference);
+        return;
+    }
+
+    // Promote to press type
+    if (last_entry.type == TouchType::Touch) {
+        type = TouchType::Press;
+    }
+}
+
+void Controller_Gesture::EndGesture(GestureProperties& gesture, GestureProperties& last_gesture,
+                                    TouchType& type, Attribute& attributes, f32 time_difference) {
+    const auto& last_entry =
+        shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17];
+    if (last_gesture.active_points != 0) {
+        switch (last_entry.type) {
+        case TouchType::Touch:
+            if (enable_press_and_tap) {
+                SetTapEvent(gesture, last_gesture, type, attributes);
+                return;
+            }
+            type = TouchType::Cancel;
+            force_update = true;
+            break;
+        case TouchType::Press:
+        case TouchType::Tap:
+        case TouchType::Swipe:
+        case TouchType::Pinch:
+        case TouchType::Rotate:
+            type = TouchType::Complete;
+            force_update = true;
+            break;
+        case TouchType::Pan:
+            EndPanEvent(gesture, last_gesture, type, time_difference);
+            break;
+        default:
+            break;
+        }
+        return;
+    }
+    if (last_entry.type == TouchType::Complete || last_entry.type == TouchType::Cancel) {
+        gesture.detection_count++;
+    }
+}
+
+void Controller_Gesture::SetTapEvent(GestureProperties& gesture, GestureProperties& last_gesture,
+                                     TouchType& type, Attribute& attributes) {
+    type = TouchType::Tap;
+    gesture = last_gesture;
+    force_update = true;
+    f32 tap_time_difference =
+        static_cast<f32>(last_update_timestamp - last_tap_timestamp) / (1000 * 1000 * 1000);
+    last_tap_timestamp = last_update_timestamp;
+    if (tap_time_difference < double_tap_delay) {
+        attributes.is_double_tap.Assign(1);
+    }
+}
+
+void Controller_Gesture::UpdatePanEvent(GestureProperties& gesture, GestureProperties& last_gesture,
+                                        TouchType& type, f32 time_difference) {
+    auto& cur_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index];
+    const auto& last_entry =
+        shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17];
+    cur_entry.delta_x = gesture.mid_point.x - last_entry.x;
+    cur_entry.delta_y = gesture.mid_point.y - last_entry.y;
+
+    cur_entry.vel_x = static_cast<f32>(cur_entry.delta_x) / time_difference;
+    cur_entry.vel_y = static_cast<f32>(cur_entry.delta_y) / time_difference;
+    last_pan_time_difference = time_difference;
+
+    // Promote to pinch type
+    if (std::abs(gesture.average_distance - last_gesture.average_distance) > pinch_threshold) {
+        type = TouchType::Pinch;
+        cur_entry.scale = gesture.average_distance / last_gesture.average_distance;
+    }
+
+    const f32 angle_between_two_lines = std::atan((gesture.angle - last_gesture.angle) /
+                                                  (1 + (gesture.angle * last_gesture.angle)));
+    // Promote to rotate type
+    if (std::abs(angle_between_two_lines) > angle_threshold) {
+        type = TouchType::Rotate;
+        cur_entry.scale = 0;
+        cur_entry.rotation_angle = angle_between_two_lines * 180.0f / Common::PI;
+    }
+}
+
+void Controller_Gesture::EndPanEvent(GestureProperties& gesture, GestureProperties& last_gesture,
+                                     TouchType& type, f32 time_difference) {
+    auto& cur_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index];
+    const auto& last_entry =
+        shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17];
+    cur_entry.vel_x =
+        static_cast<f32>(last_entry.delta_x) / (last_pan_time_difference + time_difference);
+    cur_entry.vel_y =
+        static_cast<f32>(last_entry.delta_y) / (last_pan_time_difference + time_difference);
+    const f32 curr_vel =
+        std::sqrt((cur_entry.vel_x * cur_entry.vel_x) + (cur_entry.vel_y * cur_entry.vel_y));
+
+    // Set swipe event with parameters
+    if (curr_vel > swipe_threshold) {
+        SetSwipeEvent(gesture, last_gesture, type);
+        return;
+    }
+
+    // End panning without swipe
+    type = TouchType::Complete;
+    cur_entry.vel_x = 0;
+    cur_entry.vel_y = 0;
+    force_update = true;
+}
+
+void Controller_Gesture::SetSwipeEvent(GestureProperties& gesture, GestureProperties& last_gesture,
+                                       TouchType& type) {
+    auto& cur_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index];
+    const auto& last_entry =
+        shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17];
+    type = TouchType::Swipe;
+    gesture = last_gesture;
+    force_update = true;
+    cur_entry.delta_x = last_entry.delta_x;
+    cur_entry.delta_y = last_entry.delta_y;
+    if (std::abs(cur_entry.delta_x) > std::abs(cur_entry.delta_y)) {
+        if (cur_entry.delta_x > 0) {
+            cur_entry.direction = Direction::Right;
+            return;
+        }
+        cur_entry.direction = Direction::Left;
+        return;
+    }
+    if (cur_entry.delta_y > 0) {
+        cur_entry.direction = Direction::Down;
+        return;
+    }
+    cur_entry.direction = Direction::Up;
+}
+
 void Controller_Gesture::OnLoadInputDevices() {
     touch_mouse_device = Input::CreateDevice<Input::TouchDevice>("engine:emu_window");
     touch_udp_device = Input::CreateDevice<Input::TouchDevice>("engine:cemuhookudp");
@@ -183,23 +398,33 @@ Controller_Gesture::GestureProperties Controller_Gesture::GetGestureProperties()
 
     for (size_t id = 0; id < gesture.active_points; ++id) {
         gesture.points[id].x =
-            static_cast<int>(active_fingers[id].x * Layout::ScreenUndocked::Width);
+            static_cast<s32>(active_fingers[id].x * Layout::ScreenUndocked::Width);
         gesture.points[id].y =
-            static_cast<int>(active_fingers[id].y * Layout::ScreenUndocked::Height);
+            static_cast<s32>(active_fingers[id].y * Layout::ScreenUndocked::Height);
-        gesture.mid_point.x += static_cast<int>(gesture.points[id].x / gesture.active_points);
+
-        gesture.mid_point.y += static_cast<int>(gesture.points[id].y / gesture.active_points);
+        // Hack: There is no touch in docked but games still allow it
+        if (Settings::values.use_docked_mode.GetValue()) {
+            gesture.points[id].x =
+                static_cast<s32>(active_fingers[id].x * Layout::ScreenDocked::Width);
+            gesture.points[id].y =
+                static_cast<s32>(active_fingers[id].y * Layout::ScreenDocked::Height);
+        }
+
+        gesture.mid_point.x += static_cast<s32>(gesture.points[id].x / gesture.active_points);
+        gesture.mid_point.y += static_cast<s32>(gesture.points[id].y / gesture.active_points);
     }
 
     for (size_t id = 0; id < gesture.active_points; ++id) {
-        const double distance =
+        const f32 distance = std::sqrt(Square(gesture.mid_point.x - gesture.points[id].x) +
-            std::pow(static_cast<float>(gesture.mid_point.x - gesture.points[id].x), 2) +
+                                       Square(gesture.mid_point.y - gesture.points[id].y));
-            std::pow(static_cast<float>(gesture.mid_point.y - gesture.points[id].y), 2);
+        gesture.average_distance += distance / static_cast<f32>(gesture.active_points);
-        gesture.average_distance +=
-            static_cast<float>(distance) / static_cast<float>(gesture.active_points);
     }
 
-    gesture.angle = std::atan2(static_cast<float>(gesture.mid_point.y - gesture.points[0].y),
+    gesture.angle = std::atan2(static_cast<f32>(gesture.mid_point.y - gesture.points[0].y),
-                               static_cast<float>(gesture.mid_point.x - gesture.points[0].x));
+                               static_cast<f32>(gesture.mid_point.x - gesture.points[0].x));
+
+    gesture.detection_count = last_gesture.detection_count;
+
     return gesture;
 }
 

src/core/hle/service/hid/controllers/gesture.h

@@ -1,4 +1,4 @@
-// Copyright 2018 yuzu emulator team
+// Copyright 2021 yuzu Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -7,7 +7,6 @@
 #include <array>
 #include "common/bit_field.h"
 #include "common/common_types.h"
-#include "common/swap.h"
 #include "core/frontend/input.h"
 #include "core/hle/service/hid/controllers/controller_base.h"
 
@@ -35,10 +34,10 @@ private:
 
     enum class TouchType : u32 {
         Idle,     // Nothing touching the screen
-        Complete, // Unknown. End of touch?
+        Complete, // Set at the end of a touch event
-        Cancel,   // Never triggered
+        Cancel,   // Set when the number of fingers change
-        Touch,    // Pressing without movement
+        Touch,    // A finger just touched the screen
-        Press,    // Never triggered
+        Press,    // Set if last type is touch and the finger hasn't moved
         Tap,      // Fast press then release
         Pan,      // All points moving together across the screen
         Swipe,    // Fast press movement and release of a single point
@@ -58,8 +57,8 @@ private:
         union {
             u32_le raw{};
 
-            BitField<0, 1, u32> is_new_touch;
+            BitField<4, 1, u32> is_new_touch;
-            BitField<1, 1, u32> is_double_tap;
+            BitField<8, 1, u32> is_double_tap;
         };
     };
     static_assert(sizeof(Attribute) == 4, "Attribute is an invalid size");
@@ -73,10 +72,9 @@ private:
     struct GestureState {
         s64_le sampling_number;
         s64_le sampling_number2;
-
         s64_le detection_count;
         TouchType type;
-        Direction dir;
+        Direction direction;
         s32_le x;
         s32_le y;
         s32_le delta_x;
@@ -84,8 +82,8 @@ private:
         f32 vel_x;
         f32 vel_y;
         Attribute attributes;
-        u32 scale;
+        f32 scale;
-        u32 rotation_angle;
+        f32 rotation_angle;
         s32_le point_count;
         std::array<Points, 4> points;
     };
@@ -109,10 +107,46 @@ private:
         Points mid_point{};
         s64_le detection_count{};
         u64_le delta_time{};
-        float average_distance{};
+        f32 average_distance{};
-        float angle{};
+        f32 angle{};
     };
 
+    // Reads input from all available input engines
+    void ReadTouchInput();
+
+    // Returns true if gesture state needs to be updated
+    bool ShouldUpdateGesture(const GestureProperties& gesture, f32 time_difference);
+
+    // Updates the shared memory to the next state
+    void UpdateGestureSharedMemory(u8* data, std::size_t size, GestureProperties& gesture,
+                                   f32 time_difference);
+
+    // Initializes new gesture
+    void NewGesture(GestureProperties& gesture, TouchType& type, Attribute& attributes);
+
+    // Updates existing gesture state
+    void UpdateExistingGesture(GestureProperties& gesture, TouchType& type, f32 time_difference);
+
+    // Terminates exiting gesture
+    void EndGesture(GestureProperties& gesture, GestureProperties& last_gesture, TouchType& type,
+                    Attribute& attributes, f32 time_difference);
+
+    // Set current event to a tap event
+    void SetTapEvent(GestureProperties& gesture, GestureProperties& last_gesture, TouchType& type,
+                     Attribute& attributes);
+
+    // Calculates and set the extra parameters related to a pan event
+    void UpdatePanEvent(GestureProperties& gesture, GestureProperties& last_gesture,
+                        TouchType& type, f32 time_difference);
+
+    // Terminates the pan event
+    void EndPanEvent(GestureProperties& gesture, GestureProperties& last_gesture, TouchType& type,
+                     f32 time_difference);
+
+    // Set current event to a swipe event
+    void SetSwipeEvent(GestureProperties& gesture, GestureProperties& last_gesture,
+                       TouchType& type);
+
     // Returns an unused finger id, if there is no fingers avaliable MAX_FINGERS will be returned
     std::optional<size_t> GetUnusedFingerID() const;
 
@@ -134,6 +168,11 @@ private:
     std::array<size_t, MAX_FINGERS> keyboard_finger_id;
     std::array<size_t, MAX_FINGERS> udp_finger_id;
     std::array<Finger, MAX_POINTS> fingers;
-    GestureProperties last_gesture;
+    GestureProperties last_gesture{};
+    s64_le last_update_timestamp{};
+    s64_le last_tap_timestamp{};
+    f32 last_pan_time_difference{};
+    bool force_update{false};
+    bool enable_press_and_tap{false};
 };
 } // namespace Service::HID