fix keyboard support - works for most if not all keys present in the current virtual keyboard layout

capybara

@@ -2,6 +2,7 @@
 
 from pynput.mouse import Button, Controller
 from pynput.keyboard import Controller as KeyboardController
+from pynput.keyboard import Key
 from sanic import Sanic
 from sanic.response import file
 
@@ -47,23 +48,26 @@ class MessageParser():
         return message_code, decoded_arguments
 
 
+button_code_lookup = [
+    Button.left,
+    Button.right
+]
+
+keyboard_lookup = {
+    "{space}": Key.space,
+    "{ent}": Key.enter,
+    "{backspace}": Key.backspace
+}
+
 class InputController():
     def __init__(self):
         self.mouse_controller = Controller()
         self.keyboard_controller = KeyboardController()
-        self.button_code_lookup = [
-            Button.left,
-            Button.right
-        ]
         self.parser = MessageParser()
 
-        # Keyboard key down
+        # Keyboard key press
         self.parser.add_handler("k", {
-            "char": "char"
+            "key": "str"
-        })
-        # Keyboard key up
-        self.parser.add_handler("z", {
-            "char": "char"
         })
         # Relative mouse movement
         self.parser.add_handler("r", {
@@ -91,9 +95,18 @@ class InputController():
         # HACK
         obj = None
         try:
-            obj = self.button_code_lookup[button_code]
+            obj = button_code_lookup[button_code]
         except IndexError:
-            return self.button_code_lookup[0]
+            return button_code_lookup[0]
+        return obj
+    def deserialize_key(self, key: str):
+        obj = None
+        try:
+            obj = keyboard_lookup[key]
+        except KeyError:
+            if len(key) != 1:
+                return None
+            return key
         return obj
     def process_message(self, message: str) -> bool:
         code, args = self.parser.parse(message)
@@ -117,11 +130,9 @@ class InputController():
             print("s", args["x"], args["y"])
             self.mouse_controller.scroll(args["x"], args["y"])
         elif code == "k":
-            print("k", args["char"])
+            key = self.deserialize_key(args["key"])
-            self.keyboard_controller.press(args["char"])
+            if key:
-        elif code == "z":
+                self.keyboard_controller.tap(key)
-            print("z", args["char"])
-            self.keyboard_controller.release(args["char"])
         else:
             print("got invalid code from parser (is this a bug with the MessageParser?)")
             return False

frontend/src/Keyboard.js

@@ -6,11 +6,7 @@ class KeyboardController {
         this.keyboard = null;
     }
 
-    _sendLetterPress(l) {
+    _sendKeyPress(l) {
-        this.connection.sendMessage("k", [l]);
-    }
-
-    _sendLetterRelease(l) {
         this.connection.sendMessage("k", [l]);
     }
 
@@ -60,11 +56,7 @@ class KeyboardController {
         if (button === "{numbers}" || button === "{abc}")
             return this.handleNumbers();
         
-        this._sendLetterPress(button);
+        this._sendKeyPress(button);
-    }
-
-    onKeyReleased(button) {
-        this._sendLetterRelease(button);
     }
 
     handleShift() {