switch to webpack and add partial keyboard support

This commit is contained in:
hippoz 2021-11-03 04:21:58 +02:00
parent 04664e5f9b
commit eee7bfa1ba
No known key found for this signature in database
GPG key ID: 7C52899193467641
12 changed files with 2107 additions and 326 deletions

2
.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
frontend/node_modules/
frontend/dist/

View file

@ -1,6 +1,7 @@
#!/usr/bin/python3
from pynput.mouse import Button, Controller
from pynput.keyboard import Controller as KeyboardController
from sanic import Sanic
from sanic.response import file
@ -36,6 +37,10 @@ class MessageParser():
return None, "Error parsing float argument for message (is it really a float?)"
elif argument_type == "str":
decoded_arguments[argument_name] = message_arguments[i]
elif argument_type == "char":
if len(message_arguments[i]) != 1:
return None, "Error parsing char argument due to invalid size"
decoded_arguments[argument_name] = message_arguments[i]
else:
raise ValueError("parse(): Message handler references an invalid argument type")
@ -45,12 +50,21 @@ class MessageParser():
class InputController():
def __init__(self):
self.mouse_controller = Controller()
self.keyboard_controller = KeyboardController()
self.button_code_lookup = [
Button.left,
Button.right
]
self.parser = MessageParser()
# Keyboard key down
self.parser.add_handler("k", {
"char": "char"
})
# Keyboard key up
self.parser.add_handler("z", {
"char": "char"
})
# Relative mouse movement
self.parser.add_handler("r", {
"x": "int",
@ -102,6 +116,12 @@ class InputController():
elif code == "s":
print("s", args["x"], args["y"])
self.mouse_controller.scroll(args["x"], args["y"])
elif code == "k":
print("k", args["char"])
self.keyboard_controller.press(args["char"])
elif code == "z":
print("z", args["char"])
self.keyboard_controller.release(args["char"])
else:
print("got invalid code from parser (is this a bug with the MessageParser?)")
return False
@ -112,7 +132,7 @@ class InputController():
app = Sanic("capybara")
app.ctx.input_controller = InputController()
app.static("/", "./public/index.html")
app.static("app", "frontend/dist/", resource_type="dir")
@app.websocket("/gateway")
async def gateway(req, ws):

21
frontend/package.json Normal file
View file

@ -0,0 +1,21 @@
{
"name": "capybara-frontend",
"version": "1.0.0",
"private": true,
"license": "MIT",
"scripts": {
"watch": "webpack --watch",
"build": "webpack"
},
"devDependencies": {
"css-loader": "^6.5.0",
"css-minimizer-webpack-plugin": "^3.1.1",
"html-webpack-plugin": "^5.5.0",
"mini-css-extract-plugin": "^2.4.3",
"webpack": "^5.61.0",
"webpack-cli": "^4.9.1"
},
"dependencies": {
"simple-keyboard": "^3.3.16"
}
}

View file

@ -0,0 +1,41 @@
import Logger from "./common/Logger";
class Connection {
constructor(url) {
this.ws = null;
this.log = Logger(["Connection"], ["log"]).log;
this.messageLog = Logger(["Connection", "Message"], ["log"]).log;
this.url = url;
}
connect() {
this.ws = new WebSocket(this.url);
this.ws.onerror = (e) => this.log("Error", e);
this.ws.onopen = () => {
this.log("Open");
};
this.ws.onclose = () => {
this.log("Closed - attempting to reconnect in 4000ms");
setTimeout(() => this.connect(), 4000);
}
}
sendMessage(code, params=[]) {
let message = code;
params.forEach((param, i) => {
if (i == params.length - 1)
message += param;
else
message += param + ";";
});
this.ws.send(message);
return message;
}
disconnect() {
this.ws.close();
}
}
export default Connection;

83
frontend/src/Keyboard.js Normal file
View file

@ -0,0 +1,83 @@
import SimpleKeyboard from "simple-keyboard";
class KeyboardController {
constructor(connection) {
this.connection = connection;
this.keyboard = null;
}
_sendLetterPress(l) {
this.connection.sendMessage("k", [l]);
}
_sendLetterRelease(l) {
this.connection.sendMessage("k", [l]);
}
bindTo(element) {
this.keyboard = new SimpleKeyboard(element, {
onKeyPress: this.onKeyPress.bind(this),
mergeDisplay: true,
layoutName: "default",
// refer to: https://hodgef.com/simple-keyboard/demos/?d=mobile
layout: {
default: [
"q w e r t y u i o p",
"a s d f g h j k l",
"{shift} z x c v b n m {backspace}",
"{numbers} {space} {ent}"
],
shift: [
"Q W E R T Y U I O P",
"A S D F G H J K L",
"{shift} Z X C V B N M {backspace}",
"{numbers} {space} {ent}"
],
numbers: ["1 2 3", "4 5 6", "7 8 9", "{abc} 0 {backspace}"]
},
display: {
"{numbers}": "123",
"{ent}": "return",
"{escape}": "esc ⎋",
"{tab}": "tab ⇥",
"{backspace}": "⌫",
"{capslock}": "caps lock ⇪",
"{shift}": "⇧",
"{controlleft}": "ctrl ⌃",
"{controlright}": "ctrl ⌃",
"{altleft}": "alt ⌥",
"{altright}": "alt ⌥",
"{metaleft}": "cmd ⌘",
"{metaright}": "cmd ⌘",
"{abc}": "ABC"
}
});
}
onKeyPress(button) {
if (button === "{shift}" || button === "{lock}")
return this.handleShift();
if (button === "{numbers}" || button === "{abc}")
return this.handleNumbers();
this._sendLetterPress(button);
}
onKeyReleased(button) {
this._sendLetterRelease(button);
}
handleShift() {
this.keyboard.setOptions({
layoutName: this.keyboard.options.layoutName === "default" ? "shift" : "default"
});
}
handleNumbers() {
this.keyboard.setOptions({
layoutName: this.keyboard.options.layoutName !== "numbers" ? "numbers" : "default"
});
}
}
export default KeyboardController;

180
frontend/src/Touchpad.js Normal file
View file

@ -0,0 +1,180 @@
const HOLDING_THRESHOLD_MS = 300;
const SCROLL_X_DAMPENING = 0.03;
const SCROLL_Y_DAMPENING = 0.09;
class TouchpadController {
constructor(connection) {
this.currentMoveX = 0;
this.currentMoveY = 0;
this.lastMoveX = 0;
this.lastMoveY = 0;
this.shouldResetLastMove = false;
this.isInHoldingMode = false;
this.ongoingTouches = {};
this.holdModeTimeout = null;
this.connection = connection;
}
getButtonCode(button) {
let buttonCode = 0;
if (button === "right") buttonCode = 1;
return buttonCode;
}
// takes a gesture name and a touch list
// adds the gesture name to the touch and returns false if that touch already had that gesture
_gesture(gestureName, touchList) {
for (let i = 0; i < touchList.length; i++) {
const touch = this.ongoingTouches[touchList[i].identifier];
if (touch.gestured.includes(gestureName)) return false;
touch.gestured.push(gestureName);
}
return true;
}
_sendRelativeMouseMovement(dx, dy) {
if (dx === 0 && dy === 0)
return false;
this.connection.sendMessage("r", [dx, dy]);
return true;
}
_sendRelativeMouseScroll(dx, dy) {
if (dx === 0 && dy === 0)
return false;
this.connection.sendMessage("s", [dx, dy]);
return true;
}
_sendMouseButtonDown(button="left") {
this.connection.sendMessage("d", [this.getButtonCode(button)]);
}
_sendMouseButtonUp(button="left") {
this.connection.sendMessage("u", [this.getButtonCode(button)]);
}
_sendSingleClick(button="left") {
this.connection.sendMessage("c", [this.getButtonCode(button)]);
}
bindTo(element) {
element.addEventListener("touchmove", this.onTouchMove.bind(this));
element.addEventListener("touchend", this.onTouchEnd.bind(this));
element.addEventListener("touchstart", this.onTouchStart.bind(this));
}
onTouchMove(event) {
const touches = event.changedTouches;
event.preventDefault();
event.stopPropagation();
const targetTouch = touches[0];
this.currentMoveX = targetTouch.pageX;
this.currentMoveY = targetTouch.pageY;
// When ending a touch and starting a new one in another part of the touchpad,
// the cursor "rubber bands" to that position.
// To solve this, the "last move" parameters are reset when a touch is ended
// (see onTouchEnd())
if (this.shouldResetLastMove) {
this.shouldResetLastMove = false;
this.lastMoveX = this.currentMoveX;
this.lastMoveY = this.currentMoveY;
}
const deltaX = this.currentMoveX - this.lastMoveX;
const deltaY = this.currentMoveY - this.lastMoveY;
this.lastMoveX = this.currentMoveX;
this.lastMoveY = this.currentMoveY;
// if two touches moved at the same time, assume scrolling intent
// if _sendRelativeMouseScroll or _sendRelativeMouseMovement return true, it means that the delta values are non-zero and a packet has been sent to the server
// in that case, the touches will be marked as moved
let shouldMarkTouchesAsMoved = false;
if (touches.length === 2) {
if (
this._sendRelativeMouseScroll(deltaX * SCROLL_X_DAMPENING, deltaY * SCROLL_Y_DAMPENING)
) {
shouldMarkTouchesAsMoved = true;
}
} else if (touches.length === 3 && this._gesture("GESTURE_RIGHT_CLICK", touches)) {
shouldMarkTouchesAsMoved = true;
this._sendSingleClick("right");
} else {
if (
this._sendRelativeMouseMovement(deltaX, deltaY)
) {
shouldMarkTouchesAsMoved = true;
}
}
if (shouldMarkTouchesAsMoved) {
for (let i = 0; i < touches.length; i++) {
this.ongoingTouches[touches[i].identifier].hasMoved = true;
}
}
}
onTouchEnd(event) {
const changedTouches = event.changedTouches;
event.preventDefault();
event.stopPropagation();
this.shouldResetLastMove = true;
if (changedTouches.length === 1) {
// This is a single tap - left click
if (!this.ongoingTouches[changedTouches[0].identifier].hasMoved) {
this._sendSingleClick("left");
// We were in "holding mode" and now that touch event has ended,
// thus we have to stop holding the left click button
} else if (this.isInHoldingMode) {
this._sendMouseButtonUp("left");
this.isInHoldingMode = false;
}
}
// remove all ended touches
for (let i = 0; i < changedTouches.length; i++) {
const touch = changedTouches[i];
this.ongoingTouches[touch.identifier] = null;
delete this.ongoingTouches[touch.identifier];
}
}
onTouchStart(event) {
const changedTouches = event.changedTouches;
event.preventDefault();
event.stopPropagation();
// Clear the hold mode time out if another touch begins
if (this.holdModeTimeout)
clearTimeout(this.holdModeTimeout);
// If the touch is still unmoved and held for a certain amount of time,
// we will enter "holding mode" which keeps the mouse button held while dragging
// allowing you to, for example, select text or drag a scrollbar
if (changedTouches.length === 1) {
const targetTouch = changedTouches[0];
this.holdModeTimeout = setTimeout(() => {
if (this.ongoingTouches[targetTouch.identifier] && !this.ongoingTouches[targetTouch.identifier].hasMoved) {
this.isInHoldingMode = true;
this._sendMouseButtonDown("left");
}
}, HOLDING_THRESHOLD_MS);
}
for (let i = 0; i < changedTouches.length; i++) {
const touch = changedTouches[i];
this.ongoingTouches[touch.identifier] = {
identifier: touch.identifier,
clientX: touch.clientX,
clientY: touch.clientY,
hasMoved: false,
gestured: []
};
}
}
}
export default TouchpadController;

View file

@ -0,0 +1,53 @@
const loggerOfType = (components, type='log') => (...args) => {
let str = '%c';
const style = 'color: #5e81ac; font-weight: bold;';
for (const i in components) {
const v = components[i];
if (components[i+1] === undefined) {
str += `[${v}]`;
} else {
str += `[${v}] `;
}
}
switch (type) {
case 'log': {
console.log(str, style, ...args);
break;
}
case 'error': {
console.error(str, style, ...args);
break;
}
case 'warn': {
console.warn(str, style, ...args);
break;
}
case 'genmsg': {
return str;
}
default: {
return str;
}
}
};
function Logger(components, types=['warn', 'error', 'log']) {
const loggerObj = {};
for (const type of types) {
loggerObj[type] = loggerOfType(components, type);
}
return loggerObj;
}
export function domLog(message) {
document.body.appendChild(document.createTextNode(message));
}
export default Logger;

28
frontend/src/index.js Normal file
View file

@ -0,0 +1,28 @@
import { domLog } from "./common/Logger";
import Connection from "./Connection";
import KeyboardController from "./Keyboard";
import TouchpadController from "./Touchpad";
import "simple-keyboard/build/css/index.css";
import "./styles/main.css";
window.onerror = (error) => {
domLog(`|| ERROR: ${error} ||`);
};
const touchpadDiv = document.createElement("div");
touchpadDiv.classList.add("touchpad");
document.body.appendChild(touchpadDiv);
const keyboardDiv = document.createElement("div");
keyboardDiv.classList.add("keyboard");
document.body.appendChild(keyboardDiv);
const connection = new Connection(`ws://${location.host}/gateway`);
const controller = new TouchpadController(connection);
const keyboardController = new KeyboardController(connection);
connection.connect();
controller.bindTo(touchpadDiv);
keyboardController.bindTo(keyboardDiv);

View file

@ -0,0 +1,53 @@
:root {
--body-bg-color: #2e2e2e;
--accent-bg-color: #d4d3d3;
--accent-color: #949494;
--grayed-text-color: #949494;
--button-accent-color: #3d3d3d;
--selected-bg-color: #2e2e2e;
--hover-bg-color: #4d4d4d;
--card-border-radius: 1rem;
--button-border-radius: 0.5rem;
--main-font-weight: 500;
--fonts-regular: "Noto Sans", "Liberation Sans", sans-serif;
}
body {
font-weight: var(--main-font-weight);
font-family: var(--fonts-regular);
background-color: var(--body-bg-color);
color: var(--accent-bg-color);
overflow-x: none;
margin: 0;
padding: 8px;
}
.touchpad {
height: clamp(5rem, 30rem, 55vh);
width: 100%;
background-color: var(--accent-bg-color);
border-radius: var(--card-border-radius);
}
/* for virtual keyboard */
.keyboard {
color: var(--body-bg-color);
margin-top: 28px;
border-radius: var(--card-border-radius);
background-color: var(--accent-bg-color);
}
.keyboard .hg-rows {
margin: 4px;
}
.hg-button {
margin: 2px;
padding: 12px;
}
.hg-activeButton {
background: var(--accent-color) !important;
}

View file

@ -0,0 +1,47 @@
const path = require("path");
const HtmlWebpackPlugin = require("html-webpack-plugin");
const MiniCssExtractPlugin = require("mini-css-extract-plugin");
const CssMinimizerPlugin = require("css-minimizer-webpack-plugin");
module.exports = {
mode: "development",
entry: './src/index.js',
output: {
filename: '[name].[contenthash].js',
path: path.resolve(__dirname, 'dist'),
clean: true
},
plugins: [
new HtmlWebpackPlugin({
title: "Capybara",
}),
new MiniCssExtractPlugin({
filename: "[name].[contenthash].css"
}),
],
module: {
rules: [
{
test: /\.css$/i,
use: [MiniCssExtractPlugin.loader, "css-loader"],
},
],
},
optimization: {
moduleIds: 'deterministic',
runtimeChunk: 'single',
splitChunks: {
cacheGroups: {
vendor: {
test: /[\\/]node_modules[\\/]/,
name: 'vendors',
chunks: 'all',
},
},
},
minimizer: [
`...`,
new CssMinimizerPlugin(),
]
},
};

1578
frontend/yarn.lock Normal file

File diff suppressed because it is too large Load diff

View file

@ -1,325 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="user-scalable=no, width=device-width, initial-scale=1, maximum-scale=1"> <title>Document</title>
<script>
"use strict";
const domlog = message => document.body.appendChild(document.createTextNode(message));
window.onerror = e => domlog(` ERROR - ${e} `);
const HOLDING_THRESHOLD_MS = 300;
const SCROLL_X_DAMPENING = 0.03;
const SCROLL_Y_DAMPENING = 0.09;
const loggerOfType = (components, type='log') => (...args) => {
let str = '%c';
const style = 'color: #5e81ac; font-weight: bold;';
for (const i in components) {
const v = components[i];
if (components[i+1] === undefined) {
str += `[${v}]`;
} else {
str += `[${v}] `;
}
}
switch (type) {
case 'log': {
console.log(str, style, ...args);
break;
}
case 'error': {
console.error(str, style, ...args);
break;
}
case 'warn': {
console.warn(str, style, ...args);
break;
}
case 'genmsg': {
return str;
}
default: {
return str;
}
}
};
function Logger(components, types=['warn', 'error', 'log']) {
const loggerObj = {};
for (const type of types) {
loggerObj[type] = loggerOfType(components, type);
}
return loggerObj;
}
class Connection {
constructor(url) {
this.ws = null;
this.log = Logger(["Connection"], ["log"]).log;
this.messageLog = Logger(["Connection", "Message"], ["log"]).log;
this.url = url;
}
connect() {
this.ws = new WebSocket(this.url);
this.ws.onerror = (e) => this.log("Error", e);
this.ws.onopen = () => {
this.log("Open");
};
this.ws.onclose = () => {
this.log("Closed - attempting to reconnect in 4000ms");
setTimeout(() => this.connect(), 4000);
}
}
sendMessage(code, params=[]) {
let message = code;
params.forEach((param, i) => {
if (i == params.length - 1)
message += param;
else
message += param + ";";
});
this.ws.send(message);
return message;
}
disconnect() {
this.ws.close();
}
}
class TouchpadController {
constructor(connection) {
this.currentMoveX = 0;
this.currentMoveY = 0;
this.lastMoveX = 0;
this.lastMoveY = 0;
this.shouldResetLastMove = false;
this.isInHoldingMode = false;
this.ongoingTouches = {};
this.holdModeTimeout = null;
this.log = Logger(["TouchpadController"], ["log"]).log;
this.connection = connection;
}
getButtonCode(button) {
let buttonCode = 0;
if (button === "right") buttonCode = 1;
return buttonCode;
}
// takes a gesture name and a touch list
// adds the gesture name to the touch and returns false if that touch already had that gesture
_gesture(gestureName, touchList) {
for (let i = 0; i < touchList.length; i++) {
const touch = this.ongoingTouches[touchList[i].identifier];
if (touch.gestured.includes(gestureName)) return false;
touch.gestured.push(gestureName);
}
return true;
}
_sendRelativeMouseMovement(dx, dy) {
if (dx === 0 && dy === 0)
return false;
this.connection.sendMessage("r", [dx, dy]);
return true;
}
_sendRelativeMouseScroll(dx, dy) {
if (dx === 0 && dy === 0)
return false;
this.connection.sendMessage("s", [dx, dy]);
return true;
}
_sendMouseButtonDown(button="left") {
this.connection.sendMessage("d", [this.getButtonCode(button)]);
}
_sendMouseButtonUp(button="left") {
this.connection.sendMessage("u", [this.getButtonCode(button)]);
}
_sendSingleClick(button="left") {
this.connection.sendMessage("c", [this.getButtonCode(button)]);
}
bindTo(element) {
element.addEventListener("touchmove", this.onTouchMove.bind(this));
element.addEventListener("touchend", this.onTouchEnd.bind(this));
element.addEventListener("touchstart", this.onTouchStart.bind(this));
}
onTouchMove(event) {
const touches = event.changedTouches;
event.preventDefault();
event.stopPropagation();
const targetTouch = touches[0];
this.currentMoveX = targetTouch.pageX;
this.currentMoveY = targetTouch.pageY;
// When ending a touch and starting a new one in another part of the touchpad,
// the cursor "rubber bands" to that position.
// To solve this, the "last move" parameters are reset when a touch is ended
// (see onTouchEnd())
if (this.shouldResetLastMove) {
this.shouldResetLastMove = false;
this.lastMoveX = this.currentMoveX;
this.lastMoveY = this.currentMoveY;
}
const deltaX = this.currentMoveX - this.lastMoveX;
const deltaY = this.currentMoveY - this.lastMoveY;
this.lastMoveX = this.currentMoveX;
this.lastMoveY = this.currentMoveY;
// if two touches moved at the same time, assume scrolling intent
// if _sendRelativeMouseScroll or _sendRelativeMouseMovement return true, it means that the delta values are non-zero and a packet has been sent to the server
// in that case, the touches will be marked as moved
let shouldMarkTouchesAsMoved = false;
if (touches.length === 2) {
if (
this._sendRelativeMouseScroll(deltaX * SCROLL_X_DAMPENING, deltaY * SCROLL_Y_DAMPENING)
) {
shouldMarkTouchesAsMoved = true;
}
} else if (touches.length === 3 && this._gesture("GESTURE_RIGHT_CLICK", touches)) {
shouldMarkTouchesAsMoved = true;
this._sendSingleClick("right");
} else {
if (
this._sendRelativeMouseMovement(deltaX, deltaY)
) {
shouldMarkTouchesAsMoved = true;
}
}
if (shouldMarkTouchesAsMoved) {
for (let i = 0; i < touches.length; i++) {
this.ongoingTouches[touches[i].identifier].hasMoved = true;
}
}
}
onTouchEnd(event) {
const changedTouches = event.changedTouches;
event.preventDefault();
event.stopPropagation();
this.shouldResetLastMove = true;
if (changedTouches.length === 1) {
// This is a single tap - left click
if (!this.ongoingTouches[changedTouches[0].identifier].hasMoved) {
this._sendSingleClick("left");
// We were in "holding mode" and now that touch event has ended,
// thus we have to stop holding the left click button
} else if (this.isInHoldingMode) {
this._sendMouseButtonUp("left");
this.isInHoldingMode = false;
}
}
// remove all ended touches
for (let i = 0; i < changedTouches.length; i++) {
const touch = changedTouches[i];
this.ongoingTouches[touch.identifier] = null;
delete this.ongoingTouches[touch.identifier];
}
}
onTouchStart(event) {
const changedTouches = event.changedTouches;
event.preventDefault();
event.stopPropagation();
// Clear the hold mode time out if another touch begins
if (this.holdModeTimeout)
clearTimeout(this.holdModeTimeout);
// If the touch is still unmoved and held for a certain amount of time,
// we will enter "holding mode" which keeps the mouse button held while dragging
// allowing you to, for example, select text or drag a scrollbar
if (changedTouches.length === 1) {
const targetTouch = changedTouches[0];
this.holdModeTimeout = setTimeout(() => {
if (this.ongoingTouches[targetTouch.identifier] && !this.ongoingTouches[targetTouch.identifier].hasMoved) {
this.isInHoldingMode = true;
this._sendMouseButtonDown("left");
}
}, HOLDING_THRESHOLD_MS);
}
for (let i = 0; i < changedTouches.length; i++) {
const touch = changedTouches[i];
this.ongoingTouches[touch.identifier] = {
identifier: touch.identifier,
clientX: touch.clientX,
clientY: touch.clientY,
hasMoved: false,
gestured: []
};
}
}
}
const connection = new Connection(`ws://${location.host}/gateway`);
const controller = new TouchpadController(connection);
function main() {
connection.connect();
controller.bindTo(document.querySelector(".touchpad"));
}
</script>
<style>
:root {
--body-bg-color: #2e2e2e;
--accent-bg-color: #d4d3d3;
--accent-color: #949494;
--grayed-text-color: #949494;
--button-accent-color: #3d3d3d;
--selected-bg-color: #2e2e2e;
--hover-bg-color: #4d4d4d;
--card-border-radius: 1rem;
--button-border-radius: 0.5rem;
--main-font-weight: 500;
--fonts-regular: "Noto Sans", "Liberation Sans", sans-serif;
}
body {
font-weight: var(--main-font-weight);
font-family: var(--fonts-regular);
background-color: var(--body-bg-color);
color: var(--accent-bg-color);
margin: 0;
padding: 24px;
}
.touchpad {
height: 30rem;
width: 100%;
background-color: var(--accent-bg-color);
border-radius: var(--card-border-radius);
}
</style>
</head>
<body onload="main();">
<div class="touchpad"></div>
</body>
</html>