add message attachments

.gitignore

@@ -1,5 +1,5 @@
 node_modules/
 dist/
 frontend-new/
-uploads/avatar/*.webp
+uploads/
 .env

frontend/src/components/Message.svelte

@@ -1,8 +1,19 @@
 <script>
+    import { getErrorFromResponse, methods, remoteBlobUpload, responseOk } from "../request";
     import { avatarUrl } from "../storage";
-import { overlayStore, OverlayType, setMessageInputEvent } from "../stores";
+    import { overlayStore, OverlayType, setMessageInputEvent } from "../stores";
+    import MessageAttachment from "./MessageAttachment.svelte";
 
     export let message;
+    let waitingForAttachments = 0;
+    $: {
+        const attachmentCount = message.attachments ? message.attachments.length : 0;
+        const expectedCount = message.pending_attachments || 0;
+        let delta = expectedCount - attachmentCount;
+        if (delta < 0) delta = 0;
+        waitingForAttachments = delta;
+    }
+
 
     const reply = () => {
         let replyString = "";
@@ -112,6 +123,20 @@ import { overlayStore, OverlayType, setMessageInputEvent } from "../stores";
     .message:hover .message-actions {
         display: flex;
     }
+
+    .attachments-container {
+        display: flex;
+        flex-direction: column;
+        margin-top: var(--space-xxs);
+    }
+
+    .hint-text {
+        display: block;
+        margin-top: 6px;
+        color: var(--foreground-color-3);
+        font-size: var(--h6);
+        user-select: none;
+    }
 </style>
 
 <div class="message" class:clumped={ message._clumped } class:pinged={ message._mentions }>
@@ -137,6 +162,18 @@ import { overlayStore, OverlayType, setMessageInputEvent } from "../stores";
         {/if}
         <span class="message-content" class:pending={ message._isPending }>{ message.content }</span>
 
+        {#if message.attachments && message.attachments.length}
+            <div class="attachments-container">
+                {#each message.attachments as attachment(attachment.id)}
+                    <MessageAttachment attachment={attachment} />
+                {/each}
+            </div>
+        {/if}
+
+        {#if waitingForAttachments}
+            <span class="hint-text">Waiting for {waitingForAttachments} more {`attachment${waitingForAttachments !== 1 ? 's' : ''}`}</span>
+        {/if}
+
         <div class="message-actions">
             <button class="icon-button material-icons-outlined" on:click="{ reply }" aria-label="Reply to Message">
                 reply

frontend/src/components/MessageAttachment.svelte

@@ -0,0 +1,93 @@
+<script>
+    import { attachmentUrl } from "../storage";
+
+    const AttachmentRenderAs = {
+        Invalid: 0,
+        Image: 1,
+        Video: 2,
+        Audio: 3,
+        DownloadableFile: 4,
+    };
+
+    export let attachment;
+    let renderAs;
+    $: {
+        renderAs = AttachmentRenderAs.Invalid;
+        if (attachment && attachment.type === "file" && attachment.file_mime && attachment.file) {
+            const mimeParts = attachment.file_mime.split("/");
+            const type = mimeParts[0];
+            if (type) {
+                switch (type) {
+                    case "audio": {
+                        renderAs = AttachmentRenderAs.Audio;
+                        break;
+                    }
+
+                    case "video": {
+                        renderAs = AttachmentRenderAs.Video;
+                        break;
+                    }
+
+                    case "image": {
+                        renderAs = AttachmentRenderAs.Image;
+                        break;
+                    }
+
+                    default: {
+                        renderAs = AttachmentRenderAs.DownloadableFile
+                        break;
+                    }
+                }
+            }
+        }
+    }
+</script>
+
+<style>
+    .attachment-card {
+        display: flex;
+        flex-direction: row;
+        align-items: center;
+        padding: var(--space-norm);
+        border-radius: 9999px;
+        background-color: var(--background-color-0);
+    }
+
+    .attachment {
+        margin-bottom: var(--space-xs);
+    }
+
+    .media {
+        max-width: 90%;
+    }
+
+    img {
+        background-color: var(--background-color-2);
+        border-radius: 6px;
+        height: auto;
+    }
+
+    .attachment-filename {
+        color: var(--foreground-color-3);
+        margin-right: 6px;
+    }
+
+    .attachment-card .icon-button {
+        margin-left: auto;
+    }
+</style>
+
+
+{#if renderAs === AttachmentRenderAs.Image}
+    <img loading="lazy" decoding="async" width="{ attachment.width }" height="{ attachment.height }" class="attachment media" alt="Attachment" src="{ attachmentUrl(attachment.file) }">
+{:else if renderAs === AttachmentRenderAs.Video}
+    <!-- svelte-ignore a11y-media-has-caption -->
+    <video controls="controls" class="attachment media" src="{ attachmentUrl(attachment.file) }"></video>
+{:else if renderAs === AttachmentRenderAs.DownloadableFile}
+    <div class="attachment attachment-card">
+        <div class="attachment-filename">{ attachment.file_name }</div>
+        <a class="icon-button material-icons-outlined" href="{ attachmentUrl(attachment.file) }" target="_blank">download</a>
+    </div>
+{:else}
+    <div class="attachment attachment-card">Couldn't render attachment</div>
+{/if}

frontend/src/components/MessageInput.svelte

@@ -1,14 +1,24 @@
 <script>
     import { onDestroy, onMount } from "svelte";
     import { getItem } from "../storage";
-    import { overlayStore, selectedChannel, sendMessageAction, setMessageInputEvent, smallViewport, typingStore, userInfoStore, usesKeyboardNavigation } from "../stores";
+    import { messagesStoreProvider, overlayStore, selectedChannel, sendMessageAction, setMessageInputEvent, smallViewport, typingStore, userInfoStore, usesKeyboardNavigation } from "../stores";
+    import { getErrorFromResponse, methods, remoteBlobUpload, remoteCall, responseOk } from "../request";
 
     export let channel;
     let messageInput = "";
-    let messageTextarea;
     let typingList = "?no one?";
     let typingMessage = "is typing...";
 
+    let messageTextarea;
+    let fileInput;
+
+    let files = [];
+    let loadingMessage = null;
+    let showFileListStrip = false;
+    $: {
+        showFileListStrip = !!files.length || !!loadingMessage;
+    }
+
     $: {
         const typing = $typingStore.filter(a => a.channelId === channel.id);
         const ownIndex = typing.findIndex(a => a.user.id === $userInfoStore.id);
@@ -37,13 +47,88 @@
         }
     }
 
+
+    const removeFile = (id) => {
+        const index = files.findIndex(f => f.id === id);
+        if (index !== -1) {
+            files.splice(index, 1);
+            files = files;
+        }
+    }
+
+    const selectFiles = async () => {
+        if (!fileInput || !fileInput.files) return;
+
+        for (let i = 0; i < fileInput.files.length; i++) {
+            const file = fileInput.files.item(i);
+            const fileObject = {
+                file,
+                id: Math.random()
+            };
+            files.push(fileObject);
+            files = files;
+        }
+        
+        return;
+
+    };
+
+    const uploadFiles = async (messageId) => {
+        const filesToUpload = [ ...files ];
+        files.length = 0;
+        files = files;
+        if (!filesToUpload.length) return;
+
+        loadingMessage = `Uploading ${filesToUpload.length} attachment${filesToUpload.length !== 1 ? 's' : ''}`;
+
+        for (let i = 0; i < filesToUpload.length; i++) {
+            const file = filesToUpload[i].file;
+
+            const res = await remoteBlobUpload(methods.createMessageAttachment, file, [messageId, file.name]);
+            if (!responseOk(res)) {
+                const error = getErrorFromResponse(res);
+                const message = error.validationErrors && error.validationErrors.length ? error.validationErrors[0].msg : error.message;
+                overlayStore.toast(`Failed to upload file ${file.name}: ${message}`);
+                return;
+            }
+        }
+
+        loadingMessage = null;
+    };
+
     const sendMessage = async () => {
         messageTextarea.focus();
-        sendMessageAction.emit({
+        const content = messageInput;
-            channelId: channel.id,
+
-            content: messageInput
+        if (content.trim() === "" || !userInfoStore.value)
-        });
+            return;
+
+        // optimistically add message to store
+        const optimisticMessageId = Math.floor(Math.random() * 9999999);
+        const optimisticMessage = {
+            id: optimisticMessageId,
+            content: content,
+            channel_id: channel.id,
+            author_id: userInfoStore.value.id,
+            author_username: userInfoStore.value.username,
+            created_at: Date.now().toString(),
+            _isPending: true
+        };
+        const messagesStoreForChannel = messagesStoreProvider.getStore(channel.id);
+        messagesStoreForChannel.addMessage(optimisticMessage);
+
+        const res = await remoteCall(methods.createChannelMessage, channel.id, content, optimisticMessageId, null, files.length);
+
+        if (!responseOk(res)) {
+            messagesStoreForChannel.deleteMessage({
+                id: optimisticMessageId
+            });
+            overlayStore.toast(`Couldn't send message: ${getMessageFromResponse(res)}`);
+            return;
+        }
         messageInput = "";
+
+        await uploadFiles(res.data.id);
     };
 
     const onKeydown = async (e) => {
@@ -109,23 +194,31 @@
     .message-input-container.small {
         padding-top: var(--space-sm);
     }
-    
+
-    .message-input.small {
+    .input-row {
-        padding: var(--space-xsplus);
+        display: flex;
-        padding-left: var(--space-sm);
+        flex-direction: row;
-        border-radius: 1.4em;
+        background-color: var(--background-color-3);
+        border-radius: var(--radius-md);
+        contain: content;
+        width: 100%;
+        padding: var(--space-sm);
+    }
+
+    .input-row.small {
+        border-radius: 9999px;
     }
 
     .message-input {
+        margin-left: calc(0.67 * var(--space-unit));
+        flex-grow: 1;
         width: 100%;
-        background-color: var(--background-color-3);
+        background-color: transparent;
         border: none;
         color: currentColor;
-        border-radius: var(--radius-md);
+        border-radius: 0;
-        padding: var(--space-sm);
         font-size: inherit;
         resize: none;
-        contain: strict;
     }
 
     /* TODO: is this good? */
@@ -176,22 +269,79 @@
     .typing-message {
         color: var(--foreground-color-2);
     }
+
+    .strip {
+        display: flex;
+        flex-direction: row;
+        align-items: center;
+        border-top-left-radius: var(--radius-md);
+        border-top-right-radius: var(--radius-md);
+        background-color: var(--background-color-1);
+        color: var(--foreground-color-2);
+        padding: var(--space-sm);
+    }
+
+    .file-strip-file {
+        display: flex;
+        flex-direction: row;
+        background-color: var(--background-color-2);
+        padding: var(--space-xs);
+        border-radius: var(--radius-sm);
+        margin-right: var(--space-xs);
+    }
+
+    .file-strip-file span {
+        margin-right: 5px;
+    }
+
+    .file-strip-file button {
+        display: none;
+    }
+
+    .file-strip-file:hover button {
+        display: block;
+    }
+
+    .loading-message {
+        color: var(--foreground-color-3);
+        margin-left: 8px;
+    }
 </style>
 
 <div class="message-input-container" class:small={ $smallViewport }>
+    <input type="file" style="display: none;" accept="*" name="attachment-upload" multiple={false} bind:this={fileInput} on:change={selectFiles}>
+    {#if showFileListStrip}
+        <div class="strip">
+            {#if loadingMessage}
+                <div class="spinner ui"></div>
+                <div class="loading-message">{loadingMessage}</div>
+            {:else}
+                {#each files as fileObject(fileObject.id)}
+                    <div class="file-strip-file">
+                        <span>{fileObject.file.name}</span>
+                        <button class="icon-button material-icons-outlined" on:click="{ () => removeFile(fileObject.id) }" aria-label="Remove Attachment">close</button>
+                    </div>
+                {/each}
+            {/if}
+        </div>
+    {/if}
+
     <div class="inner-input-container">
-        <textarea
+        <div class="input-row" class:small={ $smallViewport }>
-            placeholder={$smallViewport ? `Message #${channel.name}` : `Send something interesting to #${channel.name}`}
+            <button class="icon-button material-icons-outlined" on:click="{ () => fileInput.click() }" aria-label="Add Attachment">add</button>
-            type="text"
+            <textarea
-            class="message-input"
+                placeholder={$smallViewport ? `Message #${channel.name}` : `Send something interesting to #${channel.name}`}
-            rows="1"
+                type="text"
-            on:keydown={ onKeydown }
+                class="message-input"
-            on:input={ onInput }
+                rows="1"
-            bind:value={ messageInput }
+                on:keydown={ onKeydown }
-            bind:this={ messageTextarea }
+                on:input={ onInput }
-            class:small={ $smallViewport || getItem("ui:alwaysUseMobileChatBar") }
+                bind:value={ messageInput }
-            class:keyboard-nav={ $usesKeyboardNavigation }
+                bind:this={ messageTextarea }
-        />
+                class:small={ $smallViewport || getItem("ui:alwaysUseMobileChatBar") }
+                class:keyboard-nav={ $usesKeyboardNavigation }
+            />
+        </div>
         {#if $smallViewport || getItem("ui:alwaysUseMobileChatBar")}
             <button class="icon-button send-button material-icons-outlined" on:click="{ sendMessage }">
                 arrow_upward

frontend/src/request.js

@@ -27,6 +27,7 @@ export const methods = {
     getCommunity: withCacheable(method(403, true)),
     getCommunities: withCacheable(method(404, true)),
     getCommunityChannels: withCacheable(method(405, true)),
+    createMessageAttachment: method(500, true),
 };
 
 export const RPCError = {
@@ -64,7 +65,7 @@ export const RequestStatusToMessage = {
 export function getErrorFromResponse(response) {
     if (!response) return;
     if (response.status === RequestStatus.OK) return;
-    console.log(response);
+    console.error("Got error response", response);
 
     let message = RequestStatusToMessage[response.status];
     if (!message) message = "Something went wrong (unknown request error)";
@@ -192,8 +193,8 @@ export async function remoteSignal(method, ...args) {
     }, ...args);
 }
 
-export async function remoteBlobUpload({methodId, requiresAuthentication, _isSignal=false}, blob) {
+export async function remoteBlobUpload({methodId, requiresAuthentication, _isSignal=false}, blob, args=[]) {
-    const calls = [[methodId, [0, blob.size]]];
+    const calls = [[methodId, ...args, [0, blob.size]]];
 
     if (requiresAuthentication && gateway.authenticated) {
         const replies = await gateway.sendRPCRequest(calls, _isSignal, blob);

frontend/src/storage.js

@@ -2,6 +2,7 @@ const defaults = {
     "server:apiBase": `${window.location.origin || ""}/api/v1`,
     "server:gatewayBase": `${location.protocol === "https:" ? "wss" : "ws"}://${location.host}/gateway`,
     "server:avatarsBase": `${window.location.origin || ""}/uploads/avatar`,
+    "server:attachmentsBase": `${window.location.origin || ""}/uploads/attachment`,
     "auth:token": "",
     "ui:doAnimations": true,
     "ui:theme": "dark",
@@ -95,3 +96,8 @@ export function apiRoute(fragment) {
 export function avatarUrl(avatarId, size) {
     return `${getItem("server:avatarsBase")}/${avatarId}_${size}.webp`;
 }
+
+export function attachmentUrl(name) {
+    return `${getItem("server:attachmentsBase")}/${name}`;
+}
+

package.json

@@ -17,6 +17,7 @@
     "dotenv": "^16.0.1",
     "express": "^4.18.1",
     "express-validator": "^6.14.2",
+    "file-type": "^18.5.0",
     "jsonwebtoken": "^8.5.1",
     "pg": "^8.8.0",
     "sharp": "^0.31.3",

src/database/init.ts

@@ -1,4 +1,4 @@
-import { query, withClient } from ".";
+import { query } from ".";
 
 export default async function databaseInit() {
     const migrations = [
@@ -43,7 +43,32 @@ export default async function databaseInit() {
         `,
         `
             ALTER TABLE channels ADD COLUMN IF NOT EXISTS community_id INT NULL REFERENCES communities(id) ON DELETE CASCADE;
+        `,
         `
+            CREATE TABLE IF NOT EXISTS message_attachments(
+                id SERIAL PRIMARY KEY,
+                type VARCHAR(64) NOT NULL,
+                owner_id SERIAL REFERENCES users ON DELETE CASCADE,
+                message_id SERIAL REFERENCES messages ON DELETE CASCADE,
+                created_at BIGINT,
+
+                file VARCHAR(256) DEFAULT NULL,
+                file_mime VARCHAR(256) DEFAULT NULL,
+                file_size_bytes BIGINT DEFAULT NULL
+            );
+        `,
+        `
+            ALTER TABLE messages ADD COLUMN IF NOT EXISTS pending_attachments INT DEFAULT NULL;
+        `,
+        `
+            ALTER TABLE message_attachments ADD COLUMN IF NOT EXISTS width INT DEFAULT NULL;
+        `,
+        `
+            ALTER TABLE message_attachments ADD COLUMN IF NOT EXISTS height INT DEFAULT NULL;
+        `,
+        `
+            ALTER TABLE message_attachments ADD COLUMN IF NOT EXISTS file_name VARCHAR(256) DEFAULT NULL;
+        `,
     ];
 
     for (let i = 0; i < migrations.length; i++) {

src/database/templates.ts

@@ -1,4 +1,41 @@
-export const getMessageById = "SELECT messages.id, messages.content, messages.channel_id, messages.created_at, messages.author_id, messages.nick_username, users.avatar AS author_avatar, users.username AS author_username FROM messages JOIN users ON messages.author_id = users.id WHERE messages.id = $1";
+export const getMessageById = `
-export const getMessagesByChannelFirstPage = (limit: number) => `SELECT messages.id, messages.content, messages.created_at, messages.author_id, messages.nick_username, users.avatar AS author_avatar, users.username AS author_username FROM messages JOIN users ON messages.author_id = users.id WHERE messages.channel_id = $1 ORDER BY id DESC LIMIT ${limit}`;
+SELECT m.id, m.content, m.channel_id, m.created_at, m.author_id, m.nick_username, m.pending_attachments, u.avatar AS author_avatar, u.username AS author_username, 
-export const getMessagesByChannelPage = (limit: number) => `SELECT messages.id, messages.content, messages.created_at, messages.author_id, messages.nick_username, users.avatar AS author_avatar, users.username AS author_username FROM messages JOIN users ON messages.author_id = users.id WHERE messages.id < $1 AND messages.channel_id = $2 ORDER BY id DESC LIMIT ${limit}`;
+    (SELECT jsonb_agg(to_jsonb(ma)) FROM message_attachments ma WHERE ma.message_id = m.id) AS attachments
-export const getMessagesByChannelAfterPage = (limit: number) => `SELECT messages.id, messages.content, messages.created_at, messages.author_id, messages.nick_username, users.avatar AS author_avatar, users.username AS author_username FROM messages JOIN users ON messages.author_id = users.id WHERE messages.id > $1 AND messages.channel_id = $2 ORDER BY id DESC LIMIT ${limit}`;
+FROM messages m
+JOIN users u ON m.author_id = u.id
+WHERE m.id = $1
+GROUP BY m.id, u.avatar, u.username;
+`;
+
+export const getMessagesByChannelFirstPage = (limit: number) => `
+SELECT m.id, m.content, m.channel_id, m.created_at, m.author_id, m.nick_username, m.pending_attachments, u.avatar AS author_avatar, u.username AS author_username, 
+    (SELECT jsonb_agg(to_jsonb(ma)) FROM message_attachments ma WHERE ma.message_id = m.id) AS attachments
+FROM messages m
+JOIN users u ON m.author_id = u.id
+WHERE m.channel_id = $1
+GROUP BY m.id, u.avatar, u.username
+ORDER BY m.id DESC
+LIMIT ${limit};
+`;
+
+export const getMessagesByChannelPage = (limit: number) => `
+SELECT m.id, m.content, m.channel_id, m.created_at, m.author_id, m.nick_username, m.pending_attachments, u.avatar AS author_avatar, u.username AS author_username, 
+    (SELECT jsonb_agg(to_jsonb(ma)) FROM message_attachments ma WHERE ma.message_id = m.id) AS attachments
+FROM messages m
+JOIN users u ON m.author_id = u.id
+WHERE m.id < $1 AND m.channel_id = $2
+GROUP BY m.id, u.avatar, u.username
+ORDER BY m.id DESC
+LIMIT ${limit};
+`;
+
+export const getMessagesByChannelAfterPage = (limit: number) => `
+SELECT m.id, m.content, m.channel_id, m.created_at, m.author_id, m.nick_username, m.pending_attachments, u.avatar AS author_avatar, u.username AS author_username, 
+    (SELECT jsonb_agg(to_jsonb(ma)) FROM message_attachments ma WHERE ma.message_id = m.id) AS attachments
+FROM messages m
+JOIN users u ON m.author_id = u.id
+WHERE m.id > $1 AND m.channel_id = $2
+GROUP BY m.id, u.avatar, u.username
+ORDER BY m.id DESC
+LIMIT ${limit};
+`;

src/impl.ts

@@ -2,12 +2,12 @@ import { query } from "./database";
 import { dispatch } from "./gateway";
 import { GatewayPayloadType } from "./gateway/gatewaypayloadtype";
 
-export default async function sendMessage(user: User, channelId: number, optimisticId: number | null, content: string, nickUsername: string | null) {
+export default async function sendMessage(user: User, channelId: number, optimisticId: number | null, content: string, nickUsername: string | null, pendingAttachments: number) {
     const authorId = user.id;
     const createdAt = Date.now().toString();
 
-    const result = await query("INSERT INTO messages(content, channel_id, author_id, created_at, nick_username) VALUES ($1, $2, $3, $4, $5) RETURNING id", [content, channelId, authorId, createdAt, nickUsername]);
+    const result = await query("INSERT INTO messages(content, channel_id, author_id, created_at, nick_username, pending_attachments) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id", [content, channelId, authorId, createdAt, nickUsername, pendingAttachments]);
-    if (!result || result.rowCount < 1) {
+    if (!result || result.rowCount !== 1) {
         return null;
     }
 
@@ -19,12 +19,14 @@ export default async function sendMessage(user: User, channelId: number, optimis
         author_username: user.username,
         author_avatar: user.avatar,
         created_at: createdAt,
-        nick_username: nickUsername
+        nick_username: nickUsername,
+        pending_attachments: pendingAttachments,
+        attachments: null
     };
 
     dispatch(`channel:${channelId}`, (ws) => {
         let payload: any = returnObject;
-        if (ws && ws.state && ws.state.user && ws.state.user.id === user.id && optimisticId) {
+        if (ws && ws.user && ws.user.id === user.id && optimisticId) {
             payload = {
                 ...payload,
                 optimistic_id: optimisticId

src/routes/matrix/index.ts

@@ -1,10 +1,8 @@
 import express from "express";
-import { body, validationResult } from "express-validator";
-import { PoolClient } from "pg";
 import { authenticateRoute, loginAttempt } from "../../auth";
-import { query, withClient } from "../../database";
+import { query } from "../../database";
 import { getMessagesByChannelAfterPage, getMessagesByChannelFirstPage } from "../../database/templates";
-import { handle, waitForEvent } from "../../gateway";
+import { waitForEvent } from "../../gateway";
 import cors from "cors";
 import sendMessage from "../../impl";
 
@@ -336,7 +334,7 @@ router.put(
                     error: "Message body must be a string between 1 and 2000 characters"
                 });
             }
-            const message = await sendMessage(req.user, channelId, null, req.body.body, null);
+            const message = await sendMessage(req.user, channelId, null, req.body.body, null, 0);
             if (!message) {
                 return res.status(500).json({
                     errcode: "M_UNKNOWN",

src/routes/uploads.ts

@@ -0,0 +1,14 @@
+import express from "express";
+import { attachmentUploadDirectory } from "../serverconfig";
+
+const router = express.Router();
+
+router.use("/avatar", express.static("uploads/avatar"));
+
+router.get("/attachment/:id", (req, res) => {
+    res.download(req.params.id, req.params.id, {
+        root: attachmentUploadDirectory
+    });
+});
+
+export default router;

src/rpc/apis/attachments.ts

@@ -0,0 +1,122 @@
+import { bufferSlice, method, string, uint } from "../rpc";
+import { query } from "../../database";
+import { errors } from "../../errors";
+import { UploadTarget, getSafeUploadPath, sanitizeFilename, supportedImageMime } from "../../uploading";
+import sharp from "sharp";
+import { randomBytes } from "node:crypto";
+import path from "node:path";
+import { promises as fsPromises } from "node:fs";
+import { getMessageById } from "../../database/templates";
+import { uploadsMode } from "../../serverconfig";
+import { dispatch } from "../../gateway";
+import { GatewayPayloadType } from "../../gateway/gatewaypayloadtype";
+
+const fileType = eval("import('file-type')");
+
+method(
+    "createMessageAttachment",
+    [uint(), string(2, 128), bufferSlice()],
+    async (user: User, messageId: number, filenameUnsafe: string, inputBuffer: Buffer) => {
+        if (inputBuffer.byteLength >= 16777220) {
+            return { ...errors.BAD_REQUEST, detail: "Uploaded file exceeds 16MiB limit." };
+        }
+
+        const messageCheckResult = await query(getMessageById, [messageId]);
+        if (!messageCheckResult || messageCheckResult.rowCount < 1) {
+            return errors.NOT_FOUND;
+        }
+        if (messageCheckResult.rows[0].author_id !== user.id && !user.is_superuser) {
+            return errors.FORBIDDEN_DUE_TO_MISSING_PERMISSIONS;
+        }
+
+        const randomId = randomBytes(60).toString("hex");
+
+        const safeFilename = sanitizeFilename(filenameUnsafe);
+        const filePath = getSafeUploadPath(UploadTarget.Attachment, `${randomId}.${safeFilename}`);
+        if (!filePath) {
+            return { ...errors.BAD_REQUEST, detail: "Invalid filename." };
+        }
+        const actualFilename = path.basename(filePath);
+
+        const filetype = await (await fileType).fileTypeFromBuffer(inputBuffer);
+
+        let bufferToUpload = inputBuffer;
+        let width = null;
+        let height = null;
+
+        if (filetype && supportedImageMime.includes(filetype.mime)) {
+            try {
+                const { data, info } = await sharp(inputBuffer, { limitInputPixels: 8000 * 8000 })
+                    .timeout({ seconds: 5 })
+                    .webp({ quality: 80 })
+                    .toBuffer({ resolveWithObject: true });
+                
+                bufferToUpload = data;
+                width = info.width;
+                height = info.height;
+            } catch (O_o) {
+                console.error("failed to process image attachment", O_o);
+                return { ...errors.INTERNAL_ERROR, detail: "An unknown error occurred while processing your image attachment." };
+            }
+        }
+
+        let fd;
+        try {
+            fd = await fsPromises.open(filePath, "wx", uploadsMode);
+            await fd.writeFile(bufferToUpload);
+        } catch (O_o) {
+            console.error("failed to write non-image buffer attachment", O_o);
+            return { ...errors.INTERNAL_ERROR, detail: "Failed to save attachment." };
+        } finally {
+            fd?.close();
+        }
+
+        const attachmentObject = {
+            id: -1,
+            type: "file",
+            owner_id: user.id,
+            message_id: messageId,
+            created_at: Date.now().toString(),
+            file: actualFilename,
+            file_mime: filetype && filetype.mime ? filetype.mime : "application/octet-stream",
+            file_size_bytes: bufferToUpload.byteLength,
+            width,
+            height,
+            file_name: safeFilename
+        };
+
+        const createAttachmentResult = await query(
+            "INSERT INTO message_attachments(type, owner_id, message_id, created_at, file, file_mime, file_size_bytes, width, height, file_name) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id",
+            [
+                attachmentObject.type,
+                attachmentObject.owner_id,
+                attachmentObject.message_id,
+                attachmentObject.created_at,
+                attachmentObject.file,
+                attachmentObject.file_mime,
+                attachmentObject.file_size_bytes,
+                attachmentObject.width,
+                attachmentObject.height,
+                attachmentObject.file_name
+            ]
+        );
+
+        if (!createAttachmentResult || createAttachmentResult.rowCount !== 1) {
+            return errors.GOT_NO_DATABASE_DATA;
+        }
+
+        attachmentObject.id = createAttachmentResult.rows[0].id;
+
+        messageCheckResult.rows[0].attachments = [
+            ...(messageCheckResult.rows[0].attachments ?? []),
+            attachmentObject
+        ];
+
+        dispatch(`channel:${messageCheckResult.rows[0].channel_id}`, {
+            t: GatewayPayloadType.MessageUpdate,
+            d: messageCheckResult.rows[0]
+        });
+
+        return messageCheckResult.rows[0];
+    }
+);

src/rpc/apis/channels.ts

@@ -1,4 +1,3 @@
-import express from "express";
 import { channelNameRegex, method, int, string, uint, withOptional, withRegexp } from "../rpc";
 import { query } from "../../database";
 import { getMessagesByChannelFirstPage, getMessagesByChannelPage } from "../../database/templates";
@@ -111,9 +110,9 @@ method(
 
 method(
     "createChannelMessage",
-    [uint(), string(1, 4000), withOptional(uint()), withOptional(string(1, 64))],
+    [uint(), string(1, 4000), withOptional(uint()), withOptional(string(1, 64)), withOptional(uint())],
-    async (user: User, id: number, content: string, optimistic_id: number | null, nick_username: string | null) => {
+    async (user: User, id: number, content: string, optimistic_id: number | null, nick_username: string | null, pending_attachments: number | null) => {
-        return await sendMessage(user, id, optimistic_id, content, nick_username);
+        return await sendMessage(user, id, optimistic_id, content, nick_username, pending_attachments ?? 0);
     }
 );
 

src/rpc/apis/messages.ts

@@ -4,29 +4,53 @@ import { getMessageById } from "../../database/templates";
 import { errors } from "../../errors";
 import { dispatch } from "../../gateway";
 import { GatewayPayloadType } from "../../gateway/gatewaypayloadtype";
+import { unlink } from "node:fs/promises";
+import path from "node:path";
+import { UploadTarget, getSafeUploadPath } from "../../uploading";
 
 method(
     "deleteMessage",
     [uint()],
     async (user: User, id: number) => {
-        const permissionCheckResult = await query("SELECT author_id, channel_id FROM messages WHERE id = $1", [id]);
+        const messageCheckResult = await query(getMessageById, [id]);
-        if (!permissionCheckResult || permissionCheckResult.rowCount < 1) {
+        if (!messageCheckResult || messageCheckResult.rowCount < 1) {
             return errors.NOT_FOUND;
         }
-        if (permissionCheckResult.rows[0].author_id !== user.id && !user.is_superuser) {
+        if (messageCheckResult.rows[0].author_id !== user.id && !user.is_superuser) {
             return errors.FORBIDDEN_DUE_TO_MISSING_PERMISSIONS;
         }
 
+        const message = messageCheckResult.rows[0];
+
+        if (message.attachments) {
+            for (let i = 0; i < message.attachments.length; i++) {
+                const attachment = message.attachments[i];
+                if (attachment.type === "file" && attachment.file) {
+                    const fileInfo = getSafeUploadPath(UploadTarget.Attachment, attachment.file);
+                    if (fileInfo) {
+                        const resolved = path.resolve(fileInfo);
+                        try {
+                            await unlink(resolved);
+                        } catch(o_O) {
+                            console.error("Failed to unlink message attachment upon message deletion", o_O);
+                        }
+                    } else {
+                        console.error("Failed to unlink message attachment upon message deletion: got null from getSafeUploadPath. This should not happen.");
+                    }
+                }
+            }
+        }
+
         const result = await query("DELETE FROM messages WHERE id = $1", [id]);
         if (!result || result.rowCount < 1) {
             return errors.GOT_NO_DATABASE_DATA;
         }
 
-        dispatch(`channel:${permissionCheckResult.rows[0].channel_id}`, {
+        dispatch(`channel:${message.channel_id}`, {
             t: GatewayPayloadType.MessageDelete,
             d: {
                 id,
-                channel_id: permissionCheckResult.rows[0].channel_id
+                channel_id: message.channel_id
             }
         });
 

src/rpc/apis/users.ts

@@ -9,15 +9,17 @@ import { randomBytes } from "crypto";
 import { unlink } from "fs/promises";
 import { GatewayPayloadType } from "../../gateway/gatewaypayloadtype";
 import { dispatch } from "../../gateway";
+import { supportedImageMime } from "../../uploading";
+import { avatarUploadDirectory, disableAccountCreation, superuserKey } from "../../serverconfig";
+
+const fileType = eval("import('file-type')");
 
-const superuserKey = process.env.SUPERUSER_KEY ? hashSync(process.env.SUPERUSER_KEY, 10) : null;
-const avatarUploadDirectory = process.env.AVATAR_UPLOADS_DIR ?? "./uploads/avatar";
 
 methodButWarningDoesNotAuthenticate(
     "createUser",
     [withRegexp(usernameRegex, string(3, 32)), string(8, 1000)],
     async (username: string, password: string) => {
-        if (process.env.DISABLE_ACCOUNT_CREATION === "true") {
+        if (disableAccountCreation) {
             return errors.FEATURE_DISABLED;
         }
 
@@ -83,20 +85,6 @@ method(
 )
 
 
-const checkMagic = (buffer: Buffer, magic: number[]) => {
-    for (let i = 0; i < magic.length; i++) {
-        try {
-            if (buffer.readUint8(i) !== magic[i]) {
-                return false;
-            }
-        } catch(O_o) {
-            return false;
-        }
-    }
-    return true;
-};
-
-
 const profilePictureSizes = [
     16, 28, 32, 64, 80, 128, 256
 ];
@@ -110,22 +98,9 @@ method(
             return { ...errors.BAD_REQUEST, detail: "Uploaded file exceeds 3MiB limit." };
         }
 
-        // TODO: maybe get rid of this entirely and give buffer directly to `sharp`?
+        const filetype = await (await fileType).fileTypeFromBuffer(buffer);
-        const supportedFormatMagic = [
+        if (!filetype || !supportedImageMime.includes(filetype.mime)) {
-            [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A], // PNG
+            return { ...errors.BAD_REQUEST, detail: "Unsupported file format. Supported file formats are: png, jpeg, webp, avif." };
-            [0xFF, 0xD8, 0xFF], // JPEG
-            [0x52, 0x49, 0x46, 0x46] // WebP
-        ];
-        let isSupported = false;
-        for (let i = 0; i < supportedFormatMagic.length; i++) {
-            if (checkMagic(buffer, supportedFormatMagic[i])) {
-                isSupported = true;
-                break;
-            }
-        }
-
-        if (!isSupported) {
-            return { ...errors.BAD_REQUEST, detail: "Unsupported file format. Supported file formats are: png, jpeg, webp." };
         }
 
         const avatarId = randomBytes(8).toString("hex");
@@ -149,7 +124,7 @@ method(
                 try {
                     await unlink(path.resolve(path.join(avatarUploadDirectory, filenames[i])));
                 } catch(o_0) {
-                    console.error("rpc: putUserAvatar: error while removing files (upon error)", o_0);
+                    //console.error("rpc: putUserAvatar: error while removing files (upon error)", o_0);
                 }
             }
             return errors.INTERNAL_ERROR;

src/rpc/index.ts

@@ -8,6 +8,8 @@ methodGroup(300);
 import "./apis/messages";
 methodGroup(400);
 import "./apis/communities";
+methodGroup(500);
+import "./apis/attachments";
 
 
 console.log("--- begin rpc method map ---")

src/server.ts

@@ -1,15 +1,16 @@
 import express, { Application, ErrorRequestHandler, json } from "express";
 import "./rpc";
+import uploadsRouter from "./routes/uploads";
 import rpcRouter from "./routes/api/v1/rpc";
 import matrixRouter from "./routes/matrix";
 import { errors } from "./errors";
 
-const ENABLE_MATRIX_LAYER = true;
+const ENABLE_MATRIX_LAYER = false;
 
 export default function(app: Application) {
     app.use(json());
     app.use("/api/v1/rpc", rpcRouter);
-    app.use("/uploads", express.static("uploads"));
+    app.use("/uploads", uploadsRouter);
     app.use("/", express.static("frontend/public"));
     if (ENABLE_MATRIX_LAYER) {
         app.use("/", matrixRouter);

src/uploading.ts

@@ -0,0 +1,75 @@
+import path from "node:path";
+import { attachmentUploadDirectory, avatarUploadDirectory } from "./serverconfig";
+
+
+export const supportedImageMime = [
+    "image/jpeg",
+    "image/png",
+    "image/webp",
+    "image/avif",
+];
+
+
+// Thanks: https://github.com/parshap/node-sanitize-filename/blob/209c39b914c8eb48ee27bcbde64b2c7822fdf3de/index.js
+const replacements = [
+    /[\/\?<>\\:\*\|"]/g,
+    /[\x00-\x1f\x80-\x9f]/g,
+    /^\.+$/,
+    /^(con|prn|aux|nul|com[0-9]|lpt[0-9])(\..*)?$/i,
+    /[\. ]+$/,
+];
+export function sanitizeFilename(filename: string): string | null {
+    if (filename.length > 256) {
+        return null;
+    }
+    for (let i = 0; i < replacements.length; i++) {
+        filename.replace(replacements[i], "");
+    }
+    if (!filename.length) {
+        return null;
+    }
+    return filename;
+}
+
+export enum UploadTarget {
+    Avatar,
+    Attachment
+}
+
+export function getSafeUploadPath(target: UploadTarget, filenameUnsafe: string) {
+    let base;
+    switch (target) {
+        case UploadTarget.Avatar: {
+            base = avatarUploadDirectory;
+            break;
+        }
+        case UploadTarget.Attachment: {
+            base = attachmentUploadDirectory;
+            break;
+        }
+        default: {
+            return null;
+        }
+    }
+    if (typeof base !== "string" || typeof filenameUnsafe !== "string") {
+        return null;
+    }
+
+    const sanitized = sanitizeFilename(filenameUnsafe);
+    if (!sanitized) {
+        return null;
+    }
+
+    if (sanitized.indexOf('\0') !== -1 || sanitized.indexOf('%') !== -1 || sanitized.indexOf('/') !== -1 || sanitized.indexOf('..') !== -1) {
+        console.error("getSafeUploadPath: attempted path traversal or illegal path, this should not be possible.");
+        return null;
+    }
+
+    const joined = path.join(base, sanitized);
+    if (joined.indexOf(base) !== 0) {
+        console.error("getSafeUploadPath: attempted path traversal or illegal path, this should not be possible.");
+        return null;
+    }
+
+    return path.resolve(joined);
+}

tsconfig.json

@@ -7,6 +7,7 @@
     "module": "commonjs",
     "esModuleInterop": true,
     "strict": true,
+    "skipLibCheck": true,
     "outDir": "./dist",
   }
 }