Blurhash support (#701)

* Generate blurhash client side

* Make blurhash generation faster

* Simple blurhash display support

* Make image display simpler

* Support non square images

* Don't attach video blurhash to thumbnail

* Add video display support

* Ignore alt tag missing warning

Co-authored-by: Ajay Bura <32841439+ajbura@users.noreply.github.com>
This commit is contained in:
ginnyTheCat 2022-08-06 05:56:26 +02:00 committed by GitHub
parent edace32213
commit 04f910ee03
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 110 additions and 29 deletions

46
package-lock.json generated
View file

@ -14,6 +14,7 @@
"@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz", "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz",
"@tippyjs/react": "^4.2.6", "@tippyjs/react": "^4.2.6",
"babel-polyfill": "^6.26.0", "babel-polyfill": "^6.26.0",
"blurhash": "^1.1.5",
"browser-encrypt-attachment": "^0.3.0", "browser-encrypt-attachment": "^0.3.0",
"dateformat": "^5.0.3", "dateformat": "^5.0.3",
"emojibase-data": "^7.0.1", "emojibase-data": "^7.0.1",
@ -34,6 +35,7 @@
"prop-types": "^15.8.1", "prop-types": "^15.8.1",
"react": "^17.0.2", "react": "^17.0.2",
"react-autosize-textarea": "^7.1.0", "react-autosize-textarea": "^7.1.0",
"react-blurhash": "^0.1.3",
"react-dnd": "^15.1.2", "react-dnd": "^15.1.2",
"react-dnd-html5-backend": "^15.1.3", "react-dnd-html5-backend": "^15.1.3",
"react-dom": "^17.0.2", "react-dom": "^17.0.2",
@ -3014,6 +3016,8 @@
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz",
"integrity": "sha512-qOKJyNj/h+OWx7s5DePL6Zu1KeM9jPZhwBqs+7DzP6bGOvqzVCSf0xueYmVuaC/oQ/VtS2zLMLHdQFbkka+XDQ==", "integrity": "sha512-qOKJyNj/h+OWx7s5DePL6Zu1KeM9jPZhwBqs+7DzP6bGOvqzVCSf0xueYmVuaC/oQ/VtS2zLMLHdQFbkka+XDQ==",
"dev": true, "dev": true,
"optional": true,
"peer": true,
"dependencies": { "dependencies": {
"fast-deep-equal": "^3.1.1", "fast-deep-equal": "^3.1.1",
"json-schema-traverse": "^1.0.0", "json-schema-traverse": "^1.0.0",
@ -3029,7 +3033,9 @@
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
"integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
"dev": true "dev": true,
"optional": true,
"peer": true
}, },
"node_modules/ajv-keywords": { "node_modules/ajv-keywords": {
"version": "3.5.2", "version": "3.5.2",
@ -3558,6 +3564,11 @@
"integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=", "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=",
"dev": true "dev": true
}, },
"node_modules/blurhash": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/blurhash/-/blurhash-1.1.5.tgz",
"integrity": "sha512-a+LO3A2DfxTaTztsmkbLYmUzUeApi0LZuKalwbNmqAHR6HhJGMt1qSV/R3wc+w4DL28holjqO3Bg74aUGavGjg=="
},
"node_modules/bmp-js": { "node_modules/bmp-js": {
"version": "0.1.0", "version": "0.1.0",
"resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz",
@ -11540,6 +11551,15 @@
"react-dom": "^0.14.0 || ^15.0.0 || ^16.0.0" "react-dom": "^0.14.0 || ^15.0.0 || ^16.0.0"
} }
}, },
"node_modules/react-blurhash": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/react-blurhash/-/react-blurhash-0.1.3.tgz",
"integrity": "sha512-Q9lqbXg92NU6/2DoIl/cBM8YWL+Z4X66OiG4aT9ozOgjBwx104LHFCH5stf6aF+s0Q9Wf310Ul+dG+VXJltmPg==",
"peerDependencies": {
"blurhash": "^1.1.1",
"react": ">=15"
}
},
"node_modules/react-dnd": { "node_modules/react-dnd": {
"version": "15.1.2", "version": "15.1.2",
"resolved": "https://registry.npmjs.org/react-dnd/-/react-dnd-15.1.2.tgz", "resolved": "https://registry.npmjs.org/react-dnd/-/react-dnd-15.1.2.tgz",
@ -16495,15 +16515,14 @@
"resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz",
"integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==",
"dev": true, "dev": true,
"requires": { "requires": {},
"ajv": "^8.0.0"
},
"dependencies": { "dependencies": {
"ajv": { "ajv": {
"version": "8.9.0", "version": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz",
"integrity": "sha512-qOKJyNj/h+OWx7s5DePL6Zu1KeM9jPZhwBqs+7DzP6bGOvqzVCSf0xueYmVuaC/oQ/VtS2zLMLHdQFbkka+XDQ==", "integrity": "sha512-qOKJyNj/h+OWx7s5DePL6Zu1KeM9jPZhwBqs+7DzP6bGOvqzVCSf0xueYmVuaC/oQ/VtS2zLMLHdQFbkka+XDQ==",
"dev": true, "dev": true,
"optional": true,
"peer": true,
"requires": { "requires": {
"fast-deep-equal": "^3.1.1", "fast-deep-equal": "^3.1.1",
"json-schema-traverse": "^1.0.0", "json-schema-traverse": "^1.0.0",
@ -16515,7 +16534,9 @@
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
"integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
"dev": true "dev": true,
"optional": true,
"peer": true
} }
} }
}, },
@ -16950,6 +16971,11 @@
} }
} }
}, },
"blurhash": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/blurhash/-/blurhash-1.1.5.tgz",
"integrity": "sha512-a+LO3A2DfxTaTztsmkbLYmUzUeApi0LZuKalwbNmqAHR6HhJGMt1qSV/R3wc+w4DL28holjqO3Bg74aUGavGjg=="
},
"bmp-js": { "bmp-js": {
"version": "0.1.0", "version": "0.1.0",
"resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz",
@ -22933,6 +22959,12 @@
"prop-types": "^15.5.6" "prop-types": "^15.5.6"
} }
}, },
"react-blurhash": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/react-blurhash/-/react-blurhash-0.1.3.tgz",
"integrity": "sha512-Q9lqbXg92NU6/2DoIl/cBM8YWL+Z4X66OiG4aT9ozOgjBwx104LHFCH5stf6aF+s0Q9Wf310Ul+dG+VXJltmPg==",
"requires": {}
},
"react-dnd": { "react-dnd": {
"version": "15.1.2", "version": "15.1.2",
"resolved": "https://registry.npmjs.org/react-dnd/-/react-dnd-15.1.2.tgz", "resolved": "https://registry.npmjs.org/react-dnd/-/react-dnd-15.1.2.tgz",

View file

@ -20,6 +20,7 @@
"@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz", "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz",
"@tippyjs/react": "^4.2.6", "@tippyjs/react": "^4.2.6",
"babel-polyfill": "^6.26.0", "babel-polyfill": "^6.26.0",
"blurhash": "^1.1.5",
"browser-encrypt-attachment": "^0.3.0", "browser-encrypt-attachment": "^0.3.0",
"dateformat": "^5.0.3", "dateformat": "^5.0.3",
"emojibase-data": "^7.0.1", "emojibase-data": "^7.0.1",
@ -40,6 +41,7 @@
"prop-types": "^15.8.1", "prop-types": "^15.8.1",
"react": "^17.0.2", "react": "^17.0.2",
"react-autosize-textarea": "^7.1.0", "react-autosize-textarea": "^7.1.0",
"react-blurhash": "^0.1.3",
"react-dnd": "^15.1.2", "react-dnd": "^15.1.2",
"react-dnd-html5-backend": "^15.1.3", "react-dnd-html5-backend": "^15.1.3",
"react-dom": "^17.0.2", "react-dom": "^17.0.2",

View file

@ -4,6 +4,7 @@ import './Media.scss';
import encrypt from 'browser-encrypt-attachment'; import encrypt from 'browser-encrypt-attachment';
import { BlurhashCanvas } from 'react-blurhash';
import Text from '../../atoms/text/Text'; import Text from '../../atoms/text/Text';
import IconButton from '../../atoms/button/IconButton'; import IconButton from '../../atoms/button/IconButton';
import Spinner from '../../atoms/spinner/Spinner'; import Spinner from '../../atoms/spinner/Spinner';
@ -154,7 +155,7 @@ File.propTypes = {
}; };
function Image({ function Image({
name, width, height, link, file, type, name, width, height, link, file, type, blurhash,
}) { }) {
const [url, setUrl] = useState(null); const [url, setUrl] = useState(null);
@ -175,6 +176,7 @@ function Image({
<div className="file-container"> <div className="file-container">
<FileHeader name={name} link={url || link} type={type} external /> <FileHeader name={name} link={url || link} type={type} external />
<div style={{ height: width !== null ? getNativeHeight(width, height) : 'unset' }} className="image-container"> <div style={{ height: width !== null ? getNativeHeight(width, height) : 'unset' }} className="image-container">
{ blurhash && <BlurhashCanvas hash={blurhash} punch={1} />}
{ url !== null && <img src={url || link} alt={name} />} { url !== null && <img src={url || link} alt={name} />}
</div> </div>
</div> </div>
@ -185,6 +187,7 @@ Image.defaultProps = {
width: null, width: null,
height: null, height: null,
type: '', type: '',
blurhash: '',
}; };
Image.propTypes = { Image.propTypes = {
name: PropTypes.string.isRequired, name: PropTypes.string.isRequired,
@ -193,6 +196,7 @@ Image.propTypes = {
link: PropTypes.string.isRequired, link: PropTypes.string.isRequired,
file: PropTypes.shape({}), file: PropTypes.shape({}),
type: PropTypes.string, type: PropTypes.string,
blurhash: PropTypes.string,
}; };
function Sticker({ function Sticker({
@ -278,8 +282,8 @@ Audio.propTypes = {
}; };
function Video({ function Video({
name, link, thumbnail, name, link, thumbnail, thumbnailFile, thumbnailType,
width, height, file, type, thumbnailFile, thumbnailType, width, height, file, type, blurhash,
}) { }) {
const [isLoading, setIsLoading] = useState(false); const [isLoading, setIsLoading] = useState(false);
const [url, setUrl] = useState(null); const [url, setUrl] = useState(null);
@ -315,10 +319,14 @@ function Video({
<div <div
style={{ style={{
height: width !== null ? getNativeHeight(width, height) : 'unset', height: width !== null ? getNativeHeight(width, height) : 'unset',
backgroundImage: thumbUrl === null ? 'none' : `url(${thumbUrl}`,
}} }}
className="video-container" className="video-container"
> >
{ url === null && blurhash && <BlurhashCanvas hash={blurhash} punch={1} />}
{ url === null && thumbUrl !== null && (
/* eslint-disable-next-line jsx-a11y/alt-text */
<img src={thumbUrl} />
)}
{ url === null && isLoading && <Spinner size="small" /> } { url === null && isLoading && <Spinner size="small" /> }
{ url === null && !isLoading && <IconButton onClick={handlePlayVideo} tooltip="Play video" src={PlaySVG} />} { url === null && !isLoading && <IconButton onClick={handlePlayVideo} tooltip="Play video" src={PlaySVG} />}
{ url !== null && ( { url !== null && (
@ -336,20 +344,22 @@ Video.defaultProps = {
height: null, height: null,
file: null, file: null,
thumbnail: null, thumbnail: null,
type: '',
thumbnailType: null, thumbnailType: null,
thumbnailFile: null, thumbnailFile: null,
type: '',
blurhash: null,
}; };
Video.propTypes = { Video.propTypes = {
name: PropTypes.string.isRequired, name: PropTypes.string.isRequired,
link: PropTypes.string.isRequired, link: PropTypes.string.isRequired,
thumbnail: PropTypes.string, thumbnail: PropTypes.string,
thumbnailFile: PropTypes.shape({}),
thumbnailType: PropTypes.string,
width: PropTypes.number, width: PropTypes.number,
height: PropTypes.number, height: PropTypes.number,
file: PropTypes.shape({}), file: PropTypes.shape({}),
type: PropTypes.string, type: PropTypes.string,
thumbnailFile: PropTypes.shape({}), blurhash: PropTypes.string,
thumbnailType: PropTypes.string,
}; };
export { export {

View file

@ -33,6 +33,8 @@
font-size: 0; font-size: 0;
line-height: 0; line-height: 0;
position: relative;
display: flex; display: flex;
justify-content: center; justify-content: center;
align-items: center; align-items: center;
@ -42,6 +44,19 @@
background-size: cover; background-size: cover;
} }
.image-container,
.video-container {
& img,
& canvas {
position: absolute;
max-width: unset !important;
width: 100% !important;
height: 100%;
border-radius: 0 !important;
margin: 0 !important;
}
}
.sticker-container { .sticker-container {
display: inline-flex; display: inline-flex;
max-width: 128px; max-width: 128px;
@ -51,25 +66,17 @@
} }
} }
.image-container {
& img {
max-width: unset !important;
width: 100% !important;
border-radius: 0 !important;
margin: 0 !important;
}
}
.video-container { .video-container {
& .ic-btn-surface { & .ic-btn-surface {
background-color: var(--bg-surface-low); background-color: var(--bg-surface-low);
position: absolute;
} }
video { video {
width: 100% width: 100%;
} }
} }
.audio-container { .audio-container {
audio { audio {
width: 100% width: 100%;
} }
} }

View file

@ -610,6 +610,8 @@ function genMediaContent(mE) {
let msgType = mE.getContent()?.msgtype; let msgType = mE.getContent()?.msgtype;
if (mE.getType() === 'm.sticker') msgType = 'm.sticker'; if (mE.getType() === 'm.sticker') msgType = 'm.sticker';
const blurhash = mContent?.info?.['xyz.amorgan.blurhash'];
switch (msgType) { switch (msgType) {
case 'm.file': case 'm.file':
return ( return (
@ -629,6 +631,7 @@ function genMediaContent(mE) {
link={mx.mxcUrlToHttp(mediaMXC)} link={mx.mxcUrlToHttp(mediaMXC)}
file={isEncryptedFile ? mContent.file : null} file={isEncryptedFile ? mContent.file : null}
type={mContent.info?.mimetype} type={mContent.info?.mimetype}
blurhash={blurhash}
/> />
); );
case 'm.sticker': case 'm.sticker':
@ -666,6 +669,7 @@ function genMediaContent(mE) {
height={typeof mContent.info?.h === 'number' ? mContent.info?.h : null} height={typeof mContent.info?.h === 'number' ? mContent.info?.h : null}
file={isEncryptedFile ? mContent.file : null} file={isEncryptedFile ? mContent.file : null}
type={mContent.info?.mimetype} type={mContent.info?.mimetype}
blurhash={blurhash}
/> />
); );
default: default:

View file

@ -3,12 +3,34 @@ import { micromark } from 'micromark';
import { gfm, gfmHtml } from 'micromark-extension-gfm'; import { gfm, gfmHtml } from 'micromark-extension-gfm';
import encrypt from 'browser-encrypt-attachment'; import encrypt from 'browser-encrypt-attachment';
import { math } from 'micromark-extension-math'; import { math } from 'micromark-extension-math';
import { encode } from 'blurhash';
import { getShortcodeToEmoji } from '../../app/organisms/emoji-board/custom-emoji'; import { getShortcodeToEmoji } from '../../app/organisms/emoji-board/custom-emoji';
import { mathExtensionHtml, spoilerExtension, spoilerExtensionHtml } from '../../util/markdown'; import { mathExtensionHtml, spoilerExtension, spoilerExtensionHtml } from '../../util/markdown';
import { getImageDimension } from '../../util/common'; import { getImageDimension } from '../../util/common';
import cons from './cons'; import cons from './cons';
import settings from './settings'; import settings from './settings';
const blurhashField = 'xyz.amorgan.blurhash';
function encodeBlurhash(img) {
const canvas = document.createElement('canvas');
canvas.width = 100;
canvas.height = 100;
const context = canvas.getContext('2d');
context.drawImage(img, 0, 0, canvas.width, canvas.height);
const data = context.getImageData(0, 0, canvas.width, canvas.height);
return encode(data.data, data.width, data.height, 4, 4);
}
function loadImage(url) {
return new Promise((resolve, reject) => {
const img = new Image();
img.onload = () => resolve(img);
img.onerror = (err) => reject(err);
img.src = url;
});
}
function loadVideo(videoFile) { function loadVideo(videoFile) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const video = document.createElement('video'); const video = document.createElement('video');
@ -300,10 +322,11 @@ class RoomsInput extends EventEmitter {
let uploadData = null; let uploadData = null;
if (fileType === 'image') { if (fileType === 'image') {
const imgDimension = await getImageDimension(file); const img = await loadImage(URL.createObjectURL(file));
info.w = imgDimension.w; info.w = img.width;
info.h = imgDimension.h; info.h = img.height;
info[blurhashField] = encodeBlurhash(img);
content.msgtype = 'm.image'; content.msgtype = 'm.image';
content.body = file.name || 'Image'; content.body = file.name || 'Image';
@ -313,8 +336,11 @@ class RoomsInput extends EventEmitter {
try { try {
const video = await loadVideo(file); const video = await loadVideo(file);
info.w = video.videoWidth; info.w = video.videoWidth;
info.h = video.videoHeight; info.h = video.videoHeight;
info[blurhashField] = encodeBlurhash(video);
const thumbnailData = await getVideoThumbnail(video, video.videoWidth, video.videoHeight, 'image/jpeg'); const thumbnailData = await getVideoThumbnail(video, video.videoWidth, video.videoHeight, 'image/jpeg');
const thumbnailUploadData = await this.uploadFile(roomId, thumbnailData.thumbnail); const thumbnailUploadData = await this.uploadFile(roomId, thumbnailData.thumbnail);
info.thumbnail_info = thumbnailData.info; info.thumbnail_info = thumbnailData.info;