Convert paste to UTF8

controllers/pastes.js

@@ -1,6 +1,8 @@
 const config = require('config');
 const fs = require('fs').promises;
 const isBinaryFile = require('isbinaryfile').isBinaryFile;
+const detectCharacterEncoding = require('detect-character-encoding');
+const encoding = require('encoding');
 const Paste = require('../models/paste');
 
 module.exports = {
@@ -52,12 +54,12 @@ module.exports = {
 					path = ctx.request.files.paste.path;
 				}
 
-				const data = await fs.readFile(path);
-				const stat = await fs.lstat(path);
+				if (await isBinaryFile(path)) ctx.throw('Binary file');
 
-				if (await isBinaryFile(data, stat.size)) ctx.throw('Binary file');
+				const data = await fs.readFile(path);
+				const detectedEncoding = detectCharacterEncoding(data);
 
-				ctx.request.body.paste = data.toString();
+				ctx.request.body.paste = encoding.convert(data, 'utf8', detectedEncoding.encoding).toString();
 			} catch {
 				ctx.throw(400, 'Bad Paste Body');
 			} finally {

package-lock.json

@@ -867,6 +867,14 @@
       "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==",
       "dev": true
     },
+    "bindings": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
+      "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
+      "requires": {
+        "file-uri-to-path": "1.0.0"
+      }
+    },
     "bl": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.0.tgz",
@@ -2009,6 +2017,15 @@
       "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
       "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
     },
+    "detect-character-encoding": {
+      "version": "0.8.0",
+      "resolved": "https://registry.npmjs.org/detect-character-encoding/-/detect-character-encoding-0.8.0.tgz",
+      "integrity": "sha512-w+ZYVkUenSJjk1lh36wYcejy03oqZZcBXx52W6zNxyBT7tBMd8IjEglSh+pMHfTvjnPBqnziTGvX1RhitYm7wg==",
+      "requires": {
+        "bindings": "^1.3.0",
+        "nan": "^2.14.0"
+      }
+    },
     "diffie-hellman": {
       "version": "5.0.3",
       "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz",
@@ -2190,6 +2207,24 @@
       "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
       "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k="
     },
+    "encoding": {
+      "version": "0.1.13",
+      "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz",
+      "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==",
+      "requires": {
+        "iconv-lite": "^0.6.2"
+      },
+      "dependencies": {
+        "iconv-lite": {
+          "version": "0.6.2",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.2.tgz",
+          "integrity": "sha512-2y91h5OpQlolefMPmUlivelittSWy0rP+oYVpn6A7GwVHNE8AWzoYOBNmlwks3LobaJxgHCYZAnyNo2GgpNRNQ==",
+          "requires": {
+            "safer-buffer": ">= 2.1.2 < 3.0.0"
+          }
+        }
+      }
+    },
     "end-of-stream": {
       "version": "1.4.4",
       "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
@@ -3532,6 +3567,11 @@
       "integrity": "sha512-uzk64HRpUZyTGZtVuvrjP0FYxzQrBf4rojot6J65YMEbwBLB0CWm0CLojVpwpmFmxcE/lkvYICgfcGozbBq6rw==",
       "dev": true
     },
+    "file-uri-to-path": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
+      "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="
+    },
     "fill-range": {
       "version": "7.0.1",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@@ -5741,6 +5781,11 @@
         "thenify-all": "^1.0.0"
       }
     },
+    "nan": {
+      "version": "2.14.1",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz",
+      "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw=="
+    },
     "nanoid": {
       "version": "3.1.12",
       "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.12.tgz",

package.json

@@ -26,6 +26,8 @@
   "dependencies": {
     "bytes": "^3.1.0",
     "config": "^3.3.1",
+    "detect-character-encoding": "^0.8.0",
+    "encoding": "^0.1.13",
     "isbinaryfile": "^4.0.6",
     "koa": "^2.13.0",
     "koa-body": "^4.2.0",