Update stuff

This commit is contained in:
Jonathan Cremin 2018-06-02 15:50:39 +00:00
parent 0254e42b9c
commit 553ba9db9a
40 changed files with 7343 additions and 717 deletions

View file

@ -1,5 +1,5 @@
import { join } from 'path';
import parse from 'co-busboy';
import Busboy from 'busboy';
import crypto from 'crypto';
import fs from 'mz/fs';
import sizeOf from 'image-size';
@ -10,6 +10,7 @@ import { formatFile } from './format';
import resize from './resize';
import malware from './malware';
import { sniff } from './type';
import { upload as s3upload } from './s3';
import debugname from 'debug';
const debug = debugname('hostr-api:uploader');
@ -33,109 +34,8 @@ export default class Uploader {
this.receivedSize = 0;
}
*accept() {
this.upload = yield parse(this.context, {
autoFields: true,
headers: this.context.request.headers,
limits: { files: 1 },
highWaterMark: 10000000,
});
this.promise = new Promise((resolve, reject) => {
this.upload.on('error', (err) => {
this.statsd.incr('file.upload.error', 1);
debug(err);
reject();
});
this.upload.on('end', () => {
resolve();
});
});
this.tempGuid = this.tempGuid;
this.file = yield models.file.create({
id: yield createHostrId(),
name: this.upload.filename.replace(/[^a-zA-Z0-9\.\-_\s]/g, '').replace(/\s+/g, ''),
originalName: this.upload.filename,
userId: this.context.user.id,
status: 'uploading',
type: sniff(this.upload.filename),
ip: this.remoteIp,
accessedAt: null,
width: null,
height: null,
});
yield this.file.save();
}
receive() {
return new Promise((resolve) => {
this.path = join(this.file.id[0], `${this.file.id}_${this.file.name}`);
this.localStream = fs.createWriteStream(join(storePath, this.path));
this.upload.pause();
this.localStream.on('finish', () => {
resolve();
});
this.upload.on('data', (data) => {
this.receivedSize += data.length;
if (this.receivedSize > this.context.user.max_filesize) {
fs.unlink(join(storePath, this.path));
this.context.throw(413, `{"error": {"message": "The file you uploaded is too large.",
"code": 601}}`);
}
this.localStream.write(data);
this.percentComplete = Math.floor(this.receivedSize * 100 / this.expectedSize);
if (this.percentComplete > this.lastPercent && this.lastTick < Date.now() - 1000) {
const progressEvent = `{"type": "file-progress", "data":
{"id": "${this.file.id}", "complete": ${this.percentComplete}}}`;
this.context.redis.publish(`/file/${this.file.id}`, progressEvent);
this.context.redis.publish(`/user/${this.context.user.id}`, progressEvent);
this.lastTick = Date.now();
}
this.lastPercent = this.percentComplete;
this.md5sum.update(data);
});
this.upload.on('end', () => {
this.file.size = this.receivedSize;
this.file.md5 = this.md5sum.digest('hex');
this.localStream.end();
});
this.upload.resume();
});
}
acceptedEvent() {
const accepted = `{"type": "file-accepted", "data":
{"id": "${this.file.id}", "guid": "${this.tempGuid}", "href": "${baseURL}/${this.file.id}"}}`;
this.context.redis.publish(`/user/${this.context.user.id}`, accepted);
this.context.statsd.incr('file.upload.accepted', 1);
}
processingEvent() {
const processing = `{"type": "file-progress", "data":
{"id": "${this.file.id}", "complete": 100}}`;
this.context.redis.publish(`/file/${this.file.id}`, processing);
this.context.redis.publish(`/user/${this.context.user.id}`, processing);
this.context.statsd.incr('file.upload.complete', 1);
}
completeEvent() {
const complete = `{"type": "file-added", "data": ${JSON.stringify(formatFile(this.file))}}`;
this.context.redis.publish(`/file/${this.file.id}`, complete);
this.context.redis.publish(`/user/${this.context.user.id}`, complete);
}
*checkLimit() {
const count = yield models.file.count({
async checkLimit() {
const count = await models.file.count({
where: {
userId: this.context.user.id,
createdAt: {
@ -157,23 +57,96 @@ export default class Uploader {
return true;
}
*finalise() {
this.file.size = this.receivedSize;
this.file.status = 'active';
this.file.processed = 'true';
yield this.file.save();
async accept() {
return new Promise((resolve) => {
this.upload = new Busboy({
autoFields: true,
headers: this.context.request.headers,
limits: { files: 1 },
highWaterMark: 10000000,
});
this.upload.on('file', async (fieldname, file, filename, encoding, mimetype) => {
debug('FILE', fieldname, file, filename, encoding, mimetype);
this.upload.filename = filename;
this.file = await models.file.create({
id: await createHostrId(),
name: this.upload.filename.replace(/[^a-zA-Z0-9\.\-_\s]/g, '').replace(/\s+/g, ''),
originalName: this.upload.filename,
userId: this.context.user.id,
status: 'uploading',
type: sniff(this.upload.filename),
ip: this.remoteIp,
accessedAt: null,
width: null,
height: null,
});
await this.file.save();
this.path = join(this.file.id[0], `${this.file.id}_${this.file.name}`);
this.localStream = fs.createWriteStream(join(storePath, this.path));
this.localStream.on('finish', () => {
resolve();
});
file.on('data', (data) => {
this.receivedSize += data.length;
if (this.receivedSize > this.context.user.max_filesize) {
fs.unlink(join(storePath, this.path));
this.context.throw(413, `{"error": {"message": "The file you uploaded is too large.",
"code": 601}}`);
}
this.localStream.write(data);
this.percentComplete = Math.floor(this.receivedSize * 100 / this.expectedSize);
if (this.percentComplete > this.lastPercent && this.lastTick < Date.now() - 1000) {
const progressEvent = `{"type": "file-progress", "data":
{"id": "${this.file.id}", "complete": ${this.percentComplete}}}`;
this.context.redis.publish(`/file/${this.file.id}`, progressEvent);
this.context.redis.publish(`/user/${this.context.user.id}`, progressEvent);
this.lastTick = Date.now();
}
this.lastPercent = this.percentComplete;
this.md5sum.update(data);
});
debug('accepted');
const accepted = `{"type": "file-accepted", "data":
{"id": "${this.file.id}", "guid": "${this.tempGuid}", "href": "${baseURL}/${this.file.id}"}}`;
this.context.redis.publish(`/user/${this.context.user.id}`, accepted);
this.context.statsd.incr('file.upload.accepted', 1);
file.on('end', () => {
this.file.size = this.receivedSize;
this.file.md5 = this.md5sum.digest('hex');
this.localStream.end();
this.processingEvent();
});
this.localStream.on('end', () => {
s3upload(fs.createReadStream(join(storePath, this.path)), this.path);
});
});
this.context.req.pipe(this.upload);
});
}
resizeImage(upload, type, currentSize, dim) {
return resize(join(storePath, this.path), type, currentSize, dim).then((image) => {
const path = join(this.file.id[0], String(dim.width), `${this.file.id}_${this.file.name}`);
debug('Writing file');
debug(join(storePath, path));
return fs.writeFile(join(storePath, path), image).catch(debug);
}).catch(debug);
processingEvent() {
debug('processing');
const processing = `{"type": "file-progress", "data":
{"id": "${this.file.id}", "complete": 100}}`;
this.context.redis.publish(`/file/${this.file.id}`, processing);
this.context.redis.publish(`/user/${this.context.user.id}`, processing);
this.context.statsd.incr('file.upload.complete', 1);
}
*processImage(upload) {
async processImage(upload) {
return new Promise((resolve) => {
let size;
try {
@ -205,16 +178,43 @@ export default class Uploader {
});
}
resizeImage(upload, type, currentSize, dim) {
return resize(join(storePath, this.path), type, currentSize, dim).then((image) => {
const path = join(this.file.id[0], String(dim.width), `${this.file.id}_${this.file.name}`);
debug('Writing file');
debug(join(storePath, path));
return fs.writeFile(join(storePath, path), image).then(() => {
s3upload(fs.createReadStream(join(storePath, path)), path);
}).catch(debug);
}).catch(debug);
}
async finalise() {
debug('finalise');
this.file.size = this.receivedSize;
this.file.status = 'active';
this.file.processed = 'true';
await this.file.save();
this.completeEvent();
}
completeEvent() {
debug('complete');
const complete = `{"type": "file-added", "data": ${JSON.stringify(formatFile(this.file))}}`;
this.context.redis.publish(`/file/${this.file.id}`, complete);
this.context.redis.publish(`/user/${this.context.user.id}`, complete);
}
malwareScan() {
if (process.env.VIRUSTOTAL_KEY) {
// Check in the background
process.nextTick(function* scan() {
process.nextTick(async () => {
debug('Malware Scan');
const result = yield malware(this);
const result = await malware(this);
if (result) {
this.file.malwarePositives = result.positives;
this.file.save();
const fileMalware = yield models.malware.create({
const fileMalware = await models.malware.create({
fileId: this.file.id,
positives: result.positives,
virustotal: result,