Update stuff
This commit is contained in:
parent
0254e42b9c
commit
553ba9db9a
40 changed files with 7343 additions and 717 deletions
240
lib/uploader.js
240
lib/uploader.js
|
|
@ -1,5 +1,5 @@
|
|||
import { join } from 'path';
|
||||
import parse from 'co-busboy';
|
||||
import Busboy from 'busboy';
|
||||
import crypto from 'crypto';
|
||||
import fs from 'mz/fs';
|
||||
import sizeOf from 'image-size';
|
||||
|
|
@ -10,6 +10,7 @@ import { formatFile } from './format';
|
|||
import resize from './resize';
|
||||
import malware from './malware';
|
||||
import { sniff } from './type';
|
||||
import { upload as s3upload } from './s3';
|
||||
|
||||
import debugname from 'debug';
|
||||
const debug = debugname('hostr-api:uploader');
|
||||
|
|
@ -33,109 +34,8 @@ export default class Uploader {
|
|||
this.receivedSize = 0;
|
||||
}
|
||||
|
||||
*accept() {
|
||||
this.upload = yield parse(this.context, {
|
||||
autoFields: true,
|
||||
headers: this.context.request.headers,
|
||||
limits: { files: 1 },
|
||||
highWaterMark: 10000000,
|
||||
});
|
||||
|
||||
this.promise = new Promise((resolve, reject) => {
|
||||
this.upload.on('error', (err) => {
|
||||
this.statsd.incr('file.upload.error', 1);
|
||||
debug(err);
|
||||
reject();
|
||||
});
|
||||
|
||||
this.upload.on('end', () => {
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
this.tempGuid = this.tempGuid;
|
||||
this.file = yield models.file.create({
|
||||
id: yield createHostrId(),
|
||||
name: this.upload.filename.replace(/[^a-zA-Z0-9\.\-_\s]/g, '').replace(/\s+/g, ''),
|
||||
originalName: this.upload.filename,
|
||||
userId: this.context.user.id,
|
||||
status: 'uploading',
|
||||
type: sniff(this.upload.filename),
|
||||
ip: this.remoteIp,
|
||||
accessedAt: null,
|
||||
width: null,
|
||||
height: null,
|
||||
});
|
||||
yield this.file.save();
|
||||
}
|
||||
|
||||
receive() {
|
||||
return new Promise((resolve) => {
|
||||
this.path = join(this.file.id[0], `${this.file.id}_${this.file.name}`);
|
||||
this.localStream = fs.createWriteStream(join(storePath, this.path));
|
||||
|
||||
this.upload.pause();
|
||||
|
||||
this.localStream.on('finish', () => {
|
||||
resolve();
|
||||
});
|
||||
|
||||
this.upload.on('data', (data) => {
|
||||
this.receivedSize += data.length;
|
||||
if (this.receivedSize > this.context.user.max_filesize) {
|
||||
fs.unlink(join(storePath, this.path));
|
||||
this.context.throw(413, `{"error": {"message": "The file you uploaded is too large.",
|
||||
"code": 601}}`);
|
||||
}
|
||||
|
||||
this.localStream.write(data);
|
||||
|
||||
this.percentComplete = Math.floor(this.receivedSize * 100 / this.expectedSize);
|
||||
if (this.percentComplete > this.lastPercent && this.lastTick < Date.now() - 1000) {
|
||||
const progressEvent = `{"type": "file-progress", "data":
|
||||
{"id": "${this.file.id}", "complete": ${this.percentComplete}}}`;
|
||||
this.context.redis.publish(`/file/${this.file.id}`, progressEvent);
|
||||
this.context.redis.publish(`/user/${this.context.user.id}`, progressEvent);
|
||||
this.lastTick = Date.now();
|
||||
}
|
||||
this.lastPercent = this.percentComplete;
|
||||
|
||||
this.md5sum.update(data);
|
||||
});
|
||||
|
||||
this.upload.on('end', () => {
|
||||
this.file.size = this.receivedSize;
|
||||
this.file.md5 = this.md5sum.digest('hex');
|
||||
this.localStream.end();
|
||||
});
|
||||
|
||||
this.upload.resume();
|
||||
});
|
||||
}
|
||||
|
||||
acceptedEvent() {
|
||||
const accepted = `{"type": "file-accepted", "data":
|
||||
{"id": "${this.file.id}", "guid": "${this.tempGuid}", "href": "${baseURL}/${this.file.id}"}}`;
|
||||
this.context.redis.publish(`/user/${this.context.user.id}`, accepted);
|
||||
this.context.statsd.incr('file.upload.accepted', 1);
|
||||
}
|
||||
|
||||
processingEvent() {
|
||||
const processing = `{"type": "file-progress", "data":
|
||||
{"id": "${this.file.id}", "complete": 100}}`;
|
||||
this.context.redis.publish(`/file/${this.file.id}`, processing);
|
||||
this.context.redis.publish(`/user/${this.context.user.id}`, processing);
|
||||
this.context.statsd.incr('file.upload.complete', 1);
|
||||
}
|
||||
|
||||
completeEvent() {
|
||||
const complete = `{"type": "file-added", "data": ${JSON.stringify(formatFile(this.file))}}`;
|
||||
this.context.redis.publish(`/file/${this.file.id}`, complete);
|
||||
this.context.redis.publish(`/user/${this.context.user.id}`, complete);
|
||||
}
|
||||
|
||||
*checkLimit() {
|
||||
const count = yield models.file.count({
|
||||
async checkLimit() {
|
||||
const count = await models.file.count({
|
||||
where: {
|
||||
userId: this.context.user.id,
|
||||
createdAt: {
|
||||
|
|
@ -157,23 +57,96 @@ export default class Uploader {
|
|||
return true;
|
||||
}
|
||||
|
||||
*finalise() {
|
||||
this.file.size = this.receivedSize;
|
||||
this.file.status = 'active';
|
||||
this.file.processed = 'true';
|
||||
yield this.file.save();
|
||||
async accept() {
|
||||
return new Promise((resolve) => {
|
||||
this.upload = new Busboy({
|
||||
autoFields: true,
|
||||
headers: this.context.request.headers,
|
||||
limits: { files: 1 },
|
||||
highWaterMark: 10000000,
|
||||
});
|
||||
|
||||
this.upload.on('file', async (fieldname, file, filename, encoding, mimetype) => {
|
||||
debug('FILE', fieldname, file, filename, encoding, mimetype);
|
||||
|
||||
this.upload.filename = filename;
|
||||
|
||||
this.file = await models.file.create({
|
||||
id: await createHostrId(),
|
||||
name: this.upload.filename.replace(/[^a-zA-Z0-9\.\-_\s]/g, '').replace(/\s+/g, ''),
|
||||
originalName: this.upload.filename,
|
||||
userId: this.context.user.id,
|
||||
status: 'uploading',
|
||||
type: sniff(this.upload.filename),
|
||||
ip: this.remoteIp,
|
||||
accessedAt: null,
|
||||
width: null,
|
||||
height: null,
|
||||
});
|
||||
await this.file.save();
|
||||
|
||||
this.path = join(this.file.id[0], `${this.file.id}_${this.file.name}`);
|
||||
this.localStream = fs.createWriteStream(join(storePath, this.path));
|
||||
this.localStream.on('finish', () => {
|
||||
resolve();
|
||||
});
|
||||
|
||||
|
||||
file.on('data', (data) => {
|
||||
this.receivedSize += data.length;
|
||||
if (this.receivedSize > this.context.user.max_filesize) {
|
||||
fs.unlink(join(storePath, this.path));
|
||||
this.context.throw(413, `{"error": {"message": "The file you uploaded is too large.",
|
||||
"code": 601}}`);
|
||||
}
|
||||
|
||||
this.localStream.write(data);
|
||||
|
||||
this.percentComplete = Math.floor(this.receivedSize * 100 / this.expectedSize);
|
||||
if (this.percentComplete > this.lastPercent && this.lastTick < Date.now() - 1000) {
|
||||
const progressEvent = `{"type": "file-progress", "data":
|
||||
{"id": "${this.file.id}", "complete": ${this.percentComplete}}}`;
|
||||
this.context.redis.publish(`/file/${this.file.id}`, progressEvent);
|
||||
this.context.redis.publish(`/user/${this.context.user.id}`, progressEvent);
|
||||
this.lastTick = Date.now();
|
||||
}
|
||||
this.lastPercent = this.percentComplete;
|
||||
|
||||
this.md5sum.update(data);
|
||||
});
|
||||
|
||||
debug('accepted');
|
||||
const accepted = `{"type": "file-accepted", "data":
|
||||
{"id": "${this.file.id}", "guid": "${this.tempGuid}", "href": "${baseURL}/${this.file.id}"}}`;
|
||||
this.context.redis.publish(`/user/${this.context.user.id}`, accepted);
|
||||
this.context.statsd.incr('file.upload.accepted', 1);
|
||||
|
||||
file.on('end', () => {
|
||||
this.file.size = this.receivedSize;
|
||||
this.file.md5 = this.md5sum.digest('hex');
|
||||
this.localStream.end();
|
||||
this.processingEvent();
|
||||
});
|
||||
|
||||
this.localStream.on('end', () => {
|
||||
s3upload(fs.createReadStream(join(storePath, this.path)), this.path);
|
||||
});
|
||||
|
||||
});
|
||||
this.context.req.pipe(this.upload);
|
||||
});
|
||||
}
|
||||
|
||||
resizeImage(upload, type, currentSize, dim) {
|
||||
return resize(join(storePath, this.path), type, currentSize, dim).then((image) => {
|
||||
const path = join(this.file.id[0], String(dim.width), `${this.file.id}_${this.file.name}`);
|
||||
debug('Writing file');
|
||||
debug(join(storePath, path));
|
||||
return fs.writeFile(join(storePath, path), image).catch(debug);
|
||||
}).catch(debug);
|
||||
processingEvent() {
|
||||
debug('processing');
|
||||
const processing = `{"type": "file-progress", "data":
|
||||
{"id": "${this.file.id}", "complete": 100}}`;
|
||||
this.context.redis.publish(`/file/${this.file.id}`, processing);
|
||||
this.context.redis.publish(`/user/${this.context.user.id}`, processing);
|
||||
this.context.statsd.incr('file.upload.complete', 1);
|
||||
}
|
||||
|
||||
*processImage(upload) {
|
||||
async processImage(upload) {
|
||||
return new Promise((resolve) => {
|
||||
let size;
|
||||
try {
|
||||
|
|
@ -205,16 +178,43 @@ export default class Uploader {
|
|||
});
|
||||
}
|
||||
|
||||
resizeImage(upload, type, currentSize, dim) {
|
||||
return resize(join(storePath, this.path), type, currentSize, dim).then((image) => {
|
||||
const path = join(this.file.id[0], String(dim.width), `${this.file.id}_${this.file.name}`);
|
||||
debug('Writing file');
|
||||
debug(join(storePath, path));
|
||||
return fs.writeFile(join(storePath, path), image).then(() => {
|
||||
s3upload(fs.createReadStream(join(storePath, path)), path);
|
||||
}).catch(debug);
|
||||
}).catch(debug);
|
||||
}
|
||||
|
||||
async finalise() {
|
||||
debug('finalise');
|
||||
this.file.size = this.receivedSize;
|
||||
this.file.status = 'active';
|
||||
this.file.processed = 'true';
|
||||
await this.file.save();
|
||||
this.completeEvent();
|
||||
}
|
||||
|
||||
completeEvent() {
|
||||
debug('complete');
|
||||
const complete = `{"type": "file-added", "data": ${JSON.stringify(formatFile(this.file))}}`;
|
||||
this.context.redis.publish(`/file/${this.file.id}`, complete);
|
||||
this.context.redis.publish(`/user/${this.context.user.id}`, complete);
|
||||
}
|
||||
|
||||
malwareScan() {
|
||||
if (process.env.VIRUSTOTAL_KEY) {
|
||||
// Check in the background
|
||||
process.nextTick(function* scan() {
|
||||
process.nextTick(async () => {
|
||||
debug('Malware Scan');
|
||||
const result = yield malware(this);
|
||||
const result = await malware(this);
|
||||
if (result) {
|
||||
this.file.malwarePositives = result.positives;
|
||||
this.file.save();
|
||||
const fileMalware = yield models.malware.create({
|
||||
const fileMalware = await models.malware.create({
|
||||
fileId: this.file.id,
|
||||
positives: result.positives,
|
||||
virustotal: result,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue