Refactor uploader
This commit is contained in:
parent
a1a39778aa
commit
6ec5f456f3
12 changed files with 647 additions and 261 deletions
|
|
@ -1,102 +1,36 @@
|
|||
import path from 'path';
|
||||
import crypto from 'crypto';
|
||||
import fs from 'mz/fs';
|
||||
import redis from 'redis';
|
||||
|
||||
import { sniff } from '../../lib/type';
|
||||
import malware from '../../lib/malware';
|
||||
import { formatFile } from '../../lib/format';
|
||||
import { accept, processImage } from '../../lib/upload';
|
||||
|
||||
import debugname from 'debug';
|
||||
const debug = debugname('hostr-api:file');
|
||||
import Uploader from '../../lib/uploader';
|
||||
|
||||
const redisUrl = process.env.REDIS_URL;
|
||||
|
||||
const storePath = process.env.UPLOAD_STORAGE_PATH;
|
||||
|
||||
export function* post(next) {
|
||||
if (!this.request.is('multipart/*')) {
|
||||
return yield next;
|
||||
yield next;
|
||||
return;
|
||||
}
|
||||
const Files = this.db.Files;
|
||||
|
||||
const expectedSize = this.request.headers['content-length'];
|
||||
const remoteIp = this.request.headers['x-real-ip'] || this.req.connection.remoteAddress;
|
||||
const md5sum = crypto.createHash('md5');
|
||||
const uploader = new Uploader(this);
|
||||
|
||||
let lastPercent = 0;
|
||||
let percentComplete = 0;
|
||||
let lastTick = 0;
|
||||
let receivedSize = 0;
|
||||
yield uploader.accept();
|
||||
uploader.acceptedEvent();
|
||||
uploader.receive();
|
||||
yield uploader.save();
|
||||
yield uploader.promise;
|
||||
|
||||
const upload = yield accept.call(this);
|
||||
uploader.processingEvent();
|
||||
|
||||
upload.path = path.join(upload.id[0], upload.id + '_' + upload.filename);
|
||||
const localStream = fs.createWriteStream(path.join(storePath, upload.path));
|
||||
yield uploader.sendToSFTP();
|
||||
yield uploader.processImage();
|
||||
|
||||
upload.pipe(localStream);
|
||||
|
||||
upload.on('data', (data) => {
|
||||
receivedSize += data.length;
|
||||
if (receivedSize > this.user.max_filesize) {
|
||||
fs.unlink(path.join(storePath, key));
|
||||
this.throw(413, '{"error": {"message": "The file you tried to upload is too large.", "code": 601}}');
|
||||
}
|
||||
|
||||
percentComplete = Math.floor(receivedSize * 100 / expectedSize);
|
||||
if (percentComplete > lastPercent && lastTick < Date.now() - 1000) {
|
||||
const progressEvent = `{"type": "file-progress", "data": {"id": "${upload.id}", "complete": ${percentComplete}}}`;
|
||||
this.redis.publish('/file/' + upload.id, progressEvent);
|
||||
this.redis.publish('/user/' + this.user.id, progressEvent);
|
||||
lastTick = Date.now();
|
||||
}
|
||||
lastPercent = percentComplete;
|
||||
|
||||
md5sum.update(data);
|
||||
});
|
||||
|
||||
const dbFile = {
|
||||
owner: this.user.id,
|
||||
ip: remoteIp,
|
||||
'system_name': upload.id,
|
||||
'file_name': upload.filename,
|
||||
'original_name': upload.originalName,
|
||||
'file_size': receivedSize,
|
||||
'time_added': Math.ceil(Date.now() / 1000),
|
||||
status: 'active',
|
||||
'last_accessed': null,
|
||||
s3: false,
|
||||
type: sniff(upload.filename),
|
||||
};
|
||||
|
||||
yield Files.insertOne({_id: upload.id, ...dbFile});
|
||||
|
||||
yield upload.promise;
|
||||
|
||||
const completeEvent = `{"type": "file-progress", "data": {"id": "${upload.id}", "complete": 100}}`;
|
||||
this.redis.publish('/file/' + upload.id, completeEvent);
|
||||
this.redis.publish('/user/' + this.user.id, completeEvent);
|
||||
this.statsd.incr('file.upload.complete', 1);
|
||||
|
||||
const size = yield processImage(upload);
|
||||
|
||||
dbFile.width = size.width;
|
||||
dbFile.height = size.height;
|
||||
dbFile.file_size = receivedSize; // eslint-disable-line camelcase
|
||||
dbFile.status = 'active';
|
||||
dbFile.md5 = md5sum.digest('hex');
|
||||
|
||||
const formattedFile = formatFile({_id: upload.id, ...dbFile});
|
||||
|
||||
yield Files.updateOne({_id: upload.id}, {$set: dbFile});
|
||||
|
||||
const addedEvent = `{"type": "file-added", "data": ${JSON.stringify(formattedFile)}}`;
|
||||
this.redis.publish('/file/' + upload.id, addedEvent);
|
||||
this.redis.publish('/user/' + this.user.id, addedEvent);
|
||||
yield uploader.finalise();
|
||||
|
||||
this.status = 201;
|
||||
this.body = formattedFile;
|
||||
this.body = uploader.toJSON();
|
||||
|
||||
uploader.completeEvent();
|
||||
uploader.malwareScan();
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -107,7 +41,7 @@ export function* list() {
|
|||
if (this.request.query.trashed) {
|
||||
status = 'trashed';
|
||||
} else if (this.request.query.all) {
|
||||
status = {'$in': ['active', 'trashed']};
|
||||
status = { $in: ['active', 'trashed'] };
|
||||
}
|
||||
|
||||
let limit = 20;
|
||||
|
|
@ -123,13 +57,14 @@ export function* list() {
|
|||
}
|
||||
|
||||
const queryOptions = {
|
||||
limit: limit, skip: skip, sort: [['time_added', 'desc']],
|
||||
limit, skip, sort: [['time_added', 'desc']],
|
||||
hint: {
|
||||
owner: 1, status: 1, 'time_added': -1,
|
||||
owner: 1, status: 1, time_added: -1,
|
||||
},
|
||||
};
|
||||
|
||||
const userFiles = yield Files.find({owner: this.user.id, status: status}, queryOptions).toArray();
|
||||
const userFiles = yield Files.find({
|
||||
owner: this.user.id, status }, queryOptions).toArray();
|
||||
this.statsd.incr('file.list', 1);
|
||||
this.body = userFiles.map(formatFile);
|
||||
}
|
||||
|
|
@ -138,9 +73,10 @@ export function* list() {
|
|||
export function* get() {
|
||||
const Files = this.db.Files;
|
||||
const Users = this.db.Users;
|
||||
const file = yield Files.findOne({_id: this.params.id, status: {'$in': ['active', 'uploading']}});
|
||||
const file = yield Files.findOne({ _id: this.params.id,
|
||||
status: { $in: ['active', 'uploading'] } });
|
||||
this.assert(file, 404, '{"error": {"message": "File not found", "code": 604}}');
|
||||
const user = yield Users.findOne({_id: file.owner});
|
||||
const user = yield Users.findOne({ _id: file.owner });
|
||||
this.assert(user && !user.banned, 404, '{"error": {"message": "File not found", "code": 604}}');
|
||||
this.statsd.incr('file.get', 1);
|
||||
this.body = formatFile(file);
|
||||
|
|
@ -151,16 +87,18 @@ export function* put() {
|
|||
if (this.request.body.trashed) {
|
||||
const Files = this.db.Files;
|
||||
const status = this.request.body.trashed ? 'trashed' : 'active';
|
||||
yield Files.updateOne({'_id': this.params.id, owner: this.user.id}, {$set: {status: status}}, {w: 1});
|
||||
yield Files.updateOne({ _id: this.params.id, owner: this.user.id },
|
||||
{ $set: { status } }, { w: 1 });
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export function* del() {
|
||||
yield this.db.Files.updateOne({'_id': this.params.id, owner: this.db.objectId(this.user.id)}, {$set: {status: 'deleted'}}, {w: 1});
|
||||
const event = {type: 'file-deleted', data: {'id': this.params.id}};
|
||||
yield this.redis.publish('/user/' + this.user.id, JSON.stringify(event));
|
||||
yield this.redis.publish('/file/' + this.params.id, JSON.stringify(event));
|
||||
yield this.db.Files.updateOne({ _id: this.params.id, owner: this.db.objectId(this.user.id) },
|
||||
{ $set: { status: 'deleted' } }, { w: 1 });
|
||||
const event = { type: 'file-deleted', data: { id: this.params.id } };
|
||||
yield this.redis.publish(`/file/${this.params.id}`, JSON.stringify(event));
|
||||
yield this.redis.publish(`/user/${this.user.id}`, JSON.stringify(event));
|
||||
this.statsd.incr('file.delete', 1);
|
||||
this.status = 204;
|
||||
this.body = '';
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue