Refactor uploads
This commit is contained in:
parent
75a46212da
commit
f59b9a5d22
14 changed files with 297 additions and 269 deletions
|
@ -46,7 +46,9 @@ router.use('*', function* authMiddleware(next) {
|
||||||
} else {
|
} else {
|
||||||
if (!err.status) {
|
if (!err.status) {
|
||||||
debug(err);
|
debug(err);
|
||||||
this.raven.captureError(err);
|
if (this.raven) {
|
||||||
|
this.raven.captureError(err);
|
||||||
|
}
|
||||||
throw err;
|
throw err;
|
||||||
} else {
|
} else {
|
||||||
this.status = err.status;
|
this.status = err.status;
|
||||||
|
|
|
@ -1,22 +1,18 @@
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
import fs from 'fs';
|
|
||||||
import crypto from 'crypto';
|
import crypto from 'crypto';
|
||||||
import gm from 'gm';
|
import fs from 'mz/fs';
|
||||||
import redis from 'redis';
|
import redis from 'redis';
|
||||||
import parse from 'co-busboy';
|
|
||||||
import { upload as s3Upload } from '../../lib/s3';
|
|
||||||
import { sniff } from '../../lib/type';
|
import { sniff } from '../../lib/type';
|
||||||
import hostrId from '../../lib/hostr-id';
|
|
||||||
import malware from '../../lib/malware';
|
import malware from '../../lib/malware';
|
||||||
import { formatFile } from '../../lib/format';
|
import { formatFile } from '../../lib/format';
|
||||||
|
import { accept, processImage } from '../../lib/upload';
|
||||||
|
|
||||||
import debugname from 'debug';
|
import debugname from 'debug';
|
||||||
const debug = debugname('hostr-api:file');
|
const debug = debugname('hostr-api:file');
|
||||||
|
|
||||||
const redisUrl = process.env.REDIS_URL;
|
const redisUrl = process.env.REDIS_URL;
|
||||||
|
|
||||||
const baseURL = process.env.WEB_BASE_URL;
|
|
||||||
|
|
||||||
const storePath = process.env.UPLOAD_STORAGE_PATH;
|
const storePath = process.env.UPLOAD_STORAGE_PATH;
|
||||||
|
|
||||||
export function* post(next) {
|
export function* post(next) {
|
||||||
|
@ -26,9 +22,7 @@ export function* post(next) {
|
||||||
const Files = this.db.Files;
|
const Files = this.db.Files;
|
||||||
|
|
||||||
const expectedSize = this.request.headers['content-length'];
|
const expectedSize = this.request.headers['content-length'];
|
||||||
const tempGuid = this.request.headers['hostr-guid'];
|
|
||||||
const remoteIp = this.request.headers['x-real-ip'] || this.req.connection.remoteAddress;
|
const remoteIp = this.request.headers['x-real-ip'] || this.req.connection.remoteAddress;
|
||||||
|
|
||||||
const md5sum = crypto.createHash('md5');
|
const md5sum = crypto.createHash('md5');
|
||||||
|
|
||||||
let lastPercent = 0;
|
let lastPercent = 0;
|
||||||
|
@ -36,74 +30,12 @@ export function* post(next) {
|
||||||
let lastTick = 0;
|
let lastTick = 0;
|
||||||
let receivedSize = 0;
|
let receivedSize = 0;
|
||||||
|
|
||||||
// Receive upload
|
const upload = yield accept.call(this);
|
||||||
debug('Parsing upload');
|
|
||||||
const upload = yield parse(this, {autoFields: true, headers: this.request.headers, limits: { files: 1}, highWaterMark: 1000000});
|
|
||||||
|
|
||||||
// Check daily upload limit
|
upload.path = path.join(upload.id[0], upload.id + '_' + upload.filename);
|
||||||
const count = yield Files.count({owner: this.user.id, 'time_added': {'$gt': Math.ceil(Date.now() / 1000) - 86400}});
|
const localStream = fs.createWriteStream(path.join(storePath, upload.path));
|
||||||
const userLimit = this.user.daily_upload_allowance;
|
|
||||||
const underLimit = (count < userLimit || userLimit === 'unlimited');
|
|
||||||
if (!underLimit) {
|
|
||||||
this.statsd.incr('file.overlimit', 1);
|
|
||||||
}
|
|
||||||
this.assert(underLimit, 400, `{
|
|
||||||
"error": {
|
|
||||||
"message": "Daily upload limits (${this.user.daily_upload_allowance}) exceeded.",
|
|
||||||
"code": 602
|
|
||||||
}
|
|
||||||
}`);
|
|
||||||
|
|
||||||
// Clean filename for storage, keep original for display
|
|
||||||
upload.originalName = upload.filename;
|
|
||||||
upload.filename = upload.filename.replace(/[^a-zA-Z0-9\.\-\_\s]/g, '').replace(/\s+/g, '');
|
|
||||||
const fileId = yield hostrId(Files);
|
|
||||||
|
|
||||||
// Fire an event to let the frontend map the GUID it sent to the real ID. Allows immediate linking to the file
|
|
||||||
const acceptedEvent = `{"type": "file-accepted", "data": {"id": "${fileId}", "guid": "${tempGuid}", "href": "${baseURL}/${fileId}"}}`;
|
|
||||||
this.redis.publish('/user/' + this.user.id, acceptedEvent);
|
|
||||||
this.statsd.incr('file.upload.accepted', 1);
|
|
||||||
|
|
||||||
const uploadPromise = new Promise((resolve, reject) => {
|
|
||||||
upload.on('error', () => {
|
|
||||||
this.statsd.incr('file.upload.error', 1);
|
|
||||||
reject();
|
|
||||||
});
|
|
||||||
|
|
||||||
upload.on('end', () => {
|
|
||||||
resolve();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
const key = path.join(fileId[0], fileId + '_' + upload.filename);
|
|
||||||
const localStream = fs.createWriteStream(path.join(storePath, key));
|
|
||||||
|
|
||||||
upload.pipe(localStream);
|
upload.pipe(localStream);
|
||||||
upload.pipe(s3Upload(key));
|
|
||||||
|
|
||||||
const thumbsPromises = [
|
|
||||||
new Promise((resolve) => {
|
|
||||||
const small = gm(upload).resize(150, 150, '>').stream();
|
|
||||||
small.pipe(fs.createWriteStream(path.join(storePath, fileId[0], '150', fileId + '_' + upload.filename)));
|
|
||||||
small.pipe(s3Upload(path.join('150', fileId + '_' + upload.filename))).on('finish', resolve);
|
|
||||||
}),
|
|
||||||
new Promise((resolve) => {
|
|
||||||
const medium = gm(upload).resize(970, '>').stream();
|
|
||||||
medium.pipe(fs.createWriteStream(path.join(storePath, fileId[0], '970', fileId + '_' + upload.filename)));
|
|
||||||
medium.pipe(s3Upload(path.join('970', fileId + '_' + upload.filename))).on('finish', resolve);
|
|
||||||
}),
|
|
||||||
];
|
|
||||||
|
|
||||||
|
|
||||||
const dimensionsPromise = new Promise((resolve, reject) => {
|
|
||||||
gm(upload).size((err, size) => {
|
|
||||||
if (err) {
|
|
||||||
reject(err);
|
|
||||||
} else {
|
|
||||||
resolve(size);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
upload.on('data', (data) => {
|
upload.on('data', (data) => {
|
||||||
receivedSize += data.length;
|
receivedSize += data.length;
|
||||||
|
@ -114,8 +46,8 @@ export function* post(next) {
|
||||||
|
|
||||||
percentComplete = Math.floor(receivedSize * 100 / expectedSize);
|
percentComplete = Math.floor(receivedSize * 100 / expectedSize);
|
||||||
if (percentComplete > lastPercent && lastTick < Date.now() - 1000) {
|
if (percentComplete > lastPercent && lastTick < Date.now() - 1000) {
|
||||||
const progressEvent = `{"type": "file-progress", "data": {"id": "${fileId}", "complete": ${percentComplete}}}`;
|
const progressEvent = `{"type": "file-progress", "data": {"id": "${upload.id}", "complete": ${percentComplete}}}`;
|
||||||
this.redis.publish('/file/' + fileId, progressEvent);
|
this.redis.publish('/file/' + upload.id, progressEvent);
|
||||||
this.redis.publish('/user/' + this.user.id, progressEvent);
|
this.redis.publish('/user/' + this.user.id, progressEvent);
|
||||||
lastTick = Date.now();
|
lastTick = Date.now();
|
||||||
}
|
}
|
||||||
|
@ -124,17 +56,10 @@ export function* post(next) {
|
||||||
md5sum.update(data);
|
md5sum.update(data);
|
||||||
});
|
});
|
||||||
|
|
||||||
// Fire final upload progress event so users know it's now processing
|
|
||||||
const completeEvent = `{"type": "file-progress", "data": {"id": "${fileId}", "complete": 100}}`;
|
|
||||||
this.redis.publish('/file/' + fileId, completeEvent);
|
|
||||||
this.redis.publish('/user/' + this.user.id, completeEvent);
|
|
||||||
this.statsd.incr('file.upload.complete', 1);
|
|
||||||
|
|
||||||
const dbFile = {
|
const dbFile = {
|
||||||
_id: fileId,
|
|
||||||
owner: this.user.id,
|
owner: this.user.id,
|
||||||
ip: remoteIp,
|
ip: remoteIp,
|
||||||
'system_name': fileId,
|
'system_name': upload.id,
|
||||||
'file_name': upload.filename,
|
'file_name': upload.filename,
|
||||||
'original_name': upload.originalName,
|
'original_name': upload.originalName,
|
||||||
'file_size': receivedSize,
|
'file_size': receivedSize,
|
||||||
|
@ -145,49 +70,33 @@ export function* post(next) {
|
||||||
type: sniff(upload.filename),
|
type: sniff(upload.filename),
|
||||||
};
|
};
|
||||||
|
|
||||||
yield Files.insertOne(dbFile);
|
yield Files.insertOne({_id: upload.id, ...dbFile});
|
||||||
yield uploadPromise;
|
|
||||||
try {
|
|
||||||
const dimensions = yield dimensionsPromise;
|
|
||||||
dbFile.width = dimensions.width;
|
|
||||||
dbFile.height = dimensions.height;
|
|
||||||
} catch (e) {
|
|
||||||
debug('Not an image');
|
|
||||||
}
|
|
||||||
|
|
||||||
yield thumbsPromises;
|
yield upload.promise;
|
||||||
|
|
||||||
|
const completeEvent = `{"type": "file-progress", "data": {"id": "${upload.id}", "complete": 100}}`;
|
||||||
|
this.redis.publish('/file/' + upload.id, completeEvent);
|
||||||
|
this.redis.publish('/user/' + this.user.id, completeEvent);
|
||||||
|
this.statsd.incr('file.upload.complete', 1);
|
||||||
|
|
||||||
|
const size = yield processImage(upload);
|
||||||
|
|
||||||
|
dbFile.width = size.width;
|
||||||
|
dbFile.height = size.height;
|
||||||
dbFile.file_size = receivedSize; // eslint-disable-line camelcase
|
dbFile.file_size = receivedSize; // eslint-disable-line camelcase
|
||||||
dbFile.status = 'active';
|
dbFile.status = 'active';
|
||||||
dbFile.md5 = md5sum.digest('hex');
|
dbFile.md5 = md5sum.digest('hex');
|
||||||
|
|
||||||
const formattedFile = formatFile(dbFile);
|
const formattedFile = formatFile({_id: upload.id, ...dbFile});
|
||||||
|
|
||||||
delete dbFile._id;
|
yield Files.updateOne({_id: upload.id}, {$set: dbFile});
|
||||||
yield Files.updateOne({_id: fileId}, {$set: dbFile});
|
|
||||||
|
|
||||||
// Fire upload complete event
|
|
||||||
const addedEvent = `{"type": "file-added", "data": ${JSON.stringify(formattedFile)}}`;
|
const addedEvent = `{"type": "file-added", "data": ${JSON.stringify(formattedFile)}}`;
|
||||||
this.redis.publish('/file/' + fileId, addedEvent);
|
this.redis.publish('/file/' + upload.id, addedEvent);
|
||||||
this.redis.publish('/user/' + this.user.id, addedEvent);
|
this.redis.publish('/user/' + this.user.id, addedEvent);
|
||||||
|
|
||||||
this.status = 201;
|
this.status = 201;
|
||||||
this.body = formattedFile;
|
this.body = formattedFile;
|
||||||
|
|
||||||
if (process.env.VIRUSTOTAL_KEY) {
|
|
||||||
// Check in the background
|
|
||||||
process.nextTick(function* malwareScan() {
|
|
||||||
debug('Malware Scan');
|
|
||||||
const result = yield malware(dbFile);
|
|
||||||
if (result) {
|
|
||||||
yield Files.updateOne({_id: fileId}, {'$set': {malware: positive, virustotal: result}});
|
|
||||||
if (result.positive) {
|
|
||||||
this.statsd.incr('file.malware', 1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
debug('Skipping Malware Scan, VIRUSTOTAL env variable not found.');
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import createError from 'http-errors';
|
import createError from 'http-errors';
|
||||||
import { get as getFile } from './s3';
|
import { get as getFile } from './sftp';
|
||||||
|
|
||||||
import debugname from 'debug';
|
import debugname from 'debug';
|
||||||
const debug = debugname('hostr:file-stream');
|
const debug = debugname('hostr:file-stream');
|
||||||
|
@ -10,22 +10,23 @@ export default function* hostrFileStream(localPath, remotePath) {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
localRead.once('error', () => {
|
localRead.once('error', () => {
|
||||||
debug('local error');
|
debug('local error');
|
||||||
const remoteRead = getFile(remotePath);
|
const remoteFile = getFile(remotePath);
|
||||||
|
|
||||||
remoteRead.once('readable', () => {
|
remoteFile.then((remoteRead) => {
|
||||||
debug('remote readable');
|
|
||||||
const localWrite = fs.createWriteStream(localPath);
|
const localWrite = fs.createWriteStream(localPath);
|
||||||
localWrite.once('finish', () => {
|
localWrite.once('finish', () => {
|
||||||
debug('local write end');
|
debug('local write end');
|
||||||
resolve(fs.createReadStream(localPath));
|
resolve(fs.createReadStream(localPath));
|
||||||
});
|
});
|
||||||
remoteRead.pipe(localWrite);
|
remoteRead.pipe(localWrite);
|
||||||
|
|
||||||
|
remoteRead.once('error', () => {
|
||||||
|
debug('remote error');
|
||||||
|
reject(createError(404));
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
remoteRead.once('error', () => {
|
|
||||||
debug('remote error');
|
|
||||||
reject(createError(404));
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
localRead.once('readable', () => {
|
localRead.once('readable', () => {
|
||||||
debug('local readable');
|
debug('local readable');
|
||||||
|
|
|
@ -1,75 +0,0 @@
|
||||||
/**
|
|
||||||
* Module dependencies.
|
|
||||||
*/
|
|
||||||
|
|
||||||
var swig = require('swig');
|
|
||||||
var http = require('http');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Expose `error`.
|
|
||||||
*/
|
|
||||||
|
|
||||||
module.exports = error;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Error middleware.
|
|
||||||
*
|
|
||||||
* - `template` defaults to ./error.html
|
|
||||||
*
|
|
||||||
* @param {Object} opts
|
|
||||||
* @api public
|
|
||||||
*/
|
|
||||||
|
|
||||||
function error(opts) {
|
|
||||||
opts = opts || {};
|
|
||||||
|
|
||||||
// template
|
|
||||||
var path = opts.template || __dirname + '/error.html';
|
|
||||||
var render = swig.compileFile(path);
|
|
||||||
|
|
||||||
// env
|
|
||||||
var env = process.env.NODE_ENV || 'development';
|
|
||||||
|
|
||||||
return function *error(next){
|
|
||||||
try {
|
|
||||||
yield next;
|
|
||||||
if (404 == this.response.status && !this.response.body) this.throw(404);
|
|
||||||
} catch (err) {
|
|
||||||
this.status = err.status || 500;
|
|
||||||
|
|
||||||
// application
|
|
||||||
this.app.emit('error', err, this);
|
|
||||||
|
|
||||||
// accepted types
|
|
||||||
switch (this.accepts('html', 'text', 'json')) {
|
|
||||||
case 'text':
|
|
||||||
this.type = 'text/plain';
|
|
||||||
if ('development' == env) this.body = err.message
|
|
||||||
else if (err.expose) this.body = err.message
|
|
||||||
else throw err;
|
|
||||||
break;
|
|
||||||
|
|
||||||
case 'json':
|
|
||||||
this.type = 'application/json';
|
|
||||||
if ('development' == env) this.body = { error: err.message }
|
|
||||||
else if (err.expose) this.body = { error: err.message }
|
|
||||||
else this.body = { error: http.STATUS_CODES[this.status] }
|
|
||||||
break;
|
|
||||||
|
|
||||||
case 'html':
|
|
||||||
this.type = 'text/html';
|
|
||||||
this.body = render({
|
|
||||||
env: env,
|
|
||||||
ctx: this,
|
|
||||||
request: this.request,
|
|
||||||
response: this.response,
|
|
||||||
error: err.message,
|
|
||||||
stack: err.stack,
|
|
||||||
status: this.status,
|
|
||||||
code: err.code
|
|
||||||
});
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,33 +1,71 @@
|
||||||
|
import fs from 'mz/fs';
|
||||||
|
import lwip from 'lwip';
|
||||||
import debugname from 'debug';
|
import debugname from 'debug';
|
||||||
const debug = debugname('hostr-api:resize');
|
const debug = debugname('hostr-api:resize');
|
||||||
import lwip from 'lwip';
|
|
||||||
import imageType from 'image-type';
|
|
||||||
|
|
||||||
const supported = ['jpg', 'png', 'gif'];
|
|
||||||
|
|
||||||
export default function(input, size) {
|
|
||||||
debug('Resizing');
|
|
||||||
|
|
||||||
const type = imageType(input);
|
|
||||||
|
|
||||||
if (!type.ext || supported.indexOf(type.ext) < 0) {
|
|
||||||
throw new Error('Not a supported image.');
|
|
||||||
}
|
|
||||||
|
|
||||||
|
function cover(path, type, size) {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
lwip.open(input, type.ext, (errIn, image) => {
|
lwip.open(path, type, (errIn, image) => {
|
||||||
|
debug('Image Opened');
|
||||||
if (errIn) {
|
if (errIn) {
|
||||||
return reject(errIn);
|
reject(errIn);
|
||||||
}
|
}
|
||||||
|
|
||||||
image.cover(size.width, size.height, (errOut, resized) => {
|
image.cover(size.width, size.height, (errOut, resized) => {
|
||||||
|
debug('Image Resized');
|
||||||
if (errOut) {
|
if (errOut) {
|
||||||
return reject(errOut);
|
reject(errOut);
|
||||||
}
|
}
|
||||||
|
|
||||||
resized.toBuffer(type.ext, (errBuf, buffer) => {
|
resized.toBuffer(type, (errBuf, buffer) => {
|
||||||
|
debug('Image Buffered');
|
||||||
|
if (errBuf) {
|
||||||
|
reject(errBuf);
|
||||||
|
}
|
||||||
resolve(buffer);
|
resolve(buffer);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function scale(path, type, size) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
lwip.open(path, type, (errIn, image) => {
|
||||||
|
debug('Image Opened');
|
||||||
|
if (errIn) {
|
||||||
|
reject(errIn);
|
||||||
|
}
|
||||||
|
|
||||||
|
image.cover(size.width, size.height, (errOut, resized) => {
|
||||||
|
debug('Image Resized');
|
||||||
|
if (errOut) {
|
||||||
|
reject(errOut);
|
||||||
|
}
|
||||||
|
|
||||||
|
resized.toBuffer(type, (errBuf, buffer) => {
|
||||||
|
debug('Image Buffered');
|
||||||
|
if (errBuf) {
|
||||||
|
reject(errBuf);
|
||||||
|
}
|
||||||
|
resolve(buffer);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function resize(path, type, currentSize, newSize) {
|
||||||
|
debug('Resizing');
|
||||||
|
const ratio = 970 / currentSize.width;
|
||||||
|
if (newSize.width <= 150) {
|
||||||
|
debug('Cover');
|
||||||
|
return cover(path, type, newSize);
|
||||||
|
} else if (newSize.width > 970 && ratio > 1) {
|
||||||
|
debug('Scale');
|
||||||
|
newSize.height = currentSize.height * ratio;
|
||||||
|
return scale(path, type, newSize);
|
||||||
|
}
|
||||||
|
debug('Copy');
|
||||||
|
return fs.readFile(path);
|
||||||
|
}
|
||||||
|
|
11
lib/s3.js
11
lib/s3.js
|
@ -1,17 +1,20 @@
|
||||||
import aws from 'aws-sdk';
|
import aws from 'aws-sdk';
|
||||||
import s3UploadStream from 's3-upload-stream';
|
|
||||||
import debugname from 'debug';
|
import debugname from 'debug';
|
||||||
const debug = debugname('hostr:s3');
|
const debug = debugname('hostr:s3');
|
||||||
|
|
||||||
const s3 = new aws.S3();
|
const s3 = new aws.S3();
|
||||||
const s3Stream = s3UploadStream(s3);
|
|
||||||
|
|
||||||
export function get(key) {
|
export function get(key) {
|
||||||
debug('fetching from s3: %s', 'hostr_files/' + key);
|
debug('fetching from s3: %s', 'hostr_files/' + key);
|
||||||
return s3.getObject({Bucket: process.env.AWS_BUCKET, Key: 'hostr_files/' + key}).createReadStream();
|
return s3.getObject({Bucket: process.env.AWS_BUCKET, Key: 'hostr_files/' + key}).createReadStream();
|
||||||
}
|
}
|
||||||
|
|
||||||
export function upload(key) {
|
export function upload(stream, key, callback) {
|
||||||
debug('sending to s3: %s', 'hostr_files/' + key);
|
debug('sending to s3: %s', 'hostr_files/' + key);
|
||||||
return s3Stream.upload({Bucket: process.env.AWS_BUCKET, Key: 'hostr_files/' + key});
|
const params = {Bucket: process.env.AWS_BUCKET, Key: 'hostr_files/' + key, Body: stream};
|
||||||
|
const uploading = s3.upload(params);
|
||||||
|
uploading.on('error', (err) => {
|
||||||
|
console.log(err)
|
||||||
|
});
|
||||||
|
uploading.send(callback);
|
||||||
}
|
}
|
||||||
|
|
40
lib/sftp.js
Normal file
40
lib/sftp.js
Normal file
|
@ -0,0 +1,40 @@
|
||||||
|
import { dirname } from 'path';
|
||||||
|
import Client from 'ssh2-sftp-client';
|
||||||
|
import debugname from 'debug';
|
||||||
|
const debug = debugname('hostr:sftp');
|
||||||
|
|
||||||
|
export function get(remotePath) {
|
||||||
|
const sftp = new Client();
|
||||||
|
return sftp.connect({
|
||||||
|
host: process.env.SFTP_HOST,
|
||||||
|
port: process.env.SFTP_PORT,
|
||||||
|
username: process.env.SFTP_USERNAME,
|
||||||
|
password: process.env.SFTP_PASSWORD,
|
||||||
|
}).then(() => {
|
||||||
|
return sftp.get('hostr/uploads/' + remotePath, true);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function upload(localPath, remotePath) {
|
||||||
|
debug('SFTP connecting');
|
||||||
|
const sftp = new Client();
|
||||||
|
return sftp.connect({
|
||||||
|
host: process.env.SFTP_HOST,
|
||||||
|
port: process.env.SFTP_PORT,
|
||||||
|
username: process.env.SFTP_USERNAME,
|
||||||
|
password: process.env.SFTP_PASSWORD,
|
||||||
|
}).then(() => {
|
||||||
|
return sftp.put(localPath, remotePath, true).then(() => {
|
||||||
|
sftp.end();
|
||||||
|
});
|
||||||
|
}).catch(() => {
|
||||||
|
debug('Creating ' + dirname(remotePath));
|
||||||
|
return sftp.mkdir(dirname(remotePath), true).then(() => {
|
||||||
|
return sftp.put(localPath, remotePath, true).then(() => {
|
||||||
|
sftp.end();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}).then(() => {
|
||||||
|
sftp.end();
|
||||||
|
});
|
||||||
|
}
|
108
lib/upload.js
Normal file
108
lib/upload.js
Normal file
|
@ -0,0 +1,108 @@
|
||||||
|
import { join } from 'path';
|
||||||
|
import parse from 'co-busboy';
|
||||||
|
import fs from 'mz/fs';
|
||||||
|
import sizeOf from 'image-size';
|
||||||
|
import hostrId from './hostr-id';
|
||||||
|
import resize from './resize';
|
||||||
|
import { upload as sftpUpload } from './sftp';
|
||||||
|
|
||||||
|
import debugname from 'debug';
|
||||||
|
const debug = debugname('hostr-api:upload');
|
||||||
|
|
||||||
|
const storePath = process.env.UPLOAD_STORAGE_PATH;
|
||||||
|
const baseURL = process.env.WEB_BASE_URL;
|
||||||
|
const supported = ['jpg', 'png', 'gif'];
|
||||||
|
|
||||||
|
export function* checkLimit() {
|
||||||
|
const count = yield this.db.Files.count({
|
||||||
|
owner: this.user.id,
|
||||||
|
time_added: {'$gt': Math.ceil(Date.now() / 1000) - 86400},
|
||||||
|
});
|
||||||
|
const userLimit = this.user.daily_upload_allowance;
|
||||||
|
const underLimit = (count < userLimit || userLimit === 'unlimited');
|
||||||
|
if (!underLimit) {
|
||||||
|
this.statsd.incr('file.overlimit', 1);
|
||||||
|
}
|
||||||
|
this.assert(underLimit, 400, `{
|
||||||
|
"error": {
|
||||||
|
"message": "Daily upload limits (${this.user.daily_upload_allowance}) exceeded.",
|
||||||
|
"code": 602
|
||||||
|
}
|
||||||
|
}`);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function* accept() {
|
||||||
|
yield checkLimit.call(this);
|
||||||
|
|
||||||
|
const upload = yield parse(this, {
|
||||||
|
autoFields: true,
|
||||||
|
headers: this.request.headers,
|
||||||
|
limits: { files: 1},
|
||||||
|
highWaterMark: 1000000,
|
||||||
|
});
|
||||||
|
|
||||||
|
upload.promise = new Promise((resolve, reject) => {
|
||||||
|
upload.on('error', (err) => {
|
||||||
|
this.statsd.incr('file.upload.error', 1);
|
||||||
|
debug(err);
|
||||||
|
reject();
|
||||||
|
});
|
||||||
|
|
||||||
|
upload.on('end', () => {
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
upload.tempGuid = this.request.headers['hostr-guid'];
|
||||||
|
upload.originalName = upload.filename;
|
||||||
|
upload.filename = upload.filename.replace(/[^a-zA-Z0-9\.\-\_\s]/g, '').replace(/\s+/g, '');
|
||||||
|
upload.id = yield hostrId(this.db.Files);
|
||||||
|
|
||||||
|
const acceptedEvent = `{"type": "file-accepted", "data": {"id": "${upload.id}", "guid": "${upload.tempGuid}", "href": "${baseURL}/${upload.id}"}}`;
|
||||||
|
this.redis.publish('/user/' + this.user.id, acceptedEvent);
|
||||||
|
this.statsd.incr('file.upload.accepted', 1);
|
||||||
|
|
||||||
|
return upload;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resizeImage(upload, type, currentSize, newSize) {
|
||||||
|
return resize(join(storePath, upload.path), type, currentSize, newSize).then((image) => {
|
||||||
|
const path = join(upload.id[0], String(newSize.width), upload.id + '_' + upload.filename);
|
||||||
|
debug('Writing file');
|
||||||
|
debug(join(storePath, path));
|
||||||
|
return fs.writeFile(join(storePath, path), image).then(() => {
|
||||||
|
debug('Uploading file');
|
||||||
|
return sftpUpload(join(storePath, path), join('hostr', 'uploads', path));
|
||||||
|
}).catch(debug);
|
||||||
|
}).catch(debug);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function* processImage(upload) {
|
||||||
|
debug('Processing image');
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const size = sizeOf(join(storePath, upload.path));
|
||||||
|
debug('Size: ', size);
|
||||||
|
if (!size.width || supported.indexOf(size.type) < 0) {
|
||||||
|
resolve();
|
||||||
|
}
|
||||||
|
|
||||||
|
Promise.all([
|
||||||
|
resizeImage(upload, size.type, size, {width: 150, height: 150}),
|
||||||
|
resizeImage(upload, size.type, size, {width: 970}),
|
||||||
|
]).then(() => {
|
||||||
|
resolve(size);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function progressEvent() {
|
||||||
|
percentComplete = Math.floor(receivedSize * 100 / expectedSize);
|
||||||
|
if (percentComplete > lastPercent && lastTick < Date.now() - 1000) {
|
||||||
|
const progressEvent = `{"type": "file-progress", "data": {"id": "${upload.id}", "complete": ${percentComplete}}}`;
|
||||||
|
this.redis.publish('/file/' + upload.id, progressEvent);
|
||||||
|
this.redis.publish('/user/' + this.user.id, progressEvent);
|
||||||
|
lastTick = Date.now();
|
||||||
|
}
|
||||||
|
lastPercent = percentComplete;
|
||||||
|
}
|
|
@ -16,7 +16,7 @@
|
||||||
"init": "node -r babel/register -e \"require('./lib/storage')();\"",
|
"init": "node -r babel/register -e \"require('./lib/storage')();\"",
|
||||||
"jspm": "jspm install",
|
"jspm": "jspm install",
|
||||||
"start": "npm run build && node -r babel/register app.js",
|
"start": "npm run build && node -r babel/register app.js",
|
||||||
"test": "npm run test-seed && mocha -r babel/register test/**/*.spec.js && mocha -r babel/register -r co-mocha test/unit/image-resize.spec.js",
|
"test": "npm run test-seed && mocha -r babel/register test/**/*.spec.js",
|
||||||
"test-seed": "node test/fixtures/mongo-user.js && node test/fixtures/mongo-file.js",
|
"test-seed": "node test/fixtures/mongo-user.js && node test/fixtures/mongo-file.js",
|
||||||
"watch": "parallelshell \"npm run watch-js\" \"npm run watch-sass\" \"npm run watch-server\"",
|
"watch": "parallelshell \"npm run watch-js\" \"npm run watch-sass\" \"npm run watch-server\"",
|
||||||
"watch-js": "babel -Dw -m system -d web/public/build web/public/src",
|
"watch-js": "babel -Dw -m system -d web/public/build web/public/src",
|
||||||
|
@ -24,7 +24,7 @@
|
||||||
"watch-sass": "node-sass -w -r -o web/public/styles/ web/public/styles/"
|
"watch-sass": "node-sass -w -r -o web/public/styles/ web/public/styles/"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"aws-sdk": "~2.1.46",
|
"aws-sdk": "~2.3.15",
|
||||||
"babel": "~5.8.21",
|
"babel": "~5.8.21",
|
||||||
"basic-auth": "~1.0.3",
|
"basic-auth": "~1.0.3",
|
||||||
"co": "~4.6.0",
|
"co": "~4.6.0",
|
||||||
|
@ -43,6 +43,7 @@
|
||||||
"koa-bodyparser": "~2.0.1",
|
"koa-bodyparser": "~2.0.1",
|
||||||
"koa-compress": "~1.0.8",
|
"koa-compress": "~1.0.8",
|
||||||
"koa-csrf": "~2.3.0",
|
"koa-csrf": "~2.3.0",
|
||||||
|
"koa-error": "^2.0.0",
|
||||||
"koa-favicon": "~1.2.0",
|
"koa-favicon": "~1.2.0",
|
||||||
"koa-generic-session": "~1.9.0",
|
"koa-generic-session": "~1.9.0",
|
||||||
"koa-helmet": "^0.2.0",
|
"koa-helmet": "^0.2.0",
|
||||||
|
@ -57,6 +58,7 @@
|
||||||
"mime-types": "~2.1.5",
|
"mime-types": "~2.1.5",
|
||||||
"moment": "~2.10.6",
|
"moment": "~2.10.6",
|
||||||
"mongodb-promisified": "~1.0.3",
|
"mongodb-promisified": "~1.0.3",
|
||||||
|
"mz": "^2.4.0",
|
||||||
"node-fetch": "^1.3.2",
|
"node-fetch": "^1.3.2",
|
||||||
"node-sass": "~3.6.0",
|
"node-sass": "~3.6.0",
|
||||||
"node-uuid": "~1.4.3",
|
"node-uuid": "~1.4.3",
|
||||||
|
@ -65,13 +67,13 @@
|
||||||
"redis": "~1.0.0",
|
"redis": "~1.0.0",
|
||||||
"s3-upload-stream": "~1.0.7",
|
"s3-upload-stream": "~1.0.7",
|
||||||
"sendgrid": "^2.0.0",
|
"sendgrid": "^2.0.0",
|
||||||
|
"ssh2-sftp-client": "^1.0.3",
|
||||||
"statsy": "~0.2.0",
|
"statsy": "~0.2.0",
|
||||||
"stripe": "~3.7.1",
|
"stripe": "~3.7.1",
|
||||||
"swig": "~1.4.2"
|
"swig": "~1.4.2"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"babel-eslint": "^4.0.10",
|
"babel-eslint": "^4.0.10",
|
||||||
"co-mocha": "^1.1.2",
|
|
||||||
"eslint": "~1.3.0",
|
"eslint": "~1.3.0",
|
||||||
"eslint-config-airbnb": "0.0.8",
|
"eslint-config-airbnb": "0.0.8",
|
||||||
"istanbul": "~0.3.18",
|
"istanbul": "~0.3.18",
|
||||||
|
@ -79,6 +81,7 @@
|
||||||
"nodemon": "~1.4.1",
|
"nodemon": "~1.4.1",
|
||||||
"parallelshell": "~2.0.0",
|
"parallelshell": "~2.0.0",
|
||||||
"supertest": "~1.1.0",
|
"supertest": "~1.1.0",
|
||||||
|
"supertest-koa-agent": "^0.2.1",
|
||||||
"tmp": "~0.0.27"
|
"tmp": "~0.0.27"
|
||||||
},
|
},
|
||||||
"jspm": {
|
"jspm": {
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
import path from 'path';
|
||||||
import assert from 'assert';
|
import assert from 'assert';
|
||||||
import { agent } from 'supertest';
|
import { agent } from 'supertest';
|
||||||
import app from '../../app';
|
import app from '../../app';
|
||||||
|
@ -25,11 +26,11 @@ describe('hostr-api file', function file() {
|
||||||
this.timeout(30000);
|
this.timeout(30000);
|
||||||
request
|
request
|
||||||
.post('/api/file')
|
.post('/api/file')
|
||||||
.attach('file', './test/fixtures/utah-arches.jpg')
|
.attach('file', path.join(__dirname, '..', 'fixtures', 'tall.jpg'))
|
||||||
.auth('test@hostr.co', 'test-password')
|
.auth('test@hostr.co', 'test-password')
|
||||||
.expect(201)
|
.expect(201)
|
||||||
.expect((response) => {
|
.expect((response) => {
|
||||||
assert(response.body.name === 'utah-arches.jpg');
|
assert(response.body.name === 'tall.jpg');
|
||||||
id = response.body.id;
|
id = response.body.id;
|
||||||
})
|
})
|
||||||
.end(done);
|
.end(done);
|
||||||
|
@ -42,7 +43,7 @@ describe('hostr-api file', function file() {
|
||||||
.get('/api/file/' + id)
|
.get('/api/file/' + id)
|
||||||
.expect(200)
|
.expect(200)
|
||||||
.expect((response) => {
|
.expect((response) => {
|
||||||
assert(response.body.name === 'utah-arches.jpg');
|
assert(response.body.name === 'tall.jpg');
|
||||||
})
|
})
|
||||||
.end(done);
|
.end(done);
|
||||||
});
|
});
|
||||||
|
|
|
@ -1,35 +1,35 @@
|
||||||
import fs from 'fs';
|
import fs from 'mz/fs';
|
||||||
import path from 'path';
|
import { join } from 'path';
|
||||||
import assert from 'assert';
|
import assert from 'assert';
|
||||||
import tmp from 'tmp';
|
import tmp from 'tmp';
|
||||||
import resize from '../../lib/resize';
|
import resize from '../../lib/resize';
|
||||||
import imageType from 'image-type';
|
import sizeOf from 'image-size';
|
||||||
|
|
||||||
|
function testResize(path, done) {
|
||||||
|
const size = sizeOf(path);
|
||||||
|
resize(path, size.type, size, {width: 100, height: 100}).then((image) => {
|
||||||
|
const tmpFile = tmp.tmpNameSync() + '.' + size.type;
|
||||||
|
fs.writeFile(tmpFile, image).then(() => {
|
||||||
|
const newSize = sizeOf(fs.readFileSync(tmpFile));
|
||||||
|
assert(newSize.type === size.type);
|
||||||
|
done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
describe('Image resizing', () => {
|
describe('Image resizing', () => {
|
||||||
it('should resize a jpg', function* resizeImage() {
|
it('should resize a jpg', (done) => {
|
||||||
const file = fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'utah-arches.jpg'));
|
const path = join(__dirname, '..', 'fixtures', 'utah-arches.jpg');
|
||||||
const imageBuffer = yield resize(file, {height: 100, width: 100});
|
testResize(path, done);
|
||||||
const tmpFile = tmp.tmpNameSync() + '.jpg';
|
|
||||||
fs.writeFileSync(tmpFile, imageBuffer);
|
|
||||||
const type = imageType(fs.readFileSync(tmpFile));
|
|
||||||
assert(type.ext === 'jpg');
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should resize a png', function* resizeImage() {
|
it('should resize a png', (done) => {
|
||||||
const file = fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'app-icon.png'));
|
const path = join(__dirname, '..', 'fixtures', 'app-icon.png');
|
||||||
const imageBuffer = yield resize(file, {height: 100, width: 100});
|
testResize(path, done);
|
||||||
const tmpFile = tmp.tmpNameSync() + '.png';
|
|
||||||
fs.writeFileSync(tmpFile, imageBuffer);
|
|
||||||
const type = imageType(fs.readFileSync(tmpFile));
|
|
||||||
assert(type.ext === 'png');
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should resize a gif', function* resizeImage() {
|
it('should resize a gif', (done) => {
|
||||||
const file = fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'kim.gif'));
|
const path = join(__dirname, '..', 'fixtures', 'kim.gif');
|
||||||
const imageBuffer = yield resize(file, {height: 100, width: 100});
|
testResize(path, done);
|
||||||
const tmpFile = tmp.tmpNameSync() + '.gif';
|
|
||||||
fs.writeFileSync(tmpFile, imageBuffer);
|
|
||||||
const type = imageType(fs.readFileSync(tmpFile));
|
|
||||||
assert(type.ext === 'gif');
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
import path from 'path';
|
||||||
import assert from 'assert';
|
import assert from 'assert';
|
||||||
import sizeOf from 'image-size';
|
import sizeOf from 'image-size';
|
||||||
import { agent } from 'supertest';
|
import { agent } from 'supertest';
|
||||||
|
@ -12,7 +13,7 @@ describe('setup hostr-web file', function() {
|
||||||
this.timeout(30000);
|
this.timeout(30000);
|
||||||
request
|
request
|
||||||
.post('/api/file')
|
.post('/api/file')
|
||||||
.attach('file', 'test/fixtures/utah-arches.jpg')
|
.attach('file', path.join(__dirname, '..', 'fixtures', 'utah-arches.jpg'))
|
||||||
.auth('test@hostr.co', 'test-password')
|
.auth('test@hostr.co', 'test-password')
|
||||||
.expect(201)
|
.expect(201)
|
||||||
.expect(function(response) {
|
.expect(function(response) {
|
||||||
|
@ -25,7 +26,6 @@ describe('setup hostr-web file', function() {
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('hostr-web file', function() {
|
describe('hostr-web file', function() {
|
||||||
|
|
||||||
describe('when GET /file/:id/:name', function() {
|
describe('when GET /file/:id/:name', function() {
|
||||||
it('should receive an image', function(done) {
|
it('should receive an image', function(done) {
|
||||||
request
|
request
|
||||||
|
@ -40,41 +40,40 @@ describe('hostr-web file', function() {
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('when GET /file/150/:id/:name', function() {
|
describe('when GET /file/150/:id/:name', function() {
|
||||||
it('should receive a 150px wide thumbnail of the image', function(done) {
|
it('should receive a 150px wide thumbnail of the image', function() {
|
||||||
request
|
request
|
||||||
.get('/file/150/' + file.id + '/' + file.name)
|
.get('/file/150/' + file.id + '/' + file.name)
|
||||||
.expect(200)
|
.expect(200)
|
||||||
.expect('Content-type', 'image/jpeg')
|
.expect('Content-type', 'image/jpeg')
|
||||||
.expect(function(response) {
|
.expect(function(response) {
|
||||||
assert(sizeOf(response.body).width === 150);
|
const width = sizeOf(response.body).width;
|
||||||
})
|
assert(width === 150);
|
||||||
.end(done);
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('when GET /file/970/:id/:name', function() {
|
describe('when GET /file/970/:id/:name', function() {
|
||||||
it('should receive a 970px wide thumbnail of the image', function(done) {
|
it('should receive a 970px wide thumbnail of the image', function() {
|
||||||
request
|
request
|
||||||
.get('/file/970/' + file.id + '/' + file.name)
|
.get('/file/970/' + file.id + '/' + file.name)
|
||||||
.expect(200)
|
.expect(200)
|
||||||
.expect('Content-type', 'image/jpeg')
|
.expect('Content-type', 'image/jpeg')
|
||||||
.expect(function(response) {
|
.expect(function(response) {
|
||||||
assert(sizeOf(response.body).width === 970);
|
const width = sizeOf(response.body).width;
|
||||||
})
|
assert(width === 970);
|
||||||
.end(done);
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('when GET /:id', function() {
|
describe('when GET /:id', function() {
|
||||||
it('should receive some HTML', function(done) {
|
it('should receive some HTML', function() {
|
||||||
request
|
request
|
||||||
.get('/' + file.id)
|
.get('/' + file.id)
|
||||||
.expect(200)
|
.expect(200)
|
||||||
.expect('Content-type', /text\/html/) // Could include charset
|
.expect('Content-type', /text\/html/) // Could include charset
|
||||||
.expect(function(response) {
|
.expect(function(response) {
|
||||||
assert(response.text.indexOf('src="/file/970/' + file.id + '/' + file.name + '"') > -1);
|
assert(response.text.indexOf('src="/file/970/' + file.id + '/' + file.name + '"') > -1);
|
||||||
})
|
});
|
||||||
.end(done);
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
@ -5,20 +5,20 @@ const request = agent(app.listen());
|
||||||
|
|
||||||
describe('hostr-web user', function() {
|
describe('hostr-web user', function() {
|
||||||
describe('when POST /signin with invalid credentials', function() {
|
describe('when POST /signin with invalid credentials', function() {
|
||||||
it('should not redirect to /', function(done) {
|
it('should not redirect to /', function() {
|
||||||
request.get('/signin').end(function(err, response) {
|
request.get('/signin').end(function(err, response) {
|
||||||
const match = response.text.match(/name="_csrf" value="([^"]+)"/);
|
const match = response.text.match(/name="_csrf" value="([^"]+)"/);
|
||||||
const csrf = match[1];
|
const csrf = match[1];
|
||||||
request
|
request
|
||||||
.post('/signin')
|
.post('/signin')
|
||||||
.send({'email': 'test@hostr.co', 'password': 'test-passworddeded', '_csrf': csrf})
|
.send({'email': 'test@hostr.co', 'password': 'test-passworddeded', '_csrf': csrf})
|
||||||
.expect(200, done);
|
.expect(200);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('when POST /signin with valid credentials', function() {
|
describe('when POST /signin with valid credentials', function() {
|
||||||
it('should redirect to /', function(done) {
|
it('should redirect to /', function() {
|
||||||
request.get('/signin').end(function(err, response) {
|
request.get('/signin').end(function(err, response) {
|
||||||
const match = response.text.match(/name="_csrf" value="([^"]+)"/);
|
const match = response.text.match(/name="_csrf" value="([^"]+)"/);
|
||||||
const csrf = match[1];
|
const csrf = match[1];
|
||||||
|
@ -27,7 +27,7 @@ describe('hostr-web user', function() {
|
||||||
.send({'email': 'test@hostr.co', 'password': 'test-password', '_csrf': csrf})
|
.send({'email': 'test@hostr.co', 'password': 'test-password', '_csrf': csrf})
|
||||||
.expect(302)
|
.expect(302)
|
||||||
.expect('Location', '/')
|
.expect('Location', '/')
|
||||||
.end(done);
|
.end();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
@ -5,8 +5,7 @@ import views from 'koa-views';
|
||||||
import stats from 'koa-statsd';
|
import stats from 'koa-statsd';
|
||||||
import * as redis from '../lib/redis';
|
import * as redis from '../lib/redis';
|
||||||
import StatsD from 'statsy';
|
import StatsD from 'statsy';
|
||||||
// waiting for PR to be merged, can remove swig dependency when done
|
import errors from 'koa-error';
|
||||||
import errors from '../lib/koa-error';
|
|
||||||
import * as index from './routes/index';
|
import * as index from './routes/index';
|
||||||
import * as file from './routes/file';
|
import * as file from './routes/file';
|
||||||
import * as pro from './routes/pro';
|
import * as pro from './routes/pro';
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue