Refactor uploads
This commit is contained in:
parent
75a46212da
commit
f59b9a5d22
14 changed files with 297 additions and 269 deletions
|
|
@ -46,7 +46,9 @@ router.use('*', function* authMiddleware(next) {
|
|||
} else {
|
||||
if (!err.status) {
|
||||
debug(err);
|
||||
this.raven.captureError(err);
|
||||
if (this.raven) {
|
||||
this.raven.captureError(err);
|
||||
}
|
||||
throw err;
|
||||
} else {
|
||||
this.status = err.status;
|
||||
|
|
|
|||
|
|
@ -1,22 +1,18 @@
|
|||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import crypto from 'crypto';
|
||||
import gm from 'gm';
|
||||
import fs from 'mz/fs';
|
||||
import redis from 'redis';
|
||||
import parse from 'co-busboy';
|
||||
import { upload as s3Upload } from '../../lib/s3';
|
||||
|
||||
import { sniff } from '../../lib/type';
|
||||
import hostrId from '../../lib/hostr-id';
|
||||
import malware from '../../lib/malware';
|
||||
import { formatFile } from '../../lib/format';
|
||||
import { accept, processImage } from '../../lib/upload';
|
||||
|
||||
import debugname from 'debug';
|
||||
const debug = debugname('hostr-api:file');
|
||||
|
||||
const redisUrl = process.env.REDIS_URL;
|
||||
|
||||
const baseURL = process.env.WEB_BASE_URL;
|
||||
|
||||
const storePath = process.env.UPLOAD_STORAGE_PATH;
|
||||
|
||||
export function* post(next) {
|
||||
|
|
@ -26,9 +22,7 @@ export function* post(next) {
|
|||
const Files = this.db.Files;
|
||||
|
||||
const expectedSize = this.request.headers['content-length'];
|
||||
const tempGuid = this.request.headers['hostr-guid'];
|
||||
const remoteIp = this.request.headers['x-real-ip'] || this.req.connection.remoteAddress;
|
||||
|
||||
const md5sum = crypto.createHash('md5');
|
||||
|
||||
let lastPercent = 0;
|
||||
|
|
@ -36,74 +30,12 @@ export function* post(next) {
|
|||
let lastTick = 0;
|
||||
let receivedSize = 0;
|
||||
|
||||
// Receive upload
|
||||
debug('Parsing upload');
|
||||
const upload = yield parse(this, {autoFields: true, headers: this.request.headers, limits: { files: 1}, highWaterMark: 1000000});
|
||||
const upload = yield accept.call(this);
|
||||
|
||||
// Check daily upload limit
|
||||
const count = yield Files.count({owner: this.user.id, 'time_added': {'$gt': Math.ceil(Date.now() / 1000) - 86400}});
|
||||
const userLimit = this.user.daily_upload_allowance;
|
||||
const underLimit = (count < userLimit || userLimit === 'unlimited');
|
||||
if (!underLimit) {
|
||||
this.statsd.incr('file.overlimit', 1);
|
||||
}
|
||||
this.assert(underLimit, 400, `{
|
||||
"error": {
|
||||
"message": "Daily upload limits (${this.user.daily_upload_allowance}) exceeded.",
|
||||
"code": 602
|
||||
}
|
||||
}`);
|
||||
|
||||
// Clean filename for storage, keep original for display
|
||||
upload.originalName = upload.filename;
|
||||
upload.filename = upload.filename.replace(/[^a-zA-Z0-9\.\-\_\s]/g, '').replace(/\s+/g, '');
|
||||
const fileId = yield hostrId(Files);
|
||||
|
||||
// Fire an event to let the frontend map the GUID it sent to the real ID. Allows immediate linking to the file
|
||||
const acceptedEvent = `{"type": "file-accepted", "data": {"id": "${fileId}", "guid": "${tempGuid}", "href": "${baseURL}/${fileId}"}}`;
|
||||
this.redis.publish('/user/' + this.user.id, acceptedEvent);
|
||||
this.statsd.incr('file.upload.accepted', 1);
|
||||
|
||||
const uploadPromise = new Promise((resolve, reject) => {
|
||||
upload.on('error', () => {
|
||||
this.statsd.incr('file.upload.error', 1);
|
||||
reject();
|
||||
});
|
||||
|
||||
upload.on('end', () => {
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
const key = path.join(fileId[0], fileId + '_' + upload.filename);
|
||||
const localStream = fs.createWriteStream(path.join(storePath, key));
|
||||
upload.path = path.join(upload.id[0], upload.id + '_' + upload.filename);
|
||||
const localStream = fs.createWriteStream(path.join(storePath, upload.path));
|
||||
|
||||
upload.pipe(localStream);
|
||||
upload.pipe(s3Upload(key));
|
||||
|
||||
const thumbsPromises = [
|
||||
new Promise((resolve) => {
|
||||
const small = gm(upload).resize(150, 150, '>').stream();
|
||||
small.pipe(fs.createWriteStream(path.join(storePath, fileId[0], '150', fileId + '_' + upload.filename)));
|
||||
small.pipe(s3Upload(path.join('150', fileId + '_' + upload.filename))).on('finish', resolve);
|
||||
}),
|
||||
new Promise((resolve) => {
|
||||
const medium = gm(upload).resize(970, '>').stream();
|
||||
medium.pipe(fs.createWriteStream(path.join(storePath, fileId[0], '970', fileId + '_' + upload.filename)));
|
||||
medium.pipe(s3Upload(path.join('970', fileId + '_' + upload.filename))).on('finish', resolve);
|
||||
}),
|
||||
];
|
||||
|
||||
|
||||
const dimensionsPromise = new Promise((resolve, reject) => {
|
||||
gm(upload).size((err, size) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve(size);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
upload.on('data', (data) => {
|
||||
receivedSize += data.length;
|
||||
|
|
@ -114,8 +46,8 @@ export function* post(next) {
|
|||
|
||||
percentComplete = Math.floor(receivedSize * 100 / expectedSize);
|
||||
if (percentComplete > lastPercent && lastTick < Date.now() - 1000) {
|
||||
const progressEvent = `{"type": "file-progress", "data": {"id": "${fileId}", "complete": ${percentComplete}}}`;
|
||||
this.redis.publish('/file/' + fileId, progressEvent);
|
||||
const progressEvent = `{"type": "file-progress", "data": {"id": "${upload.id}", "complete": ${percentComplete}}}`;
|
||||
this.redis.publish('/file/' + upload.id, progressEvent);
|
||||
this.redis.publish('/user/' + this.user.id, progressEvent);
|
||||
lastTick = Date.now();
|
||||
}
|
||||
|
|
@ -124,17 +56,10 @@ export function* post(next) {
|
|||
md5sum.update(data);
|
||||
});
|
||||
|
||||
// Fire final upload progress event so users know it's now processing
|
||||
const completeEvent = `{"type": "file-progress", "data": {"id": "${fileId}", "complete": 100}}`;
|
||||
this.redis.publish('/file/' + fileId, completeEvent);
|
||||
this.redis.publish('/user/' + this.user.id, completeEvent);
|
||||
this.statsd.incr('file.upload.complete', 1);
|
||||
|
||||
const dbFile = {
|
||||
_id: fileId,
|
||||
owner: this.user.id,
|
||||
ip: remoteIp,
|
||||
'system_name': fileId,
|
||||
'system_name': upload.id,
|
||||
'file_name': upload.filename,
|
||||
'original_name': upload.originalName,
|
||||
'file_size': receivedSize,
|
||||
|
|
@ -145,49 +70,33 @@ export function* post(next) {
|
|||
type: sniff(upload.filename),
|
||||
};
|
||||
|
||||
yield Files.insertOne(dbFile);
|
||||
yield uploadPromise;
|
||||
try {
|
||||
const dimensions = yield dimensionsPromise;
|
||||
dbFile.width = dimensions.width;
|
||||
dbFile.height = dimensions.height;
|
||||
} catch (e) {
|
||||
debug('Not an image');
|
||||
}
|
||||
yield Files.insertOne({_id: upload.id, ...dbFile});
|
||||
|
||||
yield thumbsPromises;
|
||||
yield upload.promise;
|
||||
|
||||
const completeEvent = `{"type": "file-progress", "data": {"id": "${upload.id}", "complete": 100}}`;
|
||||
this.redis.publish('/file/' + upload.id, completeEvent);
|
||||
this.redis.publish('/user/' + this.user.id, completeEvent);
|
||||
this.statsd.incr('file.upload.complete', 1);
|
||||
|
||||
const size = yield processImage(upload);
|
||||
|
||||
dbFile.width = size.width;
|
||||
dbFile.height = size.height;
|
||||
dbFile.file_size = receivedSize; // eslint-disable-line camelcase
|
||||
dbFile.status = 'active';
|
||||
dbFile.md5 = md5sum.digest('hex');
|
||||
|
||||
const formattedFile = formatFile(dbFile);
|
||||
const formattedFile = formatFile({_id: upload.id, ...dbFile});
|
||||
|
||||
delete dbFile._id;
|
||||
yield Files.updateOne({_id: fileId}, {$set: dbFile});
|
||||
yield Files.updateOne({_id: upload.id}, {$set: dbFile});
|
||||
|
||||
// Fire upload complete event
|
||||
const addedEvent = `{"type": "file-added", "data": ${JSON.stringify(formattedFile)}}`;
|
||||
this.redis.publish('/file/' + fileId, addedEvent);
|
||||
this.redis.publish('/file/' + upload.id, addedEvent);
|
||||
this.redis.publish('/user/' + this.user.id, addedEvent);
|
||||
|
||||
this.status = 201;
|
||||
this.body = formattedFile;
|
||||
|
||||
if (process.env.VIRUSTOTAL_KEY) {
|
||||
// Check in the background
|
||||
process.nextTick(function* malwareScan() {
|
||||
debug('Malware Scan');
|
||||
const result = yield malware(dbFile);
|
||||
if (result) {
|
||||
yield Files.updateOne({_id: fileId}, {'$set': {malware: positive, virustotal: result}});
|
||||
if (result.positive) {
|
||||
this.statsd.incr('file.malware', 1);
|
||||
}
|
||||
}
|
||||
});
|
||||
} else {
|
||||
debug('Skipping Malware Scan, VIRUSTOTAL env variable not found.');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import fs from 'fs';
|
||||
import createError from 'http-errors';
|
||||
import { get as getFile } from './s3';
|
||||
import { get as getFile } from './sftp';
|
||||
|
||||
import debugname from 'debug';
|
||||
const debug = debugname('hostr:file-stream');
|
||||
|
|
@ -10,22 +10,23 @@ export default function* hostrFileStream(localPath, remotePath) {
|
|||
return new Promise((resolve, reject) => {
|
||||
localRead.once('error', () => {
|
||||
debug('local error');
|
||||
const remoteRead = getFile(remotePath);
|
||||
const remoteFile = getFile(remotePath);
|
||||
|
||||
remoteRead.once('readable', () => {
|
||||
debug('remote readable');
|
||||
remoteFile.then((remoteRead) => {
|
||||
const localWrite = fs.createWriteStream(localPath);
|
||||
localWrite.once('finish', () => {
|
||||
debug('local write end');
|
||||
resolve(fs.createReadStream(localPath));
|
||||
});
|
||||
remoteRead.pipe(localWrite);
|
||||
|
||||
remoteRead.once('error', () => {
|
||||
debug('remote error');
|
||||
reject(createError(404));
|
||||
});
|
||||
});
|
||||
|
||||
remoteRead.once('error', () => {
|
||||
debug('remote error');
|
||||
reject(createError(404));
|
||||
});
|
||||
|
||||
});
|
||||
localRead.once('readable', () => {
|
||||
debug('local readable');
|
||||
|
|
|
|||
|
|
@ -1,75 +0,0 @@
|
|||
/**
|
||||
* Module dependencies.
|
||||
*/
|
||||
|
||||
var swig = require('swig');
|
||||
var http = require('http');
|
||||
|
||||
/**
|
||||
* Expose `error`.
|
||||
*/
|
||||
|
||||
module.exports = error;
|
||||
|
||||
/**
|
||||
* Error middleware.
|
||||
*
|
||||
* - `template` defaults to ./error.html
|
||||
*
|
||||
* @param {Object} opts
|
||||
* @api public
|
||||
*/
|
||||
|
||||
function error(opts) {
|
||||
opts = opts || {};
|
||||
|
||||
// template
|
||||
var path = opts.template || __dirname + '/error.html';
|
||||
var render = swig.compileFile(path);
|
||||
|
||||
// env
|
||||
var env = process.env.NODE_ENV || 'development';
|
||||
|
||||
return function *error(next){
|
||||
try {
|
||||
yield next;
|
||||
if (404 == this.response.status && !this.response.body) this.throw(404);
|
||||
} catch (err) {
|
||||
this.status = err.status || 500;
|
||||
|
||||
// application
|
||||
this.app.emit('error', err, this);
|
||||
|
||||
// accepted types
|
||||
switch (this.accepts('html', 'text', 'json')) {
|
||||
case 'text':
|
||||
this.type = 'text/plain';
|
||||
if ('development' == env) this.body = err.message
|
||||
else if (err.expose) this.body = err.message
|
||||
else throw err;
|
||||
break;
|
||||
|
||||
case 'json':
|
||||
this.type = 'application/json';
|
||||
if ('development' == env) this.body = { error: err.message }
|
||||
else if (err.expose) this.body = { error: err.message }
|
||||
else this.body = { error: http.STATUS_CODES[this.status] }
|
||||
break;
|
||||
|
||||
case 'html':
|
||||
this.type = 'text/html';
|
||||
this.body = render({
|
||||
env: env,
|
||||
ctx: this,
|
||||
request: this.request,
|
||||
response: this.response,
|
||||
error: err.message,
|
||||
stack: err.stack,
|
||||
status: this.status,
|
||||
code: err.code
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,33 +1,71 @@
|
|||
import fs from 'mz/fs';
|
||||
import lwip from 'lwip';
|
||||
import debugname from 'debug';
|
||||
const debug = debugname('hostr-api:resize');
|
||||
import lwip from 'lwip';
|
||||
import imageType from 'image-type';
|
||||
|
||||
const supported = ['jpg', 'png', 'gif'];
|
||||
|
||||
export default function(input, size) {
|
||||
debug('Resizing');
|
||||
|
||||
const type = imageType(input);
|
||||
|
||||
if (!type.ext || supported.indexOf(type.ext) < 0) {
|
||||
throw new Error('Not a supported image.');
|
||||
}
|
||||
|
||||
function cover(path, type, size) {
|
||||
return new Promise((resolve, reject) => {
|
||||
lwip.open(input, type.ext, (errIn, image) => {
|
||||
lwip.open(path, type, (errIn, image) => {
|
||||
debug('Image Opened');
|
||||
if (errIn) {
|
||||
return reject(errIn);
|
||||
reject(errIn);
|
||||
}
|
||||
|
||||
image.cover(size.width, size.height, (errOut, resized) => {
|
||||
debug('Image Resized');
|
||||
if (errOut) {
|
||||
return reject(errOut);
|
||||
reject(errOut);
|
||||
}
|
||||
|
||||
resized.toBuffer(type.ext, (errBuf, buffer) => {
|
||||
resized.toBuffer(type, (errBuf, buffer) => {
|
||||
debug('Image Buffered');
|
||||
if (errBuf) {
|
||||
reject(errBuf);
|
||||
}
|
||||
resolve(buffer);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function scale(path, type, size) {
|
||||
return new Promise((resolve, reject) => {
|
||||
lwip.open(path, type, (errIn, image) => {
|
||||
debug('Image Opened');
|
||||
if (errIn) {
|
||||
reject(errIn);
|
||||
}
|
||||
|
||||
image.cover(size.width, size.height, (errOut, resized) => {
|
||||
debug('Image Resized');
|
||||
if (errOut) {
|
||||
reject(errOut);
|
||||
}
|
||||
|
||||
resized.toBuffer(type, (errBuf, buffer) => {
|
||||
debug('Image Buffered');
|
||||
if (errBuf) {
|
||||
reject(errBuf);
|
||||
}
|
||||
resolve(buffer);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export default function resize(path, type, currentSize, newSize) {
|
||||
debug('Resizing');
|
||||
const ratio = 970 / currentSize.width;
|
||||
if (newSize.width <= 150) {
|
||||
debug('Cover');
|
||||
return cover(path, type, newSize);
|
||||
} else if (newSize.width > 970 && ratio > 1) {
|
||||
debug('Scale');
|
||||
newSize.height = currentSize.height * ratio;
|
||||
return scale(path, type, newSize);
|
||||
}
|
||||
debug('Copy');
|
||||
return fs.readFile(path);
|
||||
}
|
||||
|
|
|
|||
11
lib/s3.js
11
lib/s3.js
|
|
@ -1,17 +1,20 @@
|
|||
import aws from 'aws-sdk';
|
||||
import s3UploadStream from 's3-upload-stream';
|
||||
import debugname from 'debug';
|
||||
const debug = debugname('hostr:s3');
|
||||
|
||||
const s3 = new aws.S3();
|
||||
const s3Stream = s3UploadStream(s3);
|
||||
|
||||
export function get(key) {
|
||||
debug('fetching from s3: %s', 'hostr_files/' + key);
|
||||
return s3.getObject({Bucket: process.env.AWS_BUCKET, Key: 'hostr_files/' + key}).createReadStream();
|
||||
}
|
||||
|
||||
export function upload(key) {
|
||||
export function upload(stream, key, callback) {
|
||||
debug('sending to s3: %s', 'hostr_files/' + key);
|
||||
return s3Stream.upload({Bucket: process.env.AWS_BUCKET, Key: 'hostr_files/' + key});
|
||||
const params = {Bucket: process.env.AWS_BUCKET, Key: 'hostr_files/' + key, Body: stream};
|
||||
const uploading = s3.upload(params);
|
||||
uploading.on('error', (err) => {
|
||||
console.log(err)
|
||||
});
|
||||
uploading.send(callback);
|
||||
}
|
||||
|
|
|
|||
40
lib/sftp.js
Normal file
40
lib/sftp.js
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
import { dirname } from 'path';
|
||||
import Client from 'ssh2-sftp-client';
|
||||
import debugname from 'debug';
|
||||
const debug = debugname('hostr:sftp');
|
||||
|
||||
export function get(remotePath) {
|
||||
const sftp = new Client();
|
||||
return sftp.connect({
|
||||
host: process.env.SFTP_HOST,
|
||||
port: process.env.SFTP_PORT,
|
||||
username: process.env.SFTP_USERNAME,
|
||||
password: process.env.SFTP_PASSWORD,
|
||||
}).then(() => {
|
||||
return sftp.get('hostr/uploads/' + remotePath, true);
|
||||
});
|
||||
}
|
||||
|
||||
export function upload(localPath, remotePath) {
|
||||
debug('SFTP connecting');
|
||||
const sftp = new Client();
|
||||
return sftp.connect({
|
||||
host: process.env.SFTP_HOST,
|
||||
port: process.env.SFTP_PORT,
|
||||
username: process.env.SFTP_USERNAME,
|
||||
password: process.env.SFTP_PASSWORD,
|
||||
}).then(() => {
|
||||
return sftp.put(localPath, remotePath, true).then(() => {
|
||||
sftp.end();
|
||||
});
|
||||
}).catch(() => {
|
||||
debug('Creating ' + dirname(remotePath));
|
||||
return sftp.mkdir(dirname(remotePath), true).then(() => {
|
||||
return sftp.put(localPath, remotePath, true).then(() => {
|
||||
sftp.end();
|
||||
});
|
||||
});
|
||||
}).then(() => {
|
||||
sftp.end();
|
||||
});
|
||||
}
|
||||
108
lib/upload.js
Normal file
108
lib/upload.js
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
import { join } from 'path';
|
||||
import parse from 'co-busboy';
|
||||
import fs from 'mz/fs';
|
||||
import sizeOf from 'image-size';
|
||||
import hostrId from './hostr-id';
|
||||
import resize from './resize';
|
||||
import { upload as sftpUpload } from './sftp';
|
||||
|
||||
import debugname from 'debug';
|
||||
const debug = debugname('hostr-api:upload');
|
||||
|
||||
const storePath = process.env.UPLOAD_STORAGE_PATH;
|
||||
const baseURL = process.env.WEB_BASE_URL;
|
||||
const supported = ['jpg', 'png', 'gif'];
|
||||
|
||||
export function* checkLimit() {
|
||||
const count = yield this.db.Files.count({
|
||||
owner: this.user.id,
|
||||
time_added: {'$gt': Math.ceil(Date.now() / 1000) - 86400},
|
||||
});
|
||||
const userLimit = this.user.daily_upload_allowance;
|
||||
const underLimit = (count < userLimit || userLimit === 'unlimited');
|
||||
if (!underLimit) {
|
||||
this.statsd.incr('file.overlimit', 1);
|
||||
}
|
||||
this.assert(underLimit, 400, `{
|
||||
"error": {
|
||||
"message": "Daily upload limits (${this.user.daily_upload_allowance}) exceeded.",
|
||||
"code": 602
|
||||
}
|
||||
}`);
|
||||
return true;
|
||||
}
|
||||
|
||||
export function* accept() {
|
||||
yield checkLimit.call(this);
|
||||
|
||||
const upload = yield parse(this, {
|
||||
autoFields: true,
|
||||
headers: this.request.headers,
|
||||
limits: { files: 1},
|
||||
highWaterMark: 1000000,
|
||||
});
|
||||
|
||||
upload.promise = new Promise((resolve, reject) => {
|
||||
upload.on('error', (err) => {
|
||||
this.statsd.incr('file.upload.error', 1);
|
||||
debug(err);
|
||||
reject();
|
||||
});
|
||||
|
||||
upload.on('end', () => {
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
upload.tempGuid = this.request.headers['hostr-guid'];
|
||||
upload.originalName = upload.filename;
|
||||
upload.filename = upload.filename.replace(/[^a-zA-Z0-9\.\-\_\s]/g, '').replace(/\s+/g, '');
|
||||
upload.id = yield hostrId(this.db.Files);
|
||||
|
||||
const acceptedEvent = `{"type": "file-accepted", "data": {"id": "${upload.id}", "guid": "${upload.tempGuid}", "href": "${baseURL}/${upload.id}"}}`;
|
||||
this.redis.publish('/user/' + this.user.id, acceptedEvent);
|
||||
this.statsd.incr('file.upload.accepted', 1);
|
||||
|
||||
return upload;
|
||||
}
|
||||
|
||||
export function resizeImage(upload, type, currentSize, newSize) {
|
||||
return resize(join(storePath, upload.path), type, currentSize, newSize).then((image) => {
|
||||
const path = join(upload.id[0], String(newSize.width), upload.id + '_' + upload.filename);
|
||||
debug('Writing file');
|
||||
debug(join(storePath, path));
|
||||
return fs.writeFile(join(storePath, path), image).then(() => {
|
||||
debug('Uploading file');
|
||||
return sftpUpload(join(storePath, path), join('hostr', 'uploads', path));
|
||||
}).catch(debug);
|
||||
}).catch(debug);
|
||||
}
|
||||
|
||||
export function* processImage(upload) {
|
||||
debug('Processing image');
|
||||
return new Promise((resolve) => {
|
||||
const size = sizeOf(join(storePath, upload.path));
|
||||
debug('Size: ', size);
|
||||
if (!size.width || supported.indexOf(size.type) < 0) {
|
||||
resolve();
|
||||
}
|
||||
|
||||
Promise.all([
|
||||
resizeImage(upload, size.type, size, {width: 150, height: 150}),
|
||||
resizeImage(upload, size.type, size, {width: 970}),
|
||||
]).then(() => {
|
||||
resolve(size);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function progressEvent() {
|
||||
percentComplete = Math.floor(receivedSize * 100 / expectedSize);
|
||||
if (percentComplete > lastPercent && lastTick < Date.now() - 1000) {
|
||||
const progressEvent = `{"type": "file-progress", "data": {"id": "${upload.id}", "complete": ${percentComplete}}}`;
|
||||
this.redis.publish('/file/' + upload.id, progressEvent);
|
||||
this.redis.publish('/user/' + this.user.id, progressEvent);
|
||||
lastTick = Date.now();
|
||||
}
|
||||
lastPercent = percentComplete;
|
||||
}
|
||||
|
|
@ -16,7 +16,7 @@
|
|||
"init": "node -r babel/register -e \"require('./lib/storage')();\"",
|
||||
"jspm": "jspm install",
|
||||
"start": "npm run build && node -r babel/register app.js",
|
||||
"test": "npm run test-seed && mocha -r babel/register test/**/*.spec.js && mocha -r babel/register -r co-mocha test/unit/image-resize.spec.js",
|
||||
"test": "npm run test-seed && mocha -r babel/register test/**/*.spec.js",
|
||||
"test-seed": "node test/fixtures/mongo-user.js && node test/fixtures/mongo-file.js",
|
||||
"watch": "parallelshell \"npm run watch-js\" \"npm run watch-sass\" \"npm run watch-server\"",
|
||||
"watch-js": "babel -Dw -m system -d web/public/build web/public/src",
|
||||
|
|
@ -24,7 +24,7 @@
|
|||
"watch-sass": "node-sass -w -r -o web/public/styles/ web/public/styles/"
|
||||
},
|
||||
"dependencies": {
|
||||
"aws-sdk": "~2.1.46",
|
||||
"aws-sdk": "~2.3.15",
|
||||
"babel": "~5.8.21",
|
||||
"basic-auth": "~1.0.3",
|
||||
"co": "~4.6.0",
|
||||
|
|
@ -43,6 +43,7 @@
|
|||
"koa-bodyparser": "~2.0.1",
|
||||
"koa-compress": "~1.0.8",
|
||||
"koa-csrf": "~2.3.0",
|
||||
"koa-error": "^2.0.0",
|
||||
"koa-favicon": "~1.2.0",
|
||||
"koa-generic-session": "~1.9.0",
|
||||
"koa-helmet": "^0.2.0",
|
||||
|
|
@ -57,6 +58,7 @@
|
|||
"mime-types": "~2.1.5",
|
||||
"moment": "~2.10.6",
|
||||
"mongodb-promisified": "~1.0.3",
|
||||
"mz": "^2.4.0",
|
||||
"node-fetch": "^1.3.2",
|
||||
"node-sass": "~3.6.0",
|
||||
"node-uuid": "~1.4.3",
|
||||
|
|
@ -65,13 +67,13 @@
|
|||
"redis": "~1.0.0",
|
||||
"s3-upload-stream": "~1.0.7",
|
||||
"sendgrid": "^2.0.0",
|
||||
"ssh2-sftp-client": "^1.0.3",
|
||||
"statsy": "~0.2.0",
|
||||
"stripe": "~3.7.1",
|
||||
"swig": "~1.4.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"babel-eslint": "^4.0.10",
|
||||
"co-mocha": "^1.1.2",
|
||||
"eslint": "~1.3.0",
|
||||
"eslint-config-airbnb": "0.0.8",
|
||||
"istanbul": "~0.3.18",
|
||||
|
|
@ -79,6 +81,7 @@
|
|||
"nodemon": "~1.4.1",
|
||||
"parallelshell": "~2.0.0",
|
||||
"supertest": "~1.1.0",
|
||||
"supertest-koa-agent": "^0.2.1",
|
||||
"tmp": "~0.0.27"
|
||||
},
|
||||
"jspm": {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import path from 'path';
|
||||
import assert from 'assert';
|
||||
import { agent } from 'supertest';
|
||||
import app from '../../app';
|
||||
|
|
@ -25,11 +26,11 @@ describe('hostr-api file', function file() {
|
|||
this.timeout(30000);
|
||||
request
|
||||
.post('/api/file')
|
||||
.attach('file', './test/fixtures/utah-arches.jpg')
|
||||
.attach('file', path.join(__dirname, '..', 'fixtures', 'tall.jpg'))
|
||||
.auth('test@hostr.co', 'test-password')
|
||||
.expect(201)
|
||||
.expect((response) => {
|
||||
assert(response.body.name === 'utah-arches.jpg');
|
||||
assert(response.body.name === 'tall.jpg');
|
||||
id = response.body.id;
|
||||
})
|
||||
.end(done);
|
||||
|
|
@ -42,7 +43,7 @@ describe('hostr-api file', function file() {
|
|||
.get('/api/file/' + id)
|
||||
.expect(200)
|
||||
.expect((response) => {
|
||||
assert(response.body.name === 'utah-arches.jpg');
|
||||
assert(response.body.name === 'tall.jpg');
|
||||
})
|
||||
.end(done);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,35 +1,35 @@
|
|||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import fs from 'mz/fs';
|
||||
import { join } from 'path';
|
||||
import assert from 'assert';
|
||||
import tmp from 'tmp';
|
||||
import resize from '../../lib/resize';
|
||||
import imageType from 'image-type';
|
||||
import sizeOf from 'image-size';
|
||||
|
||||
function testResize(path, done) {
|
||||
const size = sizeOf(path);
|
||||
resize(path, size.type, size, {width: 100, height: 100}).then((image) => {
|
||||
const tmpFile = tmp.tmpNameSync() + '.' + size.type;
|
||||
fs.writeFile(tmpFile, image).then(() => {
|
||||
const newSize = sizeOf(fs.readFileSync(tmpFile));
|
||||
assert(newSize.type === size.type);
|
||||
done();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
describe('Image resizing', () => {
|
||||
it('should resize a jpg', function* resizeImage() {
|
||||
const file = fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'utah-arches.jpg'));
|
||||
const imageBuffer = yield resize(file, {height: 100, width: 100});
|
||||
const tmpFile = tmp.tmpNameSync() + '.jpg';
|
||||
fs.writeFileSync(tmpFile, imageBuffer);
|
||||
const type = imageType(fs.readFileSync(tmpFile));
|
||||
assert(type.ext === 'jpg');
|
||||
it('should resize a jpg', (done) => {
|
||||
const path = join(__dirname, '..', 'fixtures', 'utah-arches.jpg');
|
||||
testResize(path, done);
|
||||
});
|
||||
|
||||
it('should resize a png', function* resizeImage() {
|
||||
const file = fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'app-icon.png'));
|
||||
const imageBuffer = yield resize(file, {height: 100, width: 100});
|
||||
const tmpFile = tmp.tmpNameSync() + '.png';
|
||||
fs.writeFileSync(tmpFile, imageBuffer);
|
||||
const type = imageType(fs.readFileSync(tmpFile));
|
||||
assert(type.ext === 'png');
|
||||
it('should resize a png', (done) => {
|
||||
const path = join(__dirname, '..', 'fixtures', 'app-icon.png');
|
||||
testResize(path, done);
|
||||
});
|
||||
|
||||
it('should resize a gif', function* resizeImage() {
|
||||
const file = fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'kim.gif'));
|
||||
const imageBuffer = yield resize(file, {height: 100, width: 100});
|
||||
const tmpFile = tmp.tmpNameSync() + '.gif';
|
||||
fs.writeFileSync(tmpFile, imageBuffer);
|
||||
const type = imageType(fs.readFileSync(tmpFile));
|
||||
assert(type.ext === 'gif');
|
||||
it('should resize a gif', (done) => {
|
||||
const path = join(__dirname, '..', 'fixtures', 'kim.gif');
|
||||
testResize(path, done);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import path from 'path';
|
||||
import assert from 'assert';
|
||||
import sizeOf from 'image-size';
|
||||
import { agent } from 'supertest';
|
||||
|
|
@ -12,7 +13,7 @@ describe('setup hostr-web file', function() {
|
|||
this.timeout(30000);
|
||||
request
|
||||
.post('/api/file')
|
||||
.attach('file', 'test/fixtures/utah-arches.jpg')
|
||||
.attach('file', path.join(__dirname, '..', 'fixtures', 'utah-arches.jpg'))
|
||||
.auth('test@hostr.co', 'test-password')
|
||||
.expect(201)
|
||||
.expect(function(response) {
|
||||
|
|
@ -25,7 +26,6 @@ describe('setup hostr-web file', function() {
|
|||
});
|
||||
|
||||
describe('hostr-web file', function() {
|
||||
|
||||
describe('when GET /file/:id/:name', function() {
|
||||
it('should receive an image', function(done) {
|
||||
request
|
||||
|
|
@ -40,41 +40,40 @@ describe('hostr-web file', function() {
|
|||
});
|
||||
|
||||
describe('when GET /file/150/:id/:name', function() {
|
||||
it('should receive a 150px wide thumbnail of the image', function(done) {
|
||||
it('should receive a 150px wide thumbnail of the image', function() {
|
||||
request
|
||||
.get('/file/150/' + file.id + '/' + file.name)
|
||||
.expect(200)
|
||||
.expect('Content-type', 'image/jpeg')
|
||||
.expect(function(response) {
|
||||
assert(sizeOf(response.body).width === 150);
|
||||
})
|
||||
.end(done);
|
||||
const width = sizeOf(response.body).width;
|
||||
assert(width === 150);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('when GET /file/970/:id/:name', function() {
|
||||
it('should receive a 970px wide thumbnail of the image', function(done) {
|
||||
it('should receive a 970px wide thumbnail of the image', function() {
|
||||
request
|
||||
.get('/file/970/' + file.id + '/' + file.name)
|
||||
.expect(200)
|
||||
.expect('Content-type', 'image/jpeg')
|
||||
.expect(function(response) {
|
||||
assert(sizeOf(response.body).width === 970);
|
||||
})
|
||||
.end(done);
|
||||
const width = sizeOf(response.body).width;
|
||||
assert(width === 970);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('when GET /:id', function() {
|
||||
it('should receive some HTML', function(done) {
|
||||
it('should receive some HTML', function() {
|
||||
request
|
||||
.get('/' + file.id)
|
||||
.expect(200)
|
||||
.expect('Content-type', /text\/html/) // Could include charset
|
||||
.expect(function(response) {
|
||||
assert(response.text.indexOf('src="/file/970/' + file.id + '/' + file.name + '"') > -1);
|
||||
})
|
||||
.end(done);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -5,20 +5,20 @@ const request = agent(app.listen());
|
|||
|
||||
describe('hostr-web user', function() {
|
||||
describe('when POST /signin with invalid credentials', function() {
|
||||
it('should not redirect to /', function(done) {
|
||||
it('should not redirect to /', function() {
|
||||
request.get('/signin').end(function(err, response) {
|
||||
const match = response.text.match(/name="_csrf" value="([^"]+)"/);
|
||||
const csrf = match[1];
|
||||
request
|
||||
.post('/signin')
|
||||
.send({'email': 'test@hostr.co', 'password': 'test-passworddeded', '_csrf': csrf})
|
||||
.expect(200, done);
|
||||
.expect(200);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('when POST /signin with valid credentials', function() {
|
||||
it('should redirect to /', function(done) {
|
||||
it('should redirect to /', function() {
|
||||
request.get('/signin').end(function(err, response) {
|
||||
const match = response.text.match(/name="_csrf" value="([^"]+)"/);
|
||||
const csrf = match[1];
|
||||
|
|
@ -27,7 +27,7 @@ describe('hostr-web user', function() {
|
|||
.send({'email': 'test@hostr.co', 'password': 'test-password', '_csrf': csrf})
|
||||
.expect(302)
|
||||
.expect('Location', '/')
|
||||
.end(done);
|
||||
.end();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -5,8 +5,7 @@ import views from 'koa-views';
|
|||
import stats from 'koa-statsd';
|
||||
import * as redis from '../lib/redis';
|
||||
import StatsD from 'statsy';
|
||||
// waiting for PR to be merged, can remove swig dependency when done
|
||||
import errors from '../lib/koa-error';
|
||||
import errors from 'koa-error';
|
||||
import * as index from './routes/index';
|
||||
import * as file from './routes/file';
|
||||
import * as pro from './routes/pro';
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue