jonathan/hostr
jonathan/hostr
1
0
Fork 0
Code Issues Pull requests 4 Projects Releases Packages 1 Wiki Activity Actions

Refactor uploads

Browse source
This commit is contained in:
Jonathan Cremin 2016-05-25 21:03:07 +01:00
parent 75a46212da
commit f59b9a5d22
14 changed files with 297 additions and 269 deletions
Download patch file Download diff file Expand all files Collapse all files

4
api/app.js
View file

@ -46,7 +46,9 @@ router.use('*', function* authMiddleware(next) {
} else { } else {
if (!err.status) { if (!err.status) {
debug(err); debug(err);
this.raven.captureError(err); if (this.raven) {
this.raven.captureError(err);
}
throw err; throw err;
} else { } else {
this.status = err.status; this.status = err.status;

139
api/routes/file.js
View file

@ -1,22 +1,18 @@
import path from 'path'; import path from 'path';
import fs from 'fs';
import crypto from 'crypto'; import crypto from 'crypto';
import gm from 'gm'; import fs from 'mz/fs';
import redis from 'redis'; import redis from 'redis';
import parse from 'co-busboy';
import { upload as s3Upload } from '../../lib/s3';
import { sniff } from '../../lib/type'; import { sniff } from '../../lib/type';
import hostrId from '../../lib/hostr-id';
import malware from '../../lib/malware'; import malware from '../../lib/malware';
import { formatFile } from '../../lib/format'; import { formatFile } from '../../lib/format';
import { accept, processImage } from '../../lib/upload';
import debugname from 'debug'; import debugname from 'debug';
const debug = debugname('hostr-api:file'); const debug = debugname('hostr-api:file');
const redisUrl = process.env.REDIS_URL; const redisUrl = process.env.REDIS_URL;
const baseURL = process.env.WEB_BASE_URL;
const storePath = process.env.UPLOAD_STORAGE_PATH; const storePath = process.env.UPLOAD_STORAGE_PATH;
export function* post(next) { export function* post(next) {
@ -26,9 +22,7 @@ export function* post(next) {
const Files = this.db.Files; const Files = this.db.Files;
const expectedSize = this.request.headers['content-length']; const expectedSize = this.request.headers['content-length'];
const tempGuid = this.request.headers['hostr-guid'];
const remoteIp = this.request.headers['x-real-ip'] || this.req.connection.remoteAddress; const remoteIp = this.request.headers['x-real-ip'] || this.req.connection.remoteAddress;
const md5sum = crypto.createHash('md5'); const md5sum = crypto.createHash('md5');
let lastPercent = 0; let lastPercent = 0;
@ -36,74 +30,12 @@ export function* post(next) {
let lastTick = 0; let lastTick = 0;
let receivedSize = 0; let receivedSize = 0;
// Receive upload const upload = yield accept.call(this);
debug('Parsing upload');
const upload = yield parse(this, {autoFields: true, headers: this.request.headers, limits: { files: 1}, highWaterMark: 1000000});
// Check daily upload limit upload.path = path.join(upload.id[0], upload.id + '_' + upload.filename);
const count = yield Files.count({owner: this.user.id, 'time_added': {'$gt': Math.ceil(Date.now() / 1000) - 86400}}); const localStream = fs.createWriteStream(path.join(storePath, upload.path));
const userLimit = this.user.daily_upload_allowance;
const underLimit = (count < userLimit || userLimit === 'unlimited');
if (!underLimit) {
this.statsd.incr('file.overlimit', 1);
}
this.assert(underLimit, 400, `{
"error": {
"message": "Daily upload limits (${this.user.daily_upload_allowance}) exceeded.",
"code": 602
}
}`);
// Clean filename for storage, keep original for display
upload.originalName = upload.filename;
upload.filename = upload.filename.replace(/[^a-zA-Z0-9\.\-\_\s]/g, '').replace(/\s+/g, '');
const fileId = yield hostrId(Files);
// Fire an event to let the frontend map the GUID it sent to the real ID. Allows immediate linking to the file
const acceptedEvent = `{"type": "file-accepted", "data": {"id": "${fileId}", "guid": "${tempGuid}", "href": "${baseURL}/${fileId}"}}`;
this.redis.publish('/user/' + this.user.id, acceptedEvent);
this.statsd.incr('file.upload.accepted', 1);
const uploadPromise = new Promise((resolve, reject) => {
upload.on('error', () => {
this.statsd.incr('file.upload.error', 1);
reject();
});
upload.on('end', () => {
resolve();
});
});
const key = path.join(fileId[0], fileId + '_' + upload.filename);
const localStream = fs.createWriteStream(path.join(storePath, key));
upload.pipe(localStream); upload.pipe(localStream);
upload.pipe(s3Upload(key));
const thumbsPromises = [
new Promise((resolve) => {
const small = gm(upload).resize(150, 150, '>').stream();
small.pipe(fs.createWriteStream(path.join(storePath, fileId[0], '150', fileId + '_' + upload.filename)));
small.pipe(s3Upload(path.join('150', fileId + '_' + upload.filename))).on('finish', resolve);
}),
new Promise((resolve) => {
const medium = gm(upload).resize(970, '>').stream();
medium.pipe(fs.createWriteStream(path.join(storePath, fileId[0], '970', fileId + '_' + upload.filename)));
medium.pipe(s3Upload(path.join('970', fileId + '_' + upload.filename))).on('finish', resolve);
}),
];
const dimensionsPromise = new Promise((resolve, reject) => {
gm(upload).size((err, size) => {
if (err) {
reject(err);
} else {
resolve(size);
}
});
});
upload.on('data', (data) => { upload.on('data', (data) => {
receivedSize += data.length; receivedSize += data.length;
@ -114,8 +46,8 @@ export function* post(next) {
percentComplete = Math.floor(receivedSize * 100 / expectedSize); percentComplete = Math.floor(receivedSize * 100 / expectedSize);
if (percentComplete > lastPercent && lastTick < Date.now() - 1000) { if (percentComplete > lastPercent && lastTick < Date.now() - 1000) {
const progressEvent = `{"type": "file-progress", "data": {"id": "${fileId}", "complete": ${percentComplete}}}`; const progressEvent = `{"type": "file-progress", "data": {"id": "${upload.id}", "complete": ${percentComplete}}}`;
this.redis.publish('/file/' + fileId, progressEvent); this.redis.publish('/file/' + upload.id, progressEvent);
this.redis.publish('/user/' + this.user.id, progressEvent); this.redis.publish('/user/' + this.user.id, progressEvent);
lastTick = Date.now(); lastTick = Date.now();
} }
@ -124,17 +56,10 @@ export function* post(next) {
md5sum.update(data); md5sum.update(data);
}); });
// Fire final upload progress event so users know it's now processing
const completeEvent = `{"type": "file-progress", "data": {"id": "${fileId}", "complete": 100}}`;
this.redis.publish('/file/' + fileId, completeEvent);
this.redis.publish('/user/' + this.user.id, completeEvent);
this.statsd.incr('file.upload.complete', 1);
const dbFile = { const dbFile = {
_id: fileId,
owner: this.user.id, owner: this.user.id,
ip: remoteIp, ip: remoteIp,
'system_name': fileId, 'system_name': upload.id,
'file_name': upload.filename, 'file_name': upload.filename,
'original_name': upload.originalName, 'original_name': upload.originalName,
'file_size': receivedSize, 'file_size': receivedSize,
@ -145,49 +70,33 @@ export function* post(next) {
type: sniff(upload.filename), type: sniff(upload.filename),
}; };
yield Files.insertOne(dbFile); yield Files.insertOne({_id: upload.id, ...dbFile});
yield uploadPromise;
try {
const dimensions = yield dimensionsPromise;
dbFile.width = dimensions.width;
dbFile.height = dimensions.height;
} catch (e) {
debug('Not an image');
}
yield thumbsPromises; yield upload.promise;
const completeEvent = `{"type": "file-progress", "data": {"id": "${upload.id}", "complete": 100}}`;
this.redis.publish('/file/' + upload.id, completeEvent);
this.redis.publish('/user/' + this.user.id, completeEvent);
this.statsd.incr('file.upload.complete', 1);
const size = yield processImage(upload);
dbFile.width = size.width;
dbFile.height = size.height;
dbFile.file_size = receivedSize; // eslint-disable-line camelcase dbFile.file_size = receivedSize; // eslint-disable-line camelcase
dbFile.status = 'active'; dbFile.status = 'active';
dbFile.md5 = md5sum.digest('hex'); dbFile.md5 = md5sum.digest('hex');
const formattedFile = formatFile(dbFile); const formattedFile = formatFile({_id: upload.id, ...dbFile});
delete dbFile._id; yield Files.updateOne({_id: upload.id}, {$set: dbFile});
yield Files.updateOne({_id: fileId}, {$set: dbFile});
// Fire upload complete event
const addedEvent = `{"type": "file-added", "data": ${JSON.stringify(formattedFile)}}`; const addedEvent = `{"type": "file-added", "data": ${JSON.stringify(formattedFile)}}`;
this.redis.publish('/file/' + fileId, addedEvent); this.redis.publish('/file/' + upload.id, addedEvent);
this.redis.publish('/user/' + this.user.id, addedEvent); this.redis.publish('/user/' + this.user.id, addedEvent);
this.status = 201; this.status = 201;
this.body = formattedFile; this.body = formattedFile;
if (process.env.VIRUSTOTAL_KEY) {
// Check in the background
process.nextTick(function* malwareScan() {
debug('Malware Scan');
const result = yield malware(dbFile);
if (result) {
yield Files.updateOne({_id: fileId}, {'$set': {malware: positive, virustotal: result}});
if (result.positive) {
this.statsd.incr('file.malware', 1);
}
}
});
} else {
debug('Skipping Malware Scan, VIRUSTOTAL env variable not found.');
}
} }

17
lib/hostr-file-stream.js
View file

@ -1,6 +1,6 @@
import fs from 'fs'; import fs from 'fs';
import createError from 'http-errors'; import createError from 'http-errors';
import { get as getFile } from './s3'; import { get as getFile } from './sftp';
import debugname from 'debug'; import debugname from 'debug';
const debug = debugname('hostr:file-stream'); const debug = debugname('hostr:file-stream');
@ -10,22 +10,23 @@ export default function* hostrFileStream(localPath, remotePath) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
localRead.once('error', () => { localRead.once('error', () => {
debug('local error'); debug('local error');
const remoteRead = getFile(remotePath); const remoteFile = getFile(remotePath);
remoteRead.once('readable', () => { remoteFile.then((remoteRead) => {
debug('remote readable');
const localWrite = fs.createWriteStream(localPath); const localWrite = fs.createWriteStream(localPath);
localWrite.once('finish', () => { localWrite.once('finish', () => {
debug('local write end'); debug('local write end');
resolve(fs.createReadStream(localPath)); resolve(fs.createReadStream(localPath));
}); });
remoteRead.pipe(localWrite); remoteRead.pipe(localWrite);
remoteRead.once('error', () => {
debug('remote error');
reject(createError(404));
});
}); });
remoteRead.once('error', () => {
debug('remote error');
reject(createError(404));
});
}); });
localRead.once('readable', () => { localRead.once('readable', () => {
debug('local readable'); debug('local readable');

75
lib/koa-error.js
View file

@ -1,75 +0,0 @@
/**
* Module dependencies.
*/
var swig = require('swig');
var http = require('http');
/**
* Expose `error`.
*/
module.exports = error;
/**
* Error middleware.
*
* - `template` defaults to ./error.html
*
* @param {Object} opts
* @api public
*/
function error(opts) {
opts = opts || {};
// template
var path = opts.template || __dirname + '/error.html';
var render = swig.compileFile(path);
// env
var env = process.env.NODE_ENV || 'development';
return function *error(next){
try {
yield next;
if (404 == this.response.status && !this.response.body) this.throw(404);
} catch (err) {
this.status = err.status || 500;
// application
this.app.emit('error', err, this);
// accepted types
switch (this.accepts('html', 'text', 'json')) {
case 'text':
this.type = 'text/plain';
if ('development' == env) this.body = err.message
else if (err.expose) this.body = err.message
else throw err;
break;
case 'json':
this.type = 'application/json';
if ('development' == env) this.body = { error: err.message }
else if (err.expose) this.body = { error: err.message }
else this.body = { error: http.STATUS_CODES[this.status] }
break;
case 'html':
this.type = 'text/html';
this.body = render({
env: env,
ctx: this,
request: this.request,
response: this.response,
error: err.message,
stack: err.stack,
status: this.status,
code: err.code
});
break;
}
}
}
}

72
lib/resize.js
View file

@ -1,33 +1,71 @@
import fs from 'mz/fs';
import lwip from 'lwip';
import debugname from 'debug'; import debugname from 'debug';
const debug = debugname('hostr-api:resize'); const debug = debugname('hostr-api:resize');
import lwip from 'lwip';
import imageType from 'image-type';
const supported = ['jpg', 'png', 'gif'];
export default function(input, size) {
debug('Resizing');
const type = imageType(input);
if (!type.ext || supported.indexOf(type.ext) < 0) {
throw new Error('Not a supported image.');
}
function cover(path, type, size) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
lwip.open(input, type.ext, (errIn, image) => { lwip.open(path, type, (errIn, image) => {
debug('Image Opened');
if (errIn) { if (errIn) {
return reject(errIn); reject(errIn);
} }
image.cover(size.width, size.height, (errOut, resized) => { image.cover(size.width, size.height, (errOut, resized) => {
debug('Image Resized');
if (errOut) { if (errOut) {
return reject(errOut); reject(errOut);
} }
resized.toBuffer(type.ext, (errBuf, buffer) => { resized.toBuffer(type, (errBuf, buffer) => {
debug('Image Buffered');
if (errBuf) {
reject(errBuf);
}
resolve(buffer); resolve(buffer);
}); });
}); });
}); });
}); });
} }
function scale(path, type, size) {
return new Promise((resolve, reject) => {
lwip.open(path, type, (errIn, image) => {
debug('Image Opened');
if (errIn) {
reject(errIn);
}
image.cover(size.width, size.height, (errOut, resized) => {
debug('Image Resized');
if (errOut) {
reject(errOut);
}
resized.toBuffer(type, (errBuf, buffer) => {
debug('Image Buffered');
if (errBuf) {
reject(errBuf);
}
resolve(buffer);
});
});
});
});
}
export default function resize(path, type, currentSize, newSize) {
debug('Resizing');
const ratio = 970 / currentSize.width;
if (newSize.width <= 150) {
debug('Cover');
return cover(path, type, newSize);
} else if (newSize.width > 970 && ratio > 1) {
debug('Scale');
newSize.height = currentSize.height * ratio;
return scale(path, type, newSize);
}
debug('Copy');
return fs.readFile(path);
}

11
lib/s3.js
View file

@ -1,17 +1,20 @@
import aws from 'aws-sdk'; import aws from 'aws-sdk';
import s3UploadStream from 's3-upload-stream';
import debugname from 'debug'; import debugname from 'debug';
const debug = debugname('hostr:s3'); const debug = debugname('hostr:s3');
const s3 = new aws.S3(); const s3 = new aws.S3();
const s3Stream = s3UploadStream(s3);
export function get(key) { export function get(key) {
debug('fetching from s3: %s', 'hostr_files/' + key); debug('fetching from s3: %s', 'hostr_files/' + key);
return s3.getObject({Bucket: process.env.AWS_BUCKET, Key: 'hostr_files/' + key}).createReadStream(); return s3.getObject({Bucket: process.env.AWS_BUCKET, Key: 'hostr_files/' + key}).createReadStream();
} }
export function upload(key) { export function upload(stream, key, callback) {
debug('sending to s3: %s', 'hostr_files/' + key); debug('sending to s3: %s', 'hostr_files/' + key);
return s3Stream.upload({Bucket: process.env.AWS_BUCKET, Key: 'hostr_files/' + key}); const params = {Bucket: process.env.AWS_BUCKET, Key: 'hostr_files/' + key, Body: stream};
const uploading = s3.upload(params);
uploading.on('error', (err) => {
console.log(err)
});
uploading.send(callback);
} }

40
lib/sftp.js Normal file
View file

@ -0,0 +1,40 @@
import { dirname } from 'path';
import Client from 'ssh2-sftp-client';
import debugname from 'debug';
const debug = debugname('hostr:sftp');
export function get(remotePath) {
const sftp = new Client();
return sftp.connect({
host: process.env.SFTP_HOST,
port: process.env.SFTP_PORT,
username: process.env.SFTP_USERNAME,
password: process.env.SFTP_PASSWORD,
}).then(() => {
return sftp.get('hostr/uploads/' + remotePath, true);
});
}
export function upload(localPath, remotePath) {
debug('SFTP connecting');
const sftp = new Client();
return sftp.connect({
host: process.env.SFTP_HOST,
port: process.env.SFTP_PORT,
username: process.env.SFTP_USERNAME,
password: process.env.SFTP_PASSWORD,
}).then(() => {
return sftp.put(localPath, remotePath, true).then(() => {
sftp.end();
});
}).catch(() => {
debug('Creating ' + dirname(remotePath));
return sftp.mkdir(dirname(remotePath), true).then(() => {
return sftp.put(localPath, remotePath, true).then(() => {
sftp.end();
});
});
}).then(() => {
sftp.end();
});
}

108
lib/upload.js Normal file
View file

@ -0,0 +1,108 @@
import { join } from 'path';
import parse from 'co-busboy';
import fs from 'mz/fs';
import sizeOf from 'image-size';
import hostrId from './hostr-id';
import resize from './resize';
import { upload as sftpUpload } from './sftp';
import debugname from 'debug';
const debug = debugname('hostr-api:upload');
const storePath = process.env.UPLOAD_STORAGE_PATH;
const baseURL = process.env.WEB_BASE_URL;
const supported = ['jpg', 'png', 'gif'];
export function* checkLimit() {
const count = yield this.db.Files.count({
owner: this.user.id,
time_added: {'$gt': Math.ceil(Date.now() / 1000) - 86400},
});
const userLimit = this.user.daily_upload_allowance;
const underLimit = (count < userLimit || userLimit === 'unlimited');
if (!underLimit) {
this.statsd.incr('file.overlimit', 1);
}
this.assert(underLimit, 400, `{
"error": {
"message": "Daily upload limits (${this.user.daily_upload_allowance}) exceeded.",
"code": 602
}
}`);
return true;
}
export function* accept() {
yield checkLimit.call(this);
const upload = yield parse(this, {
autoFields: true,
headers: this.request.headers,
limits: { files: 1},
highWaterMark: 1000000,
});
upload.promise = new Promise((resolve, reject) => {
upload.on('error', (err) => {
this.statsd.incr('file.upload.error', 1);
debug(err);
reject();
});
upload.on('end', () => {
resolve();
});
});
upload.tempGuid = this.request.headers['hostr-guid'];
upload.originalName = upload.filename;
upload.filename = upload.filename.replace(/[^a-zA-Z0-9\.\-\_\s]/g, '').replace(/\s+/g, '');
upload.id = yield hostrId(this.db.Files);
const acceptedEvent = `{"type": "file-accepted", "data": {"id": "${upload.id}", "guid": "${upload.tempGuid}", "href": "${baseURL}/${upload.id}"}}`;
this.redis.publish('/user/' + this.user.id, acceptedEvent);
this.statsd.incr('file.upload.accepted', 1);
return upload;
}
export function resizeImage(upload, type, currentSize, newSize) {
return resize(join(storePath, upload.path), type, currentSize, newSize).then((image) => {
const path = join(upload.id[0], String(newSize.width), upload.id + '_' + upload.filename);
debug('Writing file');
debug(join(storePath, path));
return fs.writeFile(join(storePath, path), image).then(() => {
debug('Uploading file');
return sftpUpload(join(storePath, path), join('hostr', 'uploads', path));
}).catch(debug);
}).catch(debug);
}
export function* processImage(upload) {
debug('Processing image');
return new Promise((resolve) => {
const size = sizeOf(join(storePath, upload.path));
debug('Size: ', size);
if (!size.width || supported.indexOf(size.type) < 0) {
resolve();
}
Promise.all([
resizeImage(upload, size.type, size, {width: 150, height: 150}),
resizeImage(upload, size.type, size, {width: 970}),
]).then(() => {
resolve(size);
});
});
}
export function progressEvent() {
percentComplete = Math.floor(receivedSize * 100 / expectedSize);
if (percentComplete > lastPercent && lastTick < Date.now() - 1000) {
const progressEvent = `{"type": "file-progress", "data": {"id": "${upload.id}", "complete": ${percentComplete}}}`;
this.redis.publish('/file/' + upload.id, progressEvent);
this.redis.publish('/user/' + this.user.id, progressEvent);
lastTick = Date.now();
}
lastPercent = percentComplete;
}

9
package.json
View file

@ -16,7 +16,7 @@
"init": "node -r babel/register -e \"require('./lib/storage')();\"", "init": "node -r babel/register -e \"require('./lib/storage')();\"",
"jspm": "jspm install", "jspm": "jspm install",
"start": "npm run build && node -r babel/register app.js", "start": "npm run build && node -r babel/register app.js",
"test": "npm run test-seed && mocha -r babel/register test/**/*.spec.js && mocha -r babel/register -r co-mocha test/unit/image-resize.spec.js", "test": "npm run test-seed && mocha -r babel/register test/**/*.spec.js",
"test-seed": "node test/fixtures/mongo-user.js && node test/fixtures/mongo-file.js", "test-seed": "node test/fixtures/mongo-user.js && node test/fixtures/mongo-file.js",
"watch": "parallelshell \"npm run watch-js\" \"npm run watch-sass\" \"npm run watch-server\"", "watch": "parallelshell \"npm run watch-js\" \"npm run watch-sass\" \"npm run watch-server\"",
"watch-js": "babel -Dw -m system -d web/public/build web/public/src", "watch-js": "babel -Dw -m system -d web/public/build web/public/src",
@ -24,7 +24,7 @@
"watch-sass": "node-sass -w -r -o web/public/styles/ web/public/styles/" "watch-sass": "node-sass -w -r -o web/public/styles/ web/public/styles/"
}, },
"dependencies": { "dependencies": {
"aws-sdk": "~2.1.46", "aws-sdk": "~2.3.15",
"babel": "~5.8.21", "babel": "~5.8.21",
"basic-auth": "~1.0.3", "basic-auth": "~1.0.3",
"co": "~4.6.0", "co": "~4.6.0",
@ -43,6 +43,7 @@
"koa-bodyparser": "~2.0.1", "koa-bodyparser": "~2.0.1",
"koa-compress": "~1.0.8", "koa-compress": "~1.0.8",
"koa-csrf": "~2.3.0", "koa-csrf": "~2.3.0",
"koa-error": "^2.0.0",
"koa-favicon": "~1.2.0", "koa-favicon": "~1.2.0",
"koa-generic-session": "~1.9.0", "koa-generic-session": "~1.9.0",
"koa-helmet": "^0.2.0", "koa-helmet": "^0.2.0",
@ -57,6 +58,7 @@
"mime-types": "~2.1.5", "mime-types": "~2.1.5",
"moment": "~2.10.6", "moment": "~2.10.6",
"mongodb-promisified": "~1.0.3", "mongodb-promisified": "~1.0.3",
"mz": "^2.4.0",
"node-fetch": "^1.3.2", "node-fetch": "^1.3.2",
"node-sass": "~3.6.0", "node-sass": "~3.6.0",
"node-uuid": "~1.4.3", "node-uuid": "~1.4.3",
@ -65,13 +67,13 @@
"redis": "~1.0.0", "redis": "~1.0.0",
"s3-upload-stream": "~1.0.7", "s3-upload-stream": "~1.0.7",
"sendgrid": "^2.0.0", "sendgrid": "^2.0.0",
"ssh2-sftp-client": "^1.0.3",
"statsy": "~0.2.0", "statsy": "~0.2.0",
"stripe": "~3.7.1", "stripe": "~3.7.1",
"swig": "~1.4.2" "swig": "~1.4.2"
}, },
"devDependencies": { "devDependencies": {
"babel-eslint": "^4.0.10", "babel-eslint": "^4.0.10",
"co-mocha": "^1.1.2",
"eslint": "~1.3.0", "eslint": "~1.3.0",
"eslint-config-airbnb": "0.0.8", "eslint-config-airbnb": "0.0.8",
"istanbul": "~0.3.18", "istanbul": "~0.3.18",
@ -79,6 +81,7 @@
"nodemon": "~1.4.1", "nodemon": "~1.4.1",
"parallelshell": "~2.0.0", "parallelshell": "~2.0.0",
"supertest": "~1.1.0", "supertest": "~1.1.0",
"supertest-koa-agent": "^0.2.1",
"tmp": "~0.0.27" "tmp": "~0.0.27"
}, },
"jspm": { "jspm": {

7
test/api/file.spec.js
View file

@ -1,3 +1,4 @@
import path from 'path';
import assert from 'assert'; import assert from 'assert';
import { agent } from 'supertest'; import { agent } from 'supertest';
import app from '../../app'; import app from '../../app';
@ -25,11 +26,11 @@ describe('hostr-api file', function file() {
this.timeout(30000); this.timeout(30000);
request request
.post('/api/file') .post('/api/file')
.attach('file', './test/fixtures/utah-arches.jpg') .attach('file', path.join(__dirname, '..', 'fixtures', 'tall.jpg'))
.auth('test@hostr.co', 'test-password') .auth('test@hostr.co', 'test-password')
.expect(201) .expect(201)
.expect((response) => { .expect((response) => {
assert(response.body.name === 'utah-arches.jpg'); assert(response.body.name === 'tall.jpg');
id = response.body.id; id = response.body.id;
}) })
.end(done); .end(done);
@ -42,7 +43,7 @@ describe('hostr-api file', function file() {
.get('/api/file/' + id) .get('/api/file/' + id)
.expect(200) .expect(200)
.expect((response) => { .expect((response) => {
assert(response.body.name === 'utah-arches.jpg'); assert(response.body.name === 'tall.jpg');
}) })
.end(done); .end(done);
}); });

48
test/unit/image-resize.spec.js
View file

@ -1,35 +1,35 @@
import fs from 'fs'; import fs from 'mz/fs';
import path from 'path'; import { join } from 'path';
import assert from 'assert'; import assert from 'assert';
import tmp from 'tmp'; import tmp from 'tmp';
import resize from '../../lib/resize'; import resize from '../../lib/resize';
import imageType from 'image-type'; import sizeOf from 'image-size';
function testResize(path, done) {
const size = sizeOf(path);
resize(path, size.type, size, {width: 100, height: 100}).then((image) => {
const tmpFile = tmp.tmpNameSync() + '.' + size.type;
fs.writeFile(tmpFile, image).then(() => {
const newSize = sizeOf(fs.readFileSync(tmpFile));
assert(newSize.type === size.type);
done();
});
});
}
describe('Image resizing', () => { describe('Image resizing', () => {
it('should resize a jpg', function* resizeImage() { it('should resize a jpg', (done) => {
const file = fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'utah-arches.jpg')); const path = join(__dirname, '..', 'fixtures', 'utah-arches.jpg');
const imageBuffer = yield resize(file, {height: 100, width: 100}); testResize(path, done);
const tmpFile = tmp.tmpNameSync() + '.jpg';
fs.writeFileSync(tmpFile, imageBuffer);
const type = imageType(fs.readFileSync(tmpFile));
assert(type.ext === 'jpg');
}); });
it('should resize a png', function* resizeImage() { it('should resize a png', (done) => {
const file = fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'app-icon.png')); const path = join(__dirname, '..', 'fixtures', 'app-icon.png');
const imageBuffer = yield resize(file, {height: 100, width: 100}); testResize(path, done);
const tmpFile = tmp.tmpNameSync() + '.png';
fs.writeFileSync(tmpFile, imageBuffer);
const type = imageType(fs.readFileSync(tmpFile));
assert(type.ext === 'png');
}); });
it('should resize a gif', function* resizeImage() { it('should resize a gif', (done) => {
const file = fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'kim.gif')); const path = join(__dirname, '..', 'fixtures', 'kim.gif');
const imageBuffer = yield resize(file, {height: 100, width: 100}); testResize(path, done);
const tmpFile = tmp.tmpNameSync() + '.gif';
fs.writeFileSync(tmpFile, imageBuffer);
const type = imageType(fs.readFileSync(tmpFile));
assert(type.ext === 'gif');
}); });
}); });

25
test/web/file.spec.js
View file

@ -1,3 +1,4 @@
import path from 'path';
import assert from 'assert'; import assert from 'assert';
import sizeOf from 'image-size'; import sizeOf from 'image-size';
import { agent } from 'supertest'; import { agent } from 'supertest';
@ -12,7 +13,7 @@ describe('setup hostr-web file', function() {
this.timeout(30000); this.timeout(30000);
request request
.post('/api/file') .post('/api/file')
.attach('file', 'test/fixtures/utah-arches.jpg') .attach('file', path.join(__dirname, '..', 'fixtures', 'utah-arches.jpg'))
.auth('test@hostr.co', 'test-password') .auth('test@hostr.co', 'test-password')
.expect(201) .expect(201)
.expect(function(response) { .expect(function(response) {
@ -25,7 +26,6 @@ describe('setup hostr-web file', function() {
}); });
describe('hostr-web file', function() { describe('hostr-web file', function() {
describe('when GET /file/:id/:name', function() { describe('when GET /file/:id/:name', function() {
it('should receive an image', function(done) { it('should receive an image', function(done) {
request request
@ -40,41 +40,40 @@ describe('hostr-web file', function() {
}); });
describe('when GET /file/150/:id/:name', function() { describe('when GET /file/150/:id/:name', function() {
it('should receive a 150px wide thumbnail of the image', function(done) { it('should receive a 150px wide thumbnail of the image', function() {
request request
.get('/file/150/' + file.id + '/' + file.name) .get('/file/150/' + file.id + '/' + file.name)
.expect(200) .expect(200)
.expect('Content-type', 'image/jpeg') .expect('Content-type', 'image/jpeg')
.expect(function(response) { .expect(function(response) {
assert(sizeOf(response.body).width === 150); const width = sizeOf(response.body).width;
}) assert(width === 150);
.end(done); });
}); });
}); });
describe('when GET /file/970/:id/:name', function() { describe('when GET /file/970/:id/:name', function() {
it('should receive a 970px wide thumbnail of the image', function(done) { it('should receive a 970px wide thumbnail of the image', function() {
request request
.get('/file/970/' + file.id + '/' + file.name) .get('/file/970/' + file.id + '/' + file.name)
.expect(200) .expect(200)
.expect('Content-type', 'image/jpeg') .expect('Content-type', 'image/jpeg')
.expect(function(response) { .expect(function(response) {
assert(sizeOf(response.body).width === 970); const width = sizeOf(response.body).width;
}) assert(width === 970);
.end(done); });
}); });
}); });
describe('when GET /:id', function() { describe('when GET /:id', function() {
it('should receive some HTML', function(done) { it('should receive some HTML', function() {
request request
.get('/' + file.id) .get('/' + file.id)
.expect(200) .expect(200)
.expect('Content-type', /text\/html/) // Could include charset .expect('Content-type', /text\/html/) // Could include charset
.expect(function(response) { .expect(function(response) {
assert(response.text.indexOf('src="/file/970/' + file.id + '/' + file.name + '"') > -1); assert(response.text.indexOf('src="/file/970/' + file.id + '/' + file.name + '"') > -1);
}) });
.end(done);
}); });
}); });

8
test/web/user.spec.js
View file

@ -5,20 +5,20 @@ const request = agent(app.listen());
describe('hostr-web user', function() { describe('hostr-web user', function() {
describe('when POST /signin with invalid credentials', function() { describe('when POST /signin with invalid credentials', function() {
it('should not redirect to /', function(done) { it('should not redirect to /', function() {
request.get('/signin').end(function(err, response) { request.get('/signin').end(function(err, response) {
const match = response.text.match(/name="_csrf" value="([^"]+)"/); const match = response.text.match(/name="_csrf" value="([^"]+)"/);
const csrf = match[1]; const csrf = match[1];
request request
.post('/signin') .post('/signin')
.send({'email': 'test@hostr.co', 'password': 'test-passworddeded', '_csrf': csrf}) .send({'email': 'test@hostr.co', 'password': 'test-passworddeded', '_csrf': csrf})
.expect(200, done); .expect(200);
}); });
}); });
}); });
describe('when POST /signin with valid credentials', function() { describe('when POST /signin with valid credentials', function() {
it('should redirect to /', function(done) { it('should redirect to /', function() {
request.get('/signin').end(function(err, response) { request.get('/signin').end(function(err, response) {
const match = response.text.match(/name="_csrf" value="([^"]+)"/); const match = response.text.match(/name="_csrf" value="([^"]+)"/);
const csrf = match[1]; const csrf = match[1];
@ -27,7 +27,7 @@ describe('hostr-web user', function() {
.send({'email': 'test@hostr.co', 'password': 'test-password', '_csrf': csrf}) .send({'email': 'test@hostr.co', 'password': 'test-password', '_csrf': csrf})
.expect(302) .expect(302)
.expect('Location', '/') .expect('Location', '/')
.end(done); .end();
}); });
}); });
}); });

3
web/app.js
View file

@ -5,8 +5,7 @@ import views from 'koa-views';
import stats from 'koa-statsd'; import stats from 'koa-statsd';
import * as redis from '../lib/redis'; import * as redis from '../lib/redis';
import StatsD from 'statsy'; import StatsD from 'statsy';
// waiting for PR to be merged, can remove swig dependency when done import errors from 'koa-error';
import errors from '../lib/koa-error';
import * as index from './routes/index'; import * as index from './routes/index';
import * as file from './routes/file'; import * as file from './routes/file';
import * as pro from './routes/pro'; import * as pro from './routes/pro';