jonathan/hostr
jonathan/hostr
1
0
Fork 0
Code Issues Pull requests 4 Projects Releases Packages 1 Wiki Activity Actions

Add more stat collection

Browse source
This commit is contained in:
Jonathan Cremin 2015-08-09 17:21:39 +01:00
parent 741f49ad63
commit e254114625
7 changed files with 79 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

27
api/app.js
View file

@ -16,6 +16,7 @@ import * as file from './routes/file';
import debugname from 'debug';
const debug = debugname('hostr-api');
import stats from 'koa-statsd';
import StatsD from 'statsy';
if (process.env.SENTRY_DSN) {
const ravenClient = new raven.Client(process.env.SENTRY_DSN);
@ -26,9 +27,13 @@ const app = websockify(koa());
const redisUrl = process.env.REDIS_URL || process.env.REDISTOGO_URL || 'redis://localhost:6379';
if (process.env.STATSD_HOST) {
app.use(stats({prefix: 'hostr-api', host: process.env.STATSD_HOST}));
}
let statsdOpts = {prefix: 'hostr-api', host: process.env.STATSD_HOST || 'localhost'};
let statsd = new StatsD(statsdOpts);
app.use(function*(next) {
this.statsd = statsd;
yield next;
});
app.use(stats(statsdOpts));
app.use(logger());
@ -96,6 +101,7 @@ app.use(function* (next){
}
} catch (err) {
if (err.status === 401) {
this.statsd.incr('auth.failure', 1);
this.set('WWW-Authenticate', 'Basic');
this.status = 401;
this.body = err.message;
@ -142,18 +148,9 @@ app.use(route.put('/file/:id', file.put));
app.use(route.delete('/file/:id', file.del));
if (!module.parent) {
if (process.env.LOCALHOST_KEY) {
spdy.createServer({
key: process.env.LOCALHOST_KEY,
cert: process.env.LOCALHOST_CRT
}, app.callback()).listen(4042, function() {
debug('Koa SPDY server listening on port ' + (process.env.PORT || 4042));
});
} else {
app.listen(process.env.PORT || 4042, function() {
debug('Koa HTTP server listening on port ' + (process.env.PORT || 4042));
});
}
app.listen(process.env.PORT || 4042, function() {
debug('Koa HTTP server listening on port ' + (process.env.PORT || 4042));
});
}
export default app;

20
api/routes/file.js
View file

@ -44,6 +44,9 @@ export function* post(next) {
const count = yield Files.count({owner: this.user.id, 'time_added': {'$gt': Math.ceil(Date.now() / 1000) - 86400}});
const userLimit = this.user.daily_upload_allowance;
const underLimit = (count < userLimit || userLimit === 'unlimited');
if (!underLimit) {
this.statsd.incr('file.overlimit', 1);
}
this.assert(underLimit, 400, `{
"error": {
"message": "Daily upload limits (${this.user.daily_upload_allowance}) exceeded.",
@ -56,8 +59,14 @@ export function* post(next) {
upload.filename = upload.filename.replace(/[^a-zA-Z0-9\.\-\_\s]/g, '').replace(/\s+/g, '');
const fileId = yield hostrId(Files);
// Fire an event to let the frontend map the GUID it sent to the real ID. Allows immediate linking to the file
let acceptedEvent = `{"type": "file-accepted", "data": {"id": "${fileId}", "guid": "${tempGuid}", "href": "${fileHost}/${fileId}"}}`;
this.redis.publish('/user/' + this.user.id, acceptedEvent);
this.statsd.incr('file.upload.accepted', 1);
const uploadPromise = new Promise((resolve, reject) => {
upload.on('error', () => {
this.statsd.incr('file.upload.error', 1);
reject();
});
@ -115,13 +124,11 @@ export function* post(next) {
md5sum.update(data);
});
// Fire an event to let the frontend map the GUID it sent to the real ID. Allows immediate linking to the file
let acceptedEvent = `{"type": "file-accepted", "data": {"id": "${fileId}", "guid": "${tempGuid}", "href": "${fileHost}/${fileId}"}}`;
this.redis.publish('/user/' + this.user.id, acceptedEvent);
// Fire final upload progress event so users know it's now processing
const completeEvent = `{"type": "file-progress", "data": {"id": "${fileId}", "complete": 100}}`;
this.redis.publish('/file/' + fileId, completeEvent);
this.redis.publish('/user/' + this.user.id, completeEvent);
this.statsd.incr('file.upload.complete', 1);
const dbFile = {
_id: fileId,
@ -171,6 +178,9 @@ export function* post(next) {
process.nextTick(function*() {
debug('Malware Scan');
const { positive, result } = yield malware(dbFile);
if (positive) {
this.statsd.incr('file.malware', 1);
}
yield Files.updateOne({_id: fileId}, {'$set': {malware: positive, virustotal: result}});
});
} else {
@ -209,7 +219,7 @@ export function* list() {
};
const userFiles = yield Files.find({owner: this.user.id, status: status}, queryOptions).toArray();
this.statsd.incr('file.list', 1);
this.body = userFiles.map(formatFile);
}
@ -221,6 +231,7 @@ export function* get(id) {
this.assert(file, 404, '{"error": {"message": "File not found", "code": 604}}');
const user = yield Users.findOne({_id: file.owner});
this.assert(user && !user.banned, 404, '{"error": {"message": "File not found", "code": 604}}');
this.statsd.incr('file.get', 1);
this.body = formatFile(file);
}
@ -240,6 +251,7 @@ export function* del(id) {
const event = {type: 'file-deleted', data: {'id': id}};
yield this.redis.publish('/user/' + this.user.id, JSON.stringify(event));
yield this.redis.publish('/file/' + id, JSON.stringify(event));
this.statsd.incr('file.delete', 1);
this.body = '';
}

1
package.json
View file

@ -61,6 +61,7 @@
"redis": "0.12.1",
"redis-url": "~1.2.1",
"s3-upload-stream": "^1.0.7",
"statsy": "^0.2.0",
"stripe": "^3.7.0",
"swig": "^1.4.2",
"virustotal.js": "~0.3.1"

11
web/app.js
View file

@ -25,6 +25,7 @@ const objectId = mongodb().ObjectId;
import debugname from 'debug';
const debug = debugname('hostr-web');
import stats from 'koa-statsd';
import StatsD from 'statsy';
if (process.env.SENTRY_DSN) {
const ravenClient = new raven.Client(process.env.SENTRY_DSN);
@ -35,9 +36,13 @@ const redisUrl = process.env.REDIS_URL || process.env.REDISTOGO_URL || 'redis://
const app = koa();
if (process.env.STATSD_HOST) {
app.use(stats({prefix: 'hostr-web', host: process.env.STATSD_HOST}));
}
let statsdOpts = {prefix: 'hostr-web', host: process.env.STATSD_HOST || 'localhost'};
let statsd = new StatsD(statsdOpts);
app.use(function*(next) {
this.statsd = statsd;
yield next;
});
app.use(stats(statsdOpts));
app.use(errors({template: path.join(__dirname, 'public', '404.html')}));

2
web/lib/auth.js
View file

@ -189,5 +189,7 @@ export function* activateUser(ctx, code) {
if (user) {
Users.updateOne({_id: user._id}, {'$unset': {activationCode: ''}});
yield setupSession(ctx, user);
} else {
return false;
}
}

32
web/routes/file.js
View file

@ -17,16 +17,28 @@ const userAgentCheck = function(userAgent) {
};
const hotlinkCheck = function(file, userAgent, referrer) {
debug(file, userAgent, referrer);
return !userAgentCheck(userAgent) && !file.width && !(referrer.match(/^https:\/\/hostr.co/) || referrer.match(/^http:\/\/localhost:4040/))
};
export function* get(id, name, size) {
const file = yield this.db.Files.findOne({_id: id, 'file_name': name, 'status': 'active'});
this.assert(file, 404);
if (hotlinkCheck(file, this.headers['user-agent'], this.headers['referer'])) {
this.redirect('/' + id);
return this.redirect('/' + id);
}
if (!file.width && this.request.query.warning != 'on') {
return this.redirect('/' + id);
}
if (file.malware) {
let alert = this.request.query.alert;
if (!alert || !alert.match(/i want to download malware/i)) {
return this.redirect('/' + id);
}
}
let localPath = path.join(storePath, file._id[0], file._id + '_' + file.file_name);
let remotePath = path.join(file._id[0], file._id + '_' + file.file_name);
if (size > 0) {
@ -34,9 +46,22 @@ export function* get(id, name, size) {
remotePath = path.join(size, file._id + '_' + file.file_name);
}
if (file.malware) {
this.statsd.incr('file.malware.download', 1);
}
let type = 'application/octet-stream';
if (file.width > 0) {
if (size) {
this.statsd.incr('file.view', 1);
}
type = mime.lookup(file.file_name);
} else {
this.statsd.incr('file.download', 1);
}
if (userAgentCheck(this.headers['user-agent'])) {
this.set('Content-Disposition', 'attachment; filename=' + file.file_name);
}
this.set('Content-type', type);
@ -59,8 +84,7 @@ export function* landing(id, next) {
if(userAgentCheck(this.headers['user-agent'])) {
return yield get.call(this, file._id, file.file_name);
}
this.statsd.incr('file.landing', 1);
const formattedFile = formatFile(file);
debug(formattedFile);
yield this.render('file', {file: formattedFile});
}

15
web/routes/user.js
View file

@ -4,13 +4,15 @@ export function* signin() {
if (!this.request.body.email) {
return yield this.render('signin');
}
this.statsd.incr('auth.attempt', 1);
const user = yield authenticate(this, this.request.body.email, this.request.body.password);
if(!user) {
this.statsd.incr('auth.failure', 1);
return yield this.render('signin', {error: 'Invalid login details'});
} else if (user.activationCode) {
return yield this.render('signin', {error: 'Your account hasn\'t been activated yet. Check your for an activation email.'});
} else {
this.statsd.incr('auth.success', 1);
yield setupSession(this, user);
this.redirect('/');
}
@ -37,6 +39,7 @@ export function* signup() {
} catch (e) {
return yield this.render('signup', {error: e.message});
}
this.statsd.incr('auth.signup', 1);
return yield this.render('signup', {message: 'Thanks for signing up, we\'ve sent you an email to activate your account.'});
}
@ -47,10 +50,11 @@ export function* forgot(token) {
if (this.request.body.email) {
var email = this.request.body.email;
yield sendResetToken(this, email);
this.statsd.incr('auth.reset.request', 1);
return yield this.render('forgot', {message: 'We\'ve sent an email with a link to reset your password. Be sure to check your spam folder if you it doesn\'t appear within a few minutes', token: null});
} else if (token && this.request.body.password) {
if (this.request.body.password.length < 7) {
return this.render('forgot', {error: 'Password needs to be at least 7 characters long.', token: token});
return yield this.render('forgot', {error: 'Password needs to be at least 7 characters long.', token: token});
}
const tokenUser = yield validateResetToken(this, token);
var userId = tokenUser._id;
@ -58,10 +62,12 @@ export function* forgot(token) {
yield Reset.remove({_id: userId});
const user = yield Users.findOne({_id: userId});
yield setupSession(this, user);
this.statsd.incr('auth.reset.success', 1);
this.redirect('/');
} else if (token.length) {
const tokenUser = yield validateResetToken(this, token);
if (!tokenUser) {
this.statsd.incr('auth.reset.fail', 1);
return yield this.render('forgot', {error: 'Invalid password reset token. It might be expired, or has already been used.', token: null});
} else {
return yield this.render('forgot', {token: token});
@ -73,6 +79,7 @@ export function* forgot(token) {
export function* logout() {
this.statsd.incr('auth.logout', 1);
this.cookies.set('r', {expires: new Date(1), path: '/'});
this.session = null;
this.redirect('/');
@ -80,6 +87,8 @@ export function* logout() {
export function* activate(code) {
yield activateUser(this, code);
if (yield activateUser(this, code)) {
this.statsd.incr('auth.activation', 1);
}
this.redirect('/');
}