diff --git a/api/app.js b/api/app.js index 412e005..02f5ff5 100644 --- a/api/app.js +++ b/api/app.js @@ -16,6 +16,7 @@ import * as file from './routes/file'; import debugname from 'debug'; const debug = debugname('hostr-api'); import stats from 'koa-statsd'; +import StatsD from 'statsy'; if (process.env.SENTRY_DSN) { const ravenClient = new raven.Client(process.env.SENTRY_DSN); @@ -26,9 +27,13 @@ const app = websockify(koa()); const redisUrl = process.env.REDIS_URL || process.env.REDISTOGO_URL || 'redis://localhost:6379'; -if (process.env.STATSD_HOST) { - app.use(stats({prefix: 'hostr-api', host: process.env.STATSD_HOST})); -} +let statsdOpts = {prefix: 'hostr-api', host: process.env.STATSD_HOST || 'localhost'}; +let statsd = new StatsD(statsdOpts); +app.use(function*(next) { + this.statsd = statsd; + yield next; +}); +app.use(stats(statsdOpts)); app.use(logger()); @@ -96,6 +101,7 @@ app.use(function* (next){ } } catch (err) { if (err.status === 401) { + this.statsd.incr('auth.failure', 1); this.set('WWW-Authenticate', 'Basic'); this.status = 401; this.body = err.message; @@ -142,18 +148,9 @@ app.use(route.put('/file/:id', file.put)); app.use(route.delete('/file/:id', file.del)); if (!module.parent) { - if (process.env.LOCALHOST_KEY) { - spdy.createServer({ - key: process.env.LOCALHOST_KEY, - cert: process.env.LOCALHOST_CRT - }, app.callback()).listen(4042, function() { - debug('Koa SPDY server listening on port ' + (process.env.PORT || 4042)); - }); - } else { - app.listen(process.env.PORT || 4042, function() { - debug('Koa HTTP server listening on port ' + (process.env.PORT || 4042)); - }); - } + app.listen(process.env.PORT || 4042, function() { + debug('Koa HTTP server listening on port ' + (process.env.PORT || 4042)); + }); } export default app; diff --git a/api/routes/file.js b/api/routes/file.js index 9b19baf..f14a0dd 100644 --- a/api/routes/file.js +++ b/api/routes/file.js @@ -44,6 +44,9 @@ export function* post(next) { const count = yield Files.count({owner: this.user.id, 'time_added': {'$gt': Math.ceil(Date.now() / 1000) - 86400}}); const userLimit = this.user.daily_upload_allowance; const underLimit = (count < userLimit || userLimit === 'unlimited'); + if (!underLimit) { + this.statsd.incr('file.overlimit', 1); + } this.assert(underLimit, 400, `{ "error": { "message": "Daily upload limits (${this.user.daily_upload_allowance}) exceeded.", @@ -56,8 +59,14 @@ export function* post(next) { upload.filename = upload.filename.replace(/[^a-zA-Z0-9\.\-\_\s]/g, '').replace(/\s+/g, ''); const fileId = yield hostrId(Files); + // Fire an event to let the frontend map the GUID it sent to the real ID. Allows immediate linking to the file + let acceptedEvent = `{"type": "file-accepted", "data": {"id": "${fileId}", "guid": "${tempGuid}", "href": "${fileHost}/${fileId}"}}`; + this.redis.publish('/user/' + this.user.id, acceptedEvent); + this.statsd.incr('file.upload.accepted', 1); + const uploadPromise = new Promise((resolve, reject) => { upload.on('error', () => { + this.statsd.incr('file.upload.error', 1); reject(); }); @@ -115,13 +124,11 @@ export function* post(next) { md5sum.update(data); }); - // Fire an event to let the frontend map the GUID it sent to the real ID. Allows immediate linking to the file - let acceptedEvent = `{"type": "file-accepted", "data": {"id": "${fileId}", "guid": "${tempGuid}", "href": "${fileHost}/${fileId}"}}`; - this.redis.publish('/user/' + this.user.id, acceptedEvent); // Fire final upload progress event so users know it's now processing const completeEvent = `{"type": "file-progress", "data": {"id": "${fileId}", "complete": 100}}`; this.redis.publish('/file/' + fileId, completeEvent); this.redis.publish('/user/' + this.user.id, completeEvent); + this.statsd.incr('file.upload.complete', 1); const dbFile = { _id: fileId, @@ -171,6 +178,9 @@ export function* post(next) { process.nextTick(function*() { debug('Malware Scan'); const { positive, result } = yield malware(dbFile); + if (positive) { + this.statsd.incr('file.malware', 1); + } yield Files.updateOne({_id: fileId}, {'$set': {malware: positive, virustotal: result}}); }); } else { @@ -209,7 +219,7 @@ export function* list() { }; const userFiles = yield Files.find({owner: this.user.id, status: status}, queryOptions).toArray(); - + this.statsd.incr('file.list', 1); this.body = userFiles.map(formatFile); } @@ -221,6 +231,7 @@ export function* get(id) { this.assert(file, 404, '{"error": {"message": "File not found", "code": 604}}'); const user = yield Users.findOne({_id: file.owner}); this.assert(user && !user.banned, 404, '{"error": {"message": "File not found", "code": 604}}'); + this.statsd.incr('file.get', 1); this.body = formatFile(file); } @@ -240,6 +251,7 @@ export function* del(id) { const event = {type: 'file-deleted', data: {'id': id}}; yield this.redis.publish('/user/' + this.user.id, JSON.stringify(event)); yield this.redis.publish('/file/' + id, JSON.stringify(event)); + this.statsd.incr('file.delete', 1); this.body = ''; } diff --git a/package.json b/package.json index 91b4b69..54d146f 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "redis": "0.12.1", "redis-url": "~1.2.1", "s3-upload-stream": "^1.0.7", + "statsy": "^0.2.0", "stripe": "^3.7.0", "swig": "^1.4.2", "virustotal.js": "~0.3.1" diff --git a/web/app.js b/web/app.js index 9ced6dc..3f5d2ee 100644 --- a/web/app.js +++ b/web/app.js @@ -25,6 +25,7 @@ const objectId = mongodb().ObjectId; import debugname from 'debug'; const debug = debugname('hostr-web'); import stats from 'koa-statsd'; +import StatsD from 'statsy'; if (process.env.SENTRY_DSN) { const ravenClient = new raven.Client(process.env.SENTRY_DSN); @@ -35,9 +36,13 @@ const redisUrl = process.env.REDIS_URL || process.env.REDISTOGO_URL || 'redis:// const app = koa(); -if (process.env.STATSD_HOST) { - app.use(stats({prefix: 'hostr-web', host: process.env.STATSD_HOST})); -} +let statsdOpts = {prefix: 'hostr-web', host: process.env.STATSD_HOST || 'localhost'}; +let statsd = new StatsD(statsdOpts); +app.use(function*(next) { + this.statsd = statsd; + yield next; +}); +app.use(stats(statsdOpts)); app.use(errors({template: path.join(__dirname, 'public', '404.html')})); diff --git a/web/lib/auth.js b/web/lib/auth.js index 59e8db6..db40708 100644 --- a/web/lib/auth.js +++ b/web/lib/auth.js @@ -189,5 +189,7 @@ export function* activateUser(ctx, code) { if (user) { Users.updateOne({_id: user._id}, {'$unset': {activationCode: ''}}); yield setupSession(ctx, user); + } else { + return false; } } diff --git a/web/routes/file.js b/web/routes/file.js index e154c0f..29228f4 100644 --- a/web/routes/file.js +++ b/web/routes/file.js @@ -17,16 +17,28 @@ const userAgentCheck = function(userAgent) { }; const hotlinkCheck = function(file, userAgent, referrer) { - debug(file, userAgent, referrer); return !userAgentCheck(userAgent) && !file.width && !(referrer.match(/^https:\/\/hostr.co/) || referrer.match(/^http:\/\/localhost:4040/)) }; export function* get(id, name, size) { const file = yield this.db.Files.findOne({_id: id, 'file_name': name, 'status': 'active'}); this.assert(file, 404); + if (hotlinkCheck(file, this.headers['user-agent'], this.headers['referer'])) { - this.redirect('/' + id); + return this.redirect('/' + id); } + + if (!file.width && this.request.query.warning != 'on') { + return this.redirect('/' + id); + } + + if (file.malware) { + let alert = this.request.query.alert; + if (!alert || !alert.match(/i want to download malware/i)) { + return this.redirect('/' + id); + } + } + let localPath = path.join(storePath, file._id[0], file._id + '_' + file.file_name); let remotePath = path.join(file._id[0], file._id + '_' + file.file_name); if (size > 0) { @@ -34,9 +46,22 @@ export function* get(id, name, size) { remotePath = path.join(size, file._id + '_' + file.file_name); } + if (file.malware) { + this.statsd.incr('file.malware.download', 1); + } + let type = 'application/octet-stream'; if (file.width > 0) { + if (size) { + this.statsd.incr('file.view', 1); + } type = mime.lookup(file.file_name); + } else { + this.statsd.incr('file.download', 1); + } + + if (userAgentCheck(this.headers['user-agent'])) { + this.set('Content-Disposition', 'attachment; filename=' + file.file_name); } this.set('Content-type', type); @@ -59,8 +84,7 @@ export function* landing(id, next) { if(userAgentCheck(this.headers['user-agent'])) { return yield get.call(this, file._id, file.file_name); } - + this.statsd.incr('file.landing', 1); const formattedFile = formatFile(file); - debug(formattedFile); yield this.render('file', {file: formattedFile}); } diff --git a/web/routes/user.js b/web/routes/user.js index 5a5dda9..f4b3e7b 100644 --- a/web/routes/user.js +++ b/web/routes/user.js @@ -4,13 +4,15 @@ export function* signin() { if (!this.request.body.email) { return yield this.render('signin'); } - + this.statsd.incr('auth.attempt', 1); const user = yield authenticate(this, this.request.body.email, this.request.body.password); if(!user) { + this.statsd.incr('auth.failure', 1); return yield this.render('signin', {error: 'Invalid login details'}); } else if (user.activationCode) { return yield this.render('signin', {error: 'Your account hasn\'t been activated yet. Check your for an activation email.'}); } else { + this.statsd.incr('auth.success', 1); yield setupSession(this, user); this.redirect('/'); } @@ -37,6 +39,7 @@ export function* signup() { } catch (e) { return yield this.render('signup', {error: e.message}); } + this.statsd.incr('auth.signup', 1); return yield this.render('signup', {message: 'Thanks for signing up, we\'ve sent you an email to activate your account.'}); } @@ -47,10 +50,11 @@ export function* forgot(token) { if (this.request.body.email) { var email = this.request.body.email; yield sendResetToken(this, email); + this.statsd.incr('auth.reset.request', 1); return yield this.render('forgot', {message: 'We\'ve sent an email with a link to reset your password. Be sure to check your spam folder if you it doesn\'t appear within a few minutes', token: null}); } else if (token && this.request.body.password) { if (this.request.body.password.length < 7) { - return this.render('forgot', {error: 'Password needs to be at least 7 characters long.', token: token}); + return yield this.render('forgot', {error: 'Password needs to be at least 7 characters long.', token: token}); } const tokenUser = yield validateResetToken(this, token); var userId = tokenUser._id; @@ -58,10 +62,12 @@ export function* forgot(token) { yield Reset.remove({_id: userId}); const user = yield Users.findOne({_id: userId}); yield setupSession(this, user); + this.statsd.incr('auth.reset.success', 1); this.redirect('/'); } else if (token.length) { const tokenUser = yield validateResetToken(this, token); if (!tokenUser) { + this.statsd.incr('auth.reset.fail', 1); return yield this.render('forgot', {error: 'Invalid password reset token. It might be expired, or has already been used.', token: null}); } else { return yield this.render('forgot', {token: token}); @@ -73,6 +79,7 @@ export function* forgot(token) { export function* logout() { + this.statsd.incr('auth.logout', 1); this.cookies.set('r', {expires: new Date(1), path: '/'}); this.session = null; this.redirect('/'); @@ -80,6 +87,8 @@ export function* logout() { export function* activate(code) { - yield activateUser(this, code); + if (yield activateUser(this, code)) { + this.statsd.incr('auth.activation', 1); + } this.redirect('/'); }