Add more stat collection

api/app.js

@@ -16,6 +16,7 @@ import * as file from './routes/file';
 import debugname from 'debug';
 const debug = debugname('hostr-api');
 import stats from 'koa-statsd';
+import StatsD from 'statsy';
 
 if (process.env.SENTRY_DSN) {
   const ravenClient = new raven.Client(process.env.SENTRY_DSN);
@@ -26,9 +27,13 @@ const app = websockify(koa());
 
 const redisUrl = process.env.REDIS_URL || process.env.REDISTOGO_URL || 'redis://localhost:6379';
 
-if (process.env.STATSD_HOST) {
+let statsdOpts = {prefix: 'hostr-api', host: process.env.STATSD_HOST || 'localhost'};
-  app.use(stats({prefix: 'hostr-api', host: process.env.STATSD_HOST}));
+let statsd = new StatsD(statsdOpts);
-}
+app.use(function*(next) {
+  this.statsd = statsd;
+  yield next;
+});
+app.use(stats(statsdOpts));
 
 app.use(logger());
 
@@ -96,6 +101,7 @@ app.use(function* (next){
     }
   } catch (err) {
     if (err.status === 401) {
+      this.statsd.incr('auth.failure', 1);
       this.set('WWW-Authenticate', 'Basic');
       this.status = 401;
       this.body = err.message;
@@ -142,18 +148,9 @@ app.use(route.put('/file/:id', file.put));
 app.use(route.delete('/file/:id', file.del));
 
 if (!module.parent) {
-  if (process.env.LOCALHOST_KEY) {
-    spdy.createServer({
-      key: process.env.LOCALHOST_KEY,
-      cert: process.env.LOCALHOST_CRT
-    }, app.callback()).listen(4042, function() {
-      debug('Koa SPDY server listening on port ' + (process.env.PORT || 4042));
-    });
-  } else {
   app.listen(process.env.PORT || 4042, function() {
     debug('Koa HTTP server listening on port ' + (process.env.PORT || 4042));
   });
 }
-}
 
 export default app;

api/routes/file.js

@@ -44,6 +44,9 @@ export function* post(next) {
   const count = yield Files.count({owner: this.user.id, 'time_added': {'$gt': Math.ceil(Date.now() / 1000) - 86400}});
   const userLimit = this.user.daily_upload_allowance;
   const underLimit = (count < userLimit || userLimit === 'unlimited');
+  if (!underLimit) {
+    this.statsd.incr('file.overlimit', 1);
+  }
   this.assert(underLimit, 400, `{
     "error": {
       "message": "Daily upload limits (${this.user.daily_upload_allowance}) exceeded.",
@@ -56,8 +59,14 @@ export function* post(next) {
   upload.filename = upload.filename.replace(/[^a-zA-Z0-9\.\-\_\s]/g, '').replace(/\s+/g, '');
   const fileId = yield hostrId(Files);
 
+  // Fire an event to let the frontend map the GUID it sent to the real ID. Allows immediate linking to the file
+  let acceptedEvent = `{"type": "file-accepted", "data": {"id": "${fileId}", "guid": "${tempGuid}", "href": "${fileHost}/${fileId}"}}`;
+  this.redis.publish('/user/' + this.user.id, acceptedEvent);
+  this.statsd.incr('file.upload.accepted', 1);
+
   const uploadPromise = new Promise((resolve, reject) => {
     upload.on('error', () => {
+      this.statsd.incr('file.upload.error', 1);
       reject();
     });
 
@@ -115,13 +124,11 @@ export function* post(next) {
     md5sum.update(data);
   });
 
-  // Fire an event to let the frontend map the GUID it sent to the real ID. Allows immediate linking to the file
-  let acceptedEvent = `{"type": "file-accepted", "data": {"id": "${fileId}", "guid": "${tempGuid}", "href": "${fileHost}/${fileId}"}}`;
-  this.redis.publish('/user/' + this.user.id, acceptedEvent);
   // Fire final upload progress event so users know it's now processing
   const completeEvent = `{"type": "file-progress", "data": {"id": "${fileId}", "complete": 100}}`;
   this.redis.publish('/file/' + fileId, completeEvent);
   this.redis.publish('/user/' + this.user.id, completeEvent);
+  this.statsd.incr('file.upload.complete', 1);
 
   const dbFile = {
     _id: fileId,
@@ -171,6 +178,9 @@ export function* post(next) {
     process.nextTick(function*() {
       debug('Malware Scan');
       const { positive, result } = yield malware(dbFile);
+      if (positive) {
+        this.statsd.incr('file.malware', 1);
+      }
       yield Files.updateOne({_id: fileId}, {'$set': {malware: positive, virustotal: result}});
     });
   } else {
@@ -209,7 +219,7 @@ export function* list() {
   };
 
   const userFiles = yield Files.find({owner: this.user.id, status: status}, queryOptions).toArray();
-
+  this.statsd.incr('file.list', 1);
   this.body = userFiles.map(formatFile);
 }
 
@@ -221,6 +231,7 @@ export function* get(id) {
   this.assert(file, 404, '{"error": {"message": "File not found", "code": 604}}');
   const user = yield Users.findOne({_id: file.owner});
   this.assert(user && !user.banned, 404, '{"error": {"message": "File not found", "code": 604}}');
+  this.statsd.incr('file.get', 1);
   this.body = formatFile(file);
 }
 
@@ -240,6 +251,7 @@ export function* del(id) {
   const event = {type: 'file-deleted', data: {'id': id}};
   yield this.redis.publish('/user/' + this.user.id, JSON.stringify(event));
   yield this.redis.publish('/file/' + id, JSON.stringify(event));
+  this.statsd.incr('file.delete', 1);
   this.body = '';
 }
 

package.json

@@ -61,6 +61,7 @@
     "redis": "0.12.1",
     "redis-url": "~1.2.1",
     "s3-upload-stream": "^1.0.7",
+    "statsy": "^0.2.0",
     "stripe": "^3.7.0",
     "swig": "^1.4.2",
     "virustotal.js": "~0.3.1"

web/app.js

@@ -25,6 +25,7 @@ const objectId = mongodb().ObjectId;
 import debugname from 'debug';
 const debug = debugname('hostr-web');
 import stats from 'koa-statsd';
+import StatsD from 'statsy';
 
 if (process.env.SENTRY_DSN) {
   const ravenClient = new raven.Client(process.env.SENTRY_DSN);
@@ -35,9 +36,13 @@ const redisUrl = process.env.REDIS_URL || process.env.REDISTOGO_URL || 'redis://
 
 const app = koa();
 
-if (process.env.STATSD_HOST) {
+let statsdOpts = {prefix: 'hostr-web', host: process.env.STATSD_HOST || 'localhost'};
-  app.use(stats({prefix: 'hostr-web', host: process.env.STATSD_HOST}));
+let statsd = new StatsD(statsdOpts);
-}
+app.use(function*(next) {
+  this.statsd = statsd;
+  yield next;
+});
+app.use(stats(statsdOpts));
 
 app.use(errors({template: path.join(__dirname, 'public', '404.html')}));
 

web/lib/auth.js

@@ -189,5 +189,7 @@ export function* activateUser(ctx, code) {
   if (user) {
     Users.updateOne({_id: user._id}, {'$unset': {activationCode: ''}});
     yield setupSession(ctx, user);
+  } else {
+    return false;
   }
 }

web/routes/file.js

@@ -17,16 +17,28 @@ const userAgentCheck = function(userAgent) {
 };
 
 const hotlinkCheck = function(file, userAgent, referrer) {
-  debug(file, userAgent, referrer);
   return !userAgentCheck(userAgent) && !file.width && !(referrer.match(/^https:\/\/hostr.co/) || referrer.match(/^http:\/\/localhost:4040/))
 };
 
 export function* get(id, name, size) {
   const file = yield this.db.Files.findOne({_id: id, 'file_name': name, 'status': 'active'});
   this.assert(file, 404);
+
   if (hotlinkCheck(file, this.headers['user-agent'], this.headers['referer'])) {
-    this.redirect('/' + id);
+    return this.redirect('/' + id);
   }
+
+  if (!file.width && this.request.query.warning != 'on') {
+    return this.redirect('/' + id);
+  }
+
+  if (file.malware) {
+    let alert = this.request.query.alert;
+    if (!alert || !alert.match(/i want to download malware/i)) {
+      return this.redirect('/' + id);
+    }
+  }
+
   let localPath = path.join(storePath, file._id[0], file._id + '_' + file.file_name);
   let remotePath = path.join(file._id[0], file._id + '_' + file.file_name);
   if (size > 0) {
@@ -34,9 +46,22 @@ export function* get(id, name, size) {
     remotePath = path.join(size, file._id + '_' + file.file_name);
   }
 
+  if (file.malware) {
+    this.statsd.incr('file.malware.download', 1);
+  }
+
   let type = 'application/octet-stream';
   if (file.width > 0) {
+    if (size) {
+      this.statsd.incr('file.view', 1);
+    }
     type = mime.lookup(file.file_name);
+  } else {
+    this.statsd.incr('file.download', 1);
+  }
+
+  if (userAgentCheck(this.headers['user-agent'])) {
+    this.set('Content-Disposition', 'attachment; filename=' + file.file_name);
   }
 
   this.set('Content-type', type);
@@ -59,8 +84,7 @@ export function* landing(id, next) {
   if(userAgentCheck(this.headers['user-agent'])) {
     return yield get.call(this, file._id, file.file_name);
   }
-
+  this.statsd.incr('file.landing', 1);
   const formattedFile = formatFile(file);
-  debug(formattedFile);
   yield this.render('file', {file: formattedFile});
 }

web/routes/user.js

@@ -4,13 +4,15 @@ export function* signin() {
   if (!this.request.body.email) {
     return yield this.render('signin');
   }
-
+  this.statsd.incr('auth.attempt', 1);
   const user = yield authenticate(this, this.request.body.email, this.request.body.password);
   if(!user) {
+    this.statsd.incr('auth.failure', 1);
     return yield this.render('signin', {error: 'Invalid login details'});
   } else if (user.activationCode) {
     return yield this.render('signin', {error: 'Your account hasn\'t been activated yet. Check your for an activation email.'});
   } else {
+    this.statsd.incr('auth.success', 1);
     yield setupSession(this, user);
     this.redirect('/');
   }
@@ -37,6 +39,7 @@ export function* signup() {
   } catch (e) {
     return yield this.render('signup', {error: e.message});
   }
+  this.statsd.incr('auth.signup', 1);
   return yield this.render('signup', {message: 'Thanks for signing up, we\'ve sent you an email to activate your account.'});
 }
 
@@ -47,10 +50,11 @@ export function* forgot(token) {
   if (this.request.body.email) {
     var email = this.request.body.email;
     yield sendResetToken(this, email);
+    this.statsd.incr('auth.reset.request', 1);
     return yield this.render('forgot', {message: 'We\'ve sent an email with a link to reset your password. Be sure to check your spam folder if you it doesn\'t appear within a few minutes', token: null});
   } else if (token && this.request.body.password) {
     if (this.request.body.password.length < 7) {
-      return this.render('forgot', {error: 'Password needs to be at least 7 characters long.', token: token});
+      return yield this.render('forgot', {error: 'Password needs to be at least 7 characters long.', token: token});
     }
     const tokenUser = yield validateResetToken(this, token);
     var userId = tokenUser._id;
@@ -58,10 +62,12 @@ export function* forgot(token) {
     yield Reset.remove({_id: userId});
     const user = yield Users.findOne({_id: userId});
     yield setupSession(this, user);
+    this.statsd.incr('auth.reset.success', 1);
     this.redirect('/');
   } else if (token.length) {
     const tokenUser = yield validateResetToken(this, token);
     if (!tokenUser) {
+      this.statsd.incr('auth.reset.fail', 1);
       return yield this.render('forgot', {error: 'Invalid password reset token. It might be expired, or has already been used.', token: null});
     } else {
       return yield this.render('forgot', {token: token});
@@ -73,6 +79,7 @@ export function* forgot(token) {
 
 
 export function* logout() {
+  this.statsd.incr('auth.logout', 1);
   this.cookies.set('r', {expires: new Date(1), path: '/'});
   this.session = null;
   this.redirect('/');
@@ -80,6 +87,8 @@ export function* logout() {
 
 
 export function* activate(code) {
-  yield activateUser(this, code);
+  if (yield activateUser(this, code)) {
+    this.statsd.incr('auth.activation', 1);
+  }
   this.redirect('/');
 }