Initial commit.

api/app.js

@@ -0,0 +1,154 @@
+import spdy from 'spdy';
+import koa from 'koa';
+import route from 'koa-route';
+import websockify from 'koa-websocket';
+import logger from 'koa-logger';
+import compress from 'koa-compress';
+import bodyparser from 'koa-bodyparser';
+import cors from 'kcors';
+import co from 'co';
+import redis from 'redis-url';
+import coRedis from 'co-redis';
+import raven from 'raven';
+import auth from './lib/auth';
+import mongoConnect from '../config/mongo';
+import * as user from './routes/user';
+import * as file from './routes/file';
+import debugname from 'debug';
+const debug = debugname('hostr-api');
+
+if (process.env.SENTRY_DSN) {
+  const ravenClient = new raven.Client(process.env.SENTRY_DSN);
+  ravenClient.patchGlobal();
+}
+
+const app = websockify(koa());
+
+const redisUrl = process.env.REDIS_URL || process.env.REDISTOGO_URL || 'redis://localhost:6379';
+
+app.use(logger());
+
+app.use(cors({
+  origin: '*',
+  credentials: true
+}));
+
+app.use(function* (next){
+  this.set('Server', 'Nintendo 64');
+  if(this.req.headers['x-forwarded-proto'] === 'http'){
+    return this.redirect('https://' + this.req.headers.host + this.req.url);
+  }
+  yield next;
+});
+
+const redisConn = redis.connect(redisUrl);
+let coRedisConn = {};
+
+co(function*() {
+  coRedisConn = coRedis(redisConn);
+  coRedisConn.on('error', function (err) {
+    debug('Redis error ' + err);
+  });
+}).catch(function(err) {
+  console.error(err);
+});
+
+let mongoConnecting = false;
+const mongoDeferred = {};
+mongoDeferred.promise = new Promise(function(resolve, reject) {
+  mongoDeferred.resolve = resolve;
+  mongoDeferred.reject = reject;
+});
+
+function* getMongo() {
+  if (!mongoConnecting) {
+    mongoConnecting = true;
+    const db = yield mongoConnect();
+    mongoDeferred.resolve(db);
+    return db;
+  } else {
+    return mongoDeferred.promise;
+  }
+}
+
+function* setupConnections(next){
+  this.db = yield getMongo();
+  this.redis = coRedisConn;
+  yield next;
+}
+app.ws.use(setupConnections);
+app.use(setupConnections);
+
+app.use(route.get('/', function* (){
+  this.status = 200;
+  this.body = '';
+}));
+
+app.use(function* (next){
+  try {
+    yield next;
+    if (this.response.status === 404 && !this.response.body) {
+      this.throw(404);
+    }
+  } catch (err) {
+    if (err.status === 401) {
+      this.set('WWW-Authenticate', 'Basic');
+      this.status = 401;
+      this.body = err.message;
+    } else if(err.status === 404) {
+      this.status = 404;
+      this.body = {
+        error: {
+          message: 'File not found',
+          code: 604
+        }
+      };
+    } else {
+      if (!err.status) {
+        debug(err);
+        throw err;
+      } else {
+        this.status = err.status;
+        this.body = err.message;
+      }
+    }
+  }
+  this.type = 'application/json';
+});
+
+app.use(compress());
+app.use(bodyparser());
+
+app.ws.use(route.all('/file/:id', file.events));
+app.ws.use(route.all('/user', user.events));
+
+app.use(route.get('/file/:id', file.get));
+
+// Run auth middleware before all other endpoints
+app.use(auth);
+
+app.use(route.get('/user', user.get));
+app.use(route.get('/user/token', user.token));
+app.use(route.get('/user/transaction', user.transaction));
+app.use(route.post('/user/settings', user.settings));
+app.use(route.get('/file', file.list));
+app.use(route.post('/file', file.post));
+app.use(route.put('/file/:id', file.put));
+app.use(route.delete('/file/:id', file.del));
+
+if (!module.parent) {
+  if (process.env.LOCALHOST_KEY) {
+    spdy.createServer({
+      key: process.env.LOCALHOST_KEY,
+      cert: process.env.LOCALHOST_CRT
+    }, app.callback()).listen(4042, function() {
+      debug('Koa SPDY server listening on port ' + (process.env.PORT || 4042));
+    });
+  } else {
+    app.listen(process.env.PORT || 4042, function() {
+      debug('Koa HTTP server listening on port ' + (process.env.PORT || 4042));
+    });
+  }
+}
+
+export default app;

api/lib/auth.js

@@ -0,0 +1,57 @@
+import passwords from 'passwords';
+import auth from 'basic-auth';
+import mongoSetup from 'mongodb-promisified';
+const objectID = mongoSetup().ObjectID;
+import debugname from 'debug';
+const debug = debugname('hostr-api:auth');
+
+const badLoginMsg = '{"error": {"message": "Incorrect login details.", "code": 607}}';
+
+module.exports = function* (next) {
+  const Users = this.db.Users;
+  const Files = this.db.Files;
+  const Logins = this.db.Logins;
+  let user = false;
+
+  if (this.req.headers.authorization && this.req.headers.authorization[0] === ':') {
+    debug('Logging in with token');
+    const userToken = yield this.redis.get(this.req.headers.authorization.substr(1));
+    this.assert(userToken, 401, '{"error": {"message": "Invalid token.", "code": 606}}');
+    debug('Token found');
+    user = yield Users.findOne({'_id': objectID(userToken)});
+  } else {
+
+    const authUser = auth(this);
+    this.assert(authUser, 401, badLoginMsg);
+    const remoteIp = this.req.headers['x-real-ip'] || this.req.connection.remoteAddress;
+    const count = yield Logins.count({ip: remoteIp, successful: false, at: { '$gt': Math.ceil(Date.now() / 1000) - 600}});
+    this.assert(count < 25, 401, '{"error": {"message": "Too many incorrect logins.", "code": 608}}');
+
+    yield Logins.insertOne({ip: remoteIp, at: Math.ceil(Date.now() / 1000), successful: null});
+    user = yield Users.findOne({'email': authUser.name, 'banned': {'$exists': false}, 'status': {'$ne': 'deleted'}});
+    this.assert(user, 401, badLoginMsg);
+    const authenticated = yield passwords.match(authUser.pass, user.salted_password);
+    this.assert(authenticated, 401, badLoginMsg);
+  }
+  debug('Checking user');
+  this.assert(user, 401, badLoginMsg);
+  debug('Checking user is activated');
+  this.assert(!user.activationCode, 401, '{"error": {"message": "Account has not been activated.", "code": 603}}');
+
+  const uploadedTotal = yield Files.count({owner: user._id, status: {'$ne': 'deleted'}});
+  const uploadedToday = yield Files.count({'owner': user._id, 'time_added': {'$gt': Date.now()}});
+
+  const normalisedUser = {
+    'id': user._id,
+    'email': user.email,
+    'daily_upload_allowance': user.type === 'Pro' ? 'unlimited' : 15,
+    'file_count': uploadedTotal,
+    'max_filesize': user.type === 'Pro' ? 524288000 : 20971520,
+    'plan': user.type || 'Free',
+    'uploads_today': uploadedToday
+  };
+  this.response.set('Daily-Uploads-Remaining', user.type === 'Pro' ? 'unlimited' : 15 - uploadedToday);
+  this.user = normalisedUser;
+  debug('Authenticated user: ' + this.user.email);
+  yield next;
+};

api/public/404.json

@@ -0,0 +1,6 @@
+{
+  "error": {
+  "message": "Invalid API endpoint",
+  "code": 404
+  }
+}

api/public/50x.json

@@ -0,0 +1,6 @@
+{
+  "error": {
+  "message": "An error occured on the server",
+  "code": 665
+  }
+}

api/routes/file.js

@@ -0,0 +1,259 @@
+import path from 'path';
+import fs from 'fs';
+import crypto from 'crypto';
+import gm from 'gm';
+import redis from 'redis-url';
+import parse from 'co-busboy';
+import { upload as s3Upload } from '../../lib/s3';
+import { sniff } from '../../lib/type';
+import hostrId from '../../lib/hostr-id';
+import malware from '../../lib/malware';
+import { formatFile } from '../../lib/format';
+
+import debugname from 'debug';
+const debug = debugname('hostr-api:file');
+
+const redisUrl = process.env.REDIS_URL || process.env.REDISTOGO_URL || 'redis://localhost:6379';
+
+const fileHost = process.env.FILE_HOST || 'https://localhost:4040';
+
+const storePath = process.env.STORE_PATH || path.join(process.env.HOME, '.hostr', 'uploads');
+
+export function* post(next) {
+  if (!this.request.is('multipart/*')) {
+    return yield next;
+  }
+  const Files = this.db.Files;
+
+  const expectedSize = this.request.headers['content-length'];
+  const tempGuid = this.request.headers['hostr-guid'];
+  const remoteIp = this.request.headers['x-real-ip'] || this.req.connection.remoteAddress;
+
+  const md5sum = crypto.createHash('md5');
+
+  let lastPercent = 0;
+  let percentComplete = 0;
+  let lastTick = 0;
+  let receivedSize = 0;
+
+  // Receive upload
+  debug('Parsing upload');
+  const upload = yield parse(this, {autoFields: true, headers: this.request.headers, limits: { files: 1}, highWaterMark: 1000000});
+
+  // Check daily upload limit
+  const count = yield Files.count({owner: this.user.id, 'time_added': {'$gt': Math.ceil(Date.now() / 1000) - 86400}});
+  const userLimit = this.user.daily_upload_allowance;
+  const underLimit = (count < userLimit || userLimit === 'unlimited');
+  this.assert(underLimit, 400, `{
+    "error": {
+      "message": "Daily upload limits (${this.user.daily_upload_allowance}) exceeded.",
+      "code": 602
+    }
+  }`);
+
+  // Clean filename for storage, keep original for display
+  upload.originalName = upload.filename;
+  upload.filename = upload.filename.replace(/[^a-zA-Z0-9\.\-\_\s]/g, '').replace(/\s+/g, '');
+  const fileId = yield hostrId(Files);
+
+  const uploadPromise = new Promise((resolve, reject) => {
+    upload.on('error', () => {
+      reject();
+    });
+
+    upload.on('end', () => {
+      resolve();
+    });
+  });
+
+  const key = path.join(fileId[0], fileId + '_' + upload.filename);
+  const localStream = fs.createWriteStream(path.join(storePath, key));
+
+  upload.pipe(localStream);
+  upload.pipe(s3Upload(key));
+
+  const thumbsPromises = [
+    new Promise((resolve, reject) => {
+      const small = gm(upload).resize(150, 150, '>').stream();
+      small.pipe(fs.createWriteStream(path.join(storePath, fileId[0], '150', fileId + '_' + upload.filename)));
+      small.pipe(s3Upload(path.join('150', fileId + '_' + upload.filename))).on('finish', resolve);
+    }),
+    new Promise((resolve, reject) => {
+      const medium = gm(upload).resize(970, '>').stream();
+      medium.pipe(fs.createWriteStream(path.join(storePath, fileId[0], '970', fileId + '_' + upload.filename)));
+      medium.pipe(s3Upload(path.join('970', fileId + '_' + upload.filename))).on('finish', resolve);
+    })
+  ];
+
+
+  let dimensionsPromise = new Promise((resolve, reject) => {
+    gm(upload).size((err, size) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(size);
+      }
+    });
+  });
+
+  upload.on('data', (data) => {
+    receivedSize += data.length;
+    if (receivedSize > this.user.max_filesize) {
+      fs.unlink(path.join(storePath, key));
+      this.throw(413, '{"error": {"message": "The file you tried to upload is too large.", "code": 601}}');
+    }
+
+    percentComplete = Math.floor(receivedSize * 100 / expectedSize);
+    if (percentComplete > lastPercent && lastTick < Date.now() - 1000) {
+      const progressEvent = `{type: 'file-progress', data: {id: ${fileId}, complete: ${percentComplete}}}`;
+      this.redis.publish('/file/' + fileId, progressEvent);
+      this.redis.publish('/user/' + this.user.id, progressEvent);
+      lastTick = Date.now();
+    }
+    lastPercent = percentComplete;
+
+    md5sum.update(data);
+  });
+
+  // Fire an event to let the frontend map the GUID it sent to the real ID. Allows immediate linking to the file
+  let acceptedEvent = `{type: 'file-accepted', data: {id: ${fileId}, guid: ${tempGuid}, href: ${fileHost}/${fileId}}}`;
+  this.redis.publish('/user/' + this.user.id, acceptedEvent);
+  // Fire final upload progress event so users know it's now processing
+  const completeEvent = `{type: 'file-progress', data: {id: ${fileId}, complete: 100}}`;
+  this.redis.publish('/file/' + fileId, completeEvent);
+  this.redis.publish('/user/' + this.user.id, completeEvent);
+
+  const dbFile = {
+    _id: fileId,
+    owner: this.user.id,
+    ip: remoteIp,
+    'system_name': fileId,
+    'file_name': upload.filename,
+    'original_name': upload.originalName,
+    'file_size': receivedSize,
+    'time_added': Math.ceil(Date.now() / 1000),
+    status: 'active',
+    'last_accessed': null,
+    s3: false,
+    type: sniff(upload.filename)
+  };
+
+  yield Files.insertOne(dbFile);
+  yield uploadPromise;
+  try {
+    const dimensions = yield dimensionsPromise;
+    dbFile.width = dimensions.width;
+    dbFile.height = dimensions.height;
+  } catch (e) {
+    debug('Not an image');
+  }
+
+  yield thumbsPromises;
+
+  dbFile.file_size = receivedSize;  // eslint-disable-line camelcase
+  dbFile.status = 'active';
+  dbFile.md5 = md5sum.digest('hex');
+
+  const formattedFile = formatFile(dbFile);
+
+  delete dbFile._id;
+  yield Files.updateOne({_id: fileId}, {$set: dbFile});
+
+  // Fire upload complete event
+  const addedEvent = `{type: 'file-added', data: ${formattedFile}}`;
+  this.redis.publish('/file/' + fileId, addedEvent);
+  this.redis.publish('/user/' + this.user.id, addedEvent);
+  this.status = 201;
+  this.body = formattedFile;
+
+  if (process.env.VIRUSTOTAL) {
+    // Check in the background
+    process.nextTick(function*() {
+      debug('Malware Scan');
+      const { positive, result } = yield malware(dbFile);
+      yield Files.updateOne({_id: fileId}, {'$set': {malware: positive, virustotal: result}});
+    });
+  } else {
+    debug('Skipping Malware Scan, VIRUSTOTAL env variable not found.');
+  }
+}
+
+
+export function* list() {
+  const Files = this.db.Files;
+
+  let status = 'active';
+  if (this.request.query.trashed) {
+    status = 'trashed';
+  } else if (this.request.query.all) {
+    status = {'$in': ['active', 'trashed']};
+  }
+
+  let limit = 20;
+  if (this.request.query.perpage === '0') {
+    limit = false;
+  } else if(this.request.query.perpage > 0) {
+    limit = parseInt(this.request.query.perpage / 1);
+  }
+
+  let skip = 0;
+  if (this.request.query.page) {
+    skip = parseInt(this.request.query.page - 1) * limit;
+  }
+
+  const queryOptions = {
+    limit: limit, skip: skip, sort: [['time_added', 'desc']],
+    hint: {
+      owner: 1, status: 1, 'time_added': -1
+    }
+  };
+
+  const userFiles = yield Files.find({owner: this.user.id, status: status}, queryOptions).toArray();
+
+  this.body = userFiles.map(formatFile);
+}
+
+
+export function* get(id) {
+  const Files = this.db.Files;
+  const Users = this.db.Users;
+  const file = yield Files.findOne({_id: id, status: {'$in': ['active', 'uploading']}});
+  this.assert(file, 404, '{"error": {"message": "File not found", "code": 604}}');
+  const user = yield Users.findOne({_id: file.owner});
+  this.assert(user && !user.banned, 404, '{"error": {"message": "File not found", "code": 604}}');
+  this.body = formatFile(file);
+}
+
+
+export function* put(id) {
+  if (this.request.body.trashed) {
+    const Files = this.db.Files;
+    const status = this.request.body.trashed ? 'trashed' : 'active';
+    yield Files.updateOne({'_id': id, owner: this.user.id}, {$set: {status: status}}, {w: 1});
+  }
+}
+
+
+export function* del(id) {
+  const Files = this.db.Files;
+  yield Files.updateOne({'_id': id, owner: this.user.id}, {$set: {status: 'deleted'}}, {w: 1});
+  const event = {type: 'file-deleted', data: {'id': id}};
+  yield this.redis.publish('/user/' + this.user.id, JSON.stringify(event));
+  yield this.redis.publish('/file/' + id, JSON.stringify(event));
+  this.body = '';
+}
+
+
+export function* events() {
+  const pubsub = redis.connect(redisUrl);
+  pubsub.on('ready', function() {
+    pubsub.subscribe(this.path);
+  }.bind(this));
+
+  pubsub.on('message', function(channel, message) {
+    this.websocket.send(message);
+  }.bind(this));
+  this.on('close', function() {
+    pubsub.quit();
+  });
+}

api/routes/user.js

@@ -0,0 +1,85 @@
+import uuid from 'node-uuid';
+import redis from 'redis-url';
+import co from 'co';
+import passwords from 'passwords';
+
+import debugname from 'debug';
+const debug = debugname('hostr-api:file');
+
+const redisUrl = process.env.REDIS_URL || process.env.REDISTOGO_URL || 'redis://localhost:6379';
+
+export function* get (){
+  this.body = this.user;
+}
+
+export function* token(){
+  const token = uuid.v4(); // eslint-disable-line no-shadow
+  yield this.redis.set(token, this.user.id, 'EX', 86400);
+  this.body = {token: token};
+}
+
+export function* transaction(){
+  const Transactions = this.db.Transactions;
+  const transactions = yield Transactions.find({'user_id': this.user.id}).toArray();
+
+  this.body = transactions.map(function(transaction) { // eslint-disable-line no-shadow
+    const type = transaction.paypal ? 'paypal' : 'direct';
+    return {
+      id: transaction._id,
+      amount: transaction.paypal ? transaction.amount : transaction.amount / 100,
+      date: transaction.date,
+      description: transaction.desc,
+      type: type
+    };
+  });
+}
+
+export function* settings() {
+  this.assert(this.request.body, 400, '{"error": {"message": "Current Password required to update account.", "code": 612}}');
+  this.assert(this.request.body.current_password, 400, '{"error": {"message": "Current Password required to update account.", "code": 612}}');
+  const Users = this.db.Users;
+  const user = yield Users.findOne({'_id': this.user.id});
+  this.assert(yield passwords.match(this.request.body.current_password, user.salted_password), 400, '{"error": {"message": "Incorrect password", "code": 606}}');
+  const data = {};
+  if (this.request.body.email && this.request.body.email !== user.email) {
+    data.email = this.request.body.email;
+    if (!user.activated_email) {
+      data.activated_email = user.email; // eslint-disable-line camelcase
+    }
+  }
+  if (this.request.body.new_password) {
+    this.assert(this.request.body.new_password.length >= 7, 400, '{"error": {"message": "Password must be 7 or more characters long.", "code": 606}}');
+    data.salted_password = yield passwords.hash(this.request.body.new_password); // eslint-disable-line camelcase
+  }
+  Users.updateOne({_id: user._id}, {'$set': data});
+  this.body = {};
+}
+
+export function* events() {
+  const pubsub = redis.connect(redisUrl);
+  pubsub.on('message', function(channel, message) {
+    this.websocket.send(message);
+  }.bind(this));
+  pubsub.on('ready', function () {
+    this.websocket.on('message', co.wrap(function* (message) {
+      let json;
+      try{
+        json = JSON.parse(message);
+      } catch(e) {
+        debug('Invalid JSON for socket auth');
+        this.websocket.send('Invalid authentication message. Bad JSON?');
+      }
+      const reply = yield this.redis.get(json.authorization);
+      if (reply) {
+        pubsub.subscribe('/user/' + reply);
+        debug('Subscribed to: /user/%s', reply);
+      } else {
+        this.websocket.send('Invalid authentication token.');
+      }
+    }));
+  }.bind(this));
+  this.on('close', function() {
+    debug('Socket closed');
+    pubsub.quit();
+  });
+}