Initial commit.

api/routes/file.js

@@ -0,0 +1,259 @@
+import path from 'path';
+import fs from 'fs';
+import crypto from 'crypto';
+import gm from 'gm';
+import redis from 'redis-url';
+import parse from 'co-busboy';
+import { upload as s3Upload } from '../../lib/s3';
+import { sniff } from '../../lib/type';
+import hostrId from '../../lib/hostr-id';
+import malware from '../../lib/malware';
+import { formatFile } from '../../lib/format';
+
+import debugname from 'debug';
+const debug = debugname('hostr-api:file');
+
+const redisUrl = process.env.REDIS_URL || process.env.REDISTOGO_URL || 'redis://localhost:6379';
+
+const fileHost = process.env.FILE_HOST || 'https://localhost:4040';
+
+const storePath = process.env.STORE_PATH || path.join(process.env.HOME, '.hostr', 'uploads');
+
+export function* post(next) {
+  if (!this.request.is('multipart/*')) {
+    return yield next;
+  }
+  const Files = this.db.Files;
+
+  const expectedSize = this.request.headers['content-length'];
+  const tempGuid = this.request.headers['hostr-guid'];
+  const remoteIp = this.request.headers['x-real-ip'] || this.req.connection.remoteAddress;
+
+  const md5sum = crypto.createHash('md5');
+
+  let lastPercent = 0;
+  let percentComplete = 0;
+  let lastTick = 0;
+  let receivedSize = 0;
+
+  // Receive upload
+  debug('Parsing upload');
+  const upload = yield parse(this, {autoFields: true, headers: this.request.headers, limits: { files: 1}, highWaterMark: 1000000});
+
+  // Check daily upload limit
+  const count = yield Files.count({owner: this.user.id, 'time_added': {'$gt': Math.ceil(Date.now() / 1000) - 86400}});
+  const userLimit = this.user.daily_upload_allowance;
+  const underLimit = (count < userLimit || userLimit === 'unlimited');
+  this.assert(underLimit, 400, `{
+    "error": {
+      "message": "Daily upload limits (${this.user.daily_upload_allowance}) exceeded.",
+      "code": 602
+    }
+  }`);
+
+  // Clean filename for storage, keep original for display
+  upload.originalName = upload.filename;
+  upload.filename = upload.filename.replace(/[^a-zA-Z0-9\.\-\_\s]/g, '').replace(/\s+/g, '');
+  const fileId = yield hostrId(Files);
+
+  const uploadPromise = new Promise((resolve, reject) => {
+    upload.on('error', () => {
+      reject();
+    });
+
+    upload.on('end', () => {
+      resolve();
+    });
+  });
+
+  const key = path.join(fileId[0], fileId + '_' + upload.filename);
+  const localStream = fs.createWriteStream(path.join(storePath, key));
+
+  upload.pipe(localStream);
+  upload.pipe(s3Upload(key));
+
+  const thumbsPromises = [
+    new Promise((resolve, reject) => {
+      const small = gm(upload).resize(150, 150, '>').stream();
+      small.pipe(fs.createWriteStream(path.join(storePath, fileId[0], '150', fileId + '_' + upload.filename)));
+      small.pipe(s3Upload(path.join('150', fileId + '_' + upload.filename))).on('finish', resolve);
+    }),
+    new Promise((resolve, reject) => {
+      const medium = gm(upload).resize(970, '>').stream();
+      medium.pipe(fs.createWriteStream(path.join(storePath, fileId[0], '970', fileId + '_' + upload.filename)));
+      medium.pipe(s3Upload(path.join('970', fileId + '_' + upload.filename))).on('finish', resolve);
+    })
+  ];
+
+
+  let dimensionsPromise = new Promise((resolve, reject) => {
+    gm(upload).size((err, size) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(size);
+      }
+    });
+  });
+
+  upload.on('data', (data) => {
+    receivedSize += data.length;
+    if (receivedSize > this.user.max_filesize) {
+      fs.unlink(path.join(storePath, key));
+      this.throw(413, '{"error": {"message": "The file you tried to upload is too large.", "code": 601}}');
+    }
+
+    percentComplete = Math.floor(receivedSize * 100 / expectedSize);
+    if (percentComplete > lastPercent && lastTick < Date.now() - 1000) {
+      const progressEvent = `{type: 'file-progress', data: {id: ${fileId}, complete: ${percentComplete}}}`;
+      this.redis.publish('/file/' + fileId, progressEvent);
+      this.redis.publish('/user/' + this.user.id, progressEvent);
+      lastTick = Date.now();
+    }
+    lastPercent = percentComplete;
+
+    md5sum.update(data);
+  });
+
+  // Fire an event to let the frontend map the GUID it sent to the real ID. Allows immediate linking to the file
+  let acceptedEvent = `{type: 'file-accepted', data: {id: ${fileId}, guid: ${tempGuid}, href: ${fileHost}/${fileId}}}`;
+  this.redis.publish('/user/' + this.user.id, acceptedEvent);
+  // Fire final upload progress event so users know it's now processing
+  const completeEvent = `{type: 'file-progress', data: {id: ${fileId}, complete: 100}}`;
+  this.redis.publish('/file/' + fileId, completeEvent);
+  this.redis.publish('/user/' + this.user.id, completeEvent);
+
+  const dbFile = {
+    _id: fileId,
+    owner: this.user.id,
+    ip: remoteIp,
+    'system_name': fileId,
+    'file_name': upload.filename,
+    'original_name': upload.originalName,
+    'file_size': receivedSize,
+    'time_added': Math.ceil(Date.now() / 1000),
+    status: 'active',
+    'last_accessed': null,
+    s3: false,
+    type: sniff(upload.filename)
+  };
+
+  yield Files.insertOne(dbFile);
+  yield uploadPromise;
+  try {
+    const dimensions = yield dimensionsPromise;
+    dbFile.width = dimensions.width;
+    dbFile.height = dimensions.height;
+  } catch (e) {
+    debug('Not an image');
+  }
+
+  yield thumbsPromises;
+
+  dbFile.file_size = receivedSize;  // eslint-disable-line camelcase
+  dbFile.status = 'active';
+  dbFile.md5 = md5sum.digest('hex');
+
+  const formattedFile = formatFile(dbFile);
+
+  delete dbFile._id;
+  yield Files.updateOne({_id: fileId}, {$set: dbFile});
+
+  // Fire upload complete event
+  const addedEvent = `{type: 'file-added', data: ${formattedFile}}`;
+  this.redis.publish('/file/' + fileId, addedEvent);
+  this.redis.publish('/user/' + this.user.id, addedEvent);
+  this.status = 201;
+  this.body = formattedFile;
+
+  if (process.env.VIRUSTOTAL) {
+    // Check in the background
+    process.nextTick(function*() {
+      debug('Malware Scan');
+      const { positive, result } = yield malware(dbFile);
+      yield Files.updateOne({_id: fileId}, {'$set': {malware: positive, virustotal: result}});
+    });
+  } else {
+    debug('Skipping Malware Scan, VIRUSTOTAL env variable not found.');
+  }
+}
+
+
+export function* list() {
+  const Files = this.db.Files;
+
+  let status = 'active';
+  if (this.request.query.trashed) {
+    status = 'trashed';
+  } else if (this.request.query.all) {
+    status = {'$in': ['active', 'trashed']};
+  }
+
+  let limit = 20;
+  if (this.request.query.perpage === '0') {
+    limit = false;
+  } else if(this.request.query.perpage > 0) {
+    limit = parseInt(this.request.query.perpage / 1);
+  }
+
+  let skip = 0;
+  if (this.request.query.page) {
+    skip = parseInt(this.request.query.page - 1) * limit;
+  }
+
+  const queryOptions = {
+    limit: limit, skip: skip, sort: [['time_added', 'desc']],
+    hint: {
+      owner: 1, status: 1, 'time_added': -1
+    }
+  };
+
+  const userFiles = yield Files.find({owner: this.user.id, status: status}, queryOptions).toArray();
+
+  this.body = userFiles.map(formatFile);
+}
+
+
+export function* get(id) {
+  const Files = this.db.Files;
+  const Users = this.db.Users;
+  const file = yield Files.findOne({_id: id, status: {'$in': ['active', 'uploading']}});
+  this.assert(file, 404, '{"error": {"message": "File not found", "code": 604}}');
+  const user = yield Users.findOne({_id: file.owner});
+  this.assert(user && !user.banned, 404, '{"error": {"message": "File not found", "code": 604}}');
+  this.body = formatFile(file);
+}
+
+
+export function* put(id) {
+  if (this.request.body.trashed) {
+    const Files = this.db.Files;
+    const status = this.request.body.trashed ? 'trashed' : 'active';
+    yield Files.updateOne({'_id': id, owner: this.user.id}, {$set: {status: status}}, {w: 1});
+  }
+}
+
+
+export function* del(id) {
+  const Files = this.db.Files;
+  yield Files.updateOne({'_id': id, owner: this.user.id}, {$set: {status: 'deleted'}}, {w: 1});
+  const event = {type: 'file-deleted', data: {'id': id}};
+  yield this.redis.publish('/user/' + this.user.id, JSON.stringify(event));
+  yield this.redis.publish('/file/' + id, JSON.stringify(event));
+  this.body = '';
+}
+
+
+export function* events() {
+  const pubsub = redis.connect(redisUrl);
+  pubsub.on('ready', function() {
+    pubsub.subscribe(this.path);
+  }.bind(this));
+
+  pubsub.on('message', function(channel, message) {
+    this.websocket.send(message);
+  }.bind(this));
+  this.on('close', function() {
+    pubsub.quit();
+  });
+}