hostr/lib/uploader.js

import { join } from 'path';
import parse from 'co-busboy';
import crypto from 'crypto';
import fs from 'mz/fs';
import sizeOf from 'image-size';

import models from '../models';
import createHostrId from './hostr-id';
import { formatFile } from './format';
import resize from './resize';
import malware from './malware';
import { sniff } from './type';
import { upload as sftpUpload } from './sftp';

import debugname from 'debug';
const debug = debugname('hostr-api:uploader');

const storePath = process.env.UPLOAD_STORAGE_PATH;
const baseURL = process.env.WEB_BASE_URL;
const supported = ['jpeg', 'jpg', 'png', 'gif'];


export default class Uploader {
  constructor(context) {
    this.context = context;
    this.expectedSize = context.request.headers['content-length'];
    this.tempGuid = context.request.headers['hostr-guid'];
    this.remoteIp = context.request.headers['x-real-ip'] || context.req.connection.remoteAddress;
    this.md5sum = crypto.createHash('md5');

    this.lastPercent = 0;
    this.percentComplete = 0;
    this.lastTick = 0;
    this.receivedSize = 0;
  }

  *accept() {
    this.upload = yield parse(this.context, {
      autoFields: true,
      headers: this.context.request.headers,
      limits: { files: 1 },
      highWaterMark: 10000000,
    });

    this.promise = new Promise((resolve, reject) => {
      this.upload.on('error', (err) => {
        this.statsd.incr('file.upload.error', 1);
        debug(err);
        reject();
      });

      this.upload.on('end', () => {
        resolve();
      });
    });

    this.tempGuid = this.tempGuid;
    this.file = yield models.file.create({
      id: yield createHostrId(),
      name: this.upload.filename.replace(/[^a-zA-Z0-9\.\-_\s]/g, '').replace(/\s+/g, ''),
      originalName: this.upload.filename,
      userId: this.context.user.id,
      status: 'uploading',
      type: sniff(this.upload.filename),
      ip: this.remoteIp,
      accessedAt: null,
      width: null,
      height: null,
    });
    yield this.file.save();
  }

  receive() {
    this.path = join(this.file.id[0], `${this.file.id}_${this.file.name}`);
    this.localStream = fs.createWriteStream(join(storePath, this.path));

    this.upload.pause();

    this.upload.on('data', (data) => {
      this.receivedSize += data.length;
      if (this.receivedSize > this.context.user.max_filesize) {
        fs.unlink(join(storePath, this.path));
        this.context.throw(413, `{"error": {"message": "The file you tried to upload is too large.",
        "code": 601}}`);
      }

      this.localStream.write(data);

      this.percentComplete = Math.floor(this.receivedSize * 100 / this.expectedSize);
      if (this.percentComplete > this.lastPercent && this.lastTick < Date.now() - 1000) {
        const progressEvent = `{"type": "file-progress", "data":
        {"id": "${this.file.id}", "complete": ${this.percentComplete}}}`;
        this.context.redis.publish(`/file/${this.file.id}`, progressEvent);
        this.context.redis.publish(`/user/${this.context.user.id}`, progressEvent);
        this.lastTick = Date.now();
      }
      this.lastPercent = this.percentComplete;

      this.md5sum.update(data);
    });

    this.upload.on('end', () => {
      this.file.size = this.receivedSize;
      this.file.md5 = this.md5sum.digest('hex');
      this.localStream.end();
    });

    this.upload.resume();
  }

  sendToSFTP() {
    return sftpUpload(join(storePath, this.path), join('hostr', 'uploads', this.path));
  }

  acceptedEvent() {
    const accepted = `{"type": "file-accepted", "data":
      {"id": "${this.file.id}", "guid": "${this.tempGuid}", "href": "${baseURL}/${this.file.id}"}}`;
    this.context.redis.publish(`/user/${this.context.user.id}`, accepted);
    this.context.statsd.incr('file.upload.accepted', 1);
  }

  processingEvent() {
    const processing = `{"type": "file-progress", "data":
     {"id": "${this.file.id}", "complete": 100}}`;
    this.context.redis.publish(`/file/${this.file.id}`, processing);
    this.context.redis.publish(`/user/${this.context.user.id}`, processing);
    this.context.statsd.incr('file.upload.complete', 1);
  }

  completeEvent() {
    const complete = `{"type": "file-added", "data": ${JSON.stringify(formatFile(this.file))}}`;
    this.context.redis.publish(`/file/${this.file.id}`, complete);
    this.context.redis.publish(`/user/${this.context.user.id}`, complete);
  }

  *checkLimit() {
    const count = yield models.file.count({
      where: {
        userId: this.context.user.id,
        createdAt: {
          $gt: Date.now() - 86400000,
        },
      },
    });
    const userLimit = this.context.user.daily_upload_allowance;
    const underLimit = (count < userLimit || userLimit === 'unlimited');
    if (!underLimit) {
      this.context.statsd.incr('file.overlimit', 1);
    }
    this.context.assert(underLimit, 400, `{
      "error": {
        "message": "Daily upload limits (${this.context.user.daily_upload_allowance}) exceeded.",
        "code": 602
      }
    }`);
    return true;
  }

  *finalise() {
    this.file.size = this.receivedSize;
    this.file.status = 'active';
    this.file.processed = 'true';
    yield this.file.save();
  }

  resizeImage(upload, type, currentSize, dim) {
    return resize(join(storePath, this.path), type, currentSize, dim).then((image) => {
      const path = join(this.file.id[0], String(dim.width), `${this.file.id}_${this.file.name}`);
      debug('Writing file');
      debug(join(storePath, path));
      return fs.writeFile(join(storePath, path), image).then(() => {
        debug('Uploading file');
        return sftpUpload(join(storePath, path), join('hostr', 'uploads', path));
      }).catch(debug);
    }).catch(debug);
  }

  *processImage(upload) {
    return new Promise((resolve) => {
      let size;
      try {
        if (supported.indexOf(this.path.split('.').pop().toLowerCase()) < 0) {
          resolve();
          return;
        }
        size = sizeOf(join(storePath, this.path));
      } catch (err) {
        debug(err);
        resolve();
        return;
      }

      if (!size.width || supported.indexOf(size.type) < 0) {
        resolve();
        return;
      }

      this.file.width = size.width;
      this.file.height = size.height;

      Promise.all([
        this.resizeImage(upload, size.type, size, { width: 150, height: 150 }),
        this.resizeImage(upload, size.type, size, { width: 970 }),
      ]).then(() => {
        resolve(size);
      });
    });
  }

  malwareScan() {
    if (process.env.VIRUSTOTAL_KEY) {
      // Check in the background
      process.nextTick(function* scan() {
        debug('Malware Scan');
        const result = yield malware(this);
        if (result) {
          this.file.malwarePositives = result.positives;
          this.file.save();
          const fileMalware = yield models.malware.create({
            fileId: this.file.id,
            positives: result.positives,
            virustotal: result,
          });
          fileMalware.save();
          if (result.positive > 5) {
            this.context.statsd.incr('file.malware', 1);
          }
        }
      });
    } else {
      debug('Skipping Malware Scan, VIRUSTOTAL env variable not found.');
    }
  }
}