hostr/lib/uploader.js

import { join } from 'path';
import parse from 'co-busboy';
import crypto from 'crypto';
import fs from 'mz/fs';
import sizeOf from 'image-size';

import { formatFile } from './format';
import hostrId from './hostr-id';
import resize from './resize';
import malware from './malware';
import { sniff } from './type';
import { upload as sftpUpload } from './sftp';

import debugname from 'debug';
const debug = debugname('hostr-api:uploader');

const storePath = process.env.UPLOAD_STORAGE_PATH;
const baseURL = process.env.WEB_BASE_URL;
const supported = ['jpeg', 'jpg', 'png', 'gif'];


export default class Uploader {
  constructor(context) {
    this.context = context;
    this.Files = context.db.Files;
    this.expectedSize = context.request.headers['content-length'];
    this.tempGuid = context.request.headers['hostr-guid'];
    this.remoteIp = context.request.headers['x-real-ip'] || context.req.connection.remoteAddress;
    this.md5sum = crypto.createHash('md5');

    this.lastPercent = 0;
    this.percentComplete = 0;
    this.lastTick = 0;
    this.receivedSize = 0;
  }

  *accept() {
    this.upload = yield parse(this.context, {
      autoFields: true,
      headers: this.context.request.headers,
      limits: { files: 1 },
      highWaterMark: 10000000,
    });

    this.promise = new Promise((resolve, reject) => {
      this.upload.on('error', (err) => {
        this.statsd.incr('file.upload.error', 1);
        debug(err);
        reject();
      });

      this.upload.on('end', () => {
        resolve();
      });
    });

    this.tempGuid = this.tempGuid;
    this.originalName = this.upload.filename;
    this.filename = this.upload.filename.replace(/[^a-zA-Z0-9\.\-_\s]/g, '').replace(/\s+/g, '');
    this.id = yield hostrId(this.Files);
  }

  receive() {
    this.path = join(this.id[0], `${this.id}_${this.filename}`);
    this.localStream = fs.createWriteStream(join(storePath, this.path));

    this.upload.pause();

    this.upload.on('data', (data) => {
      this.receivedSize += data.length;
      if (this.receivedSize > this.context.user.max_filesize) {
        fs.unlink(join(storePath, this.path));
        this.context.throw(413, `{"error": {"message": "The file you tried to upload is too large.",
        "code": 601}}`);
      }

      this.localStream.write(data);

      this.percentComplete = Math.floor(this.receivedSize * 100 / this.expectedSize);
      if (this.percentComplete > this.lastPercent && this.lastTick < Date.now() - 1000) {
        const progressEvent = `{"type": "file-progress", "data":
        {"id": "${this.upload.id}", "complete": ${this.percentComplete}}}`;
        this.context.redis.publish(`/file/${this.upload.id}`, progressEvent);
        this.context.redis.publish(`/user/${this.context.user.id}`, progressEvent);
        this.lastTick = Date.now();
      }
      this.lastPercent = this.percentComplete;

      this.md5sum.update(data);
    });

    this.upload.on('end', () => {
      this.localStream.end();
    });

    this.upload.resume();
  }

  sendToSFTP() {
    return sftpUpload(join(storePath, this.path), join('hostr', 'uploads', this.path));
  }

  acceptedEvent() {
    const acceptedEvent = `{"type": "file-accepted", "data":
      {"id": "${this.id}", "guid": "${this.tempGuid}", "href": "${baseURL}/${this.id}"}}`;
    this.context.redis.publish(`/user/${this.context.user.id}`, acceptedEvent);
    this.context.statsd.incr('file.upload.accepted', 1);
  }

  processingEvent() {
    const processingEvent = `{"type": "file-progress", "data":
     {"id": "${this.id}", "complete": 100}}`;
    this.context.redis.publish(`/file/${this.id}`, processingEvent);
    this.context.redis.publish(`/user/${this.context.user.id}`, processingEvent);
    this.context.statsd.incr('file.upload.complete', 1);
  }

  completeEvent() {
    const completeEvent = `{"type": "file-added", "data": ${JSON.stringify(this.toDBFormat())}}`;
    this.context.redis.publish(`/file/${this.id}`, completeEvent);
    this.context.redis.publish(`/user/${this.context.user.id}`, completeEvent);
  }

  toDBFormat() {
    const formatted = {
      owner: this.context.user.id,
      ip: this.remoteIp,
      system_name: this.id,
      file_name: this.filename,
      original_name: this.originalName,
      file_size: this.receivedSize,
      time_added: Math.ceil(Date.now() / 1000),
      status: 'active',
      last_accessed: null,
      s3: false,
      type: sniff(this.filename),
    };

    if (this.width) {
      formatted.width = this.width;
      formatted.height = this.height;
    }

    return formatted;
  }

  save() {
    return this.Files.insertOne({ _id: this.id, ...this.toDBFormat() });
  }

  toJSON() {
    return formatFile({ _id: this.id, ...this.toDBFormat() });
  }

  *checkLimit() {
    const count = yield this.Files.count({
      owner: this.context.user.id,
      time_added: { $gt: Math.ceil(Date.now() / 1000) - 86400 },
    });
    const userLimit = this.context.user.daily_upload_allowance;
    const underLimit = (count < userLimit || userLimit === 'unlimited');
    if (!underLimit) {
      this.context.statsd.incr('file.overlimit', 1);
    }
    this.context.assert(underLimit, 400, `{
      "error": {
        "message": "Daily upload limits (${this.context.user.daily_upload_allowance}) exceeded.",
        "code": 602
      }
    }`);
    return true;
  }

  *finalise() {
    const dbFile = this.toDBFormat();
    dbFile.file_size = this.receivedSize;
    dbFile.status = 'active';
    dbFile.md5 = this.md5sum.digest('hex');

    if (this.width) {
      dbFile.width = this.width;
      dbFile.height = this.height;
    }

    yield this.Files.updateOne({ _id: this.id }, { $set: dbFile });
  }

  resizeImage(upload, type, currentSize, newSize) {
    return resize(join(storePath, this.path), type, currentSize, newSize).then((image) => {
      const path = join(this.id[0], String(newSize.width), `${this.id}_${this.filename}`);
      debug('Writing file');
      debug(join(storePath, path));
      return fs.writeFile(join(storePath, path), image).then(() => {
        debug('Uploading file');
        return sftpUpload(join(storePath, path), join('hostr', 'uploads', path));
      }).catch(debug);
    }).catch(debug);
  }

  *processImage(upload) {
    return new Promise((resolve) => {
      let size;
      try {
        if (supported.indexOf(this.path.split('.').pop().toLowerCase()) < 0) {
          resolve();
          return;
        }
        size = sizeOf(join(storePath, this.path));
      } catch (err) {
        debug(err);
        resolve();
        return;
      }

      if (!size.width || supported.indexOf(size.type) < 0) {
        resolve();
        return;
      }

      this.width = size.width;
      this.height = size.height;

      Promise.all([
        this.resizeImage(upload, size.type, size, { width: 150, height: 150 }),
        this.resizeImage(upload, size.type, size, { width: 970 }),
      ]).then(() => {
        resolve(size);
      });
    });
  }

  malwareScan() {
    if (process.env.VIRUSTOTAL_KEY) {
      // Check in the background
      process.nextTick(function* scan() {
        debug('Malware Scan');
        const result = yield malware(this);
        if (result) {
          yield this.Files.updateOne({ _id: this.id },
            { $set: { malware: result.positive, virustotal: result } });
          if (result.positive) {
            this.context.statsd.incr('file.malware', 1);
          }
        }
      });
    } else {
      debug('Skipping Malware Scan, VIRUSTOTAL env variable not found.');
    }
  }
}