jonathan/hostr
jonathan/hostr
1
0
Fork 0
Code Issues Pull requests 4 Projects Releases Packages 1 Wiki Activity Actions

Merge branch 'postgres'

Browse source
This commit is contained in:
Jonathan Cremin 2016-08-07 19:16:03 +01:00
commit 305dd77f43
36 changed files with 1195 additions and 312 deletions
Download patch file Download diff file Expand all files Collapse all files

24
lib/format.js
View file

@ -22,26 +22,26 @@ export function formatSize(size) {
export function formatFile(file) {
const formattedFile = {
added: moment.unix(file.time_added).format(),
readableAdded: formatDate(file.time_added),
added: moment.unix(file.createdAt / 1000).format(),
readableAdded: formatDate(file.createdAt / 1000),
downloads: file.downloads !== undefined ? file.downloads : 0,
href: `${baseURL}/${file._id}`,
id: file._id,
name: file.file_name,
size: file.file_size,
readableSize: formatSize(file.file_size),
type: sniff(file.file_name),
href: `${baseURL}/${file.id}`,
id: file.id,
name: file.name,
size: file.size,
readableSize: formatSize(file.size),
type: sniff(file.name),
trashed: (file.status === 'trashed'),
status: file.status,
status: file.processed === true ? 'active' : 'uploading',
};
if (file.width) {
formattedFile.height = file.height;
formattedFile.width = file.width;
const ext = (file.file_name.split('.').pop().toLowerCase() === 'psd' ? '.png' : '');
const ext = (file.name.split('.').pop().toLowerCase() === 'psd' ? '.png' : '');
formattedFile.direct = {
'150x': `${baseURL}/file/150/${file._id}/${file.file_name}${ext}`,
'970x': `${baseURL}/file/970/${file._id}/${file.file_name}${ext}`,
'150x': `${baseURL}/file/150/${file.id}/${file.name}${ext}`,
'970x': `${baseURL}/file/970/${file.id}/${file.name}${ext}`,
};
}
return formattedFile;

4
lib/hostr-id.js
View file

@ -1,3 +1,5 @@
import models from '../models';
const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
function randomID() {
@ -12,7 +14,7 @@ function* checkId(Files, fileId, attempts) {
if (attempts > 10) {
return false;
}
const file = yield Files.findOne({ _id: fileId });
const file = yield models.file.findById(fileId);
if (file === null) {
return fileId;
}

6
lib/mongo.js
View file

@ -4,7 +4,7 @@ import debugname from 'debug';
const debug = debugname('hostr:mongo');
/* eslint no-param-reassign: ["error", { "props": false }] */
const configuredClient = new Promise((resolve, reject) => {
export const mongo = new Promise((resolve, reject) => {
debug('Connecting to Mongodb');
return MongoClient.connect(process.env.MONGO_URL).then((client) => {
debug('Successfully connected to Mongodb');
@ -25,10 +25,10 @@ const configuredClient = new Promise((resolve, reject) => {
debug(e);
});
export default function mongo() {
export default function () {
return function* dbMiddleware(next) {
try {
this.db = yield configuredClient;
this.db = yield mongo;
} catch (e) {
debug(e);
}

126
lib/uploader.js
View file

@ -4,8 +4,9 @@ import crypto from 'crypto';
import fs from 'mz/fs';
import sizeOf from 'image-size';
import models from '../models';
import createHostrId from './hostr-id';
import { formatFile } from './format';
import hostrId from './hostr-id';
import resize from './resize';
import malware from './malware';
import { sniff } from './type';
@ -21,7 +22,6 @@ const supported = ['jpeg', 'jpg', 'png', 'gif'];
export default class Uploader {
constructor(context) {
this.context = context;
this.Files = context.db.Files;
this.expectedSize = context.request.headers['content-length'];
this.tempGuid = context.request.headers['hostr-guid'];
this.remoteIp = context.request.headers['x-real-ip'] || context.req.connection.remoteAddress;
@ -54,13 +54,23 @@ export default class Uploader {
});
this.tempGuid = this.tempGuid;
this.originalName = this.upload.filename;
this.filename = this.upload.filename.replace(/[^a-zA-Z0-9\.\-_\s]/g, '').replace(/\s+/g, '');
this.id = yield hostrId(this.Files);
this.file = yield models.file.create({
id: yield createHostrId(),
name: this.upload.filename.replace(/[^a-zA-Z0-9\.\-_\s]/g, '').replace(/\s+/g, ''),
originalName: this.upload.filename,
userId: this.context.user.id,
status: 'uploading',
type: sniff(this.upload.filename),
ip: this.remoteIp,
accessedAt: null,
width: null,
height: null,
});
yield this.file.save();
}
receive() {
this.path = join(this.id[0], `${this.id}_${this.filename}`);
this.path = join(this.file.id[0], `${this.file.id}_${this.file.name}`);
this.localStream = fs.createWriteStream(join(storePath, this.path));
this.upload.pause();
@ -78,8 +88,8 @@ export default class Uploader {
this.percentComplete = Math.floor(this.receivedSize * 100 / this.expectedSize);
if (this.percentComplete > this.lastPercent && this.lastTick < Date.now() - 1000) {
const progressEvent = `{"type": "file-progress", "data":
{"id": "${this.upload.id}", "complete": ${this.percentComplete}}}`;
this.context.redis.publish(`/file/${this.upload.id}`, progressEvent);
{"id": "${this.file.id}", "complete": ${this.percentComplete}}}`;
this.context.redis.publish(`/file/${this.file.id}`, progressEvent);
this.context.redis.publish(`/user/${this.context.user.id}`, progressEvent);
this.lastTick = Date.now();
}
@ -89,6 +99,8 @@ export default class Uploader {
});
this.upload.on('end', () => {
this.file.size = this.receivedSize;
this.file.md5 = this.md5sum.digest('hex');
this.localStream.end();
});
@ -96,61 +108,34 @@ export default class Uploader {
}
acceptedEvent() {
const acceptedEvent = `{"type": "file-accepted", "data":
{"id": "${this.id}", "guid": "${this.tempGuid}", "href": "${baseURL}/${this.id}"}}`;
this.context.redis.publish(`/user/${this.context.user.id}`, acceptedEvent);
const accepted = `{"type": "file-accepted", "data":
{"id": "${this.file.id}", "guid": "${this.tempGuid}", "href": "${baseURL}/${this.file.id}"}}`;
this.context.redis.publish(`/user/${this.context.user.id}`, accepted);
this.context.statsd.incr('file.upload.accepted', 1);
}
processingEvent() {
const processingEvent = `{"type": "file-progress", "data":
{"id": "${this.id}", "complete": 100}}`;
this.context.redis.publish(`/file/${this.id}`, processingEvent);
this.context.redis.publish(`/user/${this.context.user.id}`, processingEvent);
const processing = `{"type": "file-progress", "data":
{"id": "${this.file.id}", "complete": 100}}`;
this.context.redis.publish(`/file/${this.file.id}`, processing);
this.context.redis.publish(`/user/${this.context.user.id}`, processing);
this.context.statsd.incr('file.upload.complete', 1);
}
completeEvent() {
const completeEvent = `{"type": "file-added", "data": ${JSON.stringify(this.toDBFormat())}}`;
this.context.redis.publish(`/file/${this.id}`, completeEvent);
this.context.redis.publish(`/user/${this.context.user.id}`, completeEvent);
}
toDBFormat() {
const formatted = {
owner: this.context.user.id,
ip: this.remoteIp,
system_name: this.id,
file_name: this.filename,
original_name: this.originalName,
file_size: this.receivedSize,
time_added: Math.ceil(Date.now() / 1000),
status: 'active',
last_accessed: null,
s3: false,
type: sniff(this.filename),
};
if (this.width) {
formatted.width = this.width;
formatted.height = this.height;
}
return formatted;
}
save() {
return this.Files.insertOne({ _id: this.id, ...this.toDBFormat() });
}
toJSON() {
return formatFile({ _id: this.id, ...this.toDBFormat() });
const complete = `{"type": "file-added", "data": ${JSON.stringify(formatFile(this.file))}}`;
this.context.redis.publish(`/file/${this.file.id}`, complete);
this.context.redis.publish(`/user/${this.context.user.id}`, complete);
}
*checkLimit() {
const count = yield this.Files.count({
owner: this.context.user.id,
time_added: { $gt: Math.ceil(Date.now() / 1000) - 86400 },
const count = yield models.file.count({
where: {
userId: this.context.user.id,
createdAt: {
$gt: Date.now() - 86400000,
},
},
});
const userLimit = this.context.user.daily_upload_allowance;
const underLimit = (count < userLimit || userLimit === 'unlimited');
@ -167,22 +152,15 @@ export default class Uploader {
}
*finalise() {
const dbFile = this.toDBFormat();
dbFile.file_size = this.receivedSize;
dbFile.status = 'active';
dbFile.md5 = this.md5sum.digest('hex');
if (this.width) {
dbFile.width = this.width;
dbFile.height = this.height;
}
yield this.Files.updateOne({ _id: this.id }, { $set: dbFile });
this.file.size = this.receivedSize;
this.file.status = 'active';
this.file.processed = 'true';
yield this.file.save();
}
resizeImage(upload, type, currentSize, newSize) {
return resize(join(storePath, this.path), type, currentSize, newSize).then((image) => {
const path = join(this.id[0], String(newSize.width), `${this.id}_${this.filename}`);
resizeImage(upload, type, currentSize, dim) {
return resize(join(storePath, this.path), type, currentSize, dim).then((image) => {
const path = join(this.file.id[0], String(dim.width), `${this.file.id}_${this.file.name}`);
debug('Writing file');
debug(join(storePath, path));
return fs.writeFile(join(storePath, path), image).catch(debug);
@ -209,8 +187,8 @@ export default class Uploader {
return;
}
this.width = size.width;
this.height = size.height;
this.file.width = size.width;
this.file.height = size.height;
Promise.all([
this.resizeImage(upload, size.type, size, { width: 150, height: 150 }),
@ -228,9 +206,15 @@ export default class Uploader {
debug('Malware Scan');
const result = yield malware(this);
if (result) {
yield this.Files.updateOne({ _id: this.id },
{ $set: { malware: result.positive, virustotal: result } });
if (result.positive) {
this.file.malwarePositives = result.positives;
this.file.save();
const fileMalware = yield models.malware.create({
fileId: this.file.id,
positives: result.positives,
virustotal: result,
});
fileMalware.save();
if (result.positive > 5) {
this.context.statsd.incr('file.malware', 1);
}
}