jonathan/hostr
jonathan/hostr
1
0
Fork 0
Code Issues Pull requests 4 Projects Releases Packages 1 Wiki Activity Actions

Postgres.

Browse source
This commit is contained in:
Jonathan Cremin 2016-06-19 10:14:47 -07:00
parent 695644c260
commit 806f42e3f8
25 changed files with 501 additions and 294 deletions
Download patch file Download diff file Expand all files Collapse all files

22
lib/format.js
View file

@ -22,15 +22,15 @@ export function formatSize(size) {
export function formatFile(file) {
const formattedFile = {
added: moment.unix(file.time_added).format(),
readableAdded: formatDate(file.time_added),
added: moment.unix(file.createdAt).format(),
readableAdded: formatDate(file.createdAt),
downloads: file.downloads !== undefined ? file.downloads : 0,
href: `${baseURL}/${file._id}`,
id: file._id,
name: file.file_name,
size: file.file_size,
readableSize: formatSize(file.file_size),
type: sniff(file.file_name),
href: `${baseURL}/${file.id}`,
id: file.id,
name: file.name,
size: file.size,
readableSize: formatSize(file.size),
type: sniff(file.name),
trashed: (file.status === 'trashed'),
status: file.status,
};
@ -38,10 +38,10 @@ export function formatFile(file) {
if (file.width) {
formattedFile.height = file.height;
formattedFile.width = file.width;
const ext = (file.file_name.split('.').pop().toLowerCase() === 'psd' ? '.png' : '');
const ext = (file.name.split('.').pop().toLowerCase() === 'psd' ? '.png' : '');
formattedFile.direct = {
'150x': `${baseURL}/file/150/${file._id}/${file.file_name}${ext}`,
'970x': `${baseURL}/file/970/${file._id}/${file.file_name}${ext}`,
'150x': `${baseURL}/file/150/${file.id}/${file.name}${ext}`,
'970x': `${baseURL}/file/970/${file.id}/${file.name}${ext}`,
};
}
return formattedFile;

4
lib/hostr-id.js
View file

@ -1,3 +1,5 @@
import models from '../models';
const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
function randomID() {
@ -12,7 +14,7 @@ function* checkId(Files, fileId, attempts) {
if (attempts > 10) {
return false;
}
const file = yield Files.findOne({ _id: fileId });
const file = yield models.file.findById(fileId);
if (file === null) {
return fileId;
}

2
lib/malware.js
View file

@ -71,7 +71,7 @@ export default function* (file) {
}
const result = yield virustotal.getFileReport(file.md5);
return {
positive: result.positives >= 5,
positives: result.positives,
result,
};
}

14
lib/resize.js
View file

@ -55,17 +55,17 @@ function scale(path, type, size) {
});
}
export default function resize(path, type, currentSize, newSize) {
export default function resize(path, type, currentSize, dim) {
debug('Resizing');
const ratio = 970 / currentSize.width;
debug(newSize.width, ratio);
if (newSize.width <= 150) {
debug(dim.width, ratio);
if (dim.width <= 150) {
debug('Cover');
return cover(path, type, newSize);
} else if (newSize.width >= 970 && ratio < 1) {
return cover(path, type, dim);
} else if (dim.width >= 970 && ratio < 1) {
debug('Scale');
newSize.height = currentSize.height * ratio; // eslint-disable-line no-param-reassign
return scale(path, type, newSize);
dim.height = currentSize.height * ratio; // eslint-disable-line no-param-reassign
return scale(path, type, dim);
}
debug('Copy');
return fs.readFile(path);

124
lib/uploader.js
View file

@ -4,8 +4,9 @@ import crypto from 'crypto';
import fs from 'mz/fs';
import sizeOf from 'image-size';
import models from '../models';
import createHostrId from './hostr-id';
import { formatFile } from './format';
import hostrId from './hostr-id';
import resize from './resize';
import malware from './malware';
import { sniff } from './type';
@ -22,7 +23,6 @@ const supported = ['jpeg', 'jpg', 'png', 'gif'];
export default class Uploader {
constructor(context) {
this.context = context;
this.Files = context.db.Files;
this.expectedSize = context.request.headers['content-length'];
this.tempGuid = context.request.headers['hostr-guid'];
this.remoteIp = context.request.headers['x-real-ip'] || context.req.connection.remoteAddress;
@ -55,13 +55,23 @@ export default class Uploader {
});
this.tempGuid = this.tempGuid;
this.originalName = this.upload.filename;
this.filename = this.upload.filename.replace(/[^a-zA-Z0-9\.\-_\s]/g, '').replace(/\s+/g, '');
this.id = yield hostrId(this.Files);
this.file = yield models.file.create({
id: yield createHostrId(),
name: this.upload.filename.replace(/[^a-zA-Z0-9\.\-_\s]/g, '').replace(/\s+/g, ''),
originalName: this.upload.filename,
userId: this.context.user.id,
status: 'uploading',
type: sniff(this.upload.filename),
ip: this.remoteIp,
accessedAt: null,
width: null,
height: null,
});
yield this.file.save();
}
receive() {
this.path = join(this.id[0], `${this.id}_${this.filename}`);
this.path = join(this.file.id[0], `${this.file.id}_${this.file.name}`);
this.localStream = fs.createWriteStream(join(storePath, this.path));
this.upload.pause();
@ -79,8 +89,8 @@ export default class Uploader {
this.percentComplete = Math.floor(this.receivedSize * 100 / this.expectedSize);
if (this.percentComplete > this.lastPercent && this.lastTick < Date.now() - 1000) {
const progressEvent = `{"type": "file-progress", "data":
{"id": "${this.upload.id}", "complete": ${this.percentComplete}}}`;
this.context.redis.publish(`/file/${this.upload.id}`, progressEvent);
{"id": "${this.file.id}", "complete": ${this.percentComplete}}}`;
this.context.redis.publish(`/file/${this.file.id}`, progressEvent);
this.context.redis.publish(`/user/${this.context.user.id}`, progressEvent);
this.lastTick = Date.now();
}
@ -90,6 +100,8 @@ export default class Uploader {
});
this.upload.on('end', () => {
this.file.size = this.receivedSize;
this.file.md5 = this.md5sum.digest('hex');
this.localStream.end();
});
@ -101,62 +113,34 @@ export default class Uploader {
}
acceptedEvent() {
const acceptedEvent = `{"type": "file-accepted", "data":
{"id": "${this.id}", "guid": "${this.tempGuid}", "href": "${baseURL}/${this.id}"}}`;
this.context.redis.publish(`/user/${this.context.user.id}`, acceptedEvent);
const accepted = `{"type": "file-accepted", "data":
{"id": "${this.file.id}", "guid": "${this.tempGuid}", "href": "${baseURL}/${this.file.id}"}}`;
this.context.redis.publish(`/user/${this.context.user.id}`, accepted);
this.context.statsd.incr('file.upload.accepted', 1);
}
processingEvent() {
const processingEvent = `{"type": "file-progress", "data":
{"id": "${this.id}", "complete": 100}}`;
this.context.redis.publish(`/file/${this.id}`, processingEvent);
this.context.redis.publish(`/user/${this.context.user.id}`, processingEvent);
const processing = `{"type": "file-progress", "data":
{"id": "${this.file.id}", "complete": 100}}`;
this.context.redis.publish(`/file/${this.file.id}`, processing);
this.context.redis.publish(`/user/${this.context.user.id}`, processing);
this.context.statsd.incr('file.upload.complete', 1);
}
completeEvent() {
const completeEvent = `{"type": "file-added", "data": ${JSON.stringify(this.toDBFormat())}}`;
this.context.redis.publish(`/file/${this.id}`, completeEvent);
this.context.redis.publish(`/user/${this.context.user.id}`, completeEvent);
}
toDBFormat() {
const formatted = {
owner: this.context.user.id,
ip: this.remoteIp,
system_name: this.id,
file_name: this.filename,
original_name: this.originalName,
file_size: this.receivedSize,
time_added: Math.ceil(Date.now() / 1000),
status: 'active',
last_accessed: null,
s3: false,
type: sniff(this.filename),
};
if (this.width) {
formatted.width = this.width;
formatted.height = this.height;
}
return formatted;
}
save() {
return this.Files.insertOne({ _id: this.id, ...this.toDBFormat() });
}
toJSON() {
return formatFile({ _id: this.id, ...this.toDBFormat() });
const complete = `{"type": "file-added", "data": ${JSON.stringify(formatFile(this.file))}}`;
this.context.redis.publish(`/file/${this.file.id}`, complete);
this.context.redis.publish(`/user/${this.context.user.id}`, complete);
}
*checkLimit() {
const count = yield this.Files.count({
owner: this.context.user.id,
time_added: { $gt: Math.ceil(Date.now() / 1000) - 86400 },
const count = yield models.file.count({
userId: this.context.user.id,
createdAt: {
$gt: Math.ceil(Date.now() / 1000) - 86400,
},
});
debug(count);
const userLimit = this.context.user.daily_upload_allowance;
const underLimit = (count < userLimit || userLimit === 'unlimited');
if (!underLimit) {
@ -172,22 +156,14 @@ export default class Uploader {
}
*finalise() {
const dbFile = this.toDBFormat();
dbFile.file_size = this.receivedSize;
dbFile.status = 'active';
dbFile.md5 = this.md5sum.digest('hex');
if (this.width) {
dbFile.width = this.width;
dbFile.height = this.height;
}
yield this.Files.updateOne({ _id: this.id }, { $set: dbFile });
this.file.size = this.receivedSize;
this.file.status = 'active';
yield this.file.save();
}
resizeImage(upload, type, currentSize, newSize) {
return resize(join(storePath, this.path), type, currentSize, newSize).then((image) => {
const path = join(this.id[0], String(newSize.width), `${this.id}_${this.filename}`);
resizeImage(upload, type, currentSize, dim) {
return resize(join(storePath, this.path), type, currentSize, dim).then((image) => {
const path = join(this.file.id[0], String(dim.width), `${this.file.id}_${this.file.name}`);
debug('Writing file');
debug(join(storePath, path));
return fs.writeFile(join(storePath, path), image).then(() => {
@ -217,8 +193,8 @@ export default class Uploader {
return;
}
this.width = size.width;
this.height = size.height;
this.file.width = size.width;
this.file.height = size.height;
Promise.all([
this.resizeImage(upload, size.type, size, { width: 150, height: 150 }),
@ -236,9 +212,15 @@ export default class Uploader {
debug('Malware Scan');
const result = yield malware(this);
if (result) {
yield this.Files.updateOne({ _id: this.id },
{ $set: { malware: result.positive, virustotal: result } });
if (result.positive) {
this.file.malwarePositives = result.positives;
this.file.save();
const fileMalware = yield models.malware.create({
fileId: this.file.id,
positives: result.positives,
virustotal: result,
});
fileMalware.save();
if (result.positive > 5) {
this.context.statsd.incr('file.malware', 1);
}
}