From 806f42e3f8d488116798386eeb3b1ac8c69ac522 Mon Sep 17 00:00:00 2001 From: Jonathan Cremin Date: Sun, 19 Jun 2016 10:14:47 -0700 Subject: [PATCH 1/4] Postgres. --- api/lib/auth.js | 79 +++++++++++++------- api/routes/file.js | 73 ++++++++++-------- api/routes/user.js | 32 ++++---- app.js | 2 + lib/format.js | 22 +++--- lib/hostr-id.js | 4 +- lib/malware.js | 2 +- lib/resize.js | 14 ++-- lib/uploader.js | 124 +++++++++++++----------------- models/activation.js | 15 ++++ models/file.js | 40 ++++++++++ models/index.js | 34 +++++++++ models/login.js | 15 ++++ models/malware.js | 15 ++++ models/remember.js | 13 ++++ models/reset.js | 13 ++++ models/transaction.js | 17 +++++ models/user.js | 21 ++++++ package.json | 4 +- test/fixtures/mongo-file.js | 11 --- test/fixtures/mongo-user.js | 15 ---- test/fixtures/user.js | 27 +++++++ web/lib/auth.js | 145 +++++++++++++++++++----------------- web/routes/file.js | 43 ++++++----- web/routes/user.js | 15 ++-- 25 files changed, 501 insertions(+), 294 deletions(-) create mode 100644 models/activation.js create mode 100644 models/file.js create mode 100644 models/index.js create mode 100644 models/login.js create mode 100644 models/malware.js create mode 100644 models/remember.js create mode 100644 models/reset.js create mode 100644 models/transaction.js create mode 100644 models/user.js delete mode 100644 test/fixtures/mongo-file.js delete mode 100644 test/fixtures/mongo-user.js create mode 100644 test/fixtures/user.js diff --git a/api/lib/auth.js b/api/lib/auth.js index 67780a1..42cf9dc 100644 --- a/api/lib/auth.js +++ b/api/lib/auth.js @@ -1,63 +1,88 @@ import passwords from 'passwords'; import auth from 'basic-auth'; +import models from '../../models'; import debugname from 'debug'; const debug = debugname('hostr-api:auth'); const badLoginMsg = '{"error": {"message": "Incorrect login details.", "code": 607}}'; export default function* (next) { - const Users = this.db.Users; - const Files = this.db.Files; - const Logins = this.db.Logins; let user = false; - + const remoteIp = this.req.headers['x-real-ip'] || this.req.connection.remoteAddress; + const login = yield models.login.create({ + ip: remoteIp, + successful: false, + }); if (this.req.headers.authorization && this.req.headers.authorization[0] === ':') { debug('Logging in with token'); const userToken = yield this.redis.get(this.req.headers.authorization.substr(1)); this.assert(userToken, 401, '{"error": {"message": "Invalid token.", "code": 606}}'); debug('Token found'); - user = yield Users.findOne({ _id: this.db.objectId(userToken) }); + user = yield models.user.findById(userToken); + if (!user) { + login.save(); + return; + } } else { const authUser = auth(this); this.assert(authUser, 401, badLoginMsg); - const remoteIp = this.req.headers['x-real-ip'] || this.req.connection.remoteAddress; - const count = yield Logins.count({ - ip: remoteIp, - successful: false, - at: { $gt: Math.ceil(Date.now() / 1000) - 600 }, + const count = yield models.login.count({ + where: { + ip: remoteIp, + successful: false, + createdAt: { + $gt: new Date(Math.ceil(Date.now()) - 600000), + }, + }, }); + this.assert(count < 25, 401, '{"error": {"message": "Too many incorrect logins.", "code": 608}}'); - yield Logins.insertOne({ ip: remoteIp, at: Math.ceil(Date.now() / 1000), successful: null }); - user = yield Users.findOne({ - email: authUser.name, - banned: { $exists: false }, - status: { $ne: 'deleted' }, + user = yield models.user.findOne({ + where: { + email: authUser.name, + activated: true, + }, }); - this.assert(user, 401, badLoginMsg); - const authenticated = yield passwords.match(authUser.pass, user.salted_password); - this.assert(authenticated, 401, badLoginMsg); + + if (!user || !(yield passwords.match(authUser.pass, user.password))) { + login.save(); + this.throw(401, badLoginMsg); + return; + } } debug('Checking user'); this.assert(user, 401, badLoginMsg); debug('Checking user is activated'); - this.assert(!user.activationCode, 401, + debug(user.activated); + this.assert(user.activated === true, 401, '{"error": {"message": "Account has not been activated.", "code": 603}}'); - const uploadedTotal = yield Files.count({ owner: user._id, status: { $ne: 'deleted' } }); - const uploadedToday = yield Files.count({ - owner: user._id, - time_added: { $gt: Math.ceil(Date.now() / 1000) - 86400 }, + login.successful = true; + yield login.save(); + + const uploadedTotal = yield models.file.count({ + where: { + userId: user.id, + }, + }); + const uploadedToday = yield models.file.count({ + where: { + userId: user.id, + createdAt: { + $gt: Math.ceil(Date.now() / 1000) - 86400, + }, + }, }); const normalisedUser = { - id: user._id, + id: user.id, email: user.email, - daily_upload_allowance: user.type === 'Pro' ? 'unlimited' : 15, + daily_upload_allowance: user.plan === 'Pro' ? 'unlimited' : 15, file_count: uploadedTotal, - max_filesize: user.type === 'Pro' ? 524288000 : 20971520, - plan: user.type || 'Free', + max_filesize: user.plan === 'Pro' ? 524288000 : 20971520, + plan: user.plan, uploads_today: uploadedToday, }; this.response.set('Daily-Uploads-Remaining', diff --git a/api/routes/file.js b/api/routes/file.js index 25e3f0d..299d11e 100644 --- a/api/routes/file.js +++ b/api/routes/file.js @@ -1,5 +1,6 @@ import redis from 'redis'; +import models from '../../models'; import { formatFile } from '../../lib/format'; import Uploader from '../../lib/uploader'; @@ -20,7 +21,6 @@ export function* post(next) { uploader.receive(); - yield uploader.save(); yield uploader.promise; uploader.processingEvent(); @@ -31,7 +31,7 @@ export function* post(next) { yield uploader.finalise(); this.status = 201; - this.body = uploader.toJSON(); + this.body = formatFile(uploader.file); uploader.completeEvent(); uploader.malwareScan(); @@ -39,48 +39,44 @@ export function* post(next) { export function* list() { - const Files = this.db.Files; - - let status = 'active'; - if (this.request.query.trashed) { - status = 'trashed'; - } else if (this.request.query.all) { - status = { $in: ['active', 'trashed'] }; - } - let limit = 20; if (this.request.query.perpage === '0') { - limit = false; + limit = 1000; } else if (this.request.query.perpage > 0) { limit = parseInt(this.request.query.perpage / 1, 10); } - let skip = 0; + let offset = 0; if (this.request.query.page) { - skip = parseInt(this.request.query.page - 1, 10) * limit; + offset = parseInt(this.request.query.page - 1, 10) * limit; } - const queryOptions = { - limit, skip, sort: [['time_added', 'desc']], - hint: { - owner: 1, status: 1, time_added: -1, + const files = yield models.file.findAll({ + where: { + userId: this.user.id, + status: 'active', }, - }; + order: '"createdAt" DESC', + offset, + limit, + }); - const userFiles = yield Files.find({ - owner: this.user.id, status }, queryOptions).toArray(); this.statsd.incr('file.list', 1); - this.body = userFiles.map(formatFile); + this.body = files.map(formatFile); } export function* get() { - const Files = this.db.Files; - const Users = this.db.Users; - const file = yield Files.findOne({ _id: this.params.id, - status: { $in: ['active', 'uploading'] } }); + const file = yield models.file.findOne({ + where: { + id: this.params.id, + status: { + $in: ['active', 'uploading'], + }, + }, + }); this.assert(file, 404, '{"error": {"message": "File not found", "code": 604}}'); - const user = yield Users.findOne({ _id: file.owner }); + const user = yield file.getUser(); this.assert(user && !user.banned, 404, '{"error": {"message": "File not found", "code": 604}}'); this.statsd.incr('file.get', 1); this.body = formatFile(file); @@ -89,17 +85,28 @@ export function* get() { export function* put() { if (this.request.body.trashed) { - const Files = this.db.Files; - const status = this.request.body.trashed ? 'trashed' : 'active'; - yield Files.updateOne({ _id: this.params.id, owner: this.user.id }, - { $set: { status } }, { w: 1 }); + const file = yield models.file.findOne({ + where: { + id: this.params.id, + userId: this.user.id, + }, + }); + file.status = this.request.body.trashed ? 'trashed' : 'active'; + yield file.save(); } } export function* del() { - yield this.db.Files.updateOne({ _id: this.params.id, owner: this.db.objectId(this.user.id) }, - { $set: { status: 'deleted' } }, { w: 1 }); + const file = yield models.file.findOne({ + where: { + id: this.params.id, + userId: this.user.id, + }, + }); + this.assert(file, 401, '{"error": {"message": "File not found", "code": 604}}'); + file.status = 'deleted'; + yield file.save(); const event = { type: 'file-deleted', data: { id: this.params.id } }; yield this.redis.publish(`/file/${this.params.id}`, JSON.stringify(event)); yield this.redis.publish(`/user/${this.user.id}`, JSON.stringify(event)); diff --git a/api/routes/user.js b/api/routes/user.js index c806551..c9b7637 100644 --- a/api/routes/user.js +++ b/api/routes/user.js @@ -2,6 +2,7 @@ import uuid from 'node-uuid'; import redis from 'redis'; import co from 'co'; import passwords from 'passwords'; +import models from '../../models'; import debugname from 'debug'; const debug = debugname('hostr-api:user'); @@ -19,17 +20,15 @@ export function* token() { } export function* transaction() { - const Transactions = this.db.Transactions; - const transactions = yield Transactions.find({ user_id: this.user.id }).toArray(); + const transactions = yield models.transaction.findAll({ userId: this.user.id }); - this.body = transactions.map((transaction) => { // eslint-disable-line no-shadow - const type = transaction.paypal ? 'paypal' : 'direct'; + this.body = transactions.map((item) => { return { - id: transaction._id, - amount: transaction.paypal ? transaction.amount : transaction.amount / 100, - date: transaction.date, - description: transaction.desc, - type, + id: item.id, + amount: item.amount / 100, + date: item.date, + description: item.description, + type: 'direct', }; }); } @@ -39,23 +38,18 @@ export function* settings() { '{"error": {"message": "Current Password required to update account.", "code": 612}}'); this.assert(this.request.body.current_password, 400, '{"error": {"message": "Current Password required to update account.", "code": 612}}'); - const Users = this.db.Users; - const user = yield Users.findOne({ _id: this.user.id }); - this.assert(yield passwords.match(this.request.body.current_password, user.salted_password), 400, + const user = yield models.user.findById(this.user.id); + this.assert(yield passwords.match(this.request.body.current_password, user.password), 400, '{"error": {"message": "Incorrect password", "code": 606}}'); - const data = {}; if (this.request.body.email && this.request.body.email !== user.email) { - data.email = this.request.body.email; - if (!user.activated_email) { - data.activated_email = user.email; // eslint-disable-line camelcase - } + user.email = this.request.body.email; } if (this.request.body.new_password) { this.assert(this.request.body.new_password.length >= 7, 400, '{"error": {"message": "Password must be 7 or more characters long.", "code": 606}}'); - data.salted_password = yield passwords.hash(this.request.body.new_password); + user.password = yield passwords.hash(this.request.body.new_password); } - Users.updateOne({ _id: user._id }, { $set: data }); + yield user.save(); this.body = {}; } diff --git a/app.js b/app.js index 4bee4cd..43b1d32 100644 --- a/app.js +++ b/app.js @@ -13,6 +13,8 @@ import * as redis from './lib/redis'; import api, { ws } from './api/app'; import web from './web/app'; +import models from './models'; + import debugname from 'debug'; const debug = debugname('hostr'); diff --git a/lib/format.js b/lib/format.js index 206921b..c02c16c 100644 --- a/lib/format.js +++ b/lib/format.js @@ -22,15 +22,15 @@ export function formatSize(size) { export function formatFile(file) { const formattedFile = { - added: moment.unix(file.time_added).format(), - readableAdded: formatDate(file.time_added), + added: moment.unix(file.createdAt).format(), + readableAdded: formatDate(file.createdAt), downloads: file.downloads !== undefined ? file.downloads : 0, - href: `${baseURL}/${file._id}`, - id: file._id, - name: file.file_name, - size: file.file_size, - readableSize: formatSize(file.file_size), - type: sniff(file.file_name), + href: `${baseURL}/${file.id}`, + id: file.id, + name: file.name, + size: file.size, + readableSize: formatSize(file.size), + type: sniff(file.name), trashed: (file.status === 'trashed'), status: file.status, }; @@ -38,10 +38,10 @@ export function formatFile(file) { if (file.width) { formattedFile.height = file.height; formattedFile.width = file.width; - const ext = (file.file_name.split('.').pop().toLowerCase() === 'psd' ? '.png' : ''); + const ext = (file.name.split('.').pop().toLowerCase() === 'psd' ? '.png' : ''); formattedFile.direct = { - '150x': `${baseURL}/file/150/${file._id}/${file.file_name}${ext}`, - '970x': `${baseURL}/file/970/${file._id}/${file.file_name}${ext}`, + '150x': `${baseURL}/file/150/${file.id}/${file.name}${ext}`, + '970x': `${baseURL}/file/970/${file.id}/${file.name}${ext}`, }; } return formattedFile; diff --git a/lib/hostr-id.js b/lib/hostr-id.js index 444d8dc..3a069da 100644 --- a/lib/hostr-id.js +++ b/lib/hostr-id.js @@ -1,3 +1,5 @@ +import models from '../models'; + const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; function randomID() { @@ -12,7 +14,7 @@ function* checkId(Files, fileId, attempts) { if (attempts > 10) { return false; } - const file = yield Files.findOne({ _id: fileId }); + const file = yield models.file.findById(fileId); if (file === null) { return fileId; } diff --git a/lib/malware.js b/lib/malware.js index 4144728..8c31510 100644 --- a/lib/malware.js +++ b/lib/malware.js @@ -71,7 +71,7 @@ export default function* (file) { } const result = yield virustotal.getFileReport(file.md5); return { - positive: result.positives >= 5, + positives: result.positives, result, }; } diff --git a/lib/resize.js b/lib/resize.js index 2e17456..740ce51 100644 --- a/lib/resize.js +++ b/lib/resize.js @@ -55,17 +55,17 @@ function scale(path, type, size) { }); } -export default function resize(path, type, currentSize, newSize) { +export default function resize(path, type, currentSize, dim) { debug('Resizing'); const ratio = 970 / currentSize.width; - debug(newSize.width, ratio); - if (newSize.width <= 150) { + debug(dim.width, ratio); + if (dim.width <= 150) { debug('Cover'); - return cover(path, type, newSize); - } else if (newSize.width >= 970 && ratio < 1) { + return cover(path, type, dim); + } else if (dim.width >= 970 && ratio < 1) { debug('Scale'); - newSize.height = currentSize.height * ratio; // eslint-disable-line no-param-reassign - return scale(path, type, newSize); + dim.height = currentSize.height * ratio; // eslint-disable-line no-param-reassign + return scale(path, type, dim); } debug('Copy'); return fs.readFile(path); diff --git a/lib/uploader.js b/lib/uploader.js index 8bb4402..e98e441 100644 --- a/lib/uploader.js +++ b/lib/uploader.js @@ -4,8 +4,9 @@ import crypto from 'crypto'; import fs from 'mz/fs'; import sizeOf from 'image-size'; +import models from '../models'; +import createHostrId from './hostr-id'; import { formatFile } from './format'; -import hostrId from './hostr-id'; import resize from './resize'; import malware from './malware'; import { sniff } from './type'; @@ -22,7 +23,6 @@ const supported = ['jpeg', 'jpg', 'png', 'gif']; export default class Uploader { constructor(context) { this.context = context; - this.Files = context.db.Files; this.expectedSize = context.request.headers['content-length']; this.tempGuid = context.request.headers['hostr-guid']; this.remoteIp = context.request.headers['x-real-ip'] || context.req.connection.remoteAddress; @@ -55,13 +55,23 @@ export default class Uploader { }); this.tempGuid = this.tempGuid; - this.originalName = this.upload.filename; - this.filename = this.upload.filename.replace(/[^a-zA-Z0-9\.\-_\s]/g, '').replace(/\s+/g, ''); - this.id = yield hostrId(this.Files); + this.file = yield models.file.create({ + id: yield createHostrId(), + name: this.upload.filename.replace(/[^a-zA-Z0-9\.\-_\s]/g, '').replace(/\s+/g, ''), + originalName: this.upload.filename, + userId: this.context.user.id, + status: 'uploading', + type: sniff(this.upload.filename), + ip: this.remoteIp, + accessedAt: null, + width: null, + height: null, + }); + yield this.file.save(); } receive() { - this.path = join(this.id[0], `${this.id}_${this.filename}`); + this.path = join(this.file.id[0], `${this.file.id}_${this.file.name}`); this.localStream = fs.createWriteStream(join(storePath, this.path)); this.upload.pause(); @@ -79,8 +89,8 @@ export default class Uploader { this.percentComplete = Math.floor(this.receivedSize * 100 / this.expectedSize); if (this.percentComplete > this.lastPercent && this.lastTick < Date.now() - 1000) { const progressEvent = `{"type": "file-progress", "data": - {"id": "${this.upload.id}", "complete": ${this.percentComplete}}}`; - this.context.redis.publish(`/file/${this.upload.id}`, progressEvent); + {"id": "${this.file.id}", "complete": ${this.percentComplete}}}`; + this.context.redis.publish(`/file/${this.file.id}`, progressEvent); this.context.redis.publish(`/user/${this.context.user.id}`, progressEvent); this.lastTick = Date.now(); } @@ -90,6 +100,8 @@ export default class Uploader { }); this.upload.on('end', () => { + this.file.size = this.receivedSize; + this.file.md5 = this.md5sum.digest('hex'); this.localStream.end(); }); @@ -101,62 +113,34 @@ export default class Uploader { } acceptedEvent() { - const acceptedEvent = `{"type": "file-accepted", "data": - {"id": "${this.id}", "guid": "${this.tempGuid}", "href": "${baseURL}/${this.id}"}}`; - this.context.redis.publish(`/user/${this.context.user.id}`, acceptedEvent); + const accepted = `{"type": "file-accepted", "data": + {"id": "${this.file.id}", "guid": "${this.tempGuid}", "href": "${baseURL}/${this.file.id}"}}`; + this.context.redis.publish(`/user/${this.context.user.id}`, accepted); this.context.statsd.incr('file.upload.accepted', 1); } processingEvent() { - const processingEvent = `{"type": "file-progress", "data": - {"id": "${this.id}", "complete": 100}}`; - this.context.redis.publish(`/file/${this.id}`, processingEvent); - this.context.redis.publish(`/user/${this.context.user.id}`, processingEvent); + const processing = `{"type": "file-progress", "data": + {"id": "${this.file.id}", "complete": 100}}`; + this.context.redis.publish(`/file/${this.file.id}`, processing); + this.context.redis.publish(`/user/${this.context.user.id}`, processing); this.context.statsd.incr('file.upload.complete', 1); } completeEvent() { - const completeEvent = `{"type": "file-added", "data": ${JSON.stringify(this.toDBFormat())}}`; - this.context.redis.publish(`/file/${this.id}`, completeEvent); - this.context.redis.publish(`/user/${this.context.user.id}`, completeEvent); - } - - toDBFormat() { - const formatted = { - owner: this.context.user.id, - ip: this.remoteIp, - system_name: this.id, - file_name: this.filename, - original_name: this.originalName, - file_size: this.receivedSize, - time_added: Math.ceil(Date.now() / 1000), - status: 'active', - last_accessed: null, - s3: false, - type: sniff(this.filename), - }; - - if (this.width) { - formatted.width = this.width; - formatted.height = this.height; - } - - return formatted; - } - - save() { - return this.Files.insertOne({ _id: this.id, ...this.toDBFormat() }); - } - - toJSON() { - return formatFile({ _id: this.id, ...this.toDBFormat() }); + const complete = `{"type": "file-added", "data": ${JSON.stringify(formatFile(this.file))}}`; + this.context.redis.publish(`/file/${this.file.id}`, complete); + this.context.redis.publish(`/user/${this.context.user.id}`, complete); } *checkLimit() { - const count = yield this.Files.count({ - owner: this.context.user.id, - time_added: { $gt: Math.ceil(Date.now() / 1000) - 86400 }, + const count = yield models.file.count({ + userId: this.context.user.id, + createdAt: { + $gt: Math.ceil(Date.now() / 1000) - 86400, + }, }); + debug(count); const userLimit = this.context.user.daily_upload_allowance; const underLimit = (count < userLimit || userLimit === 'unlimited'); if (!underLimit) { @@ -172,22 +156,14 @@ export default class Uploader { } *finalise() { - const dbFile = this.toDBFormat(); - dbFile.file_size = this.receivedSize; - dbFile.status = 'active'; - dbFile.md5 = this.md5sum.digest('hex'); - - if (this.width) { - dbFile.width = this.width; - dbFile.height = this.height; - } - - yield this.Files.updateOne({ _id: this.id }, { $set: dbFile }); + this.file.size = this.receivedSize; + this.file.status = 'active'; + yield this.file.save(); } - resizeImage(upload, type, currentSize, newSize) { - return resize(join(storePath, this.path), type, currentSize, newSize).then((image) => { - const path = join(this.id[0], String(newSize.width), `${this.id}_${this.filename}`); + resizeImage(upload, type, currentSize, dim) { + return resize(join(storePath, this.path), type, currentSize, dim).then((image) => { + const path = join(this.file.id[0], String(dim.width), `${this.file.id}_${this.file.name}`); debug('Writing file'); debug(join(storePath, path)); return fs.writeFile(join(storePath, path), image).then(() => { @@ -217,8 +193,8 @@ export default class Uploader { return; } - this.width = size.width; - this.height = size.height; + this.file.width = size.width; + this.file.height = size.height; Promise.all([ this.resizeImage(upload, size.type, size, { width: 150, height: 150 }), @@ -236,9 +212,15 @@ export default class Uploader { debug('Malware Scan'); const result = yield malware(this); if (result) { - yield this.Files.updateOne({ _id: this.id }, - { $set: { malware: result.positive, virustotal: result } }); - if (result.positive) { + this.file.malwarePositives = result.positives; + this.file.save(); + const fileMalware = yield models.malware.create({ + fileId: this.file.id, + positives: result.positives, + virustotal: result, + }); + fileMalware.save(); + if (result.positive > 5) { this.context.statsd.incr('file.malware', 1); } } diff --git a/models/activation.js b/models/activation.js new file mode 100644 index 0000000..c959919 --- /dev/null +++ b/models/activation.js @@ -0,0 +1,15 @@ +export default function (sequelize, DataTypes) { + const Activation = sequelize.define('activation', { + id: { type: DataTypes.UUID, defaultValue: DataTypes.UUIDV4, primaryKey: true }, + activated: false, + email: DataTypes.STRING, + }, { + classMethods: { + associate: (models) => { + Activation.belongsTo(models.user); + }, + }, + }); + + return Activation; +} diff --git a/models/file.js b/models/file.js new file mode 100644 index 0000000..8d81a76 --- /dev/null +++ b/models/file.js @@ -0,0 +1,40 @@ +export default function (sequelize, DataTypes) { + const File = sequelize.define('file', { + id: { type: DataTypes.STRING(12), primaryKey: true }, // eslint-disable-line new-cap + name: DataTypes.TEXT, + originalName: DataTypes.TEXT, + size: DataTypes.BIGINT, + downloads: DataTypes.BIGINT, + accessedAt: DataTypes.DATE, + status: DataTypes.ENUM('active', 'uploading', 'deleted'), // eslint-disable-line new-cap + type: DataTypes.ENUM( // eslint-disable-line new-cap + 'image', + 'audio', + 'video', + 'archive', + 'other' + ), + width: DataTypes.INTEGER, + height: DataTypes.INTEGER, + ip: 'inet', + legacyId: DataTypes.STRING(12), // eslint-disable-line new-cap + md5: DataTypes.STRING(32), // eslint-disable-line new-cap + malwarePositives: DataTypes.INTEGER, + }, { + classMethods: { + accessed: (id) => sequelize.query(` + UPDATE files + SET "downloads" = downloads + 1, "accessedAt" = NOW() + WHERE "id" = :id`, + { + replacements: { id }, + type: sequelize.QueryTypes.UPDATE, + }), + associate: (models) => { + File.belongsTo(models.user); + }, + }, + }); + + return File; +} diff --git a/models/index.js b/models/index.js new file mode 100644 index 0000000..312c669 --- /dev/null +++ b/models/index.js @@ -0,0 +1,34 @@ +import fs from 'fs'; +import path from 'path'; +import Sequelize from 'sequelize'; + +import debugname from 'debug'; +const debug = debugname('hostr:models'); + +const config = { + dialect: 'postgres', + protocol: 'postgres', + logging: debug, +}; + +const sequelize = new Sequelize(process.env.DATABASE_URL, config); +const db = {}; + +fs + .readdirSync(__dirname) + .filter((file) => (file.indexOf('.') !== 0) && (file !== 'index.js')) + .forEach((file) => { + const model = sequelize.import(path.join(__dirname, file)); + db[model.name] = model; + }); + +Object.keys(db).forEach((modelName) => { + if ('associate' in db[modelName]) { + db[modelName].associate(db); + } +}); + +db.sequelize = sequelize; +db.Sequelize = Sequelize; + +export default db; diff --git a/models/login.js b/models/login.js new file mode 100644 index 0000000..855ae63 --- /dev/null +++ b/models/login.js @@ -0,0 +1,15 @@ +export default function (sequelize, DataTypes) { + const Login = sequelize.define('login', { + id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true }, + successful: { type: DataTypes.BOOLEAN }, + ip: { type: 'inet' }, + }, { + classMethods: { + associate: (models) => { + Login.belongsTo(models.user); + }, + }, + }); + + return Login; +} diff --git a/models/malware.js b/models/malware.js new file mode 100644 index 0000000..a2d994b --- /dev/null +++ b/models/malware.js @@ -0,0 +1,15 @@ +export default function (sequelize, DataTypes) { + const Malware = sequelize.define('malware', { + fileId: { type: DataTypes.STRING(12), primaryKey: true }, // eslint-disable-line new-cap + positives: DataTypes.INTEGER, + virustotal: DataTypes.JSON, + }, { + classMethods: { + associate: (models) => { + Malware.belongsTo(models.file); + }, + }, + }); + + return Malware; +} diff --git a/models/remember.js b/models/remember.js new file mode 100644 index 0000000..2b34bac --- /dev/null +++ b/models/remember.js @@ -0,0 +1,13 @@ +export default function (sequelize, DataTypes) { + const Remember = sequelize.define('remember', { + id: { type: DataTypes.UUID, defaultValue: DataTypes.UUIDV4, primaryKey: true }, + }, { + classMethods: { + associate: (models) => { + Remember.belongsTo(models.user); + }, + }, + }); + + return Remember; +} diff --git a/models/reset.js b/models/reset.js new file mode 100644 index 0000000..dada472 --- /dev/null +++ b/models/reset.js @@ -0,0 +1,13 @@ +export default function (sequelize, DataTypes) { + const Reset = sequelize.define('reset', { + id: { type: DataTypes.UUID, defaultValue: DataTypes.UUIDV4, primaryKey: true }, + }, { + classMethods: { + associate: (models) => { + Reset.belongsTo(models.user); + }, + }, + }); + + return Reset; +} diff --git a/models/transaction.js b/models/transaction.js new file mode 100644 index 0000000..a846d66 --- /dev/null +++ b/models/transaction.js @@ -0,0 +1,17 @@ +export default function (sequelize, DataTypes) { + const Transaction = sequelize.define('transaction', { + uuid: { type: DataTypes.UUID, defaultValue: DataTypes.UUIDV4 }, + amount: DataTypes.DECIMAL, + description: DataTypes.STRING, + type: DataTypes.ENUM('direct', 'paypal'), // eslint-disable-line new-cap + ip: 'inet', + }, { + classMethods: { + associate: (models) => { + Transaction.belongsTo(models.user); + }, + }, + }); + + return Transaction; +} diff --git a/models/user.js b/models/user.js new file mode 100644 index 0000000..0f64415 --- /dev/null +++ b/models/user.js @@ -0,0 +1,21 @@ +export default function (sequelize, DataTypes) { + const User = sequelize.define('user', { + id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true }, + email: DataTypes.STRING, + password: DataTypes.STRING, + plan: DataTypes.ENUM('Free', 'Pro'), // eslint-disable-line new-cap + ip: 'inet', + activated: DataTypes.BOOLEAN, + banned: DataTypes.BOOLEAN, + deleted: DataTypes.BOOLEAN, + }, { + classMethods: { + associate: (models) => { + User.hasMany(models.file); + User.hasOne(models.activation); + }, + }, + }); + + return User; +} diff --git a/package.json b/package.json index 3a43418..fe210a8 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "lint": "eslint .", "start": "node -r babel-register app.js", "test": "npm run test-seed && mocha -r babel-register test/**/*.spec.js", - "test-seed": "node test/fixtures/mongo-user.js && node test/fixtures/mongo-file.js", + "test-seed": "babel-node test/fixtures/user.js", "watch": "parallelshell \"npm run watch-js\" \"npm run watch-sass\" \"npm run watch-server\"", "watch-js": "babel -Dw -m system -d web/public/build web/public/src", "watch-server": "nodemon -r babel-register app.js", @@ -73,6 +73,8 @@ "raven": "^0.11.0", "redis": "^2.6.1", "sendgrid": "^2.0.0", + "sequelize": "^3.23.3", + "sequelize-classes": "^0.1.12", "ssh2": "^0.5.0", "statsy": "~0.2.0", "stripe": "^4.7.0", diff --git a/test/fixtures/mongo-file.js b/test/fixtures/mongo-file.js deleted file mode 100644 index 3d11f11..0000000 --- a/test/fixtures/mongo-file.js +++ /dev/null @@ -1,11 +0,0 @@ -const MongoClient = require('mongodb').MongoClient; - -MongoClient.connect(process.env.MONGO_URL, function connect(err, db) { - const collection = db.collection('files'); - collection.createIndex({ - 'owner': 1, - 'status': 1, - 'time_added': -1, - }); - db.close(); -}); diff --git a/test/fixtures/mongo-user.js b/test/fixtures/mongo-user.js deleted file mode 100644 index 14334e2..0000000 --- a/test/fixtures/mongo-user.js +++ /dev/null @@ -1,15 +0,0 @@ -const MongoClient = require('mongodb').MongoClient; - -MongoClient.connect(process.env.MONGO_URL, function connect(err, db) { - const collection = db.collection('users'); - collection.remove({ - 'email': 'test@hostr.co', - }); - collection.save({ - 'email': 'test@hostr.co', - 'salted_password': '$pbkdf2-256-1$2$kBhIDRqFwnF/1ms6ZHfME2o2$a48e8c350d26397fcc88bf0a7a2817b1cdcd1ffffe0521a5', - 'joined': Math.ceil(Date.now() / 1000), - 'signup_ip': '127.0.0.1', - }); - db.close(); -}); diff --git a/test/fixtures/user.js b/test/fixtures/user.js new file mode 100644 index 0000000..cc3a391 --- /dev/null +++ b/test/fixtures/user.js @@ -0,0 +1,27 @@ +import models from '../../models'; + +function createUser() { + models.user.create({ + 'email': 'test@hostr.co', + 'password': '$pbkdf2-256-1$2$kBhIDRqFwnF/1ms6ZHfME2o2$a48e8c350d26397fcc88bf0a7a2817b1cdcd1ffffe0521a5', + 'ip': '127.0.0.1', + 'plan': 'Free', + 'activated': true, + }).then((user) => { + user.save().then(() => { + models.sequelize.close(); + }); + }); +} + +models.user.findOne({ + where: { + email: 'test@hostr.co', + }, +}).then((user) => { + if (user) { + user.destroy().then(createUser); + } else { + createUser(); + } +}); diff --git a/web/lib/auth.js b/web/lib/auth.js index 3eb6c51..6e9fc64 100644 --- a/web/lib/auth.js +++ b/web/lib/auth.js @@ -1,8 +1,9 @@ import crypto from 'crypto'; +import { join } from 'path'; import passwords from 'passwords'; import uuid from 'node-uuid'; import views from 'co-views'; -import { join } from 'path'; +import models from '../../models'; const render = views(join(__dirname, '..', 'views'), { default: 'ejs' }); import debugname from 'debug'; const debug = debugname('hostr-web:auth'); @@ -13,45 +14,46 @@ const from = process.env.EMAIL_FROM; const fromname = process.env.EMAIL_NAME; export function* authenticate(email, password) { - const Users = this.db.Users; - const Logins = this.db.Logins; const remoteIp = this.headers['x-real-ip'] || this.ip; if (!password || password.length < 6) { debug('No password, or password too short'); return new Error('Invalid login details'); } - const count = yield Logins.count({ - ip: remoteIp, - successful: false, - at: { $gt: Math.ceil(Date.now() / 1000) - 600 }, + const count = yield models.login.count({ + where: { + ip: remoteIp, + successful: false, + createdAt: { + $gt: Math.ceil(Date.now() / 1000) - 600, + }, + }, }); + if (count > 25) { debug('Throttling brute force'); return new Error('Invalid login details'); } - const login = { ip: remoteIp, at: Math.ceil(Date.now() / 1000), successful: null }; - yield Logins.save(login); - const user = yield Users.findOne({ + const user = yield models.user.findOne({ email: email.toLowerCase(), - banned: { $exists: false }, status: { $ne: 'deleted' }, + activated: 'true', }); + const login = yield models.login.create({ + ip: remoteIp, + successful: false, + }); + if (user) { - const verified = yield passwords.verify(password, user.salted_password); - if (verified) { + if (yield passwords.verify(password, user.password)) { debug('Password verified'); login.successful = true; - yield Logins.updateOne({ _id: login._id }, login); + yield login.save(); return user; } debug('Password invalid'); - login.successful = false; - yield Logins.updateOne({ _id: login._id }, login); - } else { - debug('Email invalid'); - login.successful = false; - yield Logins.updateOne({ _id: login._id }, login); + login.userId = user.id; } + yield login.save(); return new Error('Invalid login details'); } @@ -59,18 +61,16 @@ export function* authenticate(email, password) { export function* setupSession(user) { debug('Setting up session'); const token = uuid.v4(); - yield this.redis.set(token, user._id, 'EX', 604800); + yield this.redis.set(token, user.id, 'EX', 604800); const sessionUser = { - id: user._id, + id: user.id, email: user.email, dailyUploadAllowance: 15, maxFileSize: 20971520, - joined: user.joined, - plan: user.type || 'Free', - uploadsToday: yield this.db.Files.count({ - owner: user._id, time_added: { $gt: Math.ceil(Date.now() / 1000) - 86400 }, - }), + joined: user.createdAt, + plan: user.plan, + uploadsToday: yield models.file.count({ userId: user.id }), md5: crypto.createHash('md5').update(user.email).digest('hex'), token, }; @@ -82,39 +82,45 @@ export function* setupSession(user) { this.session.user = sessionUser; if (this.request.body.remember && this.request.body.remember === 'on') { - const Remember = this.db.Remember; - const rememberToken = uuid(); - Remember.save({ _id: rememberToken, user_id: user.id, created: new Date().getTime() }); - this.cookies.set('r', rememberToken, { maxAge: 1209600000, httpOnly: true }); + const remember = yield models.remember.create({ + id: uuid(), + userId: user.id, + }); + this.cookies.set('r', remember.id, { maxAge: 1209600000, httpOnly: true }); } debug('Session set up'); } export function* signup(email, password, ip) { - const Users = this.db.Users; - const existingUser = yield Users.findOne({ email, status: { $ne: 'deleted' } }); + const existingUser = yield models.user.findOne({ where: { email, activated: true } }); if (existingUser) { debug('Email already in use.'); throw new Error('Email already in use.'); } const cryptedPassword = yield passwords.crypt(password); - const user = { + const user = yield models.user.create({ email, - salted_password: cryptedPassword, - joined: Math.round(new Date().getTime() / 1000), - signup_ip: ip, - activationCode: uuid(), - }; - Users.insertOne(user); + password: cryptedPassword, + created: Math.round(new Date().getTime() / 1000), + ip, + activation: { + id: uuid(), + email, + }, + }, { + include: [models.activation], + }); + + yield user.save(); const html = yield render('email/inlined/activate', { - activationUrl: `${process.env.WEB_BASE_URL}/activate/${user.activationCode}`, + activationUrl: `${process.env.WEB_BASE_URL}/activate/${user.activation.id}`, }); const text = `Thanks for signing up to Hostr! Please confirm your email address by clicking the link below. -${process.env.WEB_BASE_URL}/activate/${user.activationCode} +${process.env.WEB_BASE_URL}/activate/${user.activation.id} — Jonathan Cremin, Hostr Founder `; @@ -132,21 +138,17 @@ ${process.env.WEB_BASE_URL}/activate/${user.activationCode} export function* sendResetToken(email) { - const Users = this.db.Users; - const Reset = this.db.Reset; - const user = yield Users.findOne({ email }); + const user = yield models.user.findOne({ email }); if (user) { - const token = uuid.v4(); - Reset.save({ - _id: user._id, - created: Math.round(new Date().getTime() / 1000), - token, + const reset = yield models.reset.create({ + id: uuid.v4(), + userId: user.id, }); const html = yield render('email/inlined/forgot', { - forgotUrl: `${process.env.WEB_BASE_URL}/forgot/${token}`, + forgotUrl: `${process.env.WEB_BASE_URL}/forgot/${reset.id}`, }); const text = `It seems you've forgotten your password :( -Visit ${process.env.WEB_BASE_URL}/forgot/${token} to set a new one. +Visit ${process.env.WEB_BASE_URL}/forgot/${reset.id} to set a new one. `; const mail = new sendgrid.Email({ to: user.email, @@ -165,38 +167,43 @@ Visit ${process.env.WEB_BASE_URL}/forgot/${token} to set a new one. export function* fromToken(token) { - const Users = this.db.Users; - const reply = yield this.redis.get(token); - return yield Users.findOne({ _id: reply }); + const userId = yield this.redis.get(token); + return yield models.user.findbyId(userId); } -export function* fromCookie(cookie) { - const Remember = this.db.Remember; - const Users = this.db.Users; - const remember = yield Remember.findOne({ _id: cookie }); - return yield Users.findOne({ _id: remember.user_id }); +export function* fromCookie(rememberId) { + const userId = yield models.remember.findById(rememberId); + return yield models.user.findbyId(userId); } -export function* validateResetToken() { - const Reset = this.db.Reset; - return yield Reset.findOne({ token: this.params.token }); +export function* validateResetToken(resetId) { + return yield models.reset.findbyId(resetId); } export function* updatePassword(userId, password) { - const Users = this.db.Users; const cryptedPassword = yield passwords.crypt(password); - yield Users.updateOne({ _id: userId }, { $set: { salted_password: cryptedPassword } }); + const user = yield models.user.findById(userId); + user.password = cryptedPassword; + yield user.save(); } export function* activateUser(code) { - const Users = this.db.Users; - const user = yield Users.findOne({ activationCode: code }); - if (user) { - Users.updateOne({ _id: user._id }, { $unset: { activationCode: '' } }); + debug(code); + const activation = yield models.activation.findOne({ + where: { + id: code, + }, + }); + if (activation.updatedAt.getTime() === activation.createdAt.getTime()) { + activation.activated = true; + yield activation.save(); + const user = yield activation.getUser(); + user.activated = true; + yield user.save(); yield setupSession.call(this, user); return true; } diff --git a/web/routes/file.js b/web/routes/file.js index 3091b84..b53a352 100644 --- a/web/routes/file.js +++ b/web/routes/file.js @@ -1,5 +1,6 @@ import { join } from 'path'; import mime from 'mime-types'; +import models from '../../models'; import hostrFileStream from '../../lib/hostr-file-stream'; import { formatFile } from '../../lib/format'; @@ -32,36 +33,38 @@ export function* get() { return; } - const file = yield this.db.Files.findOne({ - _id: this.params.id, - file_name: this.params.name, - status: 'active', + const file = yield models.file.findOne({ + where: { + id: this.params.id, + name: this.params.name, + status: 'active', + }, }); this.assert(file, 404); if (!hotlinkCheck(file, this.headers['user-agent'], this.headers.referer)) { - this.redirect(`/${file._id}`); + this.redirect(`/${file.id}`); return; } if (!file.width && this.request.query.warning !== 'on') { - this.redirect(`/${file._id}`); + this.redirect(`/${file.id}`); return; } if (file.malware) { const alert = this.request.query.alert; if (!alert || !alert.match(/i want to download malware/i)) { - this.redirect(`/${file._id}`); + this.redirect(`/${file.id}`); return; } } - let localPath = join(storePath, file._id[0], `${file._id}_${file.file_name}`); - let remotePath = join(file._id[0], `${file._id}_${file.file_name}`); + let localPath = join(storePath, file.id[0], `${file.id}_${file.name}`); + let remotePath = join(file.id[0], `${file.id}_${file.name}`); if (this.params.size > 0) { - localPath = join(storePath, file._id[0], this.params.size, `${file._id}_${file.file_name}`); - remotePath = join(file._id[0], this.params.size, `${file._id}_${file.file_name}`); + localPath = join(storePath, file.id[0], this.params.size, `${file.id}_${file.name}`); + remotePath = join(file.id[0], this.params.size, `${file.id}_${file.name}`); } if (file.malware) { @@ -73,13 +76,13 @@ export function* get() { if (this.params.size) { this.statsd.incr('file.view', 1); } - type = mime.lookup(file.file_name); + type = mime.lookup(file.name); } else { this.statsd.incr('file.download', 1); } if (userAgentCheck(this.headers['user-agent'])) { - this.set('Content-Disposition', `attachment; filename=${file.file_name}`); + this.set('Content-Disposition', `attachment; filename=${file.name}`); } this.set('Content-type', type); @@ -87,10 +90,7 @@ export function* get() { this.set('Cache-control', 'max-age=2592000'); if (!this.params.size || (this.params.size && this.params.size > 150)) { - this.db.Files.updateOne( - { _id: file._id }, - { $set: { last_accessed: Math.ceil(Date.now() / 1000) }, $inc: { downloads: 1 } }, - { w: 0 }); + models.file.accessed(file.id); } this.body = yield hostrFileStream(localPath, remotePath); @@ -101,10 +101,15 @@ export function* resized() { } export function* landing() { - const file = yield this.db.Files.findOne({ _id: this.params.id, status: 'active' }); + const file = yield models.file.findOne({ + where: { + id: this.params.id, + status: 'active', + }, + }); this.assert(file, 404); if (userAgentCheck(this.headers['user-agent'])) { - this.params.name = file.file_name; + this.params.name = file.name; yield get.call(this); return; } diff --git a/web/routes/user.js b/web/routes/user.js index 86e24f8..ac414dc 100644 --- a/web/routes/user.js +++ b/web/routes/user.js @@ -2,6 +2,7 @@ import { authenticate, setupSession, signup as signupUser, activateUser, sendResetToken, validateResetToken, updatePassword, } from '../lib/auth'; +import models from '../../models'; import debugname from 'debug'; const debug = debugname('hostr-web:user'); @@ -69,8 +70,6 @@ export function* signup() { export function* forgot() { - const Reset = this.db.Reset; - const Users = this.db.Users; const token = this.params.token; if (this.request.body.password) { @@ -83,16 +82,14 @@ export function* forgot() { return; } this.assertCSRF(this.request.body); - const tokenUser = yield validateResetToken.call(this, token); - const userId = tokenUser._id; - yield updatePassword.call(this, userId, this.request.body.password); - yield Reset.deleteOne({ _id: userId }); - const user = yield Users.findOne({ _id: userId }); - yield setupSession.call(this, user); + const user = yield validateResetToken(token); + yield updatePassword(user.id, this.request.body.password); + yield models.reset.deleteById(token); + yield setupSession(this, user); this.statsd.incr('auth.reset.success', 1); this.redirect('/'); } else if (token) { - const tokenUser = yield validateResetToken.call(this, token); + const tokenUser = yield validateResetToken(token); if (!tokenUser) { this.statsd.incr('auth.reset.fail', 1); yield this.render('forgot', { From de0284e48ae067a70c264d12c8fa67cf66e77a58 Mon Sep 17 00:00:00 2001 From: Jonathan Cremin Date: Sun, 19 Jun 2016 11:17:31 -0700 Subject: [PATCH 2/4] Setup schema and tests. --- .gitlab-ci.yml | 9 +++++++-- models/activation.js | 2 +- models/file.js | 5 +++++ models/login.js | 5 +++++ models/transaction.js | 5 +++++ models/user.js | 5 +++++ test/fixtures/user.js | 35 +++++++++++++++++++++-------------- 7 files changed, 49 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 83e8a1b..43d3403 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,13 @@ image: node:6 services: - - mongo + - postgres - redis +variables: + POSTGRES_DB: hostr_test + POSTGRES_USER: hostr_test + POSTGRES_PASSWORD: "" + stages: - test @@ -16,7 +21,7 @@ test: tags: - docker variables: - MONGO_URL: "mongodb://mongo:27017/hostr" + DATABASE_URL: "postgres://hostr_test@postgres:5432/hostr_test" REDIS_URL: "redis://redis:6379" DEBUG: "hostr*" diff --git a/models/activation.js b/models/activation.js index c959919..cbc849b 100644 --- a/models/activation.js +++ b/models/activation.js @@ -1,7 +1,7 @@ export default function (sequelize, DataTypes) { const Activation = sequelize.define('activation', { id: { type: DataTypes.UUID, defaultValue: DataTypes.UUIDV4, primaryKey: true }, - activated: false, + activated: { type: DataTypes.BOOLEAN, defaultValue: false, primaryKey: true }, email: DataTypes.STRING, }, { classMethods: { diff --git a/models/file.js b/models/file.js index 8d81a76..ba74754 100644 --- a/models/file.js +++ b/models/file.js @@ -21,6 +21,11 @@ export default function (sequelize, DataTypes) { md5: DataTypes.STRING(32), // eslint-disable-line new-cap malwarePositives: DataTypes.INTEGER, }, { + indexes: [ + { + fields: ['userId'], + }, + ], classMethods: { accessed: (id) => sequelize.query(` UPDATE files diff --git a/models/login.js b/models/login.js index 855ae63..ef81884 100644 --- a/models/login.js +++ b/models/login.js @@ -4,6 +4,11 @@ export default function (sequelize, DataTypes) { successful: { type: DataTypes.BOOLEAN }, ip: { type: 'inet' }, }, { + indexes: [ + { + fields: ['ip'], + }, + ], classMethods: { associate: (models) => { Login.belongsTo(models.user); diff --git a/models/transaction.js b/models/transaction.js index a846d66..2337e9d 100644 --- a/models/transaction.js +++ b/models/transaction.js @@ -6,6 +6,11 @@ export default function (sequelize, DataTypes) { type: DataTypes.ENUM('direct', 'paypal'), // eslint-disable-line new-cap ip: 'inet', }, { + indexes: [ + { + fields: ['userId'], + }, + ], classMethods: { associate: (models) => { Transaction.belongsTo(models.user); diff --git a/models/user.js b/models/user.js index 0f64415..1507bf9 100644 --- a/models/user.js +++ b/models/user.js @@ -9,6 +9,11 @@ export default function (sequelize, DataTypes) { banned: DataTypes.BOOLEAN, deleted: DataTypes.BOOLEAN, }, { + indexes: [ + { + fields: ['email'], + }, + ], classMethods: { associate: (models) => { User.hasMany(models.file); diff --git a/test/fixtures/user.js b/test/fixtures/user.js index cc3a391..6de7f59 100644 --- a/test/fixtures/user.js +++ b/test/fixtures/user.js @@ -1,27 +1,34 @@ +import co from 'co'; + import models from '../../models'; -function createUser() { - models.user.create({ +import debugname from 'debug'; +const debug = debugname('hostr:db'); + +function *createUser() { + const user = yield models.user.create({ 'email': 'test@hostr.co', 'password': '$pbkdf2-256-1$2$kBhIDRqFwnF/1ms6ZHfME2o2$a48e8c350d26397fcc88bf0a7a2817b1cdcd1ffffe0521a5', 'ip': '127.0.0.1', 'plan': 'Free', 'activated': true, - }).then((user) => { - user.save().then(() => { - models.sequelize.close(); - }); }); + yield user.save(); + yield models.sequelize.close(); } -models.user.findOne({ - where: { - email: 'test@hostr.co', - }, -}).then((user) => { +co(function *sync() { + debug('Syncing schema'); + yield models.sequelize.sync(); + debug('Schema synced'); + const user = yield models.user.findOne({ + where: { + email: 'test@hostr.co', + }, + }); if (user) { - user.destroy().then(createUser); - } else { - createUser(); + yield user.destroy(); } + debug('Creating test user'); + yield createUser(); }); From 889dc02945400ad5e341eba176d6e85fb89bc42d Mon Sep 17 00:00:00 2001 From: Jonathan Cremin Date: Sun, 7 Aug 2016 14:38:05 +0100 Subject: [PATCH 3/4] More changes for db migration --- .gitlab-ci.yml | 1 + api/app.js | 4 +- api/lib/auth.js | 4 +- api/routes/file.js | 22 +- {web => api}/routes/pro.js | 51 +++-- api/routes/user.js | 6 +- {web => api}/views/email/inlined/pro.ejs | 0 {web => api}/views/email/pro.html | 0 api/views/email/style.css | 271 +++++++++++++++++++++++ app.js | 4 - lib/format.js | 6 +- lib/malware.js | 2 +- lib/mongo.js | 6 +- lib/resize.js | 14 +- lib/uploader.js | 10 +- migrate/migrate-activations.js | 75 +++++++ migrate/migrate-files.js | 80 +++++++ migrate/migrate-logins.js | 54 +++++ migrate/migrate-malwares.js | 46 ++++ migrate/migrate-users.js | 77 +++++++ models/activation.js | 2 +- models/file.js | 5 +- models/transaction.js | 1 + models/user.js | 7 +- package.json | 5 +- test/fixtures/mongo-file.js | 11 + test/fixtures/mongo-user.js | 15 ++ web/app.js | 4 - web/lib/auth.js | 35 ++- web/public/src/app/controllers.js | 2 +- web/public/src/app/directives.js | 2 +- web/routes/file.js | 2 - web/routes/user.js | 16 +- 33 files changed, 740 insertions(+), 100 deletions(-) rename {web => api}/routes/pro.js (57%) rename {web => api}/views/email/inlined/pro.ejs (100%) rename {web => api}/views/email/pro.html (100%) create mode 100644 api/views/email/style.css create mode 100644 migrate/migrate-activations.js create mode 100644 migrate/migrate-files.js create mode 100644 migrate/migrate-logins.js create mode 100644 migrate/migrate-malwares.js create mode 100644 migrate/migrate-users.js create mode 100644 test/fixtures/mongo-file.js create mode 100644 test/fixtures/mongo-user.js diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 43d3403..644077f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,3 +30,4 @@ cache: untracked: true paths: - node_modules + - web/public/jspm_packages diff --git a/api/app.js b/api/app.js index c2f88ac..b56c59f 100644 --- a/api/app.js +++ b/api/app.js @@ -5,6 +5,7 @@ import StatsD from 'statsy'; import auth from './lib/auth'; import * as user from './routes/user'; import * as file from './routes/file'; +import * as pro from './routes/pro'; import debugname from 'debug'; const debug = debugname('hostr-api'); @@ -64,10 +65,11 @@ router.get('/user/token', auth, user.token); router.get('/token', auth, user.token); router.get('/user/transaction', auth, user.transaction); router.post('/user/settings', auth, user.settings); +router.post('/user/pro', auth, pro.create); +router.delete('/user/pro', auth, pro.cancel); router.get('/file', auth, file.list); router.post('/file', auth, file.post); router.get('/file/:id', file.get); -router.put('/file/:id', auth, file.put); router.delete('/file/:id', auth, file.del); router.delete('/file/:id', auth, file.del); diff --git a/api/lib/auth.js b/api/lib/auth.js index 42cf9dc..5fead69 100644 --- a/api/lib/auth.js +++ b/api/lib/auth.js @@ -31,7 +31,7 @@ export default function* (next) { ip: remoteIp, successful: false, createdAt: { - $gt: new Date(Math.ceil(Date.now()) - 600000), + $gt: new Date(Date.now() - 600000), }, }, }); @@ -71,7 +71,7 @@ export default function* (next) { where: { userId: user.id, createdAt: { - $gt: Math.ceil(Date.now() / 1000) - 86400, + $gt: Date.now() - 86400000, }, }, }); diff --git a/api/routes/file.js b/api/routes/file.js index 299d11e..2793c3a 100644 --- a/api/routes/file.js +++ b/api/routes/file.js @@ -54,7 +54,7 @@ export function* list() { const files = yield models.file.findAll({ where: { userId: this.user.id, - status: 'active', + processed: true, }, order: '"createdAt" DESC', offset, @@ -70,9 +70,6 @@ export function* get() { const file = yield models.file.findOne({ where: { id: this.params.id, - status: { - $in: ['active', 'uploading'], - }, }, }); this.assert(file, 404, '{"error": {"message": "File not found", "code": 604}}'); @@ -83,20 +80,6 @@ export function* get() { } -export function* put() { - if (this.request.body.trashed) { - const file = yield models.file.findOne({ - where: { - id: this.params.id, - userId: this.user.id, - }, - }); - file.status = this.request.body.trashed ? 'trashed' : 'active'; - yield file.save(); - } -} - - export function* del() { const file = yield models.file.findOne({ where: { @@ -105,8 +88,7 @@ export function* del() { }, }); this.assert(file, 401, '{"error": {"message": "File not found", "code": 604}}'); - file.status = 'deleted'; - yield file.save(); + yield file.destroy(); const event = { type: 'file-deleted', data: { id: this.params.id } }; yield this.redis.publish(`/file/${this.params.id}`, JSON.stringify(event)); yield this.redis.publish(`/user/${this.user.id}`, JSON.stringify(event)); diff --git a/web/routes/pro.js b/api/routes/pro.js similarity index 57% rename from web/routes/pro.js rename to api/routes/pro.js index a59c7bd..5fadcd8 100644 --- a/web/routes/pro.js +++ b/api/routes/pro.js @@ -6,18 +6,20 @@ const stripe = new Stripe(process.env.STRIPE_SECRET_KEY); import sendgridInit from 'sendgrid'; const sendgrid = sendgridInit(process.env.SENDGRID_KEY); +import models from '../../models'; + const from = process.env.EMAIL_FROM; const fromname = process.env.EMAIL_NAME; export function* create() { - const Users = this.db.Users; - const Transactions = this.db.Transactions; const stripeToken = this.request.body.stripeToken; + const ip = this.request.headers['x-real-ip'] || this.req.connection.remoteAddress; + const createCustomer = { card: stripeToken.id, plan: 'usd_monthly', - email: this.session.email, + email: this.user.email, }; const customer = yield stripe.customers.create(createCustomer); @@ -26,19 +28,22 @@ export function* create() { delete customer.subscriptions; - yield Users.updateOne({ _id: this.session.user.id }, - { $set: { stripe_customer: customer, type: 'Pro' } }); + const user = yield models.user.findById(this.user.id); + user.plan = 'Pro'; + yield user.save(); - const transaction = { - user_id: this.session.user.id, + const transaction = yield models.transaction.create({ + userId: this.user.id, amount: customer.subscription.plan.amount, - desc: customer.subscription.plan.name, - date: new Date(customer.subscription.plan.created * 1000), - }; + description: customer.subscription.plan.name, + data: customer, + type: 'direct', + ip, + }); - yield Transactions.insertOne(transaction); + yield transaction.save(); - this.session.user.plan = 'Pro'; + this.user.plan = 'Pro'; this.body = { status: 'active' }; const html = yield render('email/inlined/pro'); @@ -50,7 +55,7 @@ export function* create() { `; const mail = new sendgrid.Email({ - to: this.session.user.email, + to: this.user.email, subject: 'Hostr Pro', from, fromname, @@ -62,19 +67,19 @@ export function* create() { } export function* cancel() { - this.assertCSRF(); - const Users = this.db.Users; - const user = yield Users.findOne({ _id: this.session.user.id }); + const user = yield models.user.findById(this.user.id); + const transactions = yield user.getTransactions(); + const transaction = transactions[0]; - const confirmation = yield stripe.customers.cancelSubscription( - user.stripe_customer.id, - user.stripe_customer.subscription.id, - { at_period_end: true } + yield stripe.customers.cancelSubscription( + transaction.data.id, + transaction.data.subscription.id, + { at_period_end: false } ); - yield Users.updateOne({ _id: this.session.user.id }, - { $set: { 'stripe_customer.subscription': confirmation, type: 'Free' } }); + user.plan = 'Free'; + yield user.save(); - this.session.user.plan = 'Pro'; + this.user.plan = 'Free'; this.body = { status: 'inactive' }; } diff --git a/api/routes/user.js b/api/routes/user.js index c9b7637..ba410e6 100644 --- a/api/routes/user.js +++ b/api/routes/user.js @@ -20,7 +20,11 @@ export function* token() { } export function* transaction() { - const transactions = yield models.transaction.findAll({ userId: this.user.id }); + const transactions = yield models.transaction.findAll({ + where: { + userId: this.user.id, + }, + }); this.body = transactions.map((item) => { return { diff --git a/web/views/email/inlined/pro.ejs b/api/views/email/inlined/pro.ejs similarity index 100% rename from web/views/email/inlined/pro.ejs rename to api/views/email/inlined/pro.ejs diff --git a/web/views/email/pro.html b/api/views/email/pro.html similarity index 100% rename from web/views/email/pro.html rename to api/views/email/pro.html diff --git a/api/views/email/style.css b/api/views/email/style.css new file mode 100644 index 0000000..3bd01e3 --- /dev/null +++ b/api/views/email/style.css @@ -0,0 +1,271 @@ +/* ------------------------------------- + GLOBAL + A very basic CSS reset +------------------------------------- */ +* { + margin: 0; + padding: 0; + font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; + box-sizing: border-box; + font-size: 14px; +} + +img { + max-width: 100%; +} + +body { + -webkit-font-smoothing: antialiased; + -webkit-text-size-adjust: none; + width: 100% !important; + height: 100%; + line-height: 1.6; +} + +/* Let's make sure all tables have defaults */ +table td { + vertical-align: top; +} + +/* ------------------------------------- + BODY & CONTAINER +------------------------------------- */ +body { + background-color: #f6f6f6; +} + +.body-wrap { + background-color: #f6f6f6; + width: 100%; +} + +.container { + display: block !important; + max-width: 600px !important; + margin: 0 auto !important; + /* makes it centered */ + clear: both !important; +} + +.content { + max-width: 600px; + margin: 0 auto; + display: block; + padding: 20px; +} + +/* ------------------------------------- + HEADER, FOOTER, MAIN +------------------------------------- */ +.main { + background: #fff; + border: 1px solid #e9e9e9; + border-radius: 3px; +} + +.content-wrap { + padding: 20px; +} + +.content-block { + padding: 0 0 20px; +} + +.header { + width: 100%; + margin-bottom: 20px; +} + +.logo { + margin-top: 40px; + margin-bottom: 20px; +} + +.footer { + width: 100%; + clear: both; + color: #999; + padding: 20px; +} +.footer a { + color: #999; +} +.footer p, .footer a, .footer unsubscribe, .footer td { + font-size: 12px; +} + +/* ------------------------------------- + TYPOGRAPHY +------------------------------------- */ +h1, h2, h3 { + font-family: "Helvetica Neue", Helvetica, Arial, "Lucida Grande", sans-serif; + color: #000; + margin: 40px 0 0; + line-height: 1.2; + font-weight: 400; +} + +h1 { + font-size: 32px; + font-weight: 500; +} + +h2 { + font-size: 24px; +} + +h3 { + font-size: 18px; +} + +h4 { + font-size: 14px; + font-weight: 600; +} + +p, ul, ol { + margin-bottom: 10px; + font-weight: normal; +} +p li, ul li, ol li { + margin-left: 5px; + list-style-position: inside; +} + +/* ------------------------------------- + LINKS & BUTTONS +------------------------------------- */ +a { + color: #456470; + text-decoration: underline; +} + +.btn-primary { + text-decoration: none; + color: #FFF; + background-color: #456470; + border: solid #456470; + /* seems to be a bug stopping this from being applied above */ + border-color: #456470; + border-width: 10px 20px; + line-height: 2; + font-weight: bold; + text-align: center; + cursor: pointer; + display: inline-block; + border-radius: 5px; + text-transform: capitalize; +} + +/* ------------------------------------- + OTHER STYLES THAT MIGHT BE USEFUL +------------------------------------- */ +.last { + margin-bottom: 0; +} + +.first { + margin-top: 0; +} + +.aligncenter { + text-align: center; +} + +.alignright { + text-align: right; +} + +.alignleft { + text-align: left; +} + +.clear { + clear: both; +} + +/* ------------------------------------- + ALERTS + Change the class depending on warning email, good email or bad email +------------------------------------- */ +.alert { + font-size: 16px; + color: #fff; + font-weight: 500; + padding: 20px; + text-align: center; + border-radius: 3px 3px 0 0; +} +.alert a { + color: #fff; + text-decoration: none; + font-weight: 500; + font-size: 16px; +} +.alert.alert-warning { + background: #ff9f00; +} +.alert.alert-bad { + background: #d0021b; +} +.alert.alert-good { + background: #68b90f; +} + +/* ------------------------------------- + INVOICE + Styles for the billing table +------------------------------------- */ +.invoice { + margin: 40px auto; + text-align: left; + width: 80%; +} +.invoice td { + padding: 5px 0; +} +.invoice .invoice-items { + width: 100%; +} +.invoice .invoice-items td { + border-top: #eee 1px solid; +} +.invoice .invoice-items .total td { + border-top: 2px solid #333; + border-bottom: 2px solid #333; + font-weight: 700; +} + +/* ------------------------------------- + RESPONSIVE AND MOBILE FRIENDLY STYLES +------------------------------------- */ +@media only screen and (max-width: 640px) { + h1, h2, h3, h4 { + font-weight: 600 !important; + margin: 20px 0 5px !important; + } + + h1 { + font-size: 22px !important; + } + + h2 { + font-size: 18px !important; + } + + h3 { + font-size: 16px !important; + } + + .container { + width: 100% !important; + } + + .content, .content-wrapper { + padding: 10px !important; + } + + .invoice { + width: 100% !important; + } +} diff --git a/app.js b/app.js index 43b1d32..a48873e 100644 --- a/app.js +++ b/app.js @@ -8,13 +8,10 @@ import bodyparser from 'koa-bodyparser'; import websockify from 'koa-websocket'; import helmet from 'koa-helmet'; import raven from 'raven'; -import mongo from './lib/mongo'; import * as redis from './lib/redis'; import api, { ws } from './api/app'; import web from './web/app'; -import models from './models'; - import debugname from 'debug'; const debug = debugname('hostr'); @@ -52,7 +49,6 @@ app.use(function* errorMiddleware(next) { } }); -app.use(mongo()); app.use(redis.middleware()); app.use(logger()); app.use(compress()); diff --git a/lib/format.js b/lib/format.js index c02c16c..1901268 100644 --- a/lib/format.js +++ b/lib/format.js @@ -22,8 +22,8 @@ export function formatSize(size) { export function formatFile(file) { const formattedFile = { - added: moment.unix(file.createdAt).format(), - readableAdded: formatDate(file.createdAt), + added: moment.unix(file.createdAt / 1000).format(), + readableAdded: formatDate(file.createdAt / 1000), downloads: file.downloads !== undefined ? file.downloads : 0, href: `${baseURL}/${file.id}`, id: file.id, @@ -32,7 +32,7 @@ export function formatFile(file) { readableSize: formatSize(file.size), type: sniff(file.name), trashed: (file.status === 'trashed'), - status: file.status, + status: file.processed === true ? 'active' : 'uploading', }; if (file.width) { diff --git a/lib/malware.js b/lib/malware.js index 8c31510..4144728 100644 --- a/lib/malware.js +++ b/lib/malware.js @@ -71,7 +71,7 @@ export default function* (file) { } const result = yield virustotal.getFileReport(file.md5); return { - positives: result.positives, + positive: result.positives >= 5, result, }; } diff --git a/lib/mongo.js b/lib/mongo.js index eac7144..bd83f2a 100644 --- a/lib/mongo.js +++ b/lib/mongo.js @@ -4,7 +4,7 @@ import debugname from 'debug'; const debug = debugname('hostr:mongo'); /* eslint no-param-reassign: ["error", { "props": false }] */ -const configuredClient = new Promise((resolve, reject) => { +export const mongo = new Promise((resolve, reject) => { debug('Connecting to Mongodb'); return MongoClient.connect(process.env.MONGO_URL).then((client) => { debug('Successfully connected to Mongodb'); @@ -25,10 +25,10 @@ const configuredClient = new Promise((resolve, reject) => { debug(e); }); -export default function mongo() { +export default function () { return function* dbMiddleware(next) { try { - this.db = yield configuredClient; + this.db = yield mongo; } catch (e) { debug(e); } diff --git a/lib/resize.js b/lib/resize.js index 740ce51..2e17456 100644 --- a/lib/resize.js +++ b/lib/resize.js @@ -55,17 +55,17 @@ function scale(path, type, size) { }); } -export default function resize(path, type, currentSize, dim) { +export default function resize(path, type, currentSize, newSize) { debug('Resizing'); const ratio = 970 / currentSize.width; - debug(dim.width, ratio); - if (dim.width <= 150) { + debug(newSize.width, ratio); + if (newSize.width <= 150) { debug('Cover'); - return cover(path, type, dim); - } else if (dim.width >= 970 && ratio < 1) { + return cover(path, type, newSize); + } else if (newSize.width >= 970 && ratio < 1) { debug('Scale'); - dim.height = currentSize.height * ratio; // eslint-disable-line no-param-reassign - return scale(path, type, dim); + newSize.height = currentSize.height * ratio; // eslint-disable-line no-param-reassign + return scale(path, type, newSize); } debug('Copy'); return fs.readFile(path); diff --git a/lib/uploader.js b/lib/uploader.js index e98e441..832a6f7 100644 --- a/lib/uploader.js +++ b/lib/uploader.js @@ -135,12 +135,13 @@ export default class Uploader { *checkLimit() { const count = yield models.file.count({ - userId: this.context.user.id, - createdAt: { - $gt: Math.ceil(Date.now() / 1000) - 86400, + where: { + userId: this.context.user.id, + createdAt: { + $gt: Date.now() - 86400000, + }, }, }); - debug(count); const userLimit = this.context.user.daily_upload_allowance; const underLimit = (count < userLimit || userLimit === 'unlimited'); if (!underLimit) { @@ -158,6 +159,7 @@ export default class Uploader { *finalise() { this.file.size = this.receivedSize; this.file.status = 'active'; + this.file.processed = 'true'; yield this.file.save(); } diff --git a/migrate/migrate-activations.js b/migrate/migrate-activations.js new file mode 100644 index 0000000..be6adbd --- /dev/null +++ b/migrate/migrate-activations.js @@ -0,0 +1,75 @@ +import co from 'co'; + +import models from '../models'; +import { mongo } from '../lib/mongo'; + +import debugname from 'debug'; +const debug = debugname('hostr:db'); +let db; + + +co(function *sync() { + debug('Syncing schema'); + yield models.sequelize.sync(); + debug('Schema synced'); + db = yield mongo; + const users = yield db.Users.find({}, { sort: [['joined', 'asc']] }).toArray(); + for (const user of users) { + if (user.joined === '0') { + const file = yield db.Files.findOne({ + owner: user._id, + }, { + limit: 1, + sort: [['time_added', 'asc']], + }); + if (file && file.time_added > 0) { + user.createdAt = new Date(file.time_added * 1000).getTime(); + } else { + user.createdAt = new Date().getTime(); + } + } else { + user.createdAt = new Date(user.joined * 1000).getTime(); + } + } + + users.sort((a, b) => (a.createdAt < b.createdAt ? -1 : 1)); + + for (const user of users) { + if (!user.email) { + continue; + } + + const exists = yield models.user.findOne({ + where: { + email: user.email, + }, + }); + + if (exists) { + debug('User exists, continue'); + continue; + } + + const oldId = user._id.toString(); + + const newUser = yield models.user.create({ + email: user.email, + password: user.salted_password, + name: user.first_name ? `${user.first_name} ${user.last_name}` : null, + plan: user.type || 'Free', + activated: !user.activationCode, + banned: !!user.banned, + deletedAt: user.status === 'deleted' ? new Date().getTime() : null, + createdAt: user.createdAt, + updatedAt: user.createdAt, + oldId, + }); + yield newUser.save({ silent: true }); + } + models.sequelize.close(); + db.close(); +}).catch((err) => { + models.sequelize.close(); + db.close(); + debug(err); +}); diff --git a/migrate/migrate-files.js b/migrate/migrate-files.js new file mode 100644 index 0000000..e1e6424 --- /dev/null +++ b/migrate/migrate-files.js @@ -0,0 +1,80 @@ +import co from 'co'; +import validateIp from 'validate-ip'; + +import models from '../models'; +import { mongo } from '../lib/mongo'; + +import debugname from 'debug'; +const debug = debugname('hostr:db'); +let db; + + +co(function *sync() { + debug('Syncing schema'); + yield models.sequelize.sync(); + debug('Schema synced'); + db = yield mongo; + const users = yield models.user.findAll({}); + const userIds = {}; + debug('remap'); + for (const user of users) { + userIds[user._id] = user.id; + } + debug('remap done'); + let files; + try { + files = db.Files.find({}, { + sort: [['time_added', 'desc']], + skip: 0, + }); + } catch (err) { + debug(err); + } + debug('fetched files'); + + while (true) { + const file = yield files.next(); + if (!file) { + break; + } + if (!file.time_added || !file.file_size) { + continue; + } + let ip = file.ip ? file.ip.split(',').pop().trim() : null; + + if (typeof ip !== 'string' || !validateIp(ip)) { + ip = null; + } + const processed = file.status !== 'uploading'; + const accessedAt = file.last_accessed ? new Date(file.last_accessed * 1000) : null; + + yield models.file.upsert({ + id: file._id.toString(), + name: file.file_name, + originalName: file.original_name || file.file_name, + size: file.file_size, + downloads: file.downloads, + deletedAt: file.status === 'deleted' ? new Date() : null, + createdAt: new Date(file.time_added * 1000), + updatedAt: new Date(file.time_added * 1000), + accessedAt, + processed, + type: file.type !== 'file' ? file.type : 'other', + width: Number.isInteger(file.width) ? file.width : null, + height: Number.isInteger(file.height) ? file.height : null, + userId: file.owner !== undefined ? userIds[file.owner] : null, + ip, + legacyId: file.system_name !== file._id ? file.system_name : null, + md5: file.md5, + malwarePositives: file.virustotal && file.virustotal.positives > 0 ? + file.virustotal.positives : null, + }, { /* logging: false */ }); + } + + models.sequelize.close(); + db.close(); +}).catch((err) => { + models.sequelize.close(); + db.close(); + debug(err); +}); diff --git a/migrate/migrate-logins.js b/migrate/migrate-logins.js new file mode 100644 index 0000000..11af8ac --- /dev/null +++ b/migrate/migrate-logins.js @@ -0,0 +1,54 @@ +import co from 'co'; +import validateIp from 'validate-ip'; + +import models from '../models'; +import { mongo } from '../lib/mongo'; + +import debugname from 'debug'; +const debug = debugname('hostr:db'); +let db; + + +co(function *sync() { + debug('Syncing schema'); + yield models.sequelize.sync(); + debug('Schema synced'); + db = yield mongo; + const users = yield models.user.findAll({}); + const userIds = {}; + debug('remap'); + for (const user of users) { + userIds[user._id] = user.id; + } + debug('remap done'); + let logins; + try { + logins = db.Logins.find({}, { + skip: 0, + }); + } catch (err) { + debug(err); + } + debug('fetched logins'); + + while (true) { + const login = yield logins.next(); + if (!login) { + break; + } + + const newLogin = yield models.login.create({ + ip: login.ip, + createdAt: login.at * 1000, + successful: login.successful, + }, { /* logging: false */ }); + newLogin.save(); + } + + models.sequelize.close(); + db.close(); +}).catch((err) => { + models.sequelize.close(); + db.close(); + debug(err); +}); diff --git a/migrate/migrate-malwares.js b/migrate/migrate-malwares.js new file mode 100644 index 0000000..e882578 --- /dev/null +++ b/migrate/migrate-malwares.js @@ -0,0 +1,46 @@ +import co from 'co'; + +import models from '../models'; +import { mongo } from '../lib/mongo'; + +import debugname from 'debug'; +const debug = debugname('hostr:db'); +let db; + + +co(function *sync() { + debug('Syncing schema'); + yield models.sequelize.sync(); + debug('Schema synced'); + db = yield mongo; + + const files = db.Files.find({}, { + sort: [['time_added', 'desc']], + skip: 0, + }); + debug('fetched files'); + + while (true) { + const file = yield files.next(); + if (!file) { + break; + } + + if (!file.time_added || !file.file_size || !file.malware) { + continue; + } + + yield models.malware.upsert({ + fileId: file._id, + positives: file.virustotal ? file.virustotal.positives : 100, + virustotal: file.virustotal || null, + }, { /* logging: false */ }); + } + + models.sequelize.close(); + db.close(); +}).catch((err) => { + models.sequelize.close(); + db.close(); + debug(err); +}); diff --git a/migrate/migrate-users.js b/migrate/migrate-users.js new file mode 100644 index 0000000..e036dae --- /dev/null +++ b/migrate/migrate-users.js @@ -0,0 +1,77 @@ +import co from 'co'; + +import models from '../models'; +import { mongo } from '../lib/mongo'; + +import debugname from 'debug'; +const debug = debugname('hostr:db'); +let db; + + +co(function *sync() { + debug('Syncing schema'); + yield models.sequelize.sync(); + debug('Schema synced'); + db = yield mongo; + const users = yield db.Users.find({}, { sort: [['joined', 'asc']] }).toArray(); + for (const user of users) { + if (user.joined === '0') { + const file = yield db.Files.findOne({ + owner: user._id, + }, { + limit: 1, + sort: [['time_added', 'asc']], + }); + if (file && file.time_added > 0) { + user.createdAt = new Date(file.time_added * 1000).getTime(); + } else { + user.createdAt = new Date().getTime(); + } + } else { + user.createdAt = new Date(user.joined * 1000).getTime(); + } + } + + users.sort((a, b) => (a.createdAt < b.createdAt ? -1 : 1)); + + for (const user of users) { + if (!user.email) { + continue; + } + + const exists = yield models.user.findOne({ + where: { + email: user.email, + }, + }); + + if (exists) { + debug('User exists, continue'); + continue; + } + + const oldId = user._id.toString(); + + const newUser = yield models.user.create({ + email: user.email, + password: user.salted_password, + name: user.first_name ? `${user.first_name} ${user.last_name}` : null, + plan: user.type || 'Free', + activated: !user.activationCode, + banned: !!user.banned, + deletedAt: user.status === 'deleted' ? new Date().getTime() : null, + createdAt: user.createdAt, + updatedAt: user.createdAt, + oldId, + }, { + include: [models.activation], + }); + yield newUser.save({ silent: true }); + } + models.sequelize.close(); + db.close(); +}).catch((err) => { + models.sequelize.close(); + db.close(); + debug(err); +}); diff --git a/models/activation.js b/models/activation.js index cbc849b..2baa5a5 100644 --- a/models/activation.js +++ b/models/activation.js @@ -1,7 +1,7 @@ export default function (sequelize, DataTypes) { const Activation = sequelize.define('activation', { id: { type: DataTypes.UUID, defaultValue: DataTypes.UUIDV4, primaryKey: true }, - activated: { type: DataTypes.BOOLEAN, defaultValue: false, primaryKey: true }, + activatedAt: { type: DataTypes.DATE }, email: DataTypes.STRING, }, { classMethods: { diff --git a/models/file.js b/models/file.js index ba74754..566d1e3 100644 --- a/models/file.js +++ b/models/file.js @@ -6,7 +6,8 @@ export default function (sequelize, DataTypes) { size: DataTypes.BIGINT, downloads: DataTypes.BIGINT, accessedAt: DataTypes.DATE, - status: DataTypes.ENUM('active', 'uploading', 'deleted'), // eslint-disable-line new-cap + deletedAt: DataTypes.DATE, + processed: DataTypes.BOOLEAN, type: DataTypes.ENUM( // eslint-disable-line new-cap 'image', 'audio', @@ -21,6 +22,7 @@ export default function (sequelize, DataTypes) { md5: DataTypes.STRING(32), // eslint-disable-line new-cap malwarePositives: DataTypes.INTEGER, }, { + paranoid: true, indexes: [ { fields: ['userId'], @@ -37,6 +39,7 @@ export default function (sequelize, DataTypes) { }), associate: (models) => { File.belongsTo(models.user); + File.hasOne(models.malware); }, }, }); diff --git a/models/transaction.js b/models/transaction.js index 2337e9d..6ac465f 100644 --- a/models/transaction.js +++ b/models/transaction.js @@ -5,6 +5,7 @@ export default function (sequelize, DataTypes) { description: DataTypes.STRING, type: DataTypes.ENUM('direct', 'paypal'), // eslint-disable-line new-cap ip: 'inet', + data: DataTypes.JSON, }, { indexes: [ { diff --git a/models/user.js b/models/user.js index 1507bf9..ff3bb7e 100644 --- a/models/user.js +++ b/models/user.js @@ -3,12 +3,14 @@ export default function (sequelize, DataTypes) { id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true }, email: DataTypes.STRING, password: DataTypes.STRING, + name: DataTypes.STRING, plan: DataTypes.ENUM('Free', 'Pro'), // eslint-disable-line new-cap - ip: 'inet', activated: DataTypes.BOOLEAN, banned: DataTypes.BOOLEAN, - deleted: DataTypes.BOOLEAN, + deletedAt: DataTypes.DATE, + oldId: DataTypes.STRING, }, { + paranoid: true, indexes: [ { fields: ['email'], @@ -17,6 +19,7 @@ export default function (sequelize, DataTypes) { classMethods: { associate: (models) => { User.hasMany(models.file); + User.hasMany(models.transaction); User.hasOne(models.activation); }, }, diff --git a/package.json b/package.json index fe210a8..ae33be8 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "koa-csrf": "^2.5.0", "koa-error": "^2.1.0", "koa-favicon": "~1.2.0", - "koa-generic-session": "^1.10.2", + "koa-generic-session": "^1.11.0", "koa-helmet": "^1.0.0", "koa-logger": "~1.3.0", "koa-redis": "^2.1.1", @@ -78,7 +78,8 @@ "ssh2": "^0.5.0", "statsy": "~0.2.0", "stripe": "^4.7.0", - "swig": "~1.4.2" + "swig": "~1.4.2", + "validate-ip": "^1.0.1" }, "devDependencies": { "babel-eslint": "^6.0.4", diff --git a/test/fixtures/mongo-file.js b/test/fixtures/mongo-file.js new file mode 100644 index 0000000..3d11f11 --- /dev/null +++ b/test/fixtures/mongo-file.js @@ -0,0 +1,11 @@ +const MongoClient = require('mongodb').MongoClient; + +MongoClient.connect(process.env.MONGO_URL, function connect(err, db) { + const collection = db.collection('files'); + collection.createIndex({ + 'owner': 1, + 'status': 1, + 'time_added': -1, + }); + db.close(); +}); diff --git a/test/fixtures/mongo-user.js b/test/fixtures/mongo-user.js new file mode 100644 index 0000000..14334e2 --- /dev/null +++ b/test/fixtures/mongo-user.js @@ -0,0 +1,15 @@ +const MongoClient = require('mongodb').MongoClient; + +MongoClient.connect(process.env.MONGO_URL, function connect(err, db) { + const collection = db.collection('users'); + collection.remove({ + 'email': 'test@hostr.co', + }); + collection.save({ + 'email': 'test@hostr.co', + 'salted_password': '$pbkdf2-256-1$2$kBhIDRqFwnF/1ms6ZHfME2o2$a48e8c350d26397fcc88bf0a7a2817b1cdcd1ffffe0521a5', + 'joined': Math.ceil(Date.now() / 1000), + 'signup_ip': '127.0.0.1', + }); + db.close(); +}); diff --git a/web/app.js b/web/app.js index 761e3e7..eb7857d 100644 --- a/web/app.js +++ b/web/app.js @@ -8,7 +8,6 @@ import errors from 'koa-error'; import * as redis from '../lib/redis'; import * as index from './routes/index'; import * as file from './routes/file'; -import * as pro from './routes/pro'; import * as user from './routes/user'; const router = new Router(); @@ -67,9 +66,6 @@ router.get('/pricing', index.staticPage); router.get('/apps', index.staticPage); router.get('/stats', index.staticPage); -router.post('/pro/create', pro.create); -router.post('/pro/cancel', pro.cancel); - router.get('/:id', file.landing); router.get('/file/:id/:name', file.get); router.get('/file/:size/:id/:name', file.get); diff --git a/web/lib/auth.js b/web/lib/auth.js index 6e9fc64..66cfe01 100644 --- a/web/lib/auth.js +++ b/web/lib/auth.js @@ -25,7 +25,7 @@ export function* authenticate(email, password) { ip: remoteIp, successful: false, createdAt: { - $gt: Math.ceil(Date.now() / 1000) - 600, + $gt: Math.ceil(Date.now()) - 600000, }, }, }); @@ -35,26 +35,30 @@ export function* authenticate(email, password) { return new Error('Invalid login details'); } const user = yield models.user.findOne({ - email: email.toLowerCase(), - activated: 'true', + where: { + email: email.toLowerCase(), + activated: true, + }, }); + debug(user); const login = yield models.login.create({ ip: remoteIp, successful: false, }); - if (user) { + if (user && user.password) { if (yield passwords.verify(password, user.password)) { debug('Password verified'); login.successful = true; yield login.save(); + debug(user); return user; } debug('Password invalid'); login.userId = user.id; } yield login.save(); - return new Error('Invalid login details'); + return false; } @@ -93,7 +97,12 @@ export function* setupSession(user) { export function* signup(email, password, ip) { - const existingUser = yield models.user.findOne({ where: { email, activated: true } }); + const existingUser = yield models.user.findOne({ + where: { + email, + activated: true, + }, + }); if (existingUser) { debug('Email already in use.'); throw new Error('Email already in use.'); @@ -102,8 +111,8 @@ export function* signup(email, password, ip) { const user = yield models.user.create({ email, password: cryptedPassword, - created: Math.round(new Date().getTime() / 1000), ip, + plan: 'Free', activation: { id: uuid(), email, @@ -138,7 +147,11 @@ ${process.env.WEB_BASE_URL}/activate/${user.activation.id} export function* sendResetToken(email) { - const user = yield models.user.findOne({ email }); + const user = yield models.user.findOne({ + where: { + email, + }, + }); if (user) { const reset = yield models.reset.create({ id: uuid.v4(), @@ -168,18 +181,18 @@ Visit ${process.env.WEB_BASE_URL}/forgot/${reset.id} to set a new one. export function* fromToken(token) { const userId = yield this.redis.get(token); - return yield models.user.findbyId(userId); + return yield models.user.findById(userId); } export function* fromCookie(rememberId) { const userId = yield models.remember.findById(rememberId); - return yield models.user.findbyId(userId); + return yield models.user.findById(userId); } export function* validateResetToken(resetId) { - return yield models.reset.findbyId(resetId); + return yield models.reset.findById(resetId); } diff --git a/web/public/src/app/controllers.js b/web/public/src/app/controllers.js index c9465ba..6266cdb 100644 --- a/web/public/src/app/controllers.js +++ b/web/public/src/app/controllers.js @@ -66,7 +66,7 @@ export class ProController { $scope.user = UserService.get(); $scope.header = 'full'; $scope.cancel = () => { - $http.post('/pro/cancel').success(() => { + $http.delete(window.settings.apiURL + '/user/pro').success(() => { window.location.reload(true); }).error((data) => { console.error(new Error(data)); diff --git a/web/public/src/app/directives.js b/web/public/src/app/directives.js index 8ac64ce..f77f635 100644 --- a/web/public/src/app/directives.js +++ b/web/public/src/app/directives.js @@ -69,7 +69,7 @@ export function stripeSubscribe($http) { key: window.settings.stripePublic, image: '/images/stripe-128.png', token: (token) => { - $http.post('/pro/create', { + $http.post(window.settings.apiURL + '/user/pro', { stripeToken: token, }) .success((data) => { diff --git a/web/routes/file.js b/web/routes/file.js index b53a352..459540b 100644 --- a/web/routes/file.js +++ b/web/routes/file.js @@ -37,7 +37,6 @@ export function* get() { where: { id: this.params.id, name: this.params.name, - status: 'active', }, }); this.assert(file, 404); @@ -104,7 +103,6 @@ export function* landing() { const file = yield models.file.findOne({ where: { id: this.params.id, - status: 'active', }, }); this.assert(file, 404); diff --git a/web/routes/user.js b/web/routes/user.js index ac414dc..35cd59a 100644 --- a/web/routes/user.js +++ b/web/routes/user.js @@ -15,13 +15,14 @@ export function* signin() { this.statsd.incr('auth.attempt', 1); this.assertCSRF(this.request.body); const user = yield authenticate.call(this, this.request.body.email, this.request.body.password); + if (!user) { this.statsd.incr('auth.failure', 1); yield this.render('signin', { error: 'Invalid login details', csrf: this.csrf }); return; } else if (user.activationCode) { yield this.render('signin', { - error: 'Your account hasn\'t been activated yet. Check your for an activation email.', + error: 'Your account hasn\'t been activated yet. Check for an activation email.', csrf: this.csrf, }); return; @@ -83,11 +84,14 @@ export function* forgot() { } this.assertCSRF(this.request.body); const user = yield validateResetToken(token); - yield updatePassword(user.id, this.request.body.password); - yield models.reset.deleteById(token); - yield setupSession(this, user); - this.statsd.incr('auth.reset.success', 1); - this.redirect('/'); + if (user) { + yield updatePassword(user.userId, this.request.body.password); + const reset = yield models.reset.findById(token); + //reset.destroy(); + yield setupSession.call(this, user); + this.statsd.incr('auth.reset.success', 1); + this.redirect('/'); + } } else if (token) { const tokenUser = yield validateResetToken(token); if (!tokenUser) { From c7c8b3621eafa501e479dadbfd8c40d25f2547ee Mon Sep 17 00:00:00 2001 From: Jonathan Cremin Date: Sun, 7 Aug 2016 17:49:27 +0100 Subject: [PATCH 4/4] Fix user mapping for migration --- migrate/migrate-activations.js | 4 ++-- migrate/migrate-files.js | 5 ++++- migrate/migrate-users.js | 4 ++-- models/file.js | 1 + models/user.js | 2 +- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/migrate/migrate-activations.js b/migrate/migrate-activations.js index be6adbd..5e5e008 100644 --- a/migrate/migrate-activations.js +++ b/migrate/migrate-activations.js @@ -50,7 +50,7 @@ co(function *sync() { continue; } - const oldId = user._id.toString(); + const mongoId = user._id.toString(); const newUser = yield models.user.create({ email: user.email, @@ -62,7 +62,7 @@ co(function *sync() { deletedAt: user.status === 'deleted' ? new Date().getTime() : null, createdAt: user.createdAt, updatedAt: user.createdAt, - oldId, + mongoId, }); yield newUser.save({ silent: true }); } diff --git a/migrate/migrate-files.js b/migrate/migrate-files.js index e1e6424..9067b24 100644 --- a/migrate/migrate-files.js +++ b/migrate/migrate-files.js @@ -18,7 +18,7 @@ co(function *sync() { const userIds = {}; debug('remap'); for (const user of users) { - userIds[user._id] = user.id; + userIds[user.mongoId] = user.id; } debug('remap done'); let files; @@ -48,6 +48,8 @@ co(function *sync() { const processed = file.status !== 'uploading'; const accessedAt = file.last_accessed ? new Date(file.last_accessed * 1000) : null; + const mongoId = file._id.toString(); + yield models.file.upsert({ id: file._id.toString(), name: file.file_name, @@ -68,6 +70,7 @@ co(function *sync() { md5: file.md5, malwarePositives: file.virustotal && file.virustotal.positives > 0 ? file.virustotal.positives : null, + mongoId, }, { /* logging: false */ }); } diff --git a/migrate/migrate-users.js b/migrate/migrate-users.js index e036dae..88dc3da 100644 --- a/migrate/migrate-users.js +++ b/migrate/migrate-users.js @@ -50,7 +50,7 @@ co(function *sync() { continue; } - const oldId = user._id.toString(); + const mongoId = user._id.toString(); const newUser = yield models.user.create({ email: user.email, @@ -62,7 +62,7 @@ co(function *sync() { deletedAt: user.status === 'deleted' ? new Date().getTime() : null, createdAt: user.createdAt, updatedAt: user.createdAt, - oldId, + mongoId, }, { include: [models.activation], }); diff --git a/models/file.js b/models/file.js index 566d1e3..504c438 100644 --- a/models/file.js +++ b/models/file.js @@ -21,6 +21,7 @@ export default function (sequelize, DataTypes) { legacyId: DataTypes.STRING(12), // eslint-disable-line new-cap md5: DataTypes.STRING(32), // eslint-disable-line new-cap malwarePositives: DataTypes.INTEGER, + mongoId: DataTypes.STRING, }, { paranoid: true, indexes: [ diff --git a/models/user.js b/models/user.js index ff3bb7e..1b97d96 100644 --- a/models/user.js +++ b/models/user.js @@ -8,7 +8,7 @@ export default function (sequelize, DataTypes) { activated: DataTypes.BOOLEAN, banned: DataTypes.BOOLEAN, deletedAt: DataTypes.DATE, - oldId: DataTypes.STRING, + mongoId: DataTypes.STRING, }, { paranoid: true, indexes: [