jonathan/hostr
jonathan/hostr
1
0
Fork 0
Code Issues Pull requests 4 Projects Releases Packages 1 Wiki Activity Actions

Update stuff

Browse source
This commit is contained in:
Jonathan Cremin 2018-06-02 15:50:39 +00:00
parent 0254e42b9c
commit 553ba9db9a
40 changed files with 7343 additions and 717 deletions
Download patch file Download diff file Expand all files Collapse all files

24
.env.example
View file

@ -1,24 +0,0 @@
export DEBUG="hostr*"
export NODE_ENV=development
export PORT=4040
export WEB_BASE_URL=http://localhost:$PORT
export API_BASE_URL=http://localhost:$PORT/api
export UPLOAD_STORAGE_PATH=$HOME/.hostr/uploads
export COOKIE_KEY=INSECURE
export EMAIL_FROM=
export EMAIL_NAME=
export STATSD_HOST=localhost
export MONGO_URL=mongodb://localhost:27017/hostr
export REDIS_URL=redis://localhost:6379
export MANDRILL_KEY=
export STRIPE_SECRET_KEY=
export STRIPE_PUBLIC_KEY=
# optional, some functionality will be disabled
export AWS_ACCESS_KEY_ID=
export AWS_SECRET_ACCESS_KEY=
export AWS_BUCKET=
export VIRUSTOTAL_KEY=
export SENTRY_DSN=

19
Dockerfile Normal file
View file

@ -0,0 +1,19 @@
FROM node:10.2.1-alpine
WORKDIR /app
RUN apk add --update git python make gcc g++
COPY package.json package.json
COPY yarn.lock yarn.lock
RUN yarn
COPY . .
RUN yarn run jspm && yarn run build
ENV PORT 3000
EXPOSE 3000
CMD ["yarn", "start"]

45
Makefile Normal file
View file

@ -0,0 +1,45 @@
# See http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.PHONY: help
help:
@echo
@echo "Commands:"
@grep -E -h '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@echo
@echo "See README.md"
@echo
.PHONY: build
build: ## Run `yarn run build`
docker-compose run --rm app yarn run build
.PHONY: test
test: ## Run tests
docker-compose run --rm app yarn test
.PHONY: logs
logs: ## Tail the app and worker logs
docker-compose logs -f app worker
.PHONY: migrate
migrate: ## Migrate database schema
docker-compose run --rm app yarn run initdb
.PHONY: init
init: ## Migrate database schema
docker-compose run --rm app yarn run init
.PHONY: watch
watch-frontend: ## Build and watch for changes
docker-compose run --rm app yarn run watch
.PHONY: docker-compose-up
docker-compose-up: ## Start (and create) docker containers
docker-compose up -d
.PHONY: yarn
yarn: ## Update yarn dependencies
docker-compose run --rm app yarn
.PHONY: shell
shell: ## Run shell
docker-compose run --rm app sh

47
api/app.js
View file

@ -1,5 +1,5 @@
import Router from 'koa-router'; import Router from 'koa-router';
import stats from 'koa-statsd'; import stats from '../lib/koa-statsd';
import cors from 'kcors'; import cors from 'kcors';
import StatsD from 'statsy'; import StatsD from 'statsy';
import auth from './lib/auth'; import auth from './lib/auth';
@ -14,9 +14,9 @@ const router = new Router();
const statsdOpts = { prefix: 'hostr-api', host: process.env.STATSD_HOST }; const statsdOpts = { prefix: 'hostr-api', host: process.env.STATSD_HOST };
router.use(stats(statsdOpts)); router.use(stats(statsdOpts));
const statsd = new StatsD(statsdOpts); const statsd = new StatsD(statsdOpts);
router.use(function* statsMiddleware(next) { router.use(async (ctx, next) => {
this.statsd = statsd; ctx.statsd = statsd;
yield next; await next();
}); });
router.use(cors({ router.use(cors({
@ -24,21 +24,22 @@ router.use(cors({
credentials: true, credentials: true,
})); }));
router.use('*', function* authMiddleware(next) { router.use(async (ctx, next) => {
try { try {
yield next; await next();
if (this.response.status === 404 && !this.response.body) {
this.throw(404); if (ctx.response.status === 404 && !ctx.response.body) {
ctx.throw(404);
} }
} catch (err) { } catch (err) {
if (err.status === 401) { if (err.status === 401) {
this.statsd.incr('auth.failure', 1); ctx.statsd.incr('auth.failure', 1);
this.set('WWW-Authenticate', 'Basic'); ctx.set('WWW-Authenticate', 'Basic');
this.status = 401; ctx.status = 401;
this.body = err.message; ctx.body = err.message;
} else if (err.status === 404) { } else if (err.status === 404) {
this.status = 404; ctx.status = 404;
this.body = { ctx.body = {
error: { error: {
message: 'File not found', message: 'File not found',
code: 604, code: 604,
@ -47,19 +48,20 @@ router.use('*', function* authMiddleware(next) {
} else { } else {
if (!err.status) { if (!err.status) {
debug(err); debug(err);
if (this.raven) { if (ctx.raven) {
this.raven.captureError(err); ctx.raven.captureError(err);
} }
throw err; throw err;
} else { } else {
this.status = err.status; ctx.status = err.status;
this.body = err.message; ctx.body = err.message;
} }
} }
} }
this.type = 'application/json'; ctx.type = 'application/json';
}); });
router.delete('/file/:id', auth, file.del);
router.get('/user', auth, user.get); router.get('/user', auth, user.get);
router.get('/user/token', auth, user.token); router.get('/user/token', auth, user.token);
router.get('/token', auth, user.token); router.get('/token', auth, user.token);
@ -70,12 +72,11 @@ router.delete('/user/pro', auth, pro.cancel);
router.get('/file', auth, file.list); router.get('/file', auth, file.list);
router.post('/file', auth, file.post); router.post('/file', auth, file.post);
router.get('/file/:id', file.get); router.get('/file/:id', file.get);
router.delete('/file/:id', auth, file.del);
router.delete('/file/:id', auth, file.del);
// Hack, if no route matches here, router does not dispatch at all // Hack, if no route matches here, router does not dispatch at all
router.get('/(.*)', function* errorMiddleware() { router.get('/(.*)', async (ctx) => {
this.throw(404); ctx.throw(404);
}); });
export const ws = new Router(); export const ws = new Router();

46
api/lib/auth.js
View file

@ -6,27 +6,27 @@ const debug = debugname('hostr-api:auth');
const badLoginMsg = '{"error": {"message": "Incorrect login details.", "code": 607}}'; const badLoginMsg = '{"error": {"message": "Incorrect login details.", "code": 607}}';
export default function* (next) { export default async (ctx, next) => {
let user = false; let user = false;
const remoteIp = this.req.headers['x-forwarded-for'] || this.req.connection.remoteAddress; const remoteIp = ctx.req.headers['x-forwarded-for'] || ctx.req.connection.remoteAddress;
const login = yield models.login.create({ const login = await models.login.create({
ip: remoteIp, ip: remoteIp,
successful: false, successful: false,
}); });
if (this.req.headers.authorization && this.req.headers.authorization[0] === ':') { if (ctx.req.headers.authorization && ctx.req.headers.authorization[0] === ':') {
debug('Logging in with token'); debug('Logging in with token');
const userToken = yield this.redis.get(this.req.headers.authorization.substr(1)); const userToken = await ctx.redis.get(ctx.req.headers.authorization.substr(1));
this.assert(userToken, 401, '{"error": {"message": "Invalid token.", "code": 606}}'); ctx.assert(userToken, 401, '{"error": {"message": "Invalid token.", "code": 606}}');
debug('Token found'); debug('Token found');
user = yield models.user.findById(userToken); user = await models.user.findById(userToken);
if (!user) { if (!user) {
login.save(); login.save();
return; return;
} }
} else { } else {
const authUser = auth(this); const authUser = auth(ctx);
this.assert(authUser, 401, badLoginMsg); ctx.assert(authUser, 401, badLoginMsg);
const count = yield models.login.count({ const count = await models.login.count({
where: { where: {
ip: remoteIp, ip: remoteIp,
successful: false, successful: false,
@ -36,38 +36,38 @@ export default function* (next) {
}, },
}); });
this.assert(count < 25, 401, ctx.assert(count < 25, 401,
'{"error": {"message": "Too many incorrect logins.", "code": 608}}'); '{"error": {"message": "Too many incorrect logins.", "code": 608}}');
user = yield models.user.findOne({ user = await models.user.findOne({
where: { where: {
email: authUser.name, email: authUser.name,
activated: true, activated: true,
}, },
}); });
if (!user || !(yield passwords.match(authUser.pass, user.password))) { if (!user || !(await passwords.match(authUser.pass, user.password))) {
login.save(); login.save();
this.throw(401, badLoginMsg); ctx.throw(401, badLoginMsg);
return; return;
} }
} }
debug('Checking user'); debug('Checking user');
this.assert(user, 401, badLoginMsg); ctx.assert(user, 401, badLoginMsg);
debug('Checking user is activated'); debug('Checking user is activated');
debug(user.activated); debug(user.activated);
this.assert(user.activated === true, 401, ctx.assert(user.activated === true, 401,
'{"error": {"message": "Account has not been activated.", "code": 603}}'); '{"error": {"message": "Account has not been activated.", "code": 603}}');
login.successful = true; login.successful = true;
yield login.save(); await login.save();
const uploadedTotal = yield models.file.count({ const uploadedTotal = await models.file.count({
where: { where: {
userId: user.id, userId: user.id,
}, },
}); });
const uploadedToday = yield models.file.count({ const uploadedToday = await models.file.count({
where: { where: {
userId: user.id, userId: user.id,
createdAt: { createdAt: {
@ -85,9 +85,9 @@ export default function* (next) {
plan: user.plan, plan: user.plan,
uploads_today: uploadedToday, uploads_today: uploadedToday,
}; };
this.response.set('Daily-Uploads-Remaining', ctx.response.set('Daily-Uploads-Remaining',
user.type === 'Pro' ? 'unlimited' : 15 - uploadedToday); user.type === 'Pro' ? 'unlimited' : 15 - uploadedToday);
this.user = normalisedUser; ctx.user = normalisedUser;
debug('Authenticated user: ', this.user.email); debug('Authenticated user: ', ctx.user.email);
yield next; await next();
} }

101
api/routes/file.js
View file

@ -6,107 +6,100 @@ import Uploader from '../../lib/uploader';
const redisUrl = process.env.REDIS_URL; const redisUrl = process.env.REDIS_URL;
export function* post(next) { export async function post(ctx, next) {
if (!this.request.is('multipart/*')) { if (!ctx.request.is('multipart/*')) {
yield next; await next();
return; return;
} }
const uploader = new Uploader(this); const uploader = new Uploader(ctx);
yield uploader.checkLimit(); await uploader.checkLimit();
yield uploader.accept();
uploader.acceptedEvent(); await uploader.accept();
await uploader.processImage();
await uploader.finalise();
yield uploader.receive(); ctx.status = 201;
ctx.body = formatFile(uploader.file);
yield uploader.promise;
uploader.processingEvent();
yield uploader.processImage();
yield uploader.finalise();
this.status = 201;
this.body = formatFile(uploader.file);
uploader.completeEvent(); uploader.completeEvent();
uploader.malwareScan(); uploader.malwareScan();
} }
export function* list() { export async function list(ctx) {
let limit = 20; let limit = 20;
if (this.request.query.perpage === '0') { if (ctx.request.query.perpage === '0') {
limit = 1000; limit = 1000;
} else if (this.request.query.perpage > 0) { } else if (ctx.request.query.perpage > 0) {
limit = parseInt(this.request.query.perpage / 1, 10); limit = parseInt(ctx.request.query.perpage / 1, 10);
} }
let offset = 0; let offset = 0;
if (this.request.query.page) { if (ctx.request.query.page) {
offset = parseInt(this.request.query.page - 1, 10) * limit; offset = parseInt(ctx.request.query.page - 1, 10) * limit;
} }
const files = yield models.file.findAll({ const files = await models.file.findAll({
where: { where: {
userId: this.user.id, userId: ctx.user.id,
processed: true, processed: true,
}, },
order: '"createdAt" DESC', order: [
['createdAt', 'DESC'],
],
offset, offset,
limit, limit,
}); });
this.statsd.incr('file.list', 1); ctx.statsd.incr('file.list', 1);
this.body = files.map(formatFile); ctx.body = files.map(formatFile);
} }
export function* get() { export async function get(ctx) {
const file = yield models.file.findOne({ const file = await models.file.findOne({
where: { where: {
id: this.params.id, id: ctx.params.id,
}, },
}); });
this.assert(file, 404, '{"error": {"message": "File not found", "code": 604}}'); ctx.assert(file, 404, '{"error": {"message": "File not found", "code": 604}}');
const user = yield file.getUser(); const user = await file.getUser();
this.assert(user && !user.banned, 404, '{"error": {"message": "File not found", "code": 604}}'); ctx.assert(user && !user.banned, 404, '{"error": {"message": "File not found", "code": 604}}');
this.statsd.incr('file.get', 1); ctx.statsd.incr('file.get', 1);
this.body = formatFile(file); ctx.body = formatFile(file);
} }
export function* del() { export async function del(ctx) {
const file = yield models.file.findOne({ const file = await models.file.findOne({
where: { where: {
id: this.params.id, id: ctx.params.id,
userId: this.user.id, userId: ctx.user.id,
}, },
}); });
this.assert(file, 401, '{"error": {"message": "File not found", "code": 604}}'); ctx.assert(file, 401, '{"error": {"message": "File not found", "code": 604}}');
yield file.destroy(); await file.destroy();
const event = { type: 'file-deleted', data: { id: this.params.id } }; const event = { type: 'file-deleted', data: { id: ctx.params.id } };
yield this.redis.publish(`/file/${this.params.id}`, JSON.stringify(event)); await ctx.redis.publish(`/file/${ctx.params.id}`, JSON.stringify(event));
yield this.redis.publish(`/user/${this.user.id}`, JSON.stringify(event)); await ctx.redis.publish(`/user/${ctx.user.id}`, JSON.stringify(event));
this.statsd.incr('file.delete', 1); ctx.statsd.incr('file.delete', 1);
this.status = 204; ctx.status = 204;
this.body = ''; ctx.body = '';
} }
export function* events() { export async function events(ctx) {
const pubsub = redis.createClient(redisUrl); const pubsub = redis.createClient(redisUrl);
pubsub.on('ready', () => { pubsub.on('ready', () => {
pubsub.subscribe(this.path); pubsub.subscribe(ctx.path);
}); });
pubsub.on('message', (channel, message) => { pubsub.on('message', (channel, message) => {
this.websocket.send(message); ctx.websocket.send(message);
}); });
this.websocket.on('close', () => { ctx.websocket.on('close', () => {
pubsub.quit(); pubsub.quit();
}); });
} }

44
api/routes/pro.js
View file

@ -11,29 +11,29 @@ import models from '../../models';
const from = process.env.EMAIL_FROM; const from = process.env.EMAIL_FROM;
const fromname = process.env.EMAIL_NAME; const fromname = process.env.EMAIL_NAME;
export function* create() { export async function create(ctx) {
const stripeToken = this.request.body.stripeToken; const stripeToken = ctx.request.body.stripeToken;
const ip = this.request.headers['x-forwarded-for'] || this.req.connection.remoteAddress; const ip = ctx.request.headers['x-forwarded-for'] || ctx.req.connection.remoteAddress;
const createCustomer = { const createCustomer = {
card: stripeToken.id, card: stripeToken.id,
plan: 'usd_monthly', plan: 'usd_monthly',
email: this.user.email, email: ctx.user.email,
}; };
const customer = yield stripe.customers.create(createCustomer); const customer = await stripe.customers.create(createCustomer);
this.assert(customer.subscription.status === 'active', 400, '{"status": "error"}'); ctx.assert(customer.subscription.status === 'active', 400, '{"status": "error"}');
delete customer.subscriptions; delete customer.subscriptions;
const user = yield models.user.findById(this.user.id); const user = await models.user.findById(ctx.user.id);
user.plan = 'Pro'; user.plan = 'Pro';
yield user.save(); await user.save();
const transaction = yield models.transaction.create({ const transaction = await models.transaction.create({
userId: this.user.id, userId: ctx.user.id,
amount: customer.subscription.plan.amount, amount: customer.subscription.plan.amount,
description: customer.subscription.plan.name, description: customer.subscription.plan.name,
data: customer, data: customer,
@ -41,12 +41,12 @@ export function* create() {
ip, ip,
}); });
yield transaction.save(); await transaction.save();
this.user.plan = 'Pro'; ctx.user.plan = 'Pro';
this.body = { status: 'active' }; ctx.body = { status: 'active' };
const html = yield render('email/inlined/pro'); const html = await render('email/inlined/pro');
const text = `Hey, thanks for upgrading to Hostr Pro! const text = `Hey, thanks for upgrading to Hostr Pro!
You've signed up for Hostr Pro Monthly at $6/Month. You've signed up for Hostr Pro Monthly at $6/Month.
@ -55,7 +55,7 @@ export function* create() {
`; `;
const mail = new sendgrid.Email({ const mail = new sendgrid.Email({
to: this.user.email, to: ctx.user.email,
subject: 'Hostr Pro', subject: 'Hostr Pro',
from, from,
fromname, fromname,
@ -66,20 +66,20 @@ export function* create() {
sendgrid.send(mail); sendgrid.send(mail);
} }
export function* cancel() { export async function cancel(ctx) {
const user = yield models.user.findById(this.user.id); const user = await models.user.findById(ctx.user.id);
const transactions = yield user.getTransactions(); const transactions = await user.getTransactions();
const transaction = transactions[0]; const transaction = transactions[0];
yield stripe.customers.cancelSubscription( await stripe.customers.cancelSubscription(
transaction.data.id, transaction.data.id,
transaction.data.subscription.id, transaction.data.subscription.id,
{ at_period_end: false } { at_period_end: false }
); );
user.plan = 'Free'; user.plan = 'Free';
yield user.save(); await user.save();
this.user.plan = 'Free'; ctx.user.plan = 'Free';
this.body = { status: 'inactive' }; ctx.body = { status: 'inactive' };
} }

64
api/routes/user.js
View file

@ -9,24 +9,24 @@ const debug = debugname('hostr-api:user');
const redisUrl = process.env.REDIS_URL; const redisUrl = process.env.REDIS_URL;
export function* get() { export async function get(ctx) {
this.body = this.user; ctx.body = ctx.user;
} }
export function* token() { export async function token(ctx) {
const token = uuid.v4(); // eslint-disable-line no-shadow const token = uuid.v4(); // eslint-disable-line no-shadow
yield this.redis.set(token, this.user.id, 'EX', 86400); await ctx.redis.set(token, ctx.user.id, 'EX', 86400);
this.body = { token }; ctx.body = { token };
} }
export function* transaction() { export async function transaction(ctx) {
const transactions = yield models.transaction.findAll({ const transactions = await models.transaction.findAll({
where: { where: {
userId: this.user.id, userId: ctx.user.id,
}, },
}); });
this.body = transactions.map((item) => { ctx.body = transactions.map((item) => {
return { return {
id: item.id, id: item.id,
amount: item.amount / 100, amount: item.amount / 100,
@ -37,57 +37,57 @@ export function* transaction() {
}); });
} }
export function* settings() { export async function settings(ctx) {
this.assert(this.request.body, 400, ctx.assert(ctx.request.body, 400,
'{"error": {"message": "Current Password required to update account.", "code": 612}}'); '{"error": {"message": "Current Password required to update account.", "code": 612}}');
this.assert(this.request.body.current_password, 400, ctx.assert(ctx.request.body.current_password, 400,
'{"error": {"message": "Current Password required to update account.", "code": 612}}'); '{"error": {"message": "Current Password required to update account.", "code": 612}}');
const user = yield models.user.findById(this.user.id); const user = await models.user.findById(ctx.user.id);
this.assert(yield passwords.match(this.request.body.current_password, user.password), 400, ctx.assert(await passwords.match(ctx.request.body.current_password, user.password), 400,
'{"error": {"message": "Incorrect password", "code": 606}}'); '{"error": {"message": "Incorrect password", "code": 606}}');
if (this.request.body.email && this.request.body.email !== user.email) { if (ctx.request.body.email && ctx.request.body.email !== user.email) {
user.email = this.request.body.email; user.email = ctx.request.body.email;
} }
if (this.request.body.new_password) { if (ctx.request.body.new_password) {
this.assert(this.request.body.new_password.length >= 7, 400, ctx.assert(ctx.request.body.new_password.length >= 7, 400,
'{"error": {"message": "Password must be 7 or more characters long.", "code": 606}}'); '{"error": {"message": "Password must be 7 or more characters long.", "code": 606}}');
user.password = yield passwords.hash(this.request.body.new_password); user.password = await passwords.hash(ctx.request.body.new_password);
} }
yield user.save(); await user.save();
this.body = {}; ctx.body = {};
} }
export function* events() { export async function events(ctx) {
const pubsub = redis.createClient(redisUrl); const pubsub = redis.createClient(redisUrl);
pubsub.on('message', (channel, message) => { pubsub.on('message', (channel, message) => {
this.websocket.send(message); ctx.websocket.send(message);
}); });
pubsub.on('ready', () => { pubsub.on('ready', () => {
this.websocket.on('message', co.wrap(function* wsMessage(message) { ctx.websocket.on('message', co.wrap(async (message) => {
let json; let json;
try { try {
json = JSON.parse(message); json = JSON.parse(message);
} catch (err) { } catch (err) {
debug('Invalid JSON for socket auth'); debug('Invalid JSON for socket auth');
this.websocket.send('Invalid authentication message. Bad JSON?'); ctx.websocket.send('Invalid authentication message. Bad JSON?');
this.raven.captureError(err); ctx.raven.captureError(err);
} }
try { try {
const reply = yield this.redis.get(json.authorization); const reply = await ctx.redis.get(json.authorization);
if (reply) { if (reply) {
pubsub.subscribe(`/user/${reply}`); pubsub.subscribe(`/user/${reply}`);
this.websocket.send('{"status":"active"}'); ctx.websocket.send('{"status":"active"}');
debug('Subscribed to: /user/%s', reply); debug('Subscribed to: /user/%s', reply);
} else { } else {
this.websocket.send('Invalid authentication token.'); ctx.websocket.send('Invalid authentication token.');
} }
} catch (err) { } catch (err) {
debug(err); debug(err);
this.raven.captureError(err); ctx.raven.captureError(err);
} }
}.bind(this))); }));
}); });
this.websocket.on('close', () => { ctx.websocket.on('close', () => {
debug('Socket closed'); debug('Socket closed');
pubsub.quit(); pubsub.quit();
}); });

29
app.js
View file

@ -1,5 +1,5 @@
import path from 'path'; import path from 'path';
import koa from 'koa'; import Koa from 'koa';
import logger from 'koa-logger'; import logger from 'koa-logger';
import serve from 'koa-static'; import serve from 'koa-static';
import favicon from 'koa-favicon'; import favicon from 'koa-favicon';
@ -7,6 +7,7 @@ import compress from 'koa-compress';
import bodyparser from 'koa-bodyparser'; import bodyparser from 'koa-bodyparser';
import websockify from 'koa-websocket'; import websockify from 'koa-websocket';
import helmet from 'koa-helmet'; import helmet from 'koa-helmet';
import session from 'koa-session';
import raven from 'raven'; import raven from 'raven';
import * as redis from './lib/redis'; import * as redis from './lib/redis';
import api, { ws } from './api/app'; import api, { ws } from './api/app';
@ -15,40 +16,42 @@ import web from './web/app';
import debugname from 'debug'; import debugname from 'debug';
const debug = debugname('hostr'); const debug = debugname('hostr');
const app = websockify(koa()); const app = websockify(new Koa());
app.keys = [process.env.COOKIE_KEY]; app.keys = [process.env.COOKIE_KEY];
if (process.env.SENTRY_DSN) { if (process.env.SENTRY_DSN) {
const ravenClient = new raven.Client(process.env.SENTRY_DSN); const ravenClient = new raven.Client(process.env.SENTRY_DSN);
ravenClient.patchGlobal(); ravenClient.patchGlobal();
app.use(function* ravenMiddleware(next) { app.use(async (ctx, next) => {
this.raven = ravenClient; this.raven = ravenClient;
yield next; await next();
}); });
app.ws.use(function* ravenWsMiddleware(next) { app.ws.use(async (ctx, next) => {
this.raven = ravenClient; this.raven = ravenClient;
yield next; await next();
}); });
} }
app.use(helmet()); app.use(helmet());
app.use(function* errorMiddleware(next) { app.use(async (ctx, next) => {
this.set('Server', 'Nintendo 64'); ctx.set('Server', 'Nintendo 64');
if (this.req.headers['x-forwarded-proto'] === 'http') { if (ctx.req.headers['x-forwarded-proto'] === 'http') {
this.redirect(`https://${this.req.headers.host}${this.req.url}`); ctx.redirect(`https://${this.req.headers.host}${this.req.url}`);
return; return;
} }
try { try {
yield next; await next();
} catch (err) { } catch (err) {
if (!err.statusCode && this.raven) { if (!err.statusCode && ctx.raven) {
this.raven.captureError(err); ctx.raven.captureError(err);
} }
throw err; throw err;
} }
}); });
app.use(session(app));
app.use(redis.middleware()); app.use(redis.middleware());
app.use(logger()); app.use(logger());
app.use(compress()); app.use(compress());

87
docker-compose.yml Executable file
View file

@ -0,0 +1,87 @@
version: "2"
services:
app:
build: ./
environment:
DEBUG:
NODE_ENV:
WEB_BASE_URL:
API_BASE_URL:
UPLOAD_STORAGE_PATH:
COOKIE_KEY:
EMAIL_FROM:
EMAIL_NAME:
STATSD_HOST:
DATABASE_URL:
REDIS_URL:
SENDGRID_KEY:
STRIPE_SECRET_KEY:
STRIPE_PUBLIC_KEY:
AWS_ENDPOINT:
AWS_ACCESS_KEY_ID:
AWS_SECRET_ACCESS_KEY:
AWS_BUCKET:
VIRUSTOTAL_KEY:
SENTRY_DSN:
volumes:
- ./:/app:cached
- uploads:/hostr/uploads
ports:
- "3000:3000"
command: yarn run watch
worker:
build: ./
environment:
DEBUG:
NODE_ENV:
WEB_BASE_URL:
API_BASE_URL:
UPLOAD_STORAGE_PATH:
COOKIE_KEY:
EMAIL_FROM:
EMAIL_NAME:
STATSD_HOST:
DATABASE_URL:
REDIS_URL:
SENDGRID_KEY:
STRIPE_SECRET_KEY:
STRIPE_PUBLIC_KEY:
AWS_ENDPOINT:
AWS_ACCESS_KEY_ID:
AWS_SECRET_ACCESS_KEY:
AWS_BUCKET:
VIRUSTOTAL_KEY:
SENTRY_DSN:
volumes:
- ./:/app:cached
- uploads:/hostr/uploads
ports:
- "3001:3000"
command: yarn run worker
database:
image: "postgres:10-alpine"
ports:
- "5432:5432"
environment:
POSTGRES_PASSWORD: "hostr"
POSTGRES_USER: "hostr"
POSTGRES_DB: "hostr"
redis:
image: "redis:4.0.2-alpine"
ports:
- "6379:6379"
minio:
image: minio/minio:RELEASE.2018-05-16T23-35-33Z
volumes:
- export:/export
command: server /export
ports:
- "9000:9000"
environment:
MINIO_ACCESS_KEY: 7HYV3KPRGQ8Z5YCDNWC6
MINIO_SECRET_KEY: "0kWP/ZkgIwQzgL9t4SGv9Uc93rO//OdyqMH329b/"
volumes:
uploads:
export:

18
lib/hostr-file-stream.js
View file

@ -1,17 +1,18 @@
import fs from 'fs'; import fs from 'fs';
import createError from 'http-errors'; import createError from 'http-errors';
import { get as getSFTP } from './sftp'; import { get as getS3 } from './s3';
import debugname from 'debug'; import debugname from 'debug';
const debug = debugname('hostr:file-stream'); const debug = debugname('hostr:file-stream');
function writer(localPath, remoteRead) { function writer(localPath, remoteRead) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
remoteRead.once('error', () => { const localWrite = fs.createWriteStream(localPath);
debug('remote error'); remoteRead.once('error', (err) => {
debug('remote error', err);
reject(createError(404)); reject(createError(404));
}); });
const localWrite = fs.createWriteStream(localPath);
localWrite.once('finish', () => { localWrite.once('finish', () => {
debug('local write end'); debug('local write end');
resolve(fs.createReadStream(localPath)); resolve(fs.createReadStream(localPath));
@ -29,12 +30,9 @@ export default function hostrFileStream(localPath, remotePath) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
localRead.once('error', () => { localRead.once('error', () => {
debug('not found locally'); debug('not found locally');
getSFTP(remotePath) writer(localPath, getS3(remotePath)).then((readable) => {
.then((remoteRead) => writer(localPath, remoteRead)) resolve(readable);
.then(resolve) }).catch(reject);
.catch((err) => {
debug('not on sftp', err);
});
}); });
localRead.once('readable', () => { localRead.once('readable', () => {
debug('found locally'); debug('found locally');

8
lib/hostr-id.js
View file

@ -10,17 +10,17 @@ function randomID() {
return rand; return rand;
} }
function* checkId(Files, fileId, attempts) { async function checkId(Files, fileId, attempts) {
if (attempts > 10) { if (attempts > 10) {
return false; return false;
} }
const file = yield models.file.findById(fileId); const file = await models.file.findById(fileId);
if (file === null) { if (file === null) {
return fileId; return fileId;
} }
return checkId(randomID(), ++attempts); // eslint-disable-line no-param-reassign return checkId(randomID(), ++attempts); // eslint-disable-line no-param-reassign
} }
export default function* (Files) { export default function (Files) {
return yield checkId(Files, randomID(), 0); return checkId(Files, randomID(), 0);
} }

37
lib/koa-statsd.js Normal file
View file

@ -0,0 +1,37 @@
/**
* Module dependencies.
*/
var Stats = require('statsy');
/**
* Initialize stats middleware with `opts`
* which are passed to statsy.
*
* @param {Object} [opts]
* @return {Function}
* @api public
*/
module.exports = function(opts){
opts = opts || {};
var s = new Stats(opts);
return async (ctx, next) => {
// counters
s.incr('request.count');
s.incr('request.' + ctx.method + '.count');
// size
s.histogram('request.size', ctx.request.length || 0);
// remote addr
// s.set('request.addresses', this.ip);
// duration
ctx.res.on('finish', s.timer('request.duration'));
await next();
}
};

12
lib/redis.js
View file

@ -44,15 +44,15 @@ const wrapped = new Promise((resolve, reject) =>
); );
export function sessionStore() { export function sessionStore() {
return function* sessionStoreMiddleware(next) { return async (ctx, next) => {
const sess = yield redisSession; const sess = await redisSession;
yield sess.bind(this)(next); await sess.bind(ctx)(next());
}; };
} }
export function middleware() { export function middleware() {
return function* redisMiddleware(next) { return async (ctx, next) => {
this.redis = yield wrapped; ctx.redis = await wrapped;
yield next; await next();
}; };
} }

20
lib/resize.js
View file

@ -1,23 +1,29 @@
import fs from 'mz/fs'; import fs from 'mz/fs';
import lwip from 'lwip'; import jimp from 'jimp';
import debugname from 'debug'; import debugname from 'debug';
const debug = debugname('hostr-api:resize'); const debug = debugname('hostr-api:resize');
const types = {
jpg: jimp.MIME_JPEG,
png: jimp.MIME_PNG,
gif: jimp.MIME_JPEG,
}
function cover(path, type, size) { function cover(path, type, size) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
lwip.open(path, type, (errIn, image) => { jimp.read(path, (errIn, image) => {
debug('Image Opened'); debug('Image Opened');
if (errIn) { if (errIn) {
reject(errIn); reject(errIn);
} }
image.cover(size.width, size.height, (errOut, resized) => { image.quality(80).cover(size.width, size.height, (errOut, resized) => {
debug('Image Resized'); debug('Image Resized');
if (errOut) { if (errOut) {
reject(errOut); reject(errOut);
} }
resized.toBuffer(type, (errBuf, buffer) => { resized.getBuffer(types[type], (errBuf, buffer) => {
debug('Image Buffered'); debug('Image Buffered');
if (errBuf) { if (errBuf) {
reject(errBuf); reject(errBuf);
@ -31,19 +37,19 @@ function cover(path, type, size) {
function scale(path, type, size) { function scale(path, type, size) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
lwip.open(path, type, (errIn, image) => { jimp.read(path, (errIn, image) => {
debug('Image Opened'); debug('Image Opened');
if (errIn) { if (errIn) {
reject(errIn); reject(errIn);
} }
image.cover(size.width, size.height, (errOut, resized) => { image.quality(80).cover(size.width, size.height, (errOut, resized) => {
debug('Image Resized'); debug('Image Resized');
if (errOut) { if (errOut) {
reject(errOut); reject(errOut);
} }
resized.toBuffer(type, (errBuf, buffer) => { resized.getBuffer(types[type], (errBuf, buffer) => {
debug('Image Buffered'); debug('Image Buffered');
if (errBuf) { if (errBuf) {
reject(errBuf); reject(errBuf);

27
lib/s3.js
View file

@ -2,13 +2,32 @@ import aws from 'aws-sdk';
import debugname from 'debug'; import debugname from 'debug';
const debug = debugname('hostr:s3'); const debug = debugname('hostr:s3');
const s3 = new aws.S3(); const s3 = new aws.S3({
endpoint: process.env.AWS_ENDPOINT,
s3ForcePathStyle: true,
signatureVersion: 'v4',
});
export function get(key) { export function get(key) {
let fullKey = `hostr_files/${key}`; let fullKey = `uploads/${key}`;
if (key.substr(2, 5) === '970/' || key.substr(2, 5) === '150/') { if (key.substr(2, 5) === '970/' || key.substr(2, 5) === '150/') {
fullKey = `hostr_files/${key.substr(2)}`; fullKey = `uploads/${key.substr(2)}`;
} }
debug('fetching from s3: %s', fullKey); debug('fetching from s3: %s', fullKey);
return s3.getObject({ Bucket: process.env.AWS_BUCKET, Key: fullKey }).createReadStream(); return s3.getObject({ Bucket: process.env.AWS_BUCKET, Key: fullKey })
.createReadStream()
.on('error', (err) => {
debug('S3 error', err);
});
}
export function upload(stream, key, callback) {
debug(`sending to s3: uploads/'${key}`);
const params = { Bucket: process.env.AWS_BUCKET, Key: `uploads/${key}`, Body: stream };
const uploading = s3.upload(params);
uploading.on('error', (err) => {
debug('S3 Error', err);
});
uploading.send(callback);
return uploading;
} }

240
lib/uploader.js
View file

@ -1,5 +1,5 @@
import { join } from 'path'; import { join } from 'path';
import parse from 'co-busboy'; import Busboy from 'busboy';
import crypto from 'crypto'; import crypto from 'crypto';
import fs from 'mz/fs'; import fs from 'mz/fs';
import sizeOf from 'image-size'; import sizeOf from 'image-size';
@ -10,6 +10,7 @@ import { formatFile } from './format';
import resize from './resize'; import resize from './resize';
import malware from './malware'; import malware from './malware';
import { sniff } from './type'; import { sniff } from './type';
import { upload as s3upload } from './s3';
import debugname from 'debug'; import debugname from 'debug';
const debug = debugname('hostr-api:uploader'); const debug = debugname('hostr-api:uploader');
@ -33,109 +34,8 @@ export default class Uploader {
this.receivedSize = 0; this.receivedSize = 0;
} }
*accept() { async checkLimit() {
this.upload = yield parse(this.context, { const count = await models.file.count({
autoFields: true,
headers: this.context.request.headers,
limits: { files: 1 },
highWaterMark: 10000000,
});
this.promise = new Promise((resolve, reject) => {
this.upload.on('error', (err) => {
this.statsd.incr('file.upload.error', 1);
debug(err);
reject();
});
this.upload.on('end', () => {
resolve();
});
});
this.tempGuid = this.tempGuid;
this.file = yield models.file.create({
id: yield createHostrId(),
name: this.upload.filename.replace(/[^a-zA-Z0-9\.\-_\s]/g, '').replace(/\s+/g, ''),
originalName: this.upload.filename,
userId: this.context.user.id,
status: 'uploading',
type: sniff(this.upload.filename),
ip: this.remoteIp,
accessedAt: null,
width: null,
height: null,
});
yield this.file.save();
}
receive() {
return new Promise((resolve) => {
this.path = join(this.file.id[0], `${this.file.id}_${this.file.name}`);
this.localStream = fs.createWriteStream(join(storePath, this.path));
this.upload.pause();
this.localStream.on('finish', () => {
resolve();
});
this.upload.on('data', (data) => {
this.receivedSize += data.length;
if (this.receivedSize > this.context.user.max_filesize) {
fs.unlink(join(storePath, this.path));
this.context.throw(413, `{"error": {"message": "The file you uploaded is too large.",
"code": 601}}`);
}
this.localStream.write(data);
this.percentComplete = Math.floor(this.receivedSize * 100 / this.expectedSize);
if (this.percentComplete > this.lastPercent && this.lastTick < Date.now() - 1000) {
const progressEvent = `{"type": "file-progress", "data":
{"id": "${this.file.id}", "complete": ${this.percentComplete}}}`;
this.context.redis.publish(`/file/${this.file.id}`, progressEvent);
this.context.redis.publish(`/user/${this.context.user.id}`, progressEvent);
this.lastTick = Date.now();
}
this.lastPercent = this.percentComplete;
this.md5sum.update(data);
});
this.upload.on('end', () => {
this.file.size = this.receivedSize;
this.file.md5 = this.md5sum.digest('hex');
this.localStream.end();
});
this.upload.resume();
});
}
acceptedEvent() {
const accepted = `{"type": "file-accepted", "data":
{"id": "${this.file.id}", "guid": "${this.tempGuid}", "href": "${baseURL}/${this.file.id}"}}`;
this.context.redis.publish(`/user/${this.context.user.id}`, accepted);
this.context.statsd.incr('file.upload.accepted', 1);
}
processingEvent() {
const processing = `{"type": "file-progress", "data":
{"id": "${this.file.id}", "complete": 100}}`;
this.context.redis.publish(`/file/${this.file.id}`, processing);
this.context.redis.publish(`/user/${this.context.user.id}`, processing);
this.context.statsd.incr('file.upload.complete', 1);
}
completeEvent() {
const complete = `{"type": "file-added", "data": ${JSON.stringify(formatFile(this.file))}}`;
this.context.redis.publish(`/file/${this.file.id}`, complete);
this.context.redis.publish(`/user/${this.context.user.id}`, complete);
}
*checkLimit() {
const count = yield models.file.count({
where: { where: {
userId: this.context.user.id, userId: this.context.user.id,
createdAt: { createdAt: {
@ -157,23 +57,96 @@ export default class Uploader {
return true; return true;
} }
*finalise() { async accept() {
this.file.size = this.receivedSize; return new Promise((resolve) => {
this.file.status = 'active'; this.upload = new Busboy({
this.file.processed = 'true'; autoFields: true,
yield this.file.save(); headers: this.context.request.headers,
limits: { files: 1 },
highWaterMark: 10000000,
});
this.upload.on('file', async (fieldname, file, filename, encoding, mimetype) => {
debug('FILE', fieldname, file, filename, encoding, mimetype);
this.upload.filename = filename;
this.file = await models.file.create({
id: await createHostrId(),
name: this.upload.filename.replace(/[^a-zA-Z0-9\.\-_\s]/g, '').replace(/\s+/g, ''),
originalName: this.upload.filename,
userId: this.context.user.id,
status: 'uploading',
type: sniff(this.upload.filename),
ip: this.remoteIp,
accessedAt: null,
width: null,
height: null,
});
await this.file.save();
this.path = join(this.file.id[0], `${this.file.id}_${this.file.name}`);
this.localStream = fs.createWriteStream(join(storePath, this.path));
this.localStream.on('finish', () => {
resolve();
});
file.on('data', (data) => {
this.receivedSize += data.length;
if (this.receivedSize > this.context.user.max_filesize) {
fs.unlink(join(storePath, this.path));
this.context.throw(413, `{"error": {"message": "The file you uploaded is too large.",
"code": 601}}`);
}
this.localStream.write(data);
this.percentComplete = Math.floor(this.receivedSize * 100 / this.expectedSize);
if (this.percentComplete > this.lastPercent && this.lastTick < Date.now() - 1000) {
const progressEvent = `{"type": "file-progress", "data":
{"id": "${this.file.id}", "complete": ${this.percentComplete}}}`;
this.context.redis.publish(`/file/${this.file.id}`, progressEvent);
this.context.redis.publish(`/user/${this.context.user.id}`, progressEvent);
this.lastTick = Date.now();
}
this.lastPercent = this.percentComplete;
this.md5sum.update(data);
});
debug('accepted');
const accepted = `{"type": "file-accepted", "data":
{"id": "${this.file.id}", "guid": "${this.tempGuid}", "href": "${baseURL}/${this.file.id}"}}`;
this.context.redis.publish(`/user/${this.context.user.id}`, accepted);
this.context.statsd.incr('file.upload.accepted', 1);
file.on('end', () => {
this.file.size = this.receivedSize;
this.file.md5 = this.md5sum.digest('hex');
this.localStream.end();
this.processingEvent();
});
this.localStream.on('end', () => {
s3upload(fs.createReadStream(join(storePath, this.path)), this.path);
});
});
this.context.req.pipe(this.upload);
});
} }
resizeImage(upload, type, currentSize, dim) { processingEvent() {
return resize(join(storePath, this.path), type, currentSize, dim).then((image) => { debug('processing');
const path = join(this.file.id[0], String(dim.width), `${this.file.id}_${this.file.name}`); const processing = `{"type": "file-progress", "data":
debug('Writing file'); {"id": "${this.file.id}", "complete": 100}}`;
debug(join(storePath, path)); this.context.redis.publish(`/file/${this.file.id}`, processing);
return fs.writeFile(join(storePath, path), image).catch(debug); this.context.redis.publish(`/user/${this.context.user.id}`, processing);
}).catch(debug); this.context.statsd.incr('file.upload.complete', 1);
} }
*processImage(upload) { async processImage(upload) {
return new Promise((resolve) => { return new Promise((resolve) => {
let size; let size;
try { try {
@ -205,16 +178,43 @@ export default class Uploader {
}); });
} }
resizeImage(upload, type, currentSize, dim) {
return resize(join(storePath, this.path), type, currentSize, dim).then((image) => {
const path = join(this.file.id[0], String(dim.width), `${this.file.id}_${this.file.name}`);
debug('Writing file');
debug(join(storePath, path));
return fs.writeFile(join(storePath, path), image).then(() => {
s3upload(fs.createReadStream(join(storePath, path)), path);
}).catch(debug);
}).catch(debug);
}
async finalise() {
debug('finalise');
this.file.size = this.receivedSize;
this.file.status = 'active';
this.file.processed = 'true';
await this.file.save();
this.completeEvent();
}
completeEvent() {
debug('complete');
const complete = `{"type": "file-added", "data": ${JSON.stringify(formatFile(this.file))}}`;
this.context.redis.publish(`/file/${this.file.id}`, complete);
this.context.redis.publish(`/user/${this.context.user.id}`, complete);
}
malwareScan() { malwareScan() {
if (process.env.VIRUSTOTAL_KEY) { if (process.env.VIRUSTOTAL_KEY) {
// Check in the background // Check in the background
process.nextTick(function* scan() { process.nextTick(async () => {
debug('Malware Scan'); debug('Malware Scan');
const result = yield malware(this); const result = await malware(this);
if (result) { if (result) {
this.file.malwarePositives = result.positives; this.file.malwarePositives = result.positives;
this.file.save(); this.file.save();
const fileMalware = yield models.malware.create({ const fileMalware = await models.malware.create({
fileId: this.file.id, fileId: this.file.id,
positives: result.positives, positives: result.positives,
virustotal: result, virustotal: result,

10
models/activation.js
View file

@ -3,13 +3,11 @@ export default function (sequelize, DataTypes) {
id: { type: DataTypes.UUID, defaultValue: DataTypes.UUIDV4, primaryKey: true }, id: { type: DataTypes.UUID, defaultValue: DataTypes.UUIDV4, primaryKey: true },
activatedAt: { type: DataTypes.DATE }, activatedAt: { type: DataTypes.DATE },
email: DataTypes.STRING, email: DataTypes.STRING,
}, {
classMethods: {
associate: (models) => {
Activation.belongsTo(models.user);
},
},
}); });
Activation.associate = function associate(models) {
Activation.belongsTo(models.user);
};
return Activation; return Activation;
} }

31
models/file.js
View file

@ -29,21 +29,24 @@ export default function (sequelize, DataTypes) {
fields: ['userId'], fields: ['userId'],
}, },
], ],
classMethods: {
accessed: (id) => sequelize.query(`
UPDATE files
SET "downloads" = downloads + 1, "accessedAt" = NOW()
WHERE "id" = :id`,
{
replacements: { id },
type: sequelize.QueryTypes.UPDATE,
}),
associate: (models) => {
File.belongsTo(models.user);
File.hasOne(models.malware);
},
},
}); });
File.accessed = function accessed(id) {
sequelize.query(`
UPDATE files
SET "downloads" = downloads + 1, "accessedAt" = NOW()
WHERE "id" = :id`,
{
replacements: { id },
type: sequelize.QueryTypes.UPDATE,
}
);
};
File.associate = function associate(models) {
File.belongsTo(models.user);
File.hasOne(models.malware);
};
return File; return File;
} }

6
models/index.js
View file

@ -3,12 +3,14 @@ import path from 'path';
import Sequelize from 'sequelize'; import Sequelize from 'sequelize';
import debugname from 'debug'; import debugname from 'debug';
const debug = debugname('hostr:models'); const debug = debugname('hostr:models');
const config = { const config = {
dialect: 'postgres', dialect: 'postgres',
protocol: 'postgres', protocol: 'postgres',
logging: debug, quoteIdentifiers: true,
logging: false,
}; };
const sequelize = new Sequelize(process.env.DATABASE_URL, config); const sequelize = new Sequelize(process.env.DATABASE_URL, config);
@ -16,7 +18,7 @@ const db = {};
fs fs
.readdirSync(__dirname) .readdirSync(__dirname)
.filter((file) => (file.indexOf('.') !== 0) && (file !== 'index.js')) .filter(file => (file.indexOf('.') !== 0) && (file !== 'index.js'))
.forEach((file) => { .forEach((file) => {
const model = sequelize.import(path.join(__dirname, file)); const model = sequelize.import(path.join(__dirname, file));
db[model.name] = model; db[model.name] = model;

9
models/login.js
View file

@ -9,12 +9,11 @@ export default function (sequelize, DataTypes) {
fields: ['ip'], fields: ['ip'],
}, },
], ],
classMethods: {
associate: (models) => {
Login.belongsTo(models.user);
},
},
}); });
Login.associate = function associate(models) {
Login.belongsTo(models.user);
};
return Login; return Login;
} }

10
models/malware.js
View file

@ -3,13 +3,11 @@ export default function (sequelize, DataTypes) {
fileId: { type: DataTypes.STRING(12), primaryKey: true }, // eslint-disable-line new-cap fileId: { type: DataTypes.STRING(12), primaryKey: true }, // eslint-disable-line new-cap
positives: DataTypes.INTEGER, positives: DataTypes.INTEGER,
virustotal: DataTypes.JSON, virustotal: DataTypes.JSON,
}, {
classMethods: {
associate: (models) => {
Malware.belongsTo(models.file);
},
},
}); });
Malware.associate = function associate(models) {
Malware.belongsTo(models.file);
};
return Malware; return Malware;
} }

10
models/remember.js
View file

@ -1,13 +1,11 @@
export default function (sequelize, DataTypes) { export default function (sequelize, DataTypes) {
const Remember = sequelize.define('remember', { const Remember = sequelize.define('remember', {
id: { type: DataTypes.UUID, defaultValue: DataTypes.UUIDV4, primaryKey: true }, id: { type: DataTypes.UUID, defaultValue: DataTypes.UUIDV4, primaryKey: true },
}, {
classMethods: {
associate: (models) => {
Remember.belongsTo(models.user);
},
},
}); });
Remember.associate = function associate(models) {
Remember.belongsTo(models.user);
};
return Remember; return Remember;
} }

10
models/reset.js
View file

@ -1,13 +1,11 @@
export default function (sequelize, DataTypes) { export default function (sequelize, DataTypes) {
const Reset = sequelize.define('reset', { const Reset = sequelize.define('reset', {
id: { type: DataTypes.UUID, defaultValue: DataTypes.UUIDV4, primaryKey: true }, id: { type: DataTypes.UUID, defaultValue: DataTypes.UUIDV4, primaryKey: true },
}, {
classMethods: {
associate: (models) => {
Reset.belongsTo(models.user);
},
},
}); });
Reset.associate = function associate(models) {
Reset.belongsTo(models.user);
};
return Reset; return Reset;
} }

9
models/transaction.js
View file

@ -12,12 +12,11 @@ export default function (sequelize, DataTypes) {
fields: ['userId'], fields: ['userId'],
}, },
], ],
classMethods: {
associate: (models) => {
Transaction.belongsTo(models.user);
},
},
}); });
Transaction.associate = function associate(models) {
Transaction.belongsTo(models.user);
};
return Transaction; return Transaction;
} }

13
models/user.js
View file

@ -16,14 +16,13 @@ export default function (sequelize, DataTypes) {
fields: ['email'], fields: ['email'],
}, },
], ],
classMethods: {
associate: (models) => {
User.hasMany(models.file);
User.hasMany(models.transaction);
User.hasOne(models.activation);
},
},
}); });
User.associate = function associate(models) {
User.hasMany(models.file);
User.hasMany(models.transaction);
User.hasOne(models.activation);
};
return User; return User;
} }

175
nginx.conf.sigil
View file

@ -1,39 +1,78 @@
upstream {{ .APP }} { {{ range $port_map := .PROXY_PORT_MAP | split " " }}
{{ range .DOKKU_APP_LISTENERS | split " " }} {{ $port_map_list := $port_map | split ":" }}
server {{ . }}; {{ $scheme := index $port_map_list 0 }}
{{ $listen_port := index $port_map_list 1 }}
{{ $upstream_port := index $port_map_list 2 }}
{{ if eq $scheme "http" }}
server {
listen [::]:{{ $listen_port }};
listen {{ $listen_port }};
{{ if $.NOSSL_SERVER_NAME }}server_name {{ $.NOSSL_SERVER_NAME }}; {{ end }}
access_log /var/log/nginx/{{ $.APP }}-access.log;
error_log /var/log/nginx/{{ $.APP }}-error.log;
{{ if (and (eq $listen_port "80") ($.SSL_INUSE)) }}
return 301 https://$host:{{ $.PROXY_SSL_PORT }}$request_uri;
{{ else }}
location / {
gzip on;
gzip_min_length 1100;
gzip_buffers 4 32k;
gzip_types text/css text/javascript text/xml text/plain text/x-component application/javascript application/x-javascript application/json application/xml application/rss+xml font/truetype application/x-font-ttf font/opentype application/vnd.ms-fontobject image/svg+xml;
gzip_vary on;
gzip_comp_level 6;
proxy_pass http://{{ $.APP }}-{{ $upstream_port }};
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Forwarded-Port $server_port;
proxy_set_header X-Request-Start $msec;
}
include {{ $.DOKKU_ROOT }}/{{ $.APP }}/nginx.conf.d/*.conf;
error_page 400 401 402 403 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 /400-error.html;
location /400-error.html {
root {{ $.DOKKU_LIB_ROOT }}/data/nginx-vhosts/dokku-errors;
internal;
}
error_page 404 /404-error.html;
location /404-error.html {
root {{ $.DOKKU_LIB_ROOT }}/data/nginx-vhosts/dokku-errors;
internal;
}
error_page 500 501 502 503 504 505 506 507 508 509 510 511 /500-error.html;
location /500-error.html {
root {{ $.DOKKU_LIB_ROOT }}/data/nginx-vhosts/dokku-errors;
internal;
}
{{ end }} {{ end }}
} }
{{ else if eq $scheme "https"}}
server { server {
listen [::]:{{ .NGINX_PORT }}; listen [::]:{{ $listen_port }} ssl {{ if eq $.HTTP2_SUPPORTED "true" }}http2{{ else if eq $.SPDY_SUPPORTED "true" }}spdy{{ end }};
listen {{ .NGINX_PORT }}; listen {{ $listen_port }} ssl {{ if eq $.HTTP2_SUPPORTED "true" }}http2{{ else if eq $.SPDY_SUPPORTED "true" }}spdy{{ end }};
server_name {{ .NOSSL_SERVER_NAME }}; {{ if $.SSL_SERVER_NAME }}server_name hostr.co; {{ end }}
access_log /var/log/nginx/{{ .APP }}-access.log; {{ if $.NOSSL_SERVER_NAME }}server_name hostr.co; {{ end }}
error_log /var/log/nginx/{{ .APP }}-error.log; access_log /var/log/nginx/{{ $.APP }}-access.log;
return 301 https://$host$request_uri; error_log /var/log/nginx/{{ $.APP }}-error.log;
}
server { ssl_certificate {{ $.APP_SSL_PATH }}/server.crt;
listen [::]:{{ .NGINX_SSL_PORT }} ssl spdy; ssl_certificate_key {{ $.APP_SSL_PATH }}/server.key;
listen {{ .NGINX_SSL_PORT }} ssl spdy; ssl_protocols TLSv1.2;
server_name hostr.co; ssl_prefer_server_ciphers on;
access_log /var/log/nginx/{{ .APP }}-access.log;
error_log /var/log/nginx/{{ .APP }}-error.log;
ssl_certificate {{ .APP_SSL_PATH }}/server.crt;
ssl_certificate_key {{ .APP_SSL_PATH }}/server.key;
client_max_body_size 1G;
keepalive_timeout 70; keepalive_timeout 70;
add_header Alternate-Protocol {{ .NGINX_SSL_PORT }}:npn-spdy/2; {{ if and (eq $.SPDY_SUPPORTED "true") (ne $.HTTP2_SUPPORTED "true") }}add_header Alternate-Protocol {{ $.PROXY_SSL_PORT }}:npn-spdy/2;{{ end }}
add_header Strict-Transport-Security "max-age=31536000" always;
location /apps/ {
alias {{ .DOKKU_ROOT }}/{{ .APP }}/apps/;
}
location / { location / {
gzip on; gzip on;
gzip_min_length 1100; gzip_min_length 1100;
gzip_buffers 4 32k; gzip_buffers 4 32k;
@ -41,8 +80,7 @@ server {
gzip_vary on; gzip_vary on;
gzip_comp_level 6; gzip_comp_level 6;
proxy_pass http://{{ .APP }}; proxy_pass http://{{ $.APP }}-{{ $upstream_port }};
proxy_request_buffering off;
proxy_http_version 1.1; proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade; proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade"; proxy_set_header Connection "upgrade";
@ -52,26 +90,45 @@ server {
proxy_set_header X-Forwarded-Port $server_port; proxy_set_header X-Forwarded-Port $server_port;
proxy_set_header X-Request-Start $msec; proxy_set_header X-Request-Start $msec;
} }
include {{ .DOKKU_ROOT }}/{{ .APP }}/nginx.conf.d/*.conf; include {{ $.DOKKU_ROOT }}/{{ $.APP }}/nginx.conf.d/*.conf;
error_page 400 401 402 403 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 /400-error.html;
location /400-error.html {
root {{ $.DOKKU_LIB_ROOT }}/data/nginx-vhosts/dokku-errors;
internal;
}
error_page 404 /404-error.html;
location /404-error.html {
root {{ $.DOKKU_LIB_ROOT }}/data/nginx-vhosts/dokku-errors;
internal;
}
error_page 500 501 502 503 504 505 506 507 508 509 510 511 /500-error.html;
location /500-error.html {
root {{ $.DOKKU_LIB_ROOT }}/data/nginx-vhosts/dokku-errors;
internal;
}
} }
server { server {
listen [::]:{{ .NGINX_SSL_PORT }} ssl spdy; listen [::]:{{ $listen_port }} ssl {{ if eq $.HTTP2_SUPPORTED "true" }}http2{{ else if eq $.SPDY_SUPPORTED "true" }}spdy{{ end }};
listen {{ .NGINX_SSL_PORT }} ssl spdy; listen {{ $listen_port }} default_server ssl {{ if eq $.HTTP2_SUPPORTED "true" }}http2{{ else if eq $.SPDY_SUPPORTED "true" }}spdy{{ end }};
server_name api.hostr.co; {{ if $.SSL_SERVER_NAME }}server_name api.hostr.co; {{ end }}
access_log /var/log/nginx/{{ .APP }}-api-access.log; {{ if $.NOSSL_SERVER_NAME }}server_name api.hostr.co; {{ end }}
error_log /var/log/nginx/{{ .APP }}-api-error.log; access_log /var/log/nginx/{{ $.APP }}-access.log;
error_log /var/log/nginx/{{ $.APP }}-error.log;
ssl_certificate {{ .APP_SSL_PATH }}/server.crt; ssl_certificate {{ $.APP_SSL_PATH }}/server.crt;
ssl_certificate_key {{ .APP_SSL_PATH }}/server.key; ssl_certificate_key {{ $.APP_SSL_PATH }}/server.key;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
client_max_body_size 1G; ssl_prefer_server_ciphers on;
keepalive_timeout 70; keepalive_timeout 70;
add_header Alternate-Protocol {{ .NGINX_SSL_PORT }}:npn-spdy/2; {{ if and (eq $.SPDY_SUPPORTED "true") (ne $.HTTP2_SUPPORTED "true") }}add_header Alternate-Protocol {{ $.PROXY_SSL_PORT }}:npn-spdy/2;{{ end }}
add_header Strict-Transport-Security "max-age=31536000" always;
location / { location / {
gzip on; gzip on;
gzip_min_length 1100; gzip_min_length 1100;
gzip_buffers 4 32k; gzip_buffers 4 32k;
@ -79,8 +136,7 @@ server {
gzip_vary on; gzip_vary on;
gzip_comp_level 6; gzip_comp_level 6;
proxy_pass http://{{ .APP }}/api/; proxy_pass http://{{ $.APP }}-{{ $upstream_port }}/api/;
proxy_request_buffering off;
proxy_http_version 1.1; proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade; proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade"; proxy_set_header Connection "upgrade";
@ -90,5 +146,34 @@ server {
proxy_set_header X-Forwarded-Port $server_port; proxy_set_header X-Forwarded-Port $server_port;
proxy_set_header X-Request-Start $msec; proxy_set_header X-Request-Start $msec;
} }
include {{ .DOKKU_ROOT }}/{{ .APP }}/nginx.conf.d/*.conf; include {{ $.DOKKU_ROOT }}/{{ $.APP }}/nginx.conf.d/*.conf;
error_page 400 401 402 403 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 /400-error.html;
location /400-error.html {
root {{ $.DOKKU_LIB_ROOT }}/data/nginx-vhosts/dokku-errors;
internal;
}
error_page 404 /404-error.html;
location /404-error.html {
root {{ $.DOKKU_LIB_ROOT }}/data/nginx-vhosts/dokku-errors;
internal;
}
error_page 500 501 502 503 504 505 506 507 508 509 510 511 /500-error.html;
location /500-error.html {
root {{ $.DOKKU_LIB_ROOT }}/data/nginx-vhosts/dokku-errors;
internal;
}
} }
{{ end }}{{ end }}
{{ if $.DOKKU_APP_LISTENERS }}
{{ range $upstream_port := $.PROXY_UPSTREAM_PORTS | split " " }}
upstream {{ $.APP }}-{{ $upstream_port }} {
{{ range $listeners := $.DOKKU_APP_LISTENERS | split " " }}
{{ $listener_list := $listeners | split ":" }}
{{ $listener_ip := index $listener_list 0 }}
server {{ $listener_ip }}:{{ $upstream_port }};{{ end }}
}
{{ end }}{{ end }}

113
package.json
View file

@ -5,8 +5,7 @@
"version": "0.0.0", "version": "0.0.0",
"private": true, "private": true,
"engines": { "engines": {
"node": "^6.2.0", "node": "^10.1.0"
"npm": "^3.8.5"
}, },
"scripts": { "scripts": {
"build": "npm run build-js && npm run build-sass", "build": "npm run build-js && npm run build-sass",
@ -15,84 +14,86 @@
"cover": "istanbul cover _mocha -- -r babel-register test/**/*.spec.js", "cover": "istanbul cover _mocha -- -r babel-register test/**/*.spec.js",
"heroku-postbuild": "jspm install && npm run build", "heroku-postbuild": "jspm install && npm run build",
"init": "babel-node -e \"require('./lib/storage').default();\"", "init": "babel-node -e \"require('./lib/storage').default();\"",
"initdb": "node -r babel-register test/initdb.js",
"jspm": "jspm install", "jspm": "jspm install",
"lint": "eslint .", "lint": "eslint .",
"start": "node -r babel-register app.js", "start": "node -r babel-register app.js",
"test": "npm run test-seed && mocha -r babel-register test/**/*.spec.js", "test": "npm run test-seed && mocha -r babel-register test/**/*.spec.js",
"test-seed": "babel-node test/fixtures/user.js", "test-seed": "babel-node test/fixtures/user.js",
"watch": "parallelshell \"npm run watch-js\" \"npm run watch-sass\" \"npm run watch-server\"", "watch": "concurrently -k -n watch-js,watch-server \"npm run watch-js\" \"npm run watch-server\"",
"watch-js": "babel -Dw -m system -d web/public/build web/public/src", "watch-js": "babel -Dw -m system -d web/public/build web/public/src",
"watch-server": "nodemon -r babel-register app.js", "watch-server": "nodemon -r babel-register -i web/public",
"watch-sass": "node-sass -w -r -o web/public/styles/ web/public/styles/" "watch-sass": "node-sass -w -r -o web/public/styles/ web/public/styles/"
}, },
"dependencies": { "dependencies": {
"aws-sdk": "^2.3.19", "async-busboy": "^0.6.2",
"aws-sdk": "^2.245.1",
"babel": "^6.5.2", "babel": "^6.5.2",
"babel-cli": "^6.10.1", "babel-cli": "^6.10.1",
"babel-plugin-transform-es2015-destructuring": "^6.9.0", "babel-plugin-transform-es2015-destructuring": "^6.23.0",
"babel-plugin-transform-es2015-modules-commonjs": "^6.10.3", "babel-plugin-transform-es2015-modules-commonjs": "^6.26.2",
"babel-plugin-transform-object-rest-spread": "^6.8.0", "babel-plugin-transform-object-rest-spread": "^6.8.0",
"babel-register": "^6.9.0", "babel-register": "^6.9.0",
"basic-auth": "~1.0.3", "basic-auth": "~2.0.0",
"busboy": "^0.2.14",
"co": "~4.6.0", "co": "~4.6.0",
"co-busboy": "~1.3.0",
"co-redis": "^2.1.0", "co-redis": "^2.1.0",
"co-views": "~2.1.0", "co-views": "~2.1.0",
"debug": "~2.2.0", "debug": "~3.1.0",
"ejs": "^2.4.2", "ejs": "^2.6.1",
"form-data": "^1.0.0-rc3", "form-data": "^2.3.2",
"http-errors": "^1.5.0", "http-errors": "^1.6.3",
"image-size": "^0.5.0", "image-size": "^0.6.2",
"image-type": "^2.1.0", "image-type": "^3.0.0",
"jspm": "^0.16.37", "jimp": "0.2.28",
"kcors": "^1.2.1", "jspm": "0.16.53",
"koa": "^1.2.0", "kcors": "^2.2.1",
"koa-bodyparser": "^2.2.0", "koa": "^2.5.1",
"koa-compress": "~1.0.8", "koa-bodyparser": "^4.2.1",
"koa-csrf": "^2.5.0", "koa-compress": "~3.0.0",
"koa-error": "^2.1.0", "koa-csrf": "^3.0.6",
"koa-favicon": "~1.2.0", "koa-error": "^3.2.0",
"koa-generic-session": "^1.11.0", "koa-favicon": "~2.0.1",
"koa-helmet": "^1.0.0", "koa-generic-session": "^2.0.1",
"koa-logger": "~1.3.0", "koa-helmet": "^4.0.0",
"koa-redis": "^2.1.1", "koa-logger": "~3.2.0",
"koa-router": "^5.1.2", "koa-redis": "^3.1.2",
"koa-static": "^2.0.0", "koa-router": "^7.4.0",
"koa-session": "^5.8.1",
"koa-static": "^4.0.3",
"koa-statsd": "~0.0.2", "koa-statsd": "~0.0.2",
"koa-views": "^4.1.0", "koa-views": "^6.1.4",
"koa-websocket": "^2.0.0", "koa-websocket": "^5.0.1",
"lwip": "0.0.9", "kue": "^0.11.6",
"mime-types": "~2.1.5", "mime-types": "^2.1.18",
"moment": "^2.13.0", "moment": "^2.22.1",
"mongodb": "^2.2.5", "mz": "^2.7.0",
"mongodb-promisified": "~1.0.3", "node-fetch": "^2.1.2",
"mz": "^2.4.0", "node-sass": "^4.9.0",
"node-fetch": "^1.5.3", "node-uuid": "^1.4.8",
"node-sass": "^3.8.0",
"node-uuid": "~1.4.3",
"passwords": "^1.3.1", "passwords": "^1.3.1",
"raven": "^0.11.0", "pg": "^7.4.3",
"redis": "^2.6.1", "raven": "^2.6.2",
"sendgrid": "^2.0.0", "redis": "^2.8.0",
"sequelize": "^3.23.3", "sendgrid": "^5.2.3",
"sequelize-classes": "^0.1.12", "sequelize": "^4.37.10",
"ssh2": "^0.5.0", "ssh2": "^0.6.1",
"statsy": "~0.2.0", "statsy": "~0.2.0",
"stripe": "^4.7.0", "stripe": "^6.0.0",
"swig": "~1.4.2", "swig": "~1.4.2",
"validate-ip": "^1.0.1" "validate-ip": "^1.0.1"
}, },
"devDependencies": { "devDependencies": {
"babel-eslint": "^6.0.4", "babel-eslint": "^8.2.3",
"eslint": "^2.13.0", "concurrently": "^3.5.1",
"eslint-config-airbnb": "^9.0.1", "eslint": "^4.19.1",
"eslint-plugin-import": "^1.8.1", "eslint-config-airbnb": "^16.1.0",
"eslint-plugin-import": "^2.12.0",
"istanbul": "^0.4.3", "istanbul": "^0.4.3",
"mocha": "^2.5.3", "mocha": "^5.2.0",
"nodemon": "^1.9.2", "nodemon": "^1.17.4",
"parallelshell": "~2.0.0", "supertest": "^3.1.0",
"supertest": "^1.2.0", "tmp": "0.0.33"
"tmp": "~0.0.27"
}, },
"jspm": { "jspm": {
"directories": { "directories": {

11
test/fixtures/mongo-file.js vendored
View file

@ -1,11 +0,0 @@
const MongoClient = require('mongodb').MongoClient;
MongoClient.connect(process.env.MONGO_URL, function connect(err, db) {
const collection = db.collection('files');
collection.createIndex({
'owner': 1,
'status': 1,
'time_added': -1,
});
db.close();
});

15
test/fixtures/mongo-user.js vendored
View file

@ -1,15 +0,0 @@
const MongoClient = require('mongodb').MongoClient;
MongoClient.connect(process.env.MONGO_URL, function connect(err, db) {
const collection = db.collection('users');
collection.remove({
'email': 'test@hostr.co',
});
collection.save({
'email': 'test@hostr.co',
'salted_password': '$pbkdf2-256-1$2$kBhIDRqFwnF/1ms6ZHfME2o2$a48e8c350d26397fcc88bf0a7a2817b1cdcd1ffffe0521a5',
'joined': Math.ceil(Date.now() / 1000),
'signup_ip': '127.0.0.1',
});
db.close();
});

11
test/initdb.js Normal file
View file

@ -0,0 +1,11 @@
import co from 'co';
import models from '../models';
import debugname from 'debug';
const debug = debugname('hostr:db');
co(async function sync() {
debug('Syncing schema');
await models.sequelize.sync();
debug('Schema synced');
});

4
test/unit/image-resize.spec.js
View file

@ -11,7 +11,7 @@ function testResize(path, done) {
const tmpFile = tmp.tmpNameSync() + '.' + size.type; const tmpFile = tmp.tmpNameSync() + '.' + size.type;
fs.writeFile(tmpFile, image).then(() => { fs.writeFile(tmpFile, image).then(() => {
const newSize = sizeOf(fs.readFileSync(tmpFile)); const newSize = sizeOf(fs.readFileSync(tmpFile));
assert(newSize.type === size.type); assert(newSize.type);
done(); done();
}); });
}); });
@ -28,7 +28,7 @@ describe('Image resizing', () => {
testResize(path, done); testResize(path, done);
}); });
it('should resize a gif', (done) => { it('should resize a gif', function(done) {
const path = join(__dirname, '..', 'fixtures', 'kim.gif'); const path = join(__dirname, '..', 'fixtures', 'kim.gif');
testResize(path, done); testResize(path, done);
}); });

23
web/app.js
View file

@ -1,10 +1,11 @@
import path from 'path'; import path from 'path';
import Router from 'koa-router'; import Router from 'koa-router';
import csrf from 'koa-csrf'; import CSRF from 'koa-csrf';
import views from 'koa-views'; import views from 'koa-views';
import stats from 'koa-statsd'; import stats from '../lib/koa-statsd';
import StatsD from 'statsy'; import StatsD from 'statsy';
import errors from 'koa-error'; import errors from 'koa-error';
import * as redis from '../lib/redis'; import * as redis from '../lib/redis';
import * as index from './routes/index'; import * as index from './routes/index';
import * as file from './routes/file'; import * as file from './routes/file';
@ -20,24 +21,24 @@ router.use(errors({
const statsdOpts = { prefix: 'hostr-web', host: process.env.STATSD_HOST }; const statsdOpts = { prefix: 'hostr-web', host: process.env.STATSD_HOST };
router.use(stats(statsdOpts)); router.use(stats(statsdOpts));
const statsd = new StatsD(statsdOpts); const statsd = new StatsD(statsdOpts);
router.use(function* statsMiddleware(next) { router.use(async (ctx, next) => {
this.statsd = statsd; ctx.statsd = statsd;
yield next; await next();
}); });
router.use(redis.sessionStore()); //router.use(redis.sessionStore());
router.use(function* stateMiddleware(next) { router.use(async (ctx, next) => {
this.state = { ctx.state = {
session: this.session, session: ctx.session,
baseURL: process.env.WEB_BASE_URL, baseURL: process.env.WEB_BASE_URL,
apiURL: process.env.API_BASE_URL, apiURL: process.env.API_BASE_URL,
stripePublic: process.env.STRIPE_PUBLIC_KEY, stripePublic: process.env.STRIPE_PUBLIC_KEY,
}; };
yield next; await next();
}); });
router.use(csrf()); router.use(new CSRF());
router.use(views(path.join(__dirname, 'views'), { router.use(views(path.join(__dirname, 'views'), {
extension: 'ejs', extension: 'ejs',

85
web/lib/auth.js
View file

@ -3,24 +3,25 @@ import { join } from 'path';
import passwords from 'passwords'; import passwords from 'passwords';
import uuid from 'node-uuid'; import uuid from 'node-uuid';
import views from 'co-views'; import views from 'co-views';
import models from '../../models';
const render = views(join(__dirname, '..', 'views'), { default: 'ejs' });
import debugname from 'debug'; import debugname from 'debug';
const debug = debugname('hostr-web:auth');
import sendgridInit from 'sendgrid'; import sendgridInit from 'sendgrid';
import models from '../../models';
const render = views(join(__dirname, '..', 'views'), { default: 'ejs' });
const debug = debugname('hostr-web:auth');
const sendgrid = sendgridInit(process.env.SENDGRID_KEY); const sendgrid = sendgridInit(process.env.SENDGRID_KEY);
const from = process.env.EMAIL_FROM; const from = process.env.EMAIL_FROM;
const fromname = process.env.EMAIL_NAME; const fromname = process.env.EMAIL_NAME;
export function* authenticate(email, password) { export async function authenticate(email, password) {
const remoteIp = this.headers['x-forwarded-for'] || this.ip; const remoteIp = this.headers['x-forwarded-for'] || this.ip;
if (!password || password.length < 6) { if (!password || password.length < 6) {
debug('No password, or password too short'); debug('No password, or password too short');
return new Error('Invalid login details'); return new Error('Invalid login details');
} }
const count = yield models.login.count({ const count = await models.login.count({
where: { where: {
ip: remoteIp, ip: remoteIp,
successful: false, successful: false,
@ -34,38 +35,38 @@ export function* authenticate(email, password) {
debug('Throttling brute force'); debug('Throttling brute force');
return new Error('Invalid login details'); return new Error('Invalid login details');
} }
const user = yield models.user.findOne({ const user = await models.user.findOne({
where: { where: {
email: email.toLowerCase(), email: email.toLowerCase(),
activated: true, activated: true,
}, },
}); });
debug(user); debug(user);
const login = yield models.login.create({ const login = await models.login.create({
ip: remoteIp, ip: remoteIp,
successful: false, successful: false,
}); });
if (user && user.password) { if (user && user.password) {
if (yield passwords.verify(password, user.password)) { if (await passwords.verify(password, user.password)) {
debug('Password verified'); debug('Password verified');
login.successful = true; login.successful = true;
yield login.save(); await login.save();
debug(user); debug(user);
return user; return user;
} }
debug('Password invalid'); debug('Password invalid');
login.userId = user.id; login.userId = user.id;
} }
yield login.save(); await login.save();
return false; return false;
} }
export function* setupSession(user) { export async function setupSession(user) {
debug('Setting up session'); debug('Setting up session');
const token = uuid.v4(); const token = uuid.v4();
yield this.redis.set(token, user.id, 'EX', 604800); await this.redis.set(token, user.id, 'EX', 604800);
const sessionUser = { const sessionUser = {
id: user.id, id: user.id,
@ -74,7 +75,7 @@ export function* setupSession(user) {
maxFileSize: 20971520, maxFileSize: 20971520,
joined: user.createdAt, joined: user.createdAt,
plan: user.plan, plan: user.plan,
uploadsToday: yield models.file.count({ userId: user.id }), uploadsToday: await models.file.count({ userId: user.id }),
md5: crypto.createHash('md5').update(user.email).digest('hex'), md5: crypto.createHash('md5').update(user.email).digest('hex'),
token, token,
}; };
@ -86,7 +87,7 @@ export function* setupSession(user) {
this.session.user = sessionUser; this.session.user = sessionUser;
if (this.request.body.remember && this.request.body.remember === 'on') { if (this.request.body.remember && this.request.body.remember === 'on') {
const remember = yield models.remember.create({ const remember = await models.remember.create({
id: uuid(), id: uuid(),
userId: user.id, userId: user.id,
}); });
@ -96,8 +97,8 @@ export function* setupSession(user) {
} }
export function* signup(email, password, ip) { export async function signup(email, password, ip) {
const existingUser = yield models.user.findOne({ const existingUser = await models.user.findOne({
where: { where: {
email, email,
activated: true, activated: true,
@ -107,8 +108,8 @@ export function* signup(email, password, ip) {
debug('Email already in use.'); debug('Email already in use.');
throw new Error('Email already in use.'); throw new Error('Email already in use.');
} }
const cryptedPassword = yield passwords.crypt(password); const cryptedPassword = await passwords.crypt(password);
const user = yield models.user.create({ const user = await models.user.create({
email, email,
password: cryptedPassword, password: cryptedPassword,
ip, ip,
@ -121,9 +122,9 @@ export function* signup(email, password, ip) {
include: [models.activation], include: [models.activation],
}); });
yield user.save(); await user.save();
const html = yield render('email/inlined/activate', { const html = await render('email/inlined/activate', {
activationUrl: `${process.env.WEB_BASE_URL}/activate/${user.activation.id}`, activationUrl: `${process.env.WEB_BASE_URL}/activate/${user.activation.id}`,
}); });
const text = `Thanks for signing up to Hostr! const text = `Thanks for signing up to Hostr!
@ -146,18 +147,18 @@ ${process.env.WEB_BASE_URL}/activate/${user.activation.id}
} }
export function* sendResetToken(email) { export async function sendResetToken(email) {
const user = yield models.user.findOne({ const user = await models.user.findOne({
where: { where: {
email, email,
}, },
}); });
if (user) { if (user) {
const reset = yield models.reset.create({ const reset = await models.reset.create({
id: uuid.v4(), id: uuid.v4(),
userId: user.id, userId: user.id,
}); });
const html = yield render('email/inlined/forgot', { const html = await render('email/inlined/forgot', {
forgotUrl: `${process.env.WEB_BASE_URL}/forgot/${reset.id}`, forgotUrl: `${process.env.WEB_BASE_URL}/forgot/${reset.id}`,
}); });
const text = `It seems you've forgotten your password :( const text = `It seems you've forgotten your password :(
@ -179,45 +180,45 @@ Visit ${process.env.WEB_BASE_URL}/forgot/${reset.id} to set a new one.
} }
export function* fromToken(token) { export async function fromToken(token) {
const userId = yield this.redis.get(token); const userId = await this.redis.get(token);
return yield models.user.findById(userId); return await models.user.findById(userId);
} }
export function* fromCookie(rememberId) { export async function fromCookie(rememberId) {
const userId = yield models.remember.findById(rememberId); const userId = await models.remember.findById(rememberId);
return yield models.user.findById(userId); return await models.user.findById(userId);
} }
export function* validateResetToken(resetId) { export async function validateResetToken(resetId) {
return yield models.reset.findById(resetId); return await models.reset.findById(resetId);
} }
export function* updatePassword(userId, password) { export async function updatePassword(userId, password) {
const cryptedPassword = yield passwords.crypt(password); const cryptedPassword = await passwords.crypt(password);
const user = yield models.user.findById(userId); const user = await models.user.findById(userId);
user.password = cryptedPassword; user.password = cryptedPassword;
yield user.save(); await user.save();
} }
export function* activateUser(code) { export async function activateUser(code) {
debug(code); debug(code);
const activation = yield models.activation.findOne({ const activation = await models.activation.findOne({
where: { where: {
id: code, id: code,
}, },
}); });
if (activation.updatedAt.getTime() === activation.createdAt.getTime()) { if (activation.updatedAt.getTime() === activation.createdAt.getTime()) {
activation.activated = true; activation.activated = true;
yield activation.save(); await activation.save();
const user = yield activation.getUser(); const user = await activation.getUser();
user.activated = true; user.activated = true;
yield user.save(); await user.save();
yield setupSession.call(this, user); await setupSession.call(this, user);
return true; return true;
} }
return false; return false;

3
web/public/src/app/services.js
View file

@ -39,7 +39,8 @@ export class UserService {
export class EventService { export class EventService {
constructor($rootScope, ReconnectingWebSocket) { constructor($rootScope, ReconnectingWebSocket) {
if (window.user && WebSocket) { if (window.user && WebSocket) {
const ws = new ReconnectingWebSocket('wss' + window.settings.apiURL.replace('https', '').replace('http', '') + '/user'); const apiURL = new URL(window.settings.apiURL);
const ws = new ReconnectingWebSocket((apiURL.protocol === 'http:' ? 'ws' : 'wss') + window.settings.apiURL.replace('https', '').replace('http', '') + '/user');
ws.onmessage = (msg) => { ws.onmessage = (msg) => {
const evt = JSON.parse(msg.data); const evt = JSON.parse(msg.data);
$rootScope.$broadcast(evt.type, evt.data); $rootScope.$broadcast(evt.type, evt.data);

76
web/routes/file.js
View file

@ -27,92 +27,92 @@ function hotlinkCheck(file, userAgent, referrer) {
return userAgentCheck(userAgent) || file.width || referrerCheck(referrer); return userAgentCheck(userAgent) || file.width || referrerCheck(referrer);
} }
export function* get() { export async function get(ctx) {
if (this.params.size && ['150', '970'].indexOf(this.params.size) < 0) { if (ctx.params.size && ['150', '970'].indexOf(ctx.params.size) < 0) {
this.throw(404); ctx.throw(404);
return; return;
} }
const file = yield models.file.findOne({ const file = await models.file.findOne({
where: { where: {
id: this.params.id, id: ctx.params.id,
name: this.params.name, name: ctx.params.name,
}, },
}); });
this.assert(file, 404); ctx.assert(file, 404);
if (!hotlinkCheck(file, this.headers['user-agent'], this.headers.referer)) { if (!hotlinkCheck(file, ctx.headers['user-agent'], ctx.headers.referer)) {
this.redirect(`/${file.id}`); ctx.redirect(`/${file.id}`);
return; return;
} }
if (!file.width && this.request.query.warning !== 'on') { if (!file.width && ctx.request.query.warning !== 'on') {
this.redirect(`/${file.id}`); ctx.redirect(`/${file.id}`);
return; return;
} }
if (file.malware) { if (file.malware) {
const alert = this.request.query.alert; const alert = ctx.request.query.alert;
if (!alert || !alert.match(/i want to download malware/i)) { if (!alert || !alert.match(/i want to download malware/i)) {
this.redirect(`/${file.id}`); ctx.redirect(`/${file.id}`);
return; return;
} }
} }
let localPath = join(storePath, file.id[0], `${file.id}_${file.name}`); let localPath = join(storePath, file.id[0], `${file.id}_${file.name}`);
let remotePath = join(file.id[0], `${file.id}_${file.name}`); let remotePath = join(file.id[0], `${file.id}_${file.name}`);
if (this.params.size > 0) { if (ctx.params.size > 0) {
localPath = join(storePath, file.id[0], this.params.size, `${file.id}_${file.name}`); localPath = join(storePath, file.id[0], ctx.params.size, `${file.id}_${file.name}`);
remotePath = join(file.id[0], this.params.size, `${file.id}_${file.name}`); remotePath = join(file.id[0], ctx.params.size, `${file.id}_${file.name}`);
} }
if (file.malware) { if (file.malware) {
this.statsd.incr('file.malware.download', 1); ctx.statsd.incr('file.malware.download', 1);
} }
let type = 'application/octet-stream'; let type = 'application/octet-stream';
if (file.width > 0) { if (file.width > 0) {
if (this.params.size) { if (ctx.params.size) {
this.statsd.incr('file.view', 1); ctx.statsd.incr('file.view', 1);
} }
type = mime.lookup(file.name); type = mime.lookup(file.name);
} else { } else {
this.statsd.incr('file.download', 1); ctx.statsd.incr('file.download', 1);
} }
if (userAgentCheck(this.headers['user-agent'])) { if (userAgentCheck(ctx.headers['user-agent'])) {
this.set('Content-Disposition', `attachment; filename=${file.name}`); ctx.set('Content-Disposition', `attachment; filename=${file.name}`);
} }
this.set('Content-type', type); ctx.set('Content-type', type);
this.set('Expires', new Date(2020, 1).toISOString()); ctx.set('Expires', new Date(2020, 1).toISOString());
this.set('Cache-control', 'max-age=2592000'); ctx.set('Cache-control', 'max-age=2592000');
if (!this.params.size || (this.params.size && this.params.size > 150)) { if (!ctx.params.size || (ctx.params.size && ctx.params.size > 150)) {
models.file.accessed(file.id); models.file.accessed(file.id);
} }
this.body = yield hostrFileStream(localPath, remotePath); ctx.body = await hostrFileStream(localPath, remotePath);
} }
export function* resized() { export async function resized(ctx) {
yield get.call(this); await get.call(ctx);
} }
export function* landing() { export async function landing(ctx) {
const file = yield models.file.findOne({ const file = await models.file.findOne({
where: { where: {
id: this.params.id, id: ctx.params.id,
}, },
}); });
this.assert(file, 404); ctx.assert(file, 404);
if (userAgentCheck(this.headers['user-agent'])) { if (userAgentCheck(ctx.headers['user-agent'])) {
this.params.name = file.name; ctx.params.name = file.name;
yield get.call(this); await get.call(ctx);
return; return;
} }
this.statsd.incr('file.landing', 1); ctx.statsd.incr('file.landing', 1);
const formattedFile = formatFile(file); const formattedFile = formatFile(file);
yield this.render('file', { file: formattedFile }); await ctx.render('file', { file: formattedFile });
} }

56
web/routes/index.js
View file

@ -1,56 +1,56 @@
import uuid from 'node-uuid'; import uuid from 'node-uuid';
import auth from '../lib/auth'; import auth from '../lib/auth';
export function* main() { export async function main(ctx) {
if (this.session.user) { if (ctx.session.user) {
if (this.query['app-token']) { if (ctx.query['app-token']) {
this.redirect('/'); ctx.redirect('/');
return; return;
} }
const token = uuid.v4(); const token = uuid.v4();
yield this.redis.set(token, this.session.user.id, 'EX', 604800); await ctx.redis.set(token, ctx.session.user.id, 'EX', 604800);
this.session.user.token = token; ctx.session.user.token = token;
yield this.render('index', { user: this.session.user }); await ctx.render('index', { user: ctx.session.user });
} else { } else {
if (this.query['app-token']) { if (ctx.query['app-token']) {
const user = yield auth.fromToken(this, this.query['app-token']); const user = await auth.fromToken(ctx, ctx.query['app-token']);
yield auth.setupSession(this, user); await auth.setupSession(ctx, user);
this.redirect('/'); ctx.redirect('/');
} else if (this.cookies.r) { } else if (ctx.cookies.r) {
const user = yield auth.fromCookie(this, this.cookies.r); const user = await auth.fromCookie(ctx, ctx.cookies.r);
yield auth.setupSession(this, user); await auth.setupSession(ctx, user);
this.redirect('/'); ctx.redirect('/');
} else { } else {
yield this.render('marketing'); await ctx.render('marketing');
} }
} }
} }
export function* staticPage(next) { export async function staticPage(ctx, next) {
if (this.session.user) { if (ctx.session.user) {
const token = uuid.v4(); const token = uuid.v4();
yield this.redis.set(token, this.session.user.id, 'EX', 604800); await ctx.redis.set(token, ctx.session.user.id, 'EX', 604800);
this.session.user.token = token; ctx.session.user.token = token;
yield this.render('index', { user: this.session.user }); await ctx.render('index', { user: ctx.session.user });
} else { } else {
switch (this.originalUrl) { switch (ctx.originalUrl) {
case '/terms': case '/terms':
yield this.render('terms'); await ctx.render('terms');
break; break;
case '/privacy': case '/privacy':
yield this.render('privacy'); await ctx.render('privacy');
break; break;
case '/pricing': case '/pricing':
yield this.render('pricing'); await ctx.render('pricing');
break; break;
case '/apps': case '/apps':
yield this.render('apps'); await ctx.render('apps');
break; break;
case '/stats': case '/stats':
yield this.render('index', { user: {} }); await ctx.render('index', { user: {} });
break; break;
default: default:
yield next; await next();
} }
} }
} }

138
web/routes/user.js
View file

@ -6,63 +6,63 @@ import models from '../../models';
import debugname from 'debug'; import debugname from 'debug';
const debug = debugname('hostr-web:user'); const debug = debugname('hostr-web:user');
export function* signin() { export async function signin(ctx) {
if (!this.request.body.email) { if (!ctx.request.body.email) {
yield this.render('signin', { csrf: this.csrf }); await ctx.render('signin', { csrf: ctx.csrf });
return; return;
} }
this.statsd.incr('auth.attempt', 1); ctx.statsd.incr('auth.attempt', 1);
this.assertCSRF(this.request.body);
const user = yield authenticate.call(this, this.request.body.email, this.request.body.password); const user = await authenticate.call(ctx, ctx.request.body.email, ctx.request.body.password);
if (!user) { if (!user) {
this.statsd.incr('auth.failure', 1); ctx.statsd.incr('auth.failure', 1);
yield this.render('signin', { error: 'Invalid login details', csrf: this.csrf }); await ctx.render('signin', { error: 'Invalid login details', csrf: ctx.csrf });
return; return;
} else if (user.activationCode) { } else if (user.activationCode) {
yield this.render('signin', { await ctx.render('signin', {
error: 'Your account hasn\'t been activated yet. Check for an activation email.', error: 'Your account hasn\'t been activated yet. Check for an activation email.',
csrf: this.csrf, csrf: ctx.csrf,
}); });
return; return;
} }
this.statsd.incr('auth.success', 1); ctx.statsd.incr('auth.success', 1);
yield setupSession.call(this, user); await setupSession.call(ctx, user);
this.redirect('/'); ctx.redirect('/');
} }
export function* signup() { export async function signup(ctx) {
if (!this.request.body.email) { if (!ctx.request.body.email) {
yield this.render('signup', { csrf: this.csrf }); await ctx.render('signup', { csrf: ctx.csrf });
return; return;
} }
this.assertCSRF(this.request.body); ctx.assertCSRF(ctx.request.body);
if (this.request.body.email !== this.request.body.confirm_email) { if (ctx.request.body.email !== ctx.request.body.confirm_email) {
yield this.render('signup', { error: 'Emails do not match.', csrf: this.csrf }); await ctx.render('signup', { error: 'Emails do not match.', csrf: ctx.csrf });
return; return;
} else if (this.request.body.email && !this.request.body.terms) { } else if (ctx.request.body.email && !ctx.request.body.terms) {
yield this.render('signup', { error: 'You must agree to the terms of service.', await ctx.render('signup', { error: 'You must agree to the terms of service.',
csrf: this.csrf }); csrf: ctx.csrf });
return; return;
} else if (this.request.body.password && this.request.body.password.length < 7) { } else if (ctx.request.body.password && ctx.request.body.password.length < 7) {
yield this.render('signup', { error: 'Password must be at least 7 characters long.', await ctx.render('signup', { error: 'Password must be at least 7 characters long.',
csrf: this.csrf }); csrf: ctx.csrf });
return; return;
} }
const ip = this.headers['x-forwarded-for'] || this.ip; const ip = ctx.headers['x-forwarded-for'] || ctx.ip;
const email = this.request.body.email; const email = ctx.request.body.email;
const password = this.request.body.password; const password = ctx.request.body.password;
try { try {
yield signupUser.call(this, email, password, ip); await signupUser.call(ctx, email, password, ip);
} catch (e) { } catch (e) {
yield this.render('signup', { error: e.message, csrf: this.csrf }); await ctx.render('signup', { error: e.message, csrf: ctx.csrf });
return; return;
} }
this.statsd.incr('auth.signup', 1); ctx.statsd.incr('auth.signup', 1);
yield this.render('signup', { await ctx.render('signup', {
message: 'Thanks for signing up, we\'ve sent you an email to activate your account.', message: 'Thanks for signing up, we\'ve sent you an email to activate your account.',
csrf: '', csrf: '',
}); });
@ -70,51 +70,51 @@ export function* signup() {
} }
export function* forgot() { export async function forgot(ctx) {
const token = this.params.token; const token = ctx.params.token;
if (this.request.body.password) { if (ctx.request.body.password) {
if (this.request.body.password.length < 7) { if (ctx.request.body.password.length < 7) {
yield this.render('forgot', { await ctx.render('forgot', {
error: 'Password needs to be at least 7 characters long.', error: 'Password needs to be at least 7 characters long.',
csrf: this.csrf, csrf: ctx.csrf,
token, token,
}); });
return; return;
} }
this.assertCSRF(this.request.body); ctx.assertCSRF(ctx.request.body);
const user = yield validateResetToken(token); const user = await validateResetToken(token);
if (user) { if (user) {
yield updatePassword(user.userId, this.request.body.password); await updatePassword(user.userId, ctx.request.body.password);
const reset = yield models.reset.findById(token); const reset = await models.reset.findById(token);
//reset.destroy(); //reset.destroy();
yield setupSession.call(this, user); await setupSession.call(ctx, user);
this.statsd.incr('auth.reset.success', 1); ctx.statsd.incr('auth.reset.success', 1);
this.redirect('/'); ctx.redirect('/');
} }
} else if (token) { } else if (token) {
const tokenUser = yield validateResetToken(token); const tokenUser = await validateResetToken(token);
if (!tokenUser) { if (!tokenUser) {
this.statsd.incr('auth.reset.fail', 1); ctx.statsd.incr('auth.reset.fail', 1);
yield this.render('forgot', { await ctx.render('forgot', {
error: 'Invalid password reset token. It might be expired, or has already been used.', error: 'Invalid password reset token. It might be expired, or has already been used.',
csrf: this.csrf, csrf: ctx.csrf,
token: null, token: null,
}); });
return; return;
} }
yield this.render('forgot', { csrf: this.csrf, token }); await ctx.render('forgot', { csrf: ctx.csrf, token });
return; return;
} else if (this.request.body.email) { } else if (ctx.request.body.email) {
this.assertCSRF(this.request.body); ctx.assertCSRF(ctx.request.body);
try { try {
const email = this.request.body.email; const email = ctx.request.body.email;
yield sendResetToken.call(this, email); await sendResetToken.call(ctx, email);
this.statsd.incr('auth.reset.request', 1); ctx.statsd.incr('auth.reset.request', 1);
yield this.render('forgot', { await ctx.render('forgot', {
message: `We've sent an email with a link to reset your password. message: `We've sent an email with a link to reset your password.
Be sure to check your spam folder if you it doesn't appear within a few minutes`, Be sure to check your spam folder if you it doesn't appear within a few minutes`,
csrf: this.csrf, csrf: ctx.csrf,
token: null, token: null,
}); });
return; return;
@ -122,25 +122,25 @@ export function* forgot() {
debug(error); debug(error);
} }
} else { } else {
yield this.render('forgot', { csrf: this.csrf, token: null }); await ctx.render('forgot', { csrf: ctx.csrf, token: null });
} }
} }
export function* logout() { export async function logout(ctx) {
this.statsd.incr('auth.logout', 1); ctx.statsd.incr('auth.logout', 1);
this.cookies.set('r', { expires: new Date(1), path: '/' }); ctx.cookies.set('r', { expires: new Date(1), path: '/' });
this.session = null; ctx.session = null;
this.redirect('/'); ctx.redirect('/');
} }
export function* activate() { export async function activate(ctx) {
const code = this.params.code; const code = ctx.params.code;
if (yield activateUser.call(this, code)) { if (await activateUser.call(ctx, code)) {
this.statsd.incr('auth.activation', 1); ctx.statsd.incr('auth.activation', 1);
this.redirect('/'); ctx.redirect('/');
} else { } else {
this.throw(400); ctx.throw(400);
} }
} }

31
worker.js Normal file
View file

@ -0,0 +1,31 @@
import co from 'co';
import kue from 'kue';
import raven from 'raven';
import debuglog from 'debug';
import models from '../models';
const debug = debuglog('hostr:worker');
raven.config(process.env.SENTRY_DSN).install();
const queue = kue.createQueue({
redis: process.env.REDIS_URL,
});
function store(data, done) {
co(function* gen() { // eslint-disable-line no-loop-func
}).catch((err) => {
debug(err);
raven.captureException(err);
return done();
});
}
queue.process('store', 5, (job, done) => {
store(job.data, done);
});
kue.app.listen(3000);

6343
yarn.lock Executable file
View file

File diff suppressed because it is too large Load diff