Fix tests, run against ci

.envrc.example

@@ -0,0 +1,25 @@
+export DEBUG="hostr*"
+
+export NODE_ENV=development
+export PORT=4040
+export WEB_BASE_URL=http://localhost:$PORT
+export API_BASE_URL=$WEB_BASE_URL/api
+export UPLOAD_STORAGE_PATH=$HOME/.hostr/uploads
+export COOKIE_KEY=INSECURE
+export EMAIL_FROM=
+export EMAIL_NAME=
+
+export STATSD_HOST=localhost
+export DATABASE_URL=postgresql://hostr:hostr@database:5432/hostr
+export REDIS_URL=redis://localhost:6379
+export SENDGRID_KEY=
+export STRIPE_SECRET_KEY=
+export STRIPE_PUBLIC_KEY=
+
+# optional, some functionality will be disabled
+export AWS_ENDPOINT= # only for AWS-like providers, not AWS
+export AWS_ACCESS_KEY_ID=
+export AWS_SECRET_ACCESS_KEY=
+export AWS_BUCKET=
+export VIRUSTOTAL_KEY=
+export SENTRY_DSN=

.forgejo/workflows/build-image.yml

@@ -1,40 +0,0 @@
-name: ci
-
-on:
-  push:
-    branches: main
-  pull_request:
-    types: [opened, synchronize, reopened]
-jobs:
-  build-image:
-    runs-on: self-hosted
-    steps:
-      - name: Set current date as env variable
-        run: echo "NOW=$(date +'%Y%m%d-%H%M%S')" >> $GITHUB_ENV
-      - name: Fix for bad os check
-        run: echo "RUNNER_OS=Linux" >> $GITHUB_ENV
-      - name: Login to Forgejo Registry
-        uses: https://cremin.dev/actions/podman-login@v1
-        with:
-          registry: cremin.dev
-          username: ${{ github.actor }}
-          password: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
-      - name: Check out repository
-        uses: https://cremin.dev/actions/checkout@v4
-      - name: Build image
-        uses: https://cremin.dev/actions/buildah-build@v2
-        with:
-          containerfiles: ./Containerfile
-          context: ./
-          oci: true
-          layers: true
-          image: hostr
-          tags: latest ${{ github.sha }}
-      - name: Push image
-        uses: https://cremin.dev/actions/push-to-registry@v2
-        with:
-          registry: cremin.dev/jonathan
-          username: ${{ github.actor }}
-          password: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
-          image: hostr
-          tags: latest ${{ github.sha }}

.forgejo/workflows/ci.yml

@@ -0,0 +1,98 @@
+name: ci
+
+on:
+  push:
+    branches: main
+  pull_request:
+    types: [opened, synchronize, reopened]
+jobs:
+  build-image:
+    runs-on: self-hosted
+    steps:
+      - name: Set current date as env variable
+        run: echo "NOW=$(date +'%Y%m%d-%H%M%S')" >> $GITHUB_ENV
+      - name: Fix for bad os check
+        run: echo "RUNNER_OS=Linux" >> $GITHUB_ENV
+      - name: Login to Forgejo Registry
+        uses: https://cremin.dev/actions/podman-login@v1
+        with:
+          registry: cremin.dev
+          username: ${{ github.actor }}
+          password: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
+      - name: Check out repository
+        uses: https://cremin.dev/actions/checkout@v4
+      - name: Build image
+        uses: https://cremin.dev/actions/buildah-build@v2
+        with:
+          containerfiles: ./Containerfile
+          context: ./
+          oci: true
+          layers: true
+          image: hostr
+          tags: latest ${{ github.sha }}
+      - name: Push image
+        uses: https://cremin.dev/actions/push-to-registry@v2
+        with:
+          registry: cremin.dev/jonathan
+          username: ${{ github.actor }}
+          password: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
+          image: hostr
+          tags: latest ${{ github.sha }}
+  test-image:
+    runs-on: self-hosted
+    needs: build-image
+    services:
+      database:
+        image: postgres:10-alpine
+        env:
+          POSTGRES_PASSWORD: hostr
+          POSTGRES_USER: hostr
+          POSTGRES_DB: hostr
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      redis:
+        image: redis:4.0.2-alpine
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      minio:
+        image: minio/minio
+        env:
+          MINIO_ACCESS_KEY: 7HYV3KPRGQ8Z5YCDNWC6
+          MINIO_SECRET_KEY: 0kWP/ZkgIwQzgL9t4SGv9Uc93rO//OdyqMH329b/
+        ports:
+          - 9000:9000
+        options: >-
+          --health-cmd "curl -f http://localhost:9000/minio/health/live"
+          --health-interval 30s
+          --health-timeout 20s
+          --health-retries 3
+        command: server /export
+    steps:
+      - name: Check out repository
+        uses: https://cremin.dev/actions/checkout@v4
+      - name: Test image
+        env:
+          WEB_BASE_URL: http://localhost:3000
+          API_BASE_URL: http://localhost:3000/api
+          UPLOAD_STORAGE_PATH: /hostr/uploads
+          COOKIE_KEY: TESTING
+          EMAIL_FROM: jonathan@hostr.co
+          EMAIL_NAME: "Jonathan from Hostr"
+          DATABASE_URL: postgresql://hostr:hostr@database:5432/hostr
+          REDIS_URL: redis://redis:6379
+          AWS_ENDPOINT: http://minio:9000
+          AWS_ACCESS_KEY_ID: 7HYV3KPRGQ8Z5YCDNWC6
+          AWS_SECRET_ACCESS_KEY: 0kWP/ZkgIwQzgL9t4SGv9Uc93rO//OdyqMH329b/
+          AWS_BUCKET: hostr
+        run: |
+          podman run --rm --env-host -it cremin.dev/jonathan/hostr:${{ github.sha }} yarn test

.gitignore

@@ -1,4 +1,4 @@
-.env*
+.envrc
 .DS_Store
 .sass-cache/
 node_modules
@@ -7,6 +7,7 @@ jspm_packages
 npm-debug.log
 web/public/build
 web/public/styles/*.css
+web/public/styles/*.css.map
 *.gz
 minigun*.json
 test*.json

Makefile

@@ -10,36 +10,36 @@ help:
 
 .PHONY: build
 build: ## Run `yarn run build`
-	docker-compose run --rm app yarn run build
+	podman compose run --rm app yarn run build
 
 .PHONY: test
 test: ## Run tests
-	docker-compose run --rm app yarn test
+	podman compose run --rm app yarn test
 
 .PHONY: logs
 logs: ## Tail the app and worker logs
-	docker-compose logs -f app worker
+	podman compose logs -f app worker
 
 .PHONY: migrate
 migrate: ## Migrate database schema
-	docker-compose run --rm app yarn run initdb
+	podman compose run --rm app yarn run initdb
 
 .PHONY: init
 init: ## Migrate database schema
-	docker-compose run --rm app yarn run init
+	podman compose run --rm app yarn run init
 
 .PHONY: watch-frontend
 watch-frontend: ## Build and watch for changes
-	docker-compose run --rm app yarn run watch
+	podman compose run --rm app yarn run watch
 
-.PHONY: docker-compose-up
-docker-compose-up: ## Start (and create) docker containers
-	docker-compose up -d
+.PHONY: podman compose-up
+podman compose-up: ## Start (and create) docker containers
+	podman compose up -d
 
 .PHONY: yarn
 yarn: ## Update yarn dependencies
-	docker-compose run --rm app yarn
+	podman compose run --rm app yarn
 
 .PHONY: shell
 shell: ## Run shell
-	docker-compose run --rm app sh
+	podman compose run --rm app sh

web/lib/auth.js

@@ -1,29 +1,29 @@
-import crypto from 'crypto';
-import { join } from 'path';
-import passwords from 'passwords';
-import uuid from 'node-uuid';
-import views from 'co-views';
-import debugname from 'debug';
-import sendgrid from '@sendgrid/mail';
-import models from '../../models';
+import crypto from "crypto";
+import { join } from "path";
+import passwords from "passwords";
+import uuid from "node-uuid";
+import views from "co-views";
+import debugname from "debug";
+import sendgrid from "@sendgrid/mail";
+import models from "../../models";
 
-const render = views(join(__dirname, '..', 'views'), { default: 'ejs' });
-const debug = debugname('hostr-web:auth');
+const render = views(join(__dirname, "..", "views"), { default: "ejs" });
+const debug = debugname("hostr-web:auth");
 sendgrid.setApiKey(process.env.SENDGRID_KEY);
 
 const from = process.env.EMAIL_FROM;
 const fromname = process.env.EMAIL_NAME;
 
 export async function authenticate(email, password) {
-  const remoteIp = this.headers['x-forwarded-for'] || this.ip;
+  const remoteIp = this.headers["x-forwarded-for"] || this.ip;
 
   if (!password || password.length < 6) {
-    debug('No password, or password too short');
-    return new Error('Invalid login details');
+    debug("No password, or password too short");
+    return new Error("Invalid login details");
   }
   const count = await models.login.count({
     where: {
-      ip: remoteIp.split(',')[0],
+      ip: remoteIp.split(",")[0],
       successful: false,
       createdAt: {
         $gt: Math.ceil(Date.now()) - 600000,
@@ -32,8 +32,8 @@ export async function authenticate(email, password) {
   });
 
   if (count > 25) {
-    debug('Throttling brute force');
-    return new Error('Invalid login details');
+    debug("Throttling brute force");
+    return new Error("Invalid login details");
   }
   const user = await models.user.findOne({
     where: {
@@ -43,30 +43,29 @@ export async function authenticate(email, password) {
   });
 
   const login = await models.login.create({
-    ip: remoteIp.split(',')[0],
+    ip: remoteIp.split(",")[0],
     successful: false,
   });
 
   if (user && user.password) {
     login.userId = user.id;
     if (await passwords.verify(password, user.password)) {
-      debug('Password verified');
+      debug("Password verified");
       login.successful = true;
       await login.save();
       return user;
     }
-    debug('Password invalid');
+    debug("Password invalid");
   }
   await login.save();
   return false;
 }
 
-
 export async function setupSession(user) {
-  debug('Setting up session');
+  debug("Setting up session");
   const token = uuid.v4();
-  debug(user)
-  await this.redis.set(token, user.id, 'EX', 604800);
+
+  await this.redis.set(token, user.id, "EX", 604800);
 
   const sessionUser = {
     id: user.id,
@@ -76,27 +75,26 @@ export async function setupSession(user) {
     joined: user.createdAt,
     plan: user.plan,
     uploadsToday: await models.file.count({ userId: user.id }),
-    md5: crypto.createHash('md5').update(user.email).digest('hex'),
+    md5: crypto.createHash("md5").update(user.email).digest("hex"),
     token,
   };
 
-  if (sessionUser.plan === 'Pro') {
+  if (sessionUser.plan === "Pro") {
     sessionUser.maxFileSize = 524288000;
-    sessionUser.dailyUploadAllowance = 'unlimited';
+    sessionUser.dailyUploadAllowance = "unlimited";
   }
 
   this.session.user = sessionUser;
-  if (this.request.body.remember && this.request.body.remember === 'on') {
+  if (this.request.body.remember && this.request.body.remember === "on") {
     const remember = await models.remember.create({
       id: uuid(),
       userId: user.id,
     });
-    this.cookies.set('r', remember.id, { maxAge: 1209600000, httpOnly: true });
+    this.cookies.set("r", remember.id, { maxAge: 1209600000, httpOnly: true });
   }
-  debug('Session set up');
+  debug("Session set up");
 }
 
-
 export async function signup(email, password, ip) {
   const existingUser = await models.user.findOne({
     where: {
@@ -105,26 +103,29 @@ export async function signup(email, password, ip) {
     },
   });
   if (existingUser) {
-    debug('Email already in use.');
-    throw new Error('Email already in use.');
+    debug("Email already in use.");
+    throw new Error("Email already in use.");
   }
   const cryptedPassword = await passwords.crypt(password);
-  const user = await models.user.create({
-    email,
-    password: cryptedPassword,
-    ip,
-    plan: 'Free',
-    activation: {
-      id: uuid(),
+  const user = await models.user.create(
+    {
       email,
+      password: cryptedPassword,
+      ip,
+      plan: "Free",
+      activation: {
+        id: uuid(),
+        email,
+      },
     },
-  }, {
-    include: [models.activation],
-  });
+    {
+      include: [models.activation],
+    },
+  );
 
   await user.save();
 
-  const html = await render('email/inlined/activate', {
+  const html = await render("email/inlined/activate", {
     activationUrl: `${process.env.WEB_BASE_URL}/activate/${user.activation.id}`,
   });
   const text = `Thanks for signing up to Hostr!
@@ -136,18 +137,15 @@ ${process.env.WEB_BASE_URL}/activate/${user.activation.id}
 `;
   sendgrid.send({
     to: user.email,
-    subject: 'Welcome to Hostr',
+    subject: "Welcome to Hostr",
     from,
     fromname,
     html,
     text,
-    categories: [
-      'activate',
-    ],
+    categories: ["activate"],
   });
 }
 
-
 export async function sendResetToken(email) {
   const user = await models.user.findOne({
     where: {
@@ -159,7 +157,7 @@ export async function sendResetToken(email) {
       id: uuid.v4(),
       userId: user.id,
     });
-    const html = await render('email/inlined/forgot', {
+    const html = await render("email/inlined/forgot", {
       forgotUrl: `${process.env.WEB_BASE_URL}/forgot/${reset.id}`,
     });
     const text = `It seems you've forgotten your password :(
@@ -167,38 +165,32 @@ Visit  ${process.env.WEB_BASE_URL}/forgot/${reset.id} to set a new one.
 `;
     sendgrid.send({
       to: user.email,
-      from: 'jonathan@hostr.co',
-      fromname: 'Jonathan from Hostr',
-      subject: 'Hostr Password Reset',
+      from: "jonathan@hostr.co",
+      fromname: "Jonathan from Hostr",
+      subject: "Hostr Password Reset",
       html,
       text,
-      categories: [
-        'password-reset',
-      ],
+      categories: ["password-reset"],
     });
   } else {
-    throw new Error('There was an error looking up your email address.');
+    throw new Error("There was an error looking up your email address.");
   }
 }
 
-
 export async function fromToken(token) {
   const userId = await this.redis.get(token);
   return models.user.findByPk(userId);
 }
 
-
 export async function fromCookie(rememberId) {
   const userId = await models.remember.findByPk(rememberId);
   return models.user.findByPk(userId);
 }
 
-
 export async function validateResetToken(resetId) {
   return models.reset.findByPk(resetId);
 }
 
-
 export async function updatePassword(userId, password) {
   const cryptedPassword = await passwords.crypt(password);
   const user = await models.user.findByPk(userId);
@@ -206,7 +198,6 @@ export async function updatePassword(userId, password) {
   await user.save();
 }
 
-
 export async function activateUser(code) {
   const activation = await models.activation.findOne({
     where: {