Migrating to workers, renaming service

This commit is contained in:
Jonathan Cremin 2017-10-23 17:58:23 +01:00
parent 850584ff36
commit a6cd5f4266
29 changed files with 5542 additions and 611 deletions

2
.gitignore vendored
View file

@ -1,8 +1,8 @@
chrome/build
public/dist
public/views
node_modules
public/**/*.gz
.DS_Store
.env
npm-debug.log
stats.json

View file

@ -1,4 +1,4 @@
FROM node:8.2.1-alpine
FROM node:8.7.0-alpine
WORKDIR /app

26
Makefile Normal file
View file

@ -0,0 +1,26 @@
# See http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.PHONY: help
help:
@echo
@echo "Commands:"
@grep -E -h '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@echo
@echo "See README.md or https://github.com/udemy/website-django/blob/master/README.md"
@echo
.PHONY: start
start: docker-compose-up watch-frontend ## Start containers and watch frontend
.PHONY: migrate
migrate: ## Migrate database schema
docker-compose run --rm app yarn initdb
.PHONY: watch-frontend
watch-frontend: ## Build and watch frontend for changes
docker-compose run --rm app yarn watch-js
.PHONY: docker-compose-up
docker-compose-up: ## Start (and create) docker containers
docker-compose up -d

4
app.js
View file

@ -16,10 +16,9 @@ import index from './routes/index';
import recent from './routes/recent';
import search from './routes/search';
import share from './routes/share';
import itunesProxy from './routes/itunes-proxy';
import errorHandler from './lib/error-handler';
const debug = debuglog('match.audio');
const debug = debuglog('combine.fm');
process.env.VUE_ENV = 'server';
@ -57,7 +56,6 @@ app.use(route.get('/', index));
app.use(route.get('/recent', recent));
app.use(route.post('/search', search));
app.use(route.get('/:service/:type/:id.:format?', share));
app.use(route.get('/itunes/(.*)', itunesProxy));
if (!module.parent) {
app.listen(process.env.PORT || 3000, () => {

View file

@ -1,7 +1,7 @@
{
"name" : "Match Audio",
"name" : "Combine.fm",
"version" : "0.3.1",
"description" : "Match Audio makes sharing from music services better.",
"description" : "Combine.fm makes sharing from music services better.",
"background" : {
"persistent": false,
"scripts": [
@ -20,7 +20,7 @@
"48": "icon-48.png",
"128": "icon-128.png"
},
"default_title": "Find matches for this music on Match Audio"
"default_title": "Find matches for this music on Combine.fm"
},
"permissions" : [
"declarativeContent",

View file

@ -1,4 +1,4 @@
const apiUrl = 'https://match.audio';
const apiUrl = 'https://combine.fm';
chrome.runtime.onInstalled.addListener(() => {
chrome.declarativeContent.onPageChanged.removeRules(undefined, () => {

View file

@ -1,8 +1,8 @@
const apiUrl = 'https://match.audio';
const apiUrl = 'https://combine.fm';
var button = document.createElement('button');
button.className = 'share-button';
button.setAttribute('aria-label', 'Share to Match Audio');
button.setAttribute('aria-label', 'Share to Combine.fm');
var buttonContent = document.createElement('div');
buttonContent.className = 'button-content';
@ -23,14 +23,14 @@ waves.className = 'style-scope paper-ripple';
paperRipple.appendChild(waves);
var img = document.createElement('img');
img.src = 'https://match.audio/images/logo-128.png';
img.src = 'https://combine.fm/images/logo-128.png';
img.height = 48;
buttonContent.appendChild(img)
var buttonLabel = document.createElement('div');
buttonLabel.className = 'button-label';
buttonLabel.setAttribute('aria-hidden', true);
buttonLabel.innerText = 'Match Audio';
buttonLabel.innerText = 'Combine.fm';
buttonContent.appendChild(buttonLabel);
// select the target node

View file

@ -1,4 +1,4 @@
const apiUrl = 'https://match.audio';
const apiUrl = 'https://combine.fm';
function contextClick(e) {
console.log(e);
@ -29,7 +29,7 @@ window.addEventListener('click', contextClick);
// const li = document.createElement('li');
// ul.appendChild(li);
// const a = document.createElement('a');
// a.innerText = 'Open in Match Audio'
// a.innerText = 'Open in Combine.fm'
// a.href = apiUrl;
// a.target = '_blank';
// a.addEventListener('click', (e) => {

View file

@ -4,7 +4,7 @@ services:
app:
build: ./
environment:
DEBUG: "match.audio*"
DEBUG: "combine.fm*"
VUE_ENV: server
DATABASE_URL:
GOOGLE_EMAIL:
@ -19,12 +19,33 @@ services:
ports:
- "3000:3000"
command: yarn run watch-server
worker:
build: ./
environment:
DEBUG: "combine.fm*"
VUE_ENV: server
DATABASE_URL:
GOOGLE_EMAIL:
GOOGLE_PASSWORD:
XBOX_CLIENT_ID:
XBOX_CLIENT_SECRET:
YOUTUBE_KEY:
SPOTIFY_CLIENT_ID:
SPOTIFY_CLIENT_SECRET:
volumes:
- ./:/app:cached
command: yarn run worker
ports:
- "3001:3000"
database:
image: "postgres:9.6"
image: "postgres:10-alpine"
ports:
- "5432:5432"
environment:
POSTGRES_PASSWORD: "password"
POSTGRES_USER: "matchaudio"
POSTGRES_DB: "matchaudio"
POSTGRES_USER: "combinefm"
POSTGRES_DB: "combinefm"
redis:
image: "redis:4.0.2-alpine"
ports:
- "6379:6379"

View file

@ -1,6 +1,6 @@
import debuglog from 'debug';
const debug = debuglog('match.audio');
const debug = debuglog('combine.fm');
export default function (raven) {
return function* error(next) {

View file

@ -4,7 +4,7 @@ import PlayMusic from 'playmusic';
import debuglog from 'debug';
import urlMatch from './url';
const debug = debuglog('match.audio');
const debug = debuglog('combine.fm');
const pm = bluebird.promisifyAll(new PlayMusic());

View file

@ -54,8 +54,8 @@ export function* lookupId(possibleId, type, countrycode) {
streamUrl: null,
purchaseUrl: result.collectionViewUrl,
artwork: {
small: `https://match.audio/itunes/${result.artworkUrl100.replace('100x100', '200x200').replace('http://', '')}`,
large: `https://match.audio/itunes/${result.artworkUrl100.replace('100x100', '600x600').replace('http://', '')}`,
small: `${result.artworkUrl100.replace('100x100', '200x200').replace('.mzstatic.com', '-ssl.mzstatic.com').replace('http://', 'https://')}`,
large: `${result.artworkUrl100.replace('100x100', '600x600').replace('.mzstatic.com', '-ssl.mzstatic.com').replace('http://', 'https://')}`,
},
artist: {
name: result.artistName,
@ -116,8 +116,8 @@ export function* search(data) {
streamUrl: null,
purchaseUrl: result.collectionViewUrl,
artwork: {
small: `https://match.audio/itunes/${result.artworkUrl100.replace('100x100', '200x200').replace('http://', '')}`,
large: `https://match.audio/itunes/${result.artworkUrl100.replace('100x100', '600x600').replace('http://', '')}`,
small: `${result.artworkUrl100.replace('100x100', '200x200').replace('.mzstatic.com', '-ssl.mzstatic.com').replace('http://', 'https://')}`,
large: `${result.artworkUrl100.replace('100x100', '600x600').replace('.mzstatic.com', '-ssl.mzstatic.com').replace('http://', 'https://')}`,
},
artist: {
name: result.artistName,

View file

@ -5,7 +5,7 @@ import urlMatch from './url';
const spotify = new SpotifyWebApi({
clientId: process.env.SPOTIFY_CLIENT_ID,
clientSecret: process.env.SPOTIFY_CLIENT_SECRET,
redirectUri: 'https://match.audio',
redirectUri: 'https://combine.fm',
});

View file

@ -4,7 +4,7 @@ import 'superagent-bluebird-promise';
import debuglog from 'debug';
import urlMatch from './url';
const debug = debuglog('match.audio:xbox');
const debug = debuglog('combine.fm:xbox');
if (!process.env.XBOX_CLIENT_ID || !process.env.XBOX_CLIENT_SECRET) {
debug('XBOX_CLIENT_ID and XBOX_CLIENT_SECRET environment variables not found, deactivating Xbox Music.');

View file

@ -6,7 +6,7 @@ import 'superagent-bluebird-promise';
import debuglog from 'debug';
import urlMatch from './url';
const debug = debuglog('match.audio:youtube');
const debug = debuglog('combine.fm:youtube');
if (!process.env.YOUTUBE_KEY) {
debug('YOUTUBE_KEY environment variable not found, deactivating Youtube.');
@ -19,7 +19,7 @@ const credentials = {
const apiRoot = 'https://www.googleapis.com/youtube/v3';
const nodebrainz = new Nodebrainz({
userAgent: 'match-audio ( https://match.audio )',
userAgent: 'combine.fm ( https://combine.fm )',
defaultLimit: 10,
retryOn: true,
retryDelay: 3000,

View file

@ -4,7 +4,7 @@ import debuglog from 'debug';
import models from '../models';
import services from '../lib/services';
const debug = debuglog('match.audio:share');
const debug = debuglog('combine.fm:share');
export function find(music) {
return models[music.type].findOne({

View file

@ -4,7 +4,7 @@ import Sequelize from 'sequelize';
import debugname from 'debug';
const debug = debugname('match.audio:models');
const debug = debugname('combine.fm:models');
const config = {
dialect: 'postgres',

3985
package-lock.json generated Normal file

File diff suppressed because it is too large Load diff

View file

@ -1,31 +1,32 @@
{
"name": "match.audio",
"name": "combine.fm",
"version": "0.0.0",
"repository": "https://github.com/kudos/match.audio",
"license": "MIT",
"scripts": {
"build": "webpack -p --config webpack.config.js && webpack -p --config webpack.config.server.js",
"start": "node -r babel-register app.js",
"worker": "nodemon -x \"node -r babel-register\" -e js,vue -i node_modules -i chrome/ worker.js",
"test": "mocha -r co-mocha --compilers js:babel-register test/**/*.js --timeout=15000",
"watch": "parallelshell \"npm run watch-js\" \"npm run watch-server\"",
"watch-js": "parallelshell \"webpack -w -d --config webpack.config.js\" \"webpack -w -d --config webpack.config.server.js\"",
"watch-server": "nodemon -x \"node -r babel-register\" -e js,vue -i node_modules -i chrome/ app.js",
"heroku-postbuild": "npm run build"
"heroku-postbuild": "npm run build",
"initdb": "node -r babel-register test/initdb.js"
},
"engines": {
"node": "^7.10.0",
"npm": "^4.2.0"
"node": "^8.6.0"
},
"dependencies": {
"babel": "^6.1.18",
"babel-cli": "^6.3.13",
"babel-core": "^6.3.13",
"babel-loader": "^7.0.0",
"babel-cli": "^6.26.0",
"babel-core": "^6.26.0",
"babel-loader": "^7.1.2",
"babel-plugin-syntax-jsx": "^6.3.13",
"babel-plugin-syntax-object-rest-spread": "^6.13.0",
"babel-plugin-transform-es2015-arrow-functions": "^6.3.13",
"babel-plugin-transform-es2015-block-scoped-functions": "^6.1.18",
"babel-plugin-transform-es2015-block-scoping": "^6.1.18",
"babel-plugin-transform-es2015-block-scoping": "^6.26.0",
"babel-plugin-transform-es2015-classes": "^6.2.2",
"babel-plugin-transform-es2015-computed-properties": "^6.1.18",
"babel-plugin-transform-es2015-constants": "^6.1.4",
@ -33,7 +34,7 @@
"babel-plugin-transform-es2015-for-of": "^6.1.18",
"babel-plugin-transform-es2015-function-name": "^6.1.18",
"babel-plugin-transform-es2015-literals": "^6.1.18",
"babel-plugin-transform-es2015-modules-commonjs": "^6.2.0",
"babel-plugin-transform-es2015-modules-commonjs": "^6.26.0",
"babel-plugin-transform-es2015-object-super": "^6.1.18",
"babel-plugin-transform-es2015-parameters": "^6.1.18",
"babel-plugin-transform-es2015-shorthand-properties": "^6.1.18",
@ -43,17 +44,17 @@
"babel-plugin-transform-es2015-typeof-symbol": "^6.1.18",
"babel-plugin-transform-es2015-unicode-regex": "^6.1.18",
"babel-plugin-transform-object-assign": "^6.8.0",
"babel-plugin-transform-object-rest-spread": "^6.16.0",
"babel-plugin-transform-object-rest-spread": "^6.26.0",
"babel-plugin-transform-remove-strict-mode": "0.0.2",
"babel-preset-es2015": "^6.0.0",
"babel-preset-latest-minimal": "^1.1.2",
"babel-register": "^6.16.3",
"babel-register": "^6.26.0",
"bluebird": "^3.4.1",
"bulma": "^0.4.1",
"co": "~4.6.0",
"css-loader": "^0.28.1",
"debug": "^2.6.6",
"ejs": "^2.5.2",
"ejs": "^2.5.7",
"extract-text-webpack-plugin": "^2.1.0",
"file-loader": "^0.11.1",
"json-loader": "^0.5.4",
@ -68,12 +69,13 @@
"koa-static": "^2.0.0",
"koa-views": "^4.0.1",
"koa-websocket": "^2.1.0",
"kue": "^0.11.6",
"moment": "^2.14.1",
"node-uuid": "~1.4.2",
"nodebrainz": "^2.1.1",
"pg": "^6.1.0",
"playmusic": "https://github.com/jamon/playmusic.git#37e98f39c33fc5359a8a30b8c8e422161a4be9a8",
"raven": "^2.0.2",
"raven": "^2.1.2",
"sequelize": "^3.24.3",
"spotify-web-api-node": "^2.4.0",
"style-loader": "^0.17.0",
@ -91,15 +93,15 @@
},
"devDependencies": {
"babel-plugin-transform-runtime": "^6.15.0",
"babel-runtime": "^6.11.6",
"babel-runtime": "^6.26.0",
"co-mocha": "^1.2.0",
"eslint": "^3.8.0",
"eslint-config-airbnb": "^14.1.0",
"eslint-plugin-import": "^2.0.1",
"eslint": "^4.7.0",
"eslint-config-airbnb": "^15.1.0",
"eslint-plugin-import": "^2.7.0",
"istanbul": "^0.4.0",
"mocha": "^3.0.2",
"nodemon": "^1.10.2",
"parallelshell": "~2.0.0",
"should": "^11.1.0"
"mocha": "^3.5.3",
"nodemon": "^1.12.1",
"parallelshell": "^3.0.1",
"should": "^13.0.1"
}
}

View file

@ -4,12 +4,12 @@
<div class="container">
<h1 class="title has-text-centered">
<router-link to="/" exact>
match<span class="lighter">.audio</span>
<img src="/assets/images/logo-128.png"> <b>combine</b><span class="lighter">.fm</span>
</router-link>
</h1>
</div>
</div>
<div>
<div class="content">
<router-view></router-view>
</div>
<footer class="footer">
@ -21,15 +21,25 @@
</template>
<style>
@import url('https://fonts.googleapis.com/css?family=Comfortaa');
html {
position: relative;
min-height: 100%;
}
body {
color: #445470;
background: #fff;
margin: 0 0 140px;
}
.header {
font-family: 'Comfortaa', cursive;
background: #FE4365;
}
.header img {
height: 64px;
}
h1 {
padding: 25px 0;
padding: 10px 0;
}
h1 a {
color: #fff;
@ -53,7 +63,10 @@ h1 .lighter {
color: #ffacc5;
}
.footer {
margin-top: 50px;
padding-bottom: 40px;
position: absolute;
left: 0;
bottom: 0;
width: 100%;
padding: 40px 24px 40px 24px;
}
</style>

View file

@ -3,10 +3,10 @@
<search></search>
<div class="blurb">
<p>
Match Audio makes sharing from music services better. What happens when you share your favourite song on Spotify with a friend, but they don't use Spotify?
Combine.fm makes sharing from music services better. What happens when you share your favourite song on Spotify with a friend, but they don't use Spotify?
</p>
<p>
We match album and track links from Youtube, Rdio, Spotify, Deezer, Google Music, Xbox Music, Beats Music, and iTunes and give you back one link with matches we find on all of them.
We match album and track links from Youtube, Spotify, Deezer, Google Music, Xbox Music, and iTunes and give you back one link with matches we find on all of them.
</p>
</div>
<div class="recently-shared">
@ -22,10 +22,10 @@
<h2 class="title is-2">Questions?</h2>
<h3 class="title is-3">Why would I want to use this?</h3>
<p>Sometimes when people want to share music they don't know what service their friends are using. Match Audio let's you take a link from one service and expand it into a link that supports all services.</p>
<p>Sometimes when people want to share music they don't know what service their friends are using. Combine.fm let's you take a link from one service and expand it into a link that supports all services.</p>
<h3 class="title is-3">I still don't get it.</h3>
<p>That's not actually a question, but that's ok. Here's an example: I'm listening to a cool new album I found on Google Play Music. So I go to the address bar (the box that sometimes says https://www.google.com in it) and copy the link to share with my friend. But my friend uses Spotify. So first I go to Match Audio and paste the link there, then grab the Match Audio link from the address bar and send them that link instead.</p>
<p>That's not actually a question, but that's ok. Here's an example: I'm listening to a cool new album I found on Google Play Music. So I go to the address bar (the box that sometimes says https://www.google.com in it) and copy the link to share with my friend. But my friend uses Spotify. So first I go to Combine.fm and paste the link there, then grab the Combine.fm link from the address bar and send them that link instead.</p>
<h3 class="title is-3">Where do I find a link to paste in the box?</h3>
<p>Most music services have a 'share' dialog for albums and tracks in their interface. If you have them open in a web browser instead of an app, you can simply copy and paste the address bar and we'll work out the rest.</p>
@ -34,13 +34,13 @@
<p>Unfortunately not. Playlists would add a huge amount of complexity and would almost certainly cause the site to break the API limits imposed by some of the services we support.</p>
<h3 class="title is-3">Why don't you guys support Bandcamp, Amazon Music, Sony Music Unlimited ?</h3>
<p>Let me stop you there. Match Audio is open source, that means any capable programmer who wants to add other music services can look at our code and submit changes. If you're not a programmer, you can always submit a request and maybe we'll do it for you.</p>
<p>Let me stop you there. Combine.fm is open source, that means any capable programmer who wants to add other music services can look at our code and submit changes. If you're not a programmer, you can always submit a request and maybe we'll do it for you.</p>
</div>
<div>
<h2 class="title is-2">Tools</h2>
<div class="columns">
<p class="column is-half">
Download the Chrome Extension and get Match Audio links right from your address bar.
Download the Chrome Extension and get Combine.fm links right from your address bar.
</p>
<p class="column is-half">
<a href="https://chrome.google.com/webstore/detail/kjfpkmfgcflggjaldcfnoppjlpnidolk"><img src="/assets/images/chrome-web-store.png" alt="Download the Chrome Extension" /></a>
@ -72,7 +72,7 @@ export default {
recents: function () {
if (typeof document !== 'undefined') {
const recents = this.$store.state.recents;
document.title = `Match Audio • Share Music`;
document.title = `Combine.fm • Share Music`;
}
},
},

View file

@ -76,7 +76,7 @@ export default {
clearInterval(this.interval);
}
this.item = res.body;
document.title = `Match Audio${this.item.name} by ${this.item.artist.name}`;
document.title = `Combine.fm${this.item.name} by ${this.item.artist.name}`;
});
}
}

View file

@ -4,7 +4,7 @@ import services from '../lib/services';
import render from '../lib/render';
import models from '../models';
const debug = debuglog('match.audio:share');
const debug = debuglog('combinefm:share');
const recentQuery = {
include: [

View file

@ -1,14 +0,0 @@
import { parse } from 'url';
import request from 'superagent';
export default function* (next) {
const url = `http://${this.request.url.substr(8)}`;
const parsed = parse(url);
if (parsed.host.match(/mzstatic\.com/)) {
const proxyResponse = yield request.get(url);
this.set(proxyResponse.headers);
this.body = proxyResponse.body;
} else {
yield next;
}
}

View file

@ -1,9 +1,15 @@
import { parse } from 'url';
import kue from 'kue';
import lookup from '../lib/lookup';
import services from '../lib/services';
import { find, create, findMatchesAsync } from '../lib/share';
const queue = kue.createQueue({
redis: {
host: 'redis',
},
});
export default function* () {
const url = parse(this.request.body.url);
@ -18,7 +24,10 @@ export default function* () {
if (!share) {
share = yield create(music);
findMatchesAsync(share);
const job = queue.create('search', share).save((err) => {
if (!err) console.log(job.id);
});
}
share = share.toJSON();

View file

@ -1,16 +1,16 @@
<!DOCTYPE html>
<html>
<head>
<title>Match Audio &bull; <%=head.title%></title>
<title>Combine.fm &bull; <%=head.title%></title>
<link rel="stylesheet" href="/dist/<%=manifest['style/main.css']%>" />
<meta name='description' content='Match Audio matches album and track links from Youtube, Rdio, Spotify, Deezer, Google Music, Xbox Music, Beats Music, and iTunes and give you back one link with matches we find on all of them.' />
<meta name='description' content='Combine.fm matches album and track links from Youtube, Rdio, Spotify, Deezer, Google Music, Xbox Music, Beats Music, and iTunes and give you back one link with matches we find on all of them.' />
<meta name='viewport' content='width=device-width, initial-scale=1' />
<meta name='theme-color' content='#FE4365' />
<meta name='twitter:card' content='<%=typeof share == 'undefined' ? 'summary': 'summary_large_image'%>' />
<meta name='twitter:site' content='@MatchAudio' />
<meta name='twitter:title' property='og:title' content='Match Audio &bull; <%=head.title%>' />
<meta name='twitter:description' property='og:description' content='Match Audio matches album and track links from Youtube, Rdio, Spotify, Deezer, Google Music, Xbox Music, Beats Music, and iTunes and give you back one link with matches we find on all of them.' />
<meta name='twitter:description' property='og:description' content='Combine.fm matches album and track links from Youtube, Rdio, Spotify, Deezer, Google Music, Xbox Music, Beats Music, and iTunes and give you back one link with matches we find on all of them.' />
<meta name='twitter:image:src' property='og:image' content='<%=head.image%>' />
<meta property='og:url' content='<%=head.shareUrl%>' />
<link rel='shortcut icon' href='/assets/images/favicon.png' />

64
worker.js Normal file
View file

@ -0,0 +1,64 @@
import co from 'co';
import kue from 'kue';
import raven from 'raven';
import models from './models';
import services from './lib/services';
import { find, create, findMatchesAsync } from './lib/share';
raven.config(process.env.SENTRY_DSN).install();
const queue = kue.createQueue({
redis: {
host: 'redis',
},
});
function search(share, done) {
for (const service of services) {
if (service.id === share.service) {
continue; // eslint-disable-line no-continue
}
co(function* gen() { // eslint-disable-line no-loop-func
const match = yield service.search(share);
if (match.id) {
models.match.create({
trackId: share.type === 'track' ? share.id : null,
albumId: share.type === 'album' ? share.id : null,
externalId: match.id.toString(),
service: match.service,
name: match.name,
streamUrl: match.streamUrl,
purchaseUrl: match.purchaseUrl,
artworkSmall: match.artwork.small,
artworkLarge: match.artwork.large,
});
} else {
models.match.create({
trackId: share.type === 'track' ? share.id : null,
albumId: share.type === 'album' ? share.id : null,
externalId: null,
service: match.service,
name: null,
streamUrl: null,
purchaseUrl: null,
artworkSmall: null,
artworkLarge: null,
});
}
}).catch((err) => {
raven.captureException(err);
});
}
return done();
}
queue.process('search', (job, done) => {
search(job.data, done);
});
kue.app.listen(3000);

1879
yarn.lock

File diff suppressed because it is too large Load diff