From 62b7121301c471b227f823740d3d0c54136ea6bf Mon Sep 17 00:00:00 2001 From: Jonathan Cremin Date: Mon, 23 Oct 2017 21:05:58 +0100 Subject: [PATCH] Fix matching from share, add fix script --- docker-compose.yml | 15 ++------ fixmissing.js | 87 ++++++++++++++++++++++++++++++++++++++++++++++ public/src/app.vue | 2 +- routes/index.js | 2 +- routes/search.js | 11 ++++-- routes/share.js | 18 ++++++++-- worker.js | 77 ++++++++++++++++++++-------------------- 7 files changed, 152 insertions(+), 60 deletions(-) create mode 100644 fixmissing.js diff --git a/docker-compose.yml b/docker-compose.yml index 4337929..97ece12 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ services: DEBUG: "combine.fm*" VUE_ENV: server DATABASE_URL: + REDIS_URL: GOOGLE_EMAIL: GOOGLE_PASSWORD: XBOX_CLIENT_ID: @@ -20,20 +21,8 @@ services: - "3000:3000" command: yarn run watch-server worker: + extends: app build: ./ - environment: - DEBUG: "combine.fm*" - VUE_ENV: server - DATABASE_URL: - GOOGLE_EMAIL: - GOOGLE_PASSWORD: - XBOX_CLIENT_ID: - XBOX_CLIENT_SECRET: - YOUTUBE_KEY: - SPOTIFY_CLIENT_ID: - SPOTIFY_CLIENT_SECRET: - volumes: - - ./:/app:cached command: yarn run worker ports: - "3001:3000" diff --git a/fixmissing.js b/fixmissing.js new file mode 100644 index 0000000..c0e1755 --- /dev/null +++ b/fixmissing.js @@ -0,0 +1,87 @@ +import co from 'co'; +import debuglog from 'debug'; + +import models from './models'; +import services from './lib/services'; + +const debug = debuglog('combine.fm:fixmissing'); + +debug('Fixing missing'); + +const serviceIds = []; + +for (const service of services) { + serviceIds.push(service.id); +} + +const query = { + include: [ + { model: models.artist }, + { model: models.match }, + ], + order: [ + ['updatedAt', 'DESC'], + ], +}; + +function* search(data, done) { + const share = data.share; + const service = services.find(item => data.service.id === item.id); + + debug(`Matching ${share.name} on ${data.service.id}`) + + const match = yield service.search(share); + + debug(`Match found for ${share.name} on ${data.service.id}`); + + if (match.id) { + models.match.create({ + trackId: share.type === 'track' ? share.id : null, + albumId: share.type === 'album' ? share.id : null, + externalId: match.id.toString(), + service: match.service, + name: match.name, + streamUrl: match.streamUrl, + purchaseUrl: match.purchaseUrl, + artworkSmall: match.artwork.small, + artworkLarge: match.artwork.large, + }); + } else { + models.match.create({ + trackId: share.type === 'track' ? share.id : null, + albumId: share.type === 'album' ? share.id : null, + externalId: null, + service: match.service, + name: null, + streamUrl: null, + purchaseUrl: null, + artworkSmall: null, + artworkLarge: null, + }); + } +} + +co(function* () { + const albums = yield models.album.findAll(query); + for (const item of albums) { + const unmatched = serviceIds; + for (const match of item.matches) { + unmatched = unmatched.filter(id => match.service !== id); + } + if (unmatched.length > 0) { + debug(`Matching ${unmatched.join(', ')}`); + for (const toMatch of unmatched) { + yield search({ share: item, service: { id: toMatch } }); + } + } else { + debug(`No broken matches for ${item.name}`) + } + } +}).catch(function (err) { + debug(err.stack); +}); + + + + + diff --git a/public/src/app.vue b/public/src/app.vue index b6f4367..6692aa9 100644 --- a/public/src/app.vue +++ b/public/src/app.vue @@ -41,7 +41,7 @@ body { height: 64px; } h1 { - text-align: left; + text-align: center; padding: 10px 0; } h1 a { diff --git a/routes/index.js b/routes/index.js index ccaa0bf..cd87636 100644 --- a/routes/index.js +++ b/routes/index.js @@ -4,7 +4,7 @@ import services from '../lib/services'; import render from '../lib/render'; import models from '../models'; -const debug = debuglog('combinefm:share'); +const debug = debuglog('combine.fm:share'); const recentQuery = { include: [ diff --git a/routes/search.js b/routes/search.js index 2dc2d20..7400a01 100644 --- a/routes/search.js +++ b/routes/search.js @@ -23,9 +23,14 @@ export default function* () { if (!share) { share = yield create(music); - const job = queue.create('search', share).save((err) => { - if (!err) console.log(job.id); - }); + for (const service of services) { + if (service.id === share.service) { + continue; // eslint-disable-line no-continue + } + const job = queue.create('search', {share: share, service: service}).save((err) => { + if (!err) console.log(job.id); + }); + } } share = share.toJSON(); diff --git a/routes/share.js b/routes/share.js index afe1a8b..c891e4d 100644 --- a/routes/share.js +++ b/routes/share.js @@ -1,7 +1,13 @@ +import kue from 'kue'; + import services from '../lib/services'; import render from '../lib/render'; import models from '../models'; -import { find, create, findMatchesAsync } from '../lib/share'; +import { find, create } from '../lib/share'; + +const queue = kue.createQueue({ + redis: process.env.REDIS_URL, +}); export default function* (serviceId, type, itemId, format) { this.assert(type === 'album' || type === 'track', 400, { error: 'Invalid type' }); @@ -26,7 +32,15 @@ export default function* (serviceId, type, itemId, format) { if (!share) { share = yield create(music); - findMatchesAsync(share); + + for (const service of services) { + if (service.id === share.service) { + continue; // eslint-disable-line no-continue + } + const job = queue.create('search', {share: share, service: service}).save((err) => { + if (!err) console.log(job.id); + }); + } } } diff --git a/worker.js b/worker.js index 35cf22f..22a1d7f 100644 --- a/worker.js +++ b/worker.js @@ -4,7 +4,6 @@ import raven from 'raven'; import models from './models'; import services from './lib/services'; -import { find, create, findMatchesAsync } from './lib/share'; raven.config(process.env.SENTRY_DSN).install(); @@ -12,46 +11,44 @@ const queue = kue.createQueue({ redis: process.env.REDIS_URL, }); -function search(share, done) { - for (const service of services) { - if (service.id === share.service) { - continue; // eslint-disable-line no-continue +function search(data, done) { + const share = data.share; + const service = services.find(item => data.service.id === item.id); + console.log(service); + co(function* gen() { // eslint-disable-line no-loop-func + const match = yield service.search(share); + + if (match.id) { + models.match.create({ + trackId: share.type === 'track' ? share.id : null, + albumId: share.type === 'album' ? share.id : null, + externalId: match.id.toString(), + service: match.service, + name: match.name, + streamUrl: match.streamUrl, + purchaseUrl: match.purchaseUrl, + artworkSmall: match.artwork.small, + artworkLarge: match.artwork.large, + }); + } else { + models.match.create({ + trackId: share.type === 'track' ? share.id : null, + albumId: share.type === 'album' ? share.id : null, + externalId: null, + service: match.service, + name: null, + streamUrl: null, + purchaseUrl: null, + artworkSmall: null, + artworkLarge: null, + }); } - co(function* gen() { // eslint-disable-line no-loop-func - const match = yield service.search(share); - - if (match.id) { - models.match.create({ - trackId: share.type === 'track' ? share.id : null, - albumId: share.type === 'album' ? share.id : null, - externalId: match.id.toString(), - service: match.service, - name: match.name, - streamUrl: match.streamUrl, - purchaseUrl: match.purchaseUrl, - artworkSmall: match.artwork.small, - artworkLarge: match.artwork.large, - }); - } else { - models.match.create({ - trackId: share.type === 'track' ? share.id : null, - albumId: share.type === 'album' ? share.id : null, - externalId: null, - service: match.service, - name: null, - streamUrl: null, - purchaseUrl: null, - artworkSmall: null, - artworkLarge: null, - }); - } - }).catch((err) => { - raven.captureException(err); - }); - } - - - return done(); + return done(); + }).catch((err) => { + console.log(err); + raven.captureException(err); + return done(); + }); } queue.process('search', (job, done) => {