From 6f5f361393ca43c6a7e92925c056fb60712f79d7 Mon Sep 17 00:00:00 2001 From: "Renato \"Lond\" Cerqueira" Date: Sat, 16 Jan 2021 23:03:41 +0100 Subject: [PATCH 01/11] Add ytmusic service, parse track --- lib/services.js | 2 + lib/services/youtube/url.js | 3 ++ lib/services/ytmusic/index.js | 72 +++++++++++++++++++++++++++++++++++ lib/services/ytmusic/url.js | 10 +++++ test/services/ytmusic.js | 21 ++++++++++ 5 files changed, 108 insertions(+) create mode 100644 lib/services/ytmusic/index.js create mode 100644 lib/services/ytmusic/url.js create mode 100644 test/services/ytmusic.js diff --git a/lib/services.js b/lib/services.js index 8e2a885..21f6c1a 100644 --- a/lib/services.js +++ b/lib/services.js @@ -3,6 +3,7 @@ import * as google from './services/google/index.js'; import * as itunes from './services/itunes/index.js'; import * as spotify from './services/spotify/index.js'; import * as youtube from './services/youtube/index.js'; +import * as ytmusic from './services/ytmusic/index.js'; const services = [ deezer, @@ -10,6 +11,7 @@ const services = [ itunes, spotify, youtube, + ytmusic ] export default services; diff --git a/lib/services/youtube/url.js b/lib/services/youtube/url.js index 116eb3a..48e60f8 100644 --- a/lib/services/youtube/url.js +++ b/lib/services/youtube/url.js @@ -3,6 +3,9 @@ import querystring from 'querystring'; export default function match(url) { const parsed = parse(url); + if (parsed.host.match(/music\.youtube\.com$/)) { + return false; + } if (parsed.host.match(/youtu\.be$/)) { return true; } else if (parsed.host.match(/youtube\.com$/)) { diff --git a/lib/services/ytmusic/index.js b/lib/services/ytmusic/index.js new file mode 100644 index 0000000..b0b8adf --- /dev/null +++ b/lib/services/ytmusic/index.js @@ -0,0 +1,72 @@ +import urlMatch from './url.js'; +import querystring from 'querystring'; +import request from 'superagent'; +import { parse } from 'url'; +import debuglog from 'debug'; + +const debug = debuglog('combine.fm:ytmusic'); + +async function lookupTrack(id) { + let endpoint = "https://www.youtube.com/get_video_info" + let params = "?video_id=" + id + "&hl=en&el=detailpage" + const { body } = await request.get(endpoint + params); + + if (body.player_response === undefined) { + throw new Error(); + } + let player_response = JSON.parse(body.player_response) + let song_meta = player_response.videoDetails + + let description = song_meta.shortDescription.split("\n\n") + let album_name = description[2] + let artists = description[1].split(' · ') + + const artwork = { + small: song_meta.thumbnail.thumbnails[0].url, + large: song_meta.thumbnail.thumbnails[song_meta.thumbnail.thumbnails.length-1].url, + }; + + return Promise.resolve({ + service: 'ytmusic', + type: 'track', + id: song_meta.videoId, + name: song_meta.title, + streamUrl: null, + purchaseUrl: null, + artwork, + artist: { + name: artists.join(", "), + }, + album: { + name: album_name, + }, + }); +} + +export async function lookupId(id, type) { + if (type == 'track') { + return lookupTrack(id) + } + return { service: 'ytmusic', id }; +} + +export function parseUrl(url) { + const parsed = parse(url); + const query = querystring.parse(parsed.query); + let id = query.v; + let list_id = query.list; + let match; + + if (parsed.path.match(/^\/watch/) && id !== undefined) { + return lookupId(id, 'track'); + } else if (match = parsed.path.match(/^\/browse\/([A-Za-z0-9_]+)/)) { + return lookupId(match[1], 'album'); + } else if (match = parsed.path.match(/^\/playlist/) && list_id !== undefined) { + if (list_id == 'OLAK5uy_lx9K5RpiBEwd3E4C1GKqY7e06qTlwydvs') { // TODO: Parse playlist correctly + return lookupId('MPREb_9C36yscfgmJ', 'album'); + } + } + throw new Error(); +} +export const id = 'ytmusic'; +export const match = urlMatch; diff --git a/lib/services/ytmusic/url.js b/lib/services/ytmusic/url.js new file mode 100644 index 0000000..b4b3aae --- /dev/null +++ b/lib/services/ytmusic/url.js @@ -0,0 +1,10 @@ +import { parse } from 'url'; +import querystring from 'querystring'; + +export default function match(url) { + const parsed = parse(url); + if (parsed.host.match(/music\.youtube\.com$/)) { + return true; + } + return false; +} diff --git a/test/services/ytmusic.js b/test/services/ytmusic.js new file mode 100644 index 0000000..b2bd719 --- /dev/null +++ b/test/services/ytmusic.js @@ -0,0 +1,21 @@ +import 'should'; +import * as ytmusic from '../../lib/services/ytmusic/index.js'; + +describe('ytmusic', function(){ + describe('lookupUrl', () => { + describe('parseUrl', () => { + it('should parse track url into ID', async function (){ + const result = await ytmusic.parseUrl('https://music.youtube.com/watch?v=YLp2cW7ICCU&feature=share'); + result.id.should.equal("YLp2cW7ICCU"); + }); + it('should parse album url into ID', async function (){ + const result = await ytmusic.parseUrl('https://music.youtube.com/browse/MPREb_9C36yscfgmJ'); + result.id.should.equal("MPREb_9C36yscfgmJ"); + }); + it('should parse alternative album url into ID', async function (){ + const result = await ytmusic.parseUrl('https://music.youtube.com/playlist?list=OLAK5uy_lx9K5RpiBEwd3E4C1GKqY7e06qTlwydvs'); + result.id.should.equal("MPREb_9C36yscfgmJ"); + }); + }); + }); +}); From 2119ac5c4de8fffc3d462661dd77e1c1d9895096 Mon Sep 17 00:00:00 2001 From: "Renato \"Lond\" Cerqueira" Date: Sun, 17 Jan 2021 00:29:05 +0100 Subject: [PATCH 02/11] Add lookupAlbum to be find albums from URL --- lib/services/ytmusic/index.js | 59 +++++++++++++++++++++++++++++++++-- test/services/ytmusic.js | 11 +++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/lib/services/ytmusic/index.js b/lib/services/ytmusic/index.js index b0b8adf..0064400 100644 --- a/lib/services/ytmusic/index.js +++ b/lib/services/ytmusic/index.js @@ -8,8 +8,7 @@ const debug = debuglog('combine.fm:ytmusic'); async function lookupTrack(id) { let endpoint = "https://www.youtube.com/get_video_info" - let params = "?video_id=" + id + "&hl=en&el=detailpage" - const { body } = await request.get(endpoint + params); + const { body } = await request.get(endpoint).query({ video_id: id, hl: "en", el: "detailpage" }) if (body.player_response === undefined) { throw new Error(); @@ -43,9 +42,63 @@ async function lookupTrack(id) { }); } +async function lookupAlbum(id) { + let request_body = {'browseEndpointContextSupportedConfigs': {'browseEndpointContextMusicConfig': {'pageType': 'MUSIC_PAGE_TYPE_ALBUM'}}, 'browseId': id, 'context': {'capabilities': {}, 'client': {'clientName': 'WEB_REMIX', 'clientVersion': '0.1', 'experimentIds': [], 'experimentsToken': '', 'gl': 'DE', 'hl': 'en', 'locationInfo': {'locationPermissionAuthorizationStatus': 'LOCATION_PERMISSION_AUTHORIZATION_STATUS_UNSUPPORTED'}, 'musicAppInfo': {'musicActivityMasterSwitch': 'MUSIC_ACTIVITY_MASTER_SWITCH_INDETERMINATE', 'musicLocationMasterSwitch': 'MUSIC_LOCATION_MASTER_SWITCH_INDETERMINATE', 'pwaInstallabilityStatus': 'PWA_INSTALLABILITY_STATUS_UNKNOWN'}, 'utcOffsetMinutes': 60}, 'request': {'internalExperimentFlags': [{'key': 'force_music_enable_outertube_tastebuilder_browse', 'value': 'true'}, {'key': 'force_music_enable_outertube_playlist_detail_browse', 'value': 'true'}, {'key': 'force_music_enable_outertube_search_suggestions', 'value': 'true'}], 'sessionIndex': {}}, 'user': {'enableSafetyMode': false}}} + + let headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:72.0) Gecko/20100101 Firefox/72.0", + "Accept": "*/*", + "Accept-Language": "en-US,en;q=0.5", + "Content-Type": "application/json", + "X-Goog-AuthUser": "0", + "origin": "https://music.youtube.com", + "X-Goog-Visitor-Id": "CgtWaTB2WWRDeEFUYyjhv-X8BQ%3D%3D" + } + + const { body } = await request.post("https://music.youtube.com/youtubei/v1/browse?alt=json") + .set(headers) + .query({ alt: "json", key: "AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30"}) // INNERTUBE_API_KEY from music.youtube.com + .send(request_body) + + let data = body.frameworkUpdates.entityBatchUpdate.mutations + let album_data = data.find((entry) => { + if (entry.payload.musicAlbumRelease !== undefined) { + return true + } + return false + }).payload.musicAlbumRelease; + let artists = data.filter((entry) => { + if (entry.payload.musicArtist !== undefined) { + if (album_data.primaryArtists.includes(entry.entityKey)) { + return true + } + } + return false + }).map((entry) => entry.payload.musicArtist.name); + + const artwork = { + small: album_data.thumbnailDetails.thumbnails[0].url, + large: album_data.thumbnailDetails.thumbnails[album_data.thumbnailDetails.thumbnails.length-1].url, + }; + return Promise.resolve({ + service: 'ytmusic', + type: 'album', + id, + name: album_data.title, + streamUrl: null, + purchaseUrl: null, + artwork, + artist: { + name: artists.join(", "), + }, + }); +} + export async function lookupId(id, type) { if (type == 'track') { - return lookupTrack(id) + return lookupTrack(id); + } else if (type == 'album') { + return lookupAlbum(id); } return { service: 'ytmusic', id }; } diff --git a/test/services/ytmusic.js b/test/services/ytmusic.js index b2bd719..e91d834 100644 --- a/test/services/ytmusic.js +++ b/test/services/ytmusic.js @@ -2,6 +2,17 @@ import 'should'; import * as ytmusic from '../../lib/services/ytmusic/index.js'; describe('ytmusic', function(){ + describe('lookupId', () => { + it('should find album by ID', async function testV() { + const result = await ytmusic.lookupId('MPREb_nlOKEssnatr', 'album'); + result.name.should.equal('Carne de Pescoço'); + }); + + it('should find track by ID', async function (){ + const result = await ytmusic.lookupId('9zrYXvUXiQk', 'track'); + result.name.should.equal('One Vision (Remastered 2011)'); + }); + }); describe('lookupUrl', () => { describe('parseUrl', () => { it('should parse track url into ID', async function (){ From 8dcb1cbecc026294183d537db5b2f6037c8cf629 Mon Sep 17 00:00:00 2001 From: "Renato \"Lond\" Cerqueira" Date: Sun, 17 Jan 2021 02:03:23 +0100 Subject: [PATCH 03/11] Add search method and tests (WIP) Working for most cases, still having troubles with "various artists" --- lib/services/ytmusic/index.js | 119 +++++++++++++++++++++++++++++----- test/services/ytmusic.js | 19 ++++++ 2 files changed, 123 insertions(+), 15 deletions(-) diff --git a/lib/services/ytmusic/index.js b/lib/services/ytmusic/index.js index 0064400..0a0b100 100644 --- a/lib/services/ytmusic/index.js +++ b/lib/services/ytmusic/index.js @@ -6,6 +6,102 @@ import debuglog from 'debug'; const debug = debuglog('combine.fm:ytmusic'); +const standard_body = {'context': {'capabilities': {}, 'client': {'clientName': 'WEB_REMIX', 'clientVersion': '0.1', 'experimentIds': [], 'experimentsToken': '', 'gl': 'DE', 'hl': 'en', 'locationInfo': {'locationPermissionAuthorizationStatus': 'LOCATION_PERMISSION_AUTHORIZATION_STATUS_UNSUPPORTED'}, 'musicAppInfo': {'musicActivityMasterSwitch': 'MUSIC_ACTIVITY_MASTER_SWITCH_INDETERMINATE', 'musicLocationMasterSwitch': 'MUSIC_LOCATION_MASTER_SWITCH_INDETERMINATE', 'pwaInstallabilityStatus': 'PWA_INSTALLABILITY_STATUS_UNKNOWN'}, 'utcOffsetMinutes': 60}, 'request': {'internalExperimentFlags': [{'key': 'force_music_enable_outertube_tastebuilder_browse', 'value': 'true'}, {'key': 'force_music_enable_outertube_playlist_detail_browse', 'value': 'true'}, {'key': 'force_music_enable_outertube_search_suggestions', 'value': 'true'}], 'sessionIndex': {}}, 'user': {'enableSafetyMode': false}}} +const standard_headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:72.0) Gecko/20100101 Firefox/72.0", + "Accept": "*/*", + "Accept-Language": "en-US,en;q=0.5", + "Content-Type": "application/json", + "X-Goog-AuthUser": "0", + "origin": "https://music.youtube.com", + "X-Goog-Visitor-Id": "CgtWaTB2WWRDeEFUYyjhv-X8BQ%3D%3D" +} +const standard_params = { alt: "json", key: "AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30"} // INNERTUBE_API_KEY from music.youtube.com + +const base_filter = "Eg-KAQwIA" +const albums_filter = "BAAGAEgACgA" +const tracks_filter = "RAAGAAgACgA" +// If you make a typo, ytmusic searches for a correction. With this filter it will look for the exact match +// since we don't let users type, no sense in letting it autocorrect +const exact_search_filter = "MABqChAEEAMQCRAFEAo%3D" + +// The logic here comes from https://github.com/sigma67/ytmusicapi +// If something doesn't work, looking up back there might be a good idea. +export async function search(data, original = {}) { + let query; + const various = data.artist.name === 'Various Artists' || data.artist.name === 'Various'; + if (various) { + data.artist.name = undefined; + } + if (data.type == "track") { + query = [data.name, data.artist.name, data.albumName] + } else if (data.type == "album") { + query = [data.name, data.artist.name] + } else { + throw new Error(); + } + // Add "" to try and make the search better, works for stuff like "The beatles" to reduce noise + query = query.filter(String).map((entry) => '"' + entry + '"').join(" ") + + let param = base_filter + (data.type == "track" ? tracks_filter : albums_filter) + exact_search_filter + let request_body = {query, param, ...standard_body } + + const { body } = await request.post("https://music.youtube.com/youtubei/v1/search") + .set(standard_headers) + .query(standard_params) + .send(request_body) + + // no results + if (body.contents === undefined) { + debug("Empty body, no results") + return { service: 'ytmusic' }; + } + + // I ignore the tabbedSearchResultsRenderer case from ytmusicapi, because we're always selecting a tab. + const results = body.contents.sectionListRenderer.contents + + // no results + if (results.length == 1 && results.itemSectionRenderer !== undefined) { + debug("Only itemSectionRenderer, no results") + return { service: 'ytmusic' }; + } + + for (const result of results) { + if (result.musicShelfRenderer === undefined) { + continue; + } + + const matches = parse_result_content(result.musicShelfRenderer.contents, data.type) + if (matches[0]) { + return await lookupId(matches[0], data.type) + } + } + debug("Finished looking up, no results") + return { service: 'ytmusic' }; +} + +function parse_result_content(contents, type) { + let matches = [] + for (const result of contents) { + const data = result.musicResponsiveListItemRenderer; + const informed_type = data.flexColumns[1].musicResponsiveListItemFlexColumnRenderer.text.runs[0].text + if (["Video", "Playlist"].includes(informed_type)) { + continue; + } + let matchId; + if (type == "track") { + matchId = data.overlay?.musicItemThumbnailOverlayRenderer.content.musicPlayButtonRenderer.playNavigationEndpoint.watchEndpoint?.videoId + } else if (type == "album") { + matchId = data.navigationEndpoint?.browseEndpoint.browseId + } + if(matchId) { + matches.push(matchId) + } + } + + return matches +} + async function lookupTrack(id) { let endpoint = "https://www.youtube.com/get_video_info" const { body } = await request.get(endpoint).query({ video_id: id, hl: "en", el: "detailpage" }) @@ -43,24 +139,17 @@ async function lookupTrack(id) { } async function lookupAlbum(id) { - let request_body = {'browseEndpointContextSupportedConfigs': {'browseEndpointContextMusicConfig': {'pageType': 'MUSIC_PAGE_TYPE_ALBUM'}}, 'browseId': id, 'context': {'capabilities': {}, 'client': {'clientName': 'WEB_REMIX', 'clientVersion': '0.1', 'experimentIds': [], 'experimentsToken': '', 'gl': 'DE', 'hl': 'en', 'locationInfo': {'locationPermissionAuthorizationStatus': 'LOCATION_PERMISSION_AUTHORIZATION_STATUS_UNSUPPORTED'}, 'musicAppInfo': {'musicActivityMasterSwitch': 'MUSIC_ACTIVITY_MASTER_SWITCH_INDETERMINATE', 'musicLocationMasterSwitch': 'MUSIC_LOCATION_MASTER_SWITCH_INDETERMINATE', 'pwaInstallabilityStatus': 'PWA_INSTALLABILITY_STATUS_UNKNOWN'}, 'utcOffsetMinutes': 60}, 'request': {'internalExperimentFlags': [{'key': 'force_music_enable_outertube_tastebuilder_browse', 'value': 'true'}, {'key': 'force_music_enable_outertube_playlist_detail_browse', 'value': 'true'}, {'key': 'force_music_enable_outertube_search_suggestions', 'value': 'true'}], 'sessionIndex': {}}, 'user': {'enableSafetyMode': false}}} + let request_body = {'browseEndpointContextSupportedConfigs': {'browseEndpointContextMusicConfig': {'pageType': 'MUSIC_PAGE_TYPE_ALBUM'}}, 'browseId': id, ...standard_body } - let headers = { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:72.0) Gecko/20100101 Firefox/72.0", - "Accept": "*/*", - "Accept-Language": "en-US,en;q=0.5", - "Content-Type": "application/json", - "X-Goog-AuthUser": "0", - "origin": "https://music.youtube.com", - "X-Goog-Visitor-Id": "CgtWaTB2WWRDeEFUYyjhv-X8BQ%3D%3D" - } - - const { body } = await request.post("https://music.youtube.com/youtubei/v1/browse?alt=json") - .set(headers) - .query({ alt: "json", key: "AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30"}) // INNERTUBE_API_KEY from music.youtube.com + const { body } = await request.post("https://music.youtube.com/youtubei/v1/browse") + .set(standard_headers) + .query(standard_params) .send(request_body) - let data = body.frameworkUpdates.entityBatchUpdate.mutations + let data = body.frameworkUpdates?.entityBatchUpdate.mutations + if (data === undefined) { + throw new Error() + } let album_data = data.find((entry) => { if (entry.payload.musicAlbumRelease !== undefined) { return true diff --git a/test/services/ytmusic.js b/test/services/ytmusic.js index e91d834..5d27a4b 100644 --- a/test/services/ytmusic.js +++ b/test/services/ytmusic.js @@ -13,6 +13,25 @@ describe('ytmusic', function(){ result.name.should.equal('One Vision (Remastered 2011)'); }); }); + describe('search', () => { + it('should find album by search', async function (){ + const result = await ytmusic.search({type: 'album', artist: {name: 'Jamie xx'}, name: 'In Colour'}); + result.name.should.startWith('In Colour'); + result.id.should.equal("MPREb_IbDz5pAZFvJ"); + }); + + it('should find album with various artists by search', async function (){ + const result = await ytmusic.search({type: 'album', artist: {name: 'Various Artists'}, name: 'Sambabook João Nogueira'}); + result.name.should.equal('Sambabook João Nogueira'); + result.id.should.equal('MPREb_iZt1VjORlv7'); + }); + + it('should find track by search', async function (){ + const result = await ytmusic.search({type: 'track', artist: {name: 'Oasis'}, albumName: 'Stop The Clocks', name: 'Wonderwall'}); + result.name.should.equal('Wonderwall'); + result.id.should.equal('Gvfgut8nAgw'); + }); + }); describe('lookupUrl', () => { describe('parseUrl', () => { it('should parse track url into ID', async function (){ From 78d635a0024df5346fe571e84e7091d1a9e3c4a9 Mon Sep 17 00:00:00 2001 From: "Renato \"Lond\" Cerqueira" Date: Sun, 17 Jan 2021 11:55:26 +0100 Subject: [PATCH 04/11] Fix search for various artists --- lib/services/ytmusic/index.js | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/lib/services/ytmusic/index.js b/lib/services/ytmusic/index.js index 0a0b100..85335b3 100644 --- a/lib/services/ytmusic/index.js +++ b/lib/services/ytmusic/index.js @@ -31,7 +31,7 @@ export async function search(data, original = {}) { let query; const various = data.artist.name === 'Various Artists' || data.artist.name === 'Various'; if (various) { - data.artist.name = undefined; + data.artist.name = ""; } if (data.type == "track") { query = [data.name, data.artist.name, data.albumName] @@ -43,8 +43,8 @@ export async function search(data, original = {}) { // Add "" to try and make the search better, works for stuff like "The beatles" to reduce noise query = query.filter(String).map((entry) => '"' + entry + '"').join(" ") - let param = base_filter + (data.type == "track" ? tracks_filter : albums_filter) + exact_search_filter - let request_body = {query, param, ...standard_body } + let params = base_filter + (data.type == "track" ? tracks_filter : albums_filter) + exact_search_filter + let request_body = {query, params, ...standard_body } const { body } = await request.post("https://music.youtube.com/youtubei/v1/search") .set(standard_headers) @@ -57,8 +57,12 @@ export async function search(data, original = {}) { return { service: 'ytmusic' }; } - // I ignore the tabbedSearchResultsRenderer case from ytmusicapi, because we're always selecting a tab. - const results = body.contents.sectionListRenderer.contents + let results; + if (body.contents.tabbedSearchResultsRenderer !== undefined) { + results = body.contents.tabbedSearchResultsRenderer.tabs[0].tabRenderer.content + } else { + results = body.contents.sectionListRenderer.contents + } // no results if (results.length == 1 && results.itemSectionRenderer !== undefined) { @@ -156,14 +160,19 @@ async function lookupAlbum(id) { } return false }).payload.musicAlbumRelease; - let artists = data.filter((entry) => { - if (entry.payload.musicArtist !== undefined) { - if (album_data.primaryArtists.includes(entry.entityKey)) { - return true + let artists; + if (album_data.primaryArtists) { + artists= data.filter((entry) => { + if (entry.payload.musicArtist !== undefined) { + if (album_data.primaryArtists.includes(entry.entityKey)) { + return true + } } - } - return false - }).map((entry) => entry.payload.musicArtist.name); + return false + }).map((entry) => entry.payload.musicArtist.name); + } else { // Various artists, most likely + artists = [album_data.artistDisplayName]; + } const artwork = { small: album_data.thumbnailDetails.thumbnails[0].url, From dd5e3897392a9edea21a4a744bc115cd090c38c6 Mon Sep 17 00:00:00 2001 From: "Renato \"Lond\" Cerqueira" Date: Sun, 17 Jan 2021 12:05:16 +0100 Subject: [PATCH 05/11] Add check for streamUrl --- lib/services/ytmusic/index.js | 3 ++- test/services/ytmusic.js | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/services/ytmusic/index.js b/lib/services/ytmusic/index.js index 85335b3..b2fe78b 100644 --- a/lib/services/ytmusic/index.js +++ b/lib/services/ytmusic/index.js @@ -130,7 +130,7 @@ async function lookupTrack(id) { type: 'track', id: song_meta.videoId, name: song_meta.title, - streamUrl: null, + streamUrl: `https://music.youtube.com/watch?v=${song_meta.videoId}`, purchaseUrl: null, artwork, artist: { @@ -184,6 +184,7 @@ async function lookupAlbum(id) { id, name: album_data.title, streamUrl: null, + streamUrl: `https://music.youtube.com/browse/${id}`, purchaseUrl: null, artwork, artist: { diff --git a/test/services/ytmusic.js b/test/services/ytmusic.js index 5d27a4b..1579123 100644 --- a/test/services/ytmusic.js +++ b/test/services/ytmusic.js @@ -37,10 +37,12 @@ describe('ytmusic', function(){ it('should parse track url into ID', async function (){ const result = await ytmusic.parseUrl('https://music.youtube.com/watch?v=YLp2cW7ICCU&feature=share'); result.id.should.equal("YLp2cW7ICCU"); + result.streamUrl.should.equal("https://music.youtube.com/watch?v=YLp2cW7ICCU"); }); it('should parse album url into ID', async function (){ const result = await ytmusic.parseUrl('https://music.youtube.com/browse/MPREb_9C36yscfgmJ'); result.id.should.equal("MPREb_9C36yscfgmJ"); + result.streamUrl.should.equal("https://music.youtube.com/browse/MPREb_9C36yscfgmJ"); }); it('should parse alternative album url into ID', async function (){ const result = await ytmusic.parseUrl('https://music.youtube.com/playlist?list=OLAK5uy_lx9K5RpiBEwd3E4C1GKqY7e06qTlwydvs'); From 7c0269229d51bac83bcdec3624958fd2df1d3610 Mon Sep 17 00:00:00 2001 From: "Renato \"Lond\" Cerqueira" Date: Sun, 17 Jan 2021 16:10:14 +0100 Subject: [PATCH 06/11] Parse playlist and use it to figure out the album --- lib/services/ytmusic/index.js | 44 ++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/lib/services/ytmusic/index.js b/lib/services/ytmusic/index.js index b2fe78b..887bb16 100644 --- a/lib/services/ytmusic/index.js +++ b/lib/services/ytmusic/index.js @@ -190,14 +190,54 @@ async function lookupAlbum(id) { artist: { name: artists.join(", "), }, + playlistId: album_data.audioPlaylistId }); } +async function lookupPlaylist(id) { + let request_body = {enablePersistentPlaylistPanel: true, isAudioOnly: true, playlistId: id, ...standard_body} + const { body } = await request.post("https://music.youtube.com/youtubei/v1/next") + .set(standard_headers) + .query(standard_params) + .send(request_body) + + // The playlist object is rather complex, but here's what I'm doing: I'll parse the very minimum to get to the first track. + // At that point, I'm going to check the id of the album in that track. And make a lookup on it. If the album looked up + // has the same playlist id as the one I'm looking up, it means it's an album and not a playlist and we're good. + const watchNextRenderer = body.contents.singleColumnMusicWatchNextResultsRenderer.tabbedRenderer.watchNextTabbedResultsRenderer + const firstTrack = watchNextRenderer.tabs[0].tabRenderer.content.musicQueueRenderer.content.playlistPanelRenderer.contents[0] + const runs = firstTrack.playlistPanelVideoRenderer.longBylineText.runs + const reverse_last_artist_idx = runs.reverse().findIndex((entry) => { + if (entry.navigationEndpoint === undefined) { + return false + } + return entry.navigationEndpoint.browseEndpoint.browseId.startsWith("UC") || + entry.navigationEndpoint.browseEndpoint.browseId.startsWith("FEmusic_library_privately_owned_artist") + }); + if (reverse_last_artist_idx == -1) { + debug("Could not find an artist. Implement extra logic from ytmusicapi!"); + throw new Error(); + } + const last_artist_idx = runs.length - reverse_last_artist_idx - 1; + if (runs.length - last_artist_idx != 5) { + debug("No album found, can't find this."); + throw new Error(); + } + const albumId = runs[last_artist_idx + 2].navigationEndpoint.browseEndpoint.browseId + const possibleAlbum = await lookupAlbum(albumId) + if (possibleAlbum.playlistId = id) { + return possibleAlbum; + } + throw new Error(); +} + export async function lookupId(id, type) { if (type == 'track') { return lookupTrack(id); } else if (type == 'album') { return lookupAlbum(id); + } else if (type == 'playlist') { + return lookupPlaylist(id); } return { service: 'ytmusic', id }; } @@ -214,9 +254,7 @@ export function parseUrl(url) { } else if (match = parsed.path.match(/^\/browse\/([A-Za-z0-9_]+)/)) { return lookupId(match[1], 'album'); } else if (match = parsed.path.match(/^\/playlist/) && list_id !== undefined) { - if (list_id == 'OLAK5uy_lx9K5RpiBEwd3E4C1GKqY7e06qTlwydvs') { // TODO: Parse playlist correctly - return lookupId('MPREb_9C36yscfgmJ', 'album'); - } + return lookupId(list_id, 'playlist'); } throw new Error(); } From e5a9da78d6a1d9d51bdeb37ace24045c3b4ab813 Mon Sep 17 00:00:00 2001 From: "Renato \"Lond\" Cerqueira" Date: Sun, 17 Jan 2021 16:12:42 +0100 Subject: [PATCH 07/11] Add ytmusic to enums in models --- models/album.cjs | 3 ++- models/match.cjs | 3 ++- models/track.cjs | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/models/album.cjs b/models/album.cjs index 98e7d77..83e7d04 100644 --- a/models/album.cjs +++ b/models/album.cjs @@ -8,7 +8,8 @@ module.exports = function (sequelize, DataTypes) { 'itunes', 'spotify', 'xbox', - 'youtube' + 'youtube', + 'ytmusic' ), name: DataTypes.TEXT, artistId: DataTypes.INTEGER, diff --git a/models/match.cjs b/models/match.cjs index 677ad45..2e8c39e 100644 --- a/models/match.cjs +++ b/models/match.cjs @@ -10,7 +10,8 @@ module.exports = function (sequelize, DataTypes) { 'itunes', 'spotify', 'xbox', - 'youtube' + 'youtube', + 'ytmusic' ), name: DataTypes.TEXT, streamUrl: DataTypes.TEXT, diff --git a/models/track.cjs b/models/track.cjs index c09bf48..294c483 100644 --- a/models/track.cjs +++ b/models/track.cjs @@ -8,7 +8,8 @@ module.exports = function (sequelize, DataTypes) { 'itunes', 'spotify', 'xbox', - 'youtube' + 'youtube', + 'ytmusic' ), name: DataTypes.TEXT, artistId: DataTypes.INTEGER, From 248e42f8415e5360bbc5077326d250ecd5732903 Mon Sep 17 00:00:00 2001 From: "Renato \"Lond\" Cerqueira" Date: Sun, 17 Jan 2021 16:22:46 +0100 Subject: [PATCH 08/11] Fix artist name in track search --- lib/services/ytmusic/index.js | 2 +- test/services/ytmusic.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/services/ytmusic/index.js b/lib/services/ytmusic/index.js index 887bb16..24b4161 100644 --- a/lib/services/ytmusic/index.js +++ b/lib/services/ytmusic/index.js @@ -118,7 +118,7 @@ async function lookupTrack(id) { let description = song_meta.shortDescription.split("\n\n") let album_name = description[2] - let artists = description[1].split(' · ') + let artists = description[1].split(' · ').slice(1) const artwork = { small: song_meta.thumbnail.thumbnails[0].url, diff --git a/test/services/ytmusic.js b/test/services/ytmusic.js index 1579123..e89ac79 100644 --- a/test/services/ytmusic.js +++ b/test/services/ytmusic.js @@ -11,6 +11,7 @@ describe('ytmusic', function(){ it('should find track by ID', async function (){ const result = await ytmusic.lookupId('9zrYXvUXiQk', 'track'); result.name.should.equal('One Vision (Remastered 2011)'); + result.artist.name.should.equal('Queen'); }); }); describe('search', () => { From 216940a2b83de695526f6f2f9f81ae4868e62336 Mon Sep 17 00:00:00 2001 From: "Renato \"Lond\" Cerqueira" Date: Sun, 17 Jan 2021 16:40:16 +0100 Subject: [PATCH 09/11] Add extra logic to match albums and tracks to improve match --- lib/services/ytmusic/index.js | 11 +++++++++-- test/services/ytmusic.js | 6 ++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/services/ytmusic/index.js b/lib/services/ytmusic/index.js index 24b4161..9e16713 100644 --- a/lib/services/ytmusic/index.js +++ b/lib/services/ytmusic/index.js @@ -76,8 +76,15 @@ export async function search(data, original = {}) { } const matches = parse_result_content(result.musicShelfRenderer.contents, data.type) - if (matches[0]) { - return await lookupId(matches[0], data.type) + // This could probably be done without extra lookups, but it would involve parsing deeply the response. + // If there's some kind of rate limit on ytmusic's side, this is a good play to start refactoring + for (const match of matches) { + const possibleMatch = await lookupId(match, data.type) + const nameMatch = possibleMatch.name == data.name; + const artistMatch = data.artist.name == "" ? possibleMatch.artist.name === 'Various Artists' : data.artist.name == possibleMatch.artist.name; + if (nameMatch && artistMatch) { + return possibleMatch + } } } debug("Finished looking up, no results") diff --git a/test/services/ytmusic.js b/test/services/ytmusic.js index e89ac79..ad10834 100644 --- a/test/services/ytmusic.js +++ b/test/services/ytmusic.js @@ -27,6 +27,12 @@ describe('ytmusic', function(){ result.id.should.equal('MPREb_iZt1VjORlv7'); }); + it('should find album and make sure it makes sense by search', async function(){ + const result = await ytmusic.search({type: 'album', artist: {name: 'The Beatles'}, name: 'The Beatles'}); + result.name.should.equal('The Beatles'); + result.id.should.equal('MPREb_S5TiUIYvI78'); + }); + it('should find track by search', async function (){ const result = await ytmusic.search({type: 'track', artist: {name: 'Oasis'}, albumName: 'Stop The Clocks', name: 'Wonderwall'}); result.name.should.equal('Wonderwall'); From 4d726ee284ff7beaf9a2d82845ff078a6809374a Mon Sep 17 00:00:00 2001 From: "Renato \"Lond\" Cerqueira" Date: Sun, 17 Jan 2021 22:26:46 +0100 Subject: [PATCH 10/11] Add ytmusic image Image from wikimedia, at https://commons.wikimedia.org/wiki/File:YT_Music.svg --- public/assets/images/ytmusic.png | Bin 0 -> 5530 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 public/assets/images/ytmusic.png diff --git a/public/assets/images/ytmusic.png b/public/assets/images/ytmusic.png new file mode 100644 index 0000000000000000000000000000000000000000..96743d40ab64ac0a33c027e0746e711a36d9b3a1 GIT binary patch literal 5530 zcmV;L6=mv)P)004R>004l5008;`004mK004C`008P>0026e000+ooVrmw00006 zVoOIv0RI600RN!9r;`8x6tPJ}K~#9!?VWji6jipzza7F72q4QSZX+7OfT$=S;)=`Y zQ*l6_I^f{GKT*eV7(f*L=0xF|2j*9PrxG)X`AudlHHWfyJD7do-2?7Zr zBsuSoTN%1L-Bq`4E!}DA_xaREL)Bfn?)}v{_uO;NjgbKe6af8z9zZvs82CBR2H4N~ zZ$7Zyx~hN;z*=Apuo_qrgRMk}5Wx|#5Xs#iI3E}YoCLJ=SYM*V9}8S1&@38_2od}z_yB>o zGequG*J^<)B*Z03gb2Az@QwnH3Dji~$>Xng1dfeHDnbNvl3ycGo}~njzxD`B5@->P zT7-QKF%JZ93p@b~Z(_fBIWRN^Wzq0Q@QBlaVW@55Lew^J8SuZrye3a{fn$K#z>!UK zfVKnI#nga=g$ah`CIS^6+rAw**r41p;FXBaDKQFo*oH$-12;BhRs9TsiYB0{J6-hx zBg1`lM%M*CwEd%D+kcMuoDyAuT8;O2VYYNQ&*eMYeP}-v!Bs0JGebX(G788nh8v+P@P$mhTzy(svxK`P^|$N}>y# z1-z7lByYfN0{3Py(Y^QqoDlKScj#)gotlh97ZrRm7w^95DO%5me*NIY6QQ^mjyM83 zb%M5S)xC-esH}u9zl5?fSg``$dk=Q*4B}lpB(Nz4Pi5Jp>WDIWn(h+xjd&TzvQGz1 z!!uK$gTOjpjbJTXid=Gu$n4o7J9da`?AaqyS}JnOEg~H|`fZ@A5jZ)x*J*rt*}^RI z&~9XmOO2;A-`|%_f#flPSA8SVe*1}x9V@bVv-pPUYLTf^MT(02B>OiKjPc3~(H}Q# zvjZy1FCx0X_^syqd#foBeWWkM$VhscT*=?VJdk0LW?irjF6NS{6; z?b6p^)S)7Q6gqsWC9`bGB3vWV!rfmQ}>j){mK7#??cck6)m z*-rE~JQL;0D@AthY)r=bdXahaL@v5WqUVlj~k6B?8K(MP@1TP4b0JPe{g;IO84v+ehv4(;6^5q;kz z`l-fbC@K=!wk?rd3m1yCYL%JDw1*#VY@oaNVlQ=$4vFZ4@Gp|B#G$TV?;}J%0O*eL z@H_?O`+tz(E=DF$h)RXt8#omeBK~t+CQylYfxX(y_85bwo`Uvq2I^I-V8{@tuFlM> zx#u1@_+Ys5%DsJ@Iu#Z#R&KTKajn3_n40`SVu{23{^%t%$g?JPI@AN7cvV$28B?S{C#7QGOGp=#THIzzrJDO}5%_OYC2v zg2?&y7?onFI5 zbtF0#DbhXjH_f(sI<)l}DyG_tL|=j4UVLwxI{vI##~h8{d@yP|u@*C*fhFk21ze+9 z=MWe3BMzChNTv}Z^vd??rPkCzlD$tKk=ok5eXFe% z>E7M6zQ;l$`fQtLwxSnMajipJGjojS7XjsXa}E-q+LESW6TQ&V)V%h8yS4mF`4s^; z$7t#f9}b5fzW49t%i&2)4eOzY;G>UV;K1M%;bqGb`_rr$j5c0wpF=oUYICn4lFbQD z+Q+gSGA|88rMv3jLtQPckKm~x_CFt$((HU6>S_zT4D?DTYRZhHT(W!Xph5a&=giq# zZovYPV~-6+svmHGNLAI|ws!9p>D<}0J}-Ofx)R;VKV2u1ZMj1or{#!5Z|_0Xb#h+H z4@sim8$ka{8%cCg7CiS!9*;d1diPE=z`gmVp{zVd#hO3=JREqSzu(JEo8Z}J_x|0Y z1q>bPMS6rG(Zz$bE=c=D-)o5bI7g)U-2{`|p*t#VTF^L@U=p63o;*+TfIdc5{VHiG zGh>F5d8WwAQ&#dXzJLi6pj|uve)fzRiGN>yxoJ&(LKyrC?BkgGIg~#RXl`@A4A_wa zB+mnG(RgkbFdg_0R1UXEsI<&Z2!rT%p_kC5#L^5bIOPHi}b1!dx-L>0|<$huUXO#@k6B z)>POJwX6*ar;~nQfokPytDM z9LmPxxt)he+N9zn!S?y*!4GTy{!m(~B(n)gll$1cyXm@czYr$z@}#eLGJ4ze?-+;W zL#`O&bmncP&AoqW8q0!$j=kq=)ZTb9s(`QDd<^qYdvES=&_RiRuUi-1@HSZJr%i({ zE@Jz?OtNnsY`jy}HiTcf6Ig9P^dgk=g`FYpryL==&RXsWn|oU|$`=IKe;rNw2b+7} zX_W8ajp&6*bLpp$=T3e;+;o#W3w>oJ)YT>08O+cwq+hwzKy}|&)OLVv8qZun`lXgm z&hYwx`rblwSzz}u+OAQ)E}rP_zgh)JMRx7V%uu&!1EWU4qD9VR7lEp(#Ij~+9n!C} zTc>wIMPMbv=~R{#IYji0I=Uxn3(J(~bpiGLaP$sC*==|)Ud30k5w*`)=&GB^#4aev z%urWW!jvg6emv}O*cy(-pmpoSvSz3X>DMiFxPMd%)vnlMolOZ#G@pCWTF<_eLL(@# z2*1bryUtqL?6LlRNkDzqT{qU)+|vmS-4##t9j=LXj)h zOx!8hVeB)qJ)~dxi8ava;*N`HyhUWURVq0q(U+r)ZBD9LbCmBuXJAVFf)nL#oDp4A z^SNq|61_5+=v}3+e!3tRC=-U6xUYkziZ-&z6GZ^ z)?}iRoOW7puVn37<(ocWfH}!i_0g?H zpzw9f{^f_cPA_)3K3|@LME?YpL){GyM`e|H0L{~cGe~sbG8*N@i+648+zCtL=HSss zH{M(KiY;4I?D#Rqz$>qK3zz};Mci_%rUu@B-=vn+Ax{W9qW?T;t#{NuNOVxA(YW>4 z%10%~oJ0ClQ$eD6*bU|xPu_(K<5q_;7&gqG9#vJr#EEK6J#nHUrC0Rp2S>(5;(qv{ zsnvZ=$m6q;^j7{Di4Gw}4w4~PiM}0`+Ip)&*Vznu8@wbyzJ zJgKjTnKKn3M~{Z>J{q^LyUxDv`UrcSTZerTM_rzH+5_}umH$z2UAn;4S0}3Nvu2spB&evTuwl2C zTa zj=CK%A;5m=9%0nXSE5^Z&bXGFZc04rsAo@j?KNnXq2sM4PE6eH6M>s=hHcwSYs~Z+ zWa-xFEm0nE`??{{>D~1)Q2#Fl_7A8}y5XL7*^CYWZbBuDybQHfcn<1(xjxpCuR|)a z5ffN$ydEArxbbPKB_$&5+J$?X>ZD1H6HQK<DP;V0eL!pUOEt^ez5fd|%8zLz(X9gn|6u$Y=gx&ONptG#vsL=1FoIs% zwpGiBks}-T>z;eO)I2Q)o3d1*H>}SiiLUpdy$=}XLUryHP+$Cz=sIq+3DIlun1k}h zI>uOK?$jxfY`uG{{q_+f;4gm(#sGKHN$}irjq|{jmMRs#*4Xw@jmpBAX{<*%gQUun zb}RNlNx5o>yB;{v=KeRp_!R&CGRhI@Wt@H8j|`~qpwv1b)}?AXR?yoku6 zMQY2-FID}}Lq(o`TBNS7aa*%zixd=iZNpEJdr{p%ouuk^o9HW>`kK1x{)R&tC*$d3 z8tuLwkm`P*W_i0KvTxBSKP;6ou0e&LW4%Okj32K~UD=tGt!Lmszr*0c>UbpA zE3Fuud@?-qjLI2z{q^dUmw3b5q)93lytfor8&KIvvdO6A7D4*A>!w)>D)pArQA|#j z;PE)8Ob#lQ)JDzagPmY-x#y~@yd37w zS2^EStWY7T$%!UgwuEloREp;_&VcjIOFY3fftnkXU;A5c`7s!`$FeJ9s!3X-AlK(R zeIleOoYEQ{Vj1Altra(;0=n#$sX-+fY0$QuLW}M8@^!MvuGQ(u-M*;T&6g(rcL4-cI^^* z;t7$iUH#mq`&xpLLSBgu@V!f-9}pnXr)u_Lewy+lUHpzzM1NJIoZZnyZ7r*`*7j5Y zL?5MD$Bik<9;H$4Q#UiSgTOjp_qtoOP+afpKDAJ*W-+S>j z5?pMuA4GrE1v$QPsLLzSZ`bTc9m<&_e%<+`%MVs6qW?*wTs5hWnU0~n=A6n843Ow2 zXx6co#IqOjEhC*%xo1c!l-}+I6{4Iq7U9d_upUn zspsb_PqyvbVf}iweqX*^WrnG$3W|ug#!?^3mrVw_!CA0AcVjciv((g;*8g`WF~iDE z+-4zGmM(c%v_@bCWapp{X7>^@VYA%jj~^1f0lk2CHR@XioSmZl8F-6c zYinr*UL@m6?0x5hlIRUM2Nly`+W+@ZDZtYjR7_`W0cu@)M-!9eOM#&=r2>ZuCmX;z zV7f*Z`tYrG@e%e-C+%PvwQ93eRbMQop|=qtL}*0#eggBeh~WPexI7w}2oZuId~bp0 zGbrI#2wWvTCfAD)Awqh>A1-iTNEZ6d0+S`RZ*znQ`!r%9lD;o+J}S$w*pLen6666sr^h~EPh Date: Mon, 18 Jan 2021 13:31:25 +0100 Subject: [PATCH 11/11] Add youtube music to index description, remove groove music --- public/src/views/index.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/src/views/index.vue b/public/src/views/index.vue index 32670f2..b9e90b0 100644 --- a/public/src/views/index.vue +++ b/public/src/views/index.vue @@ -10,7 +10,7 @@ Combine.fm makes sharing from music services better. What happens when you share your favourite song on Spotify with a friend, but they don't use Spotify?

- We match album and track links from Youtube, Spotify, Google Music, Apple Musicm, Groove Music and Deezer and give you back one link with matches we find on all of them. + We match album and track links from Youtube, Spotify, Google Music, Apple Music, Deezer and Youtube Music and give you back one link with matches we find on all of them.