Skip to content

feat(douban): support music subjects #1764

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
lenovobenben wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat(douban): support music subjects #1764

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions cli-manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8560,10 +8560,11 @@
"type": "str",
"default": "movie",
"required": false,
"help": "条目类型（movie=电影, book=图书）",
"help": "条目类型（movie=电影, book=图书, music=音乐）",
"choices": [
"movie",
"book"
"book",
"music"
]
}
],
Expand All @@ -8584,6 +8585,11 @@
"series",
"isbn10",
"isbn13",
"artists",
"albumType",
"media",
"barcode",
"discCount",
"year",
"rating",
"ratingCount",
Expand All @@ -8593,6 +8599,8 @@
"country",
"duration",
"summary",
"tracks",
"cover",
"url"
],
"type": "js",
Expand Down
9 changes: 8 additions & 1 deletion clis/douban/subject.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ cli({
navigateBefore: false,
args: [
{ name: 'id', required: true, positional: true, help: '豆瓣条目 ID' },
{ name: 'type', default: 'movie', choices: ['movie', 'book'], help: '条目类型（movie=电影, book=图书）' },
{ name: 'type', default: 'movie', choices: ['movie', 'book', 'music'], help: '条目类型（movie=电影, book=图书, music=音乐）' },
],
columns: [
'id',
Expand All @@ -31,6 +31,11 @@ cli({
'series',
'isbn10',
'isbn13',
'artists',
'albumType',
'media',
'barcode',
'discCount',
'year',
'rating',
'ratingCount',
Expand All @@ -40,6 +45,8 @@ cli({
'country',
'duration',
'summary',
'tracks',
'cover',
'url',
],
func: async (page, args) => [await loadDoubanSubjectDetail(page, args.id, args.type)],
Expand Down
6 changes: 6 additions & 0 deletions clis/douban/subject.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,10 @@ describe('douban subject command', () => {
expect(command).toBeDefined();
expect(command?.navigateBefore).toBe(false);
});

it('accepts music subject type', () => {
const command = getRegistry().get('douban/subject');
const typeArg = command?.args.find((arg) => arg.name === 'type');
expect(typeArg?.choices).toEqual(['movie', 'book', 'music']);
});
});
74 changes: 73 additions & 1 deletion clis/douban/utils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,32 @@ export function normalizeDoubanBookSubject(raw) {
url: firstNonEmpty([raw?.url]),
};
}
export function normalizeDoubanMusicSubject(raw) {
const info = parseDoubanBookInfoText(raw?.infoText);
return {
id: normalizeDoubanSubjectId(raw?.id),
type: 'music',
title: firstNonEmpty([raw?.title]),
subtitle: firstNonEmpty([raw?.subtitle, info['又名']]),
artists: splitDoubanPeople(firstNonEmpty([info['表演者'], info['作者']])),
genres: firstNonEmpty([info['流派']]),
albumType: firstNonEmpty([info['专辑类型']]),
media: firstNonEmpty([info['介质']]),
publishDate: firstNonEmpty([info['发行时间'], info['出版年']]),
publishYear: extractDoubanPublishYear(firstNonEmpty([info['发行时间'], info['出版年']])),
publisher: firstNonEmpty([info['出版者'], info['出版社']]),
discCount: parseDoubanPageCount(info['唱片数']),
barcode: firstNonEmpty([info['条形码'], info['ISBN']]),
rating: parseDoubanRating(raw?.rating),
ratingCount: parseDoubanCount(raw?.ratingCount),
summary: normalizeText(raw?.summary).replace(/\s*投诉$/, ''),
tracks: Array.isArray(raw?.tracks)
? raw.tracks.map((track) => normalizeText(track)).filter(Boolean)
: [],
cover: firstNonEmpty([raw?.cover]),
url: firstNonEmpty([raw?.url]),
};
}
async function loadDoubanMovieSubject(page, subjectId) {
const normalizedId = normalizeDoubanSubjectId(subjectId);
const data = await withDetachedRetry(async () => {
Expand Down Expand Up @@ -300,11 +326,57 @@ async function loadDoubanBookSubject(page, subjectId) {
});
return normalizeDoubanBookSubject(data);
}
async function loadDoubanMusicSubject(page, subjectId) {
const normalizedId = normalizeDoubanSubjectId(subjectId);
const data = await withDetachedRetry(async () => {
await page.goto(`https://music.douban.com/subject/${normalizedId}/`, { waitUntil: 'load', settleMs: 1500 });
await ensureDoubanReady(page);
await page.wait({ selector: 'h1, #info', timeout: 8 }).catch(() => { });
return page.evaluate(`
(() => {
const normalize = (value) => (value || '').replace(/\\s+/g, ' ').trim();
const sectionText = (headingPattern) => {
const headings = Array.from(document.querySelectorAll('h2'));
const heading = headings.find((node) => headingPattern.test(normalize(node.textContent)));
let next = heading?.nextElementSibling || null;
while (next) {
if (next.matches?.('.indent, .track-items, #link-report')) {
const text = normalize(next.innerText || next.textContent || '');
if (text) return text;
}
next = next.nextElementSibling;
}
return '';
};
const tracks = Array.from(document.querySelectorAll('.track-items li'))
.map((node) => normalize(node.textContent))
.filter(Boolean);
return {
id: ${JSON.stringify(normalizedId)},
title: normalize(document.querySelector('h1 span')?.textContent || document.querySelector('h1')?.textContent || ''),
subtitle: '',
infoText: document.querySelector('#info')?.innerText || document.querySelector('#info')?.textContent || '',
rating: normalize(document.querySelector('strong.rating_num, strong[property="v:average"]')?.textContent || ''),
ratingCount: normalize(document.querySelector('a.rating_people > span, span[property="v:votes"]')?.textContent || ''),
summary: sectionText(/简介/),
tracks,
cover: document.querySelector('#mainpic img')?.getAttribute('src') || '',
url: location.href,
};
})()
`);
});
return normalizeDoubanMusicSubject(data);
}
export async function loadDoubanSubjectDetail(page, subjectId, subjectType = 'movie') {
const type = String(subjectType || 'movie').trim() === 'book' ? 'book' : 'movie';
const rawType = String(subjectType || 'movie').trim();
const type = rawType === 'book' || rawType === 'music' ? rawType : 'movie';
if (type === 'book') {
return loadDoubanBookSubject(page, subjectId);
}
if (type === 'music') {
return loadDoubanMusicSubject(page, subjectId);
}
return loadDoubanMovieSubject(page, subjectId);
}
export async function loadDoubanSubjectPhotos(page, subjectId, options = {}) {
Expand Down
100 changes: 100 additions & 0 deletions clis/douban/utils.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
loadDoubanSubjectDetail,
loadDoubanSubjectPhotos,
normalizeDoubanBookSubject,
normalizeDoubanMusicSubject,
normalizeDoubanSubjectId,
promoteDoubanPhotoUrl,
resolveDoubanPhotoAssetUrl,
Expand Down Expand Up @@ -276,6 +277,51 @@ ISBN: 9787544270871
});
});

it('normalizes douban music subject raw data into structured fields', () => {
const normalized = normalizeDoubanMusicSubject({
id: '26812952',
title: '周杰伦的床边故事',
infoText: `
又名: Jay Chou's Bedtime Stories
表演者: 周杰伦
流派: 流行
专辑类型: 专辑
介质: CD
发行时间: 2016-06-24
出版者: 杰威尔音乐
唱片数: 1
条形码: 9787799452302
`,
rating: '8.1',
ratingCount: '30853',
summary: '周杰伦2016年最新专辑 投诉',
tracks: ['床边故事', '告白气球'],
cover: 'https://img9.doubanio.com/view/subject/m/public/s28836865.jpg',
url: 'https://music.douban.com/subject/26812952/',
});
expect(normalized).toMatchObject({
id: '26812952',
type: 'music',
title: '周杰伦的床边故事',
subtitle: "Jay Chou's Bedtime Stories",
artists: ['周杰伦'],
genres: '流行',
albumType: '专辑',
media: 'CD',
publishDate: '2016-06-24',
publishYear: '2016',
publisher: '杰威尔音乐',
discCount: 1,
barcode: '9787799452302',
rating: 8.1,
ratingCount: 30853,
summary: '周杰伦2016年最新专辑',
tracks: ['床边故事', '告白气球'],
cover: 'https://img9.doubanio.com/view/subject/m/public/s28836865.jpg',
url: 'https://music.douban.com/subject/26812952/',
});
});

it('parses movie-hot rows with real chart fields only', async () => {
const items = [
createFakeMovieHotItem({
Expand Down Expand Up @@ -364,6 +410,60 @@ ISBN: 9787544270871
});
});

it('loads music subject details from music.douban.com when type=music', async () => {
const page = {
goto: vi.fn().mockResolvedValue(undefined),
wait: vi.fn().mockResolvedValue(undefined),
evaluate: vi.fn()
.mockResolvedValueOnce({ blocked: false, title: '周杰伦的床边故事 (豆瓣)', href: 'https://music.douban.com/subject/26812952/' })
.mockResolvedValueOnce({
id: '26812952',
title: '周杰伦的床边故事',
subtitle: '',
infoText: `
表演者: 周杰伦
流派: 流行
专辑类型: 专辑
介质: CD
发行时间: 2016-06-24
出版者: 杰威尔音乐
唱片数: 1
条形码: 9787799452302
`,
rating: '8.1',
ratingCount: '30853',
summary: '周杰伦2016年最新专辑',
tracks: ['床边故事', '告白气球'],
cover: 'https://img9.doubanio.com/view/subject/m/public/s28836865.jpg',
url: 'https://music.douban.com/subject/26812952/',
}),
};
const detail = await loadDoubanSubjectDetail(page, '26812952', 'music');
expect(page.goto).toHaveBeenCalledWith('https://music.douban.com/subject/26812952/', {
waitUntil: 'load',
settleMs: 1500,
});
expect(page.wait).toHaveBeenCalledWith({ selector: 'h1, #info', timeout: 8 });
expect(detail).toMatchObject({
id: '26812952',
type: 'music',
title: '周杰伦的床边故事',
artists: ['周杰伦'],
genres: '流行',
albumType: '专辑',
media: 'CD',
publishDate: '2016-06-24',
publishYear: '2016',
publisher: '杰威尔音乐',
discCount: 1,
barcode: '9787799452302',
rating: 8.1,
ratingCount: 30853,
tracks: ['床边故事', '告白气球'],
url: 'https://music.douban.com/subject/26812952/',
});
});

it('retries transient detached navigation errors when loading douban search results', async () => {
const page = {
goto: vi.fn()
Expand Down
Loading