From 5c345efa1da147dbf8203076920dddcaa1a53668 Mon Sep 17 00:00:00 2001 From: wanxp <977741432@qq.com> Date: Sat, 23 Sep 2023 09:21:46 +0800 Subject: [PATCH] 1. fix sync movie fetch wrong type 2. fix can not get the comment of book or movie --- src/org/wanxp/constant/Constsant.ts | 164 ++++++++++++++---- src/org/wanxp/constant/DefaultSettings.ts | 4 +- src/org/wanxp/constant/DoubanUserState.ts | 29 ++++ .../wanxp/douban/component/DoubanSyncModal.ts | 1 + .../data/handler/DoubanAbstractLoadHandler.ts | 42 ++++- .../data/handler/DoubanBookLoadHandler.ts | 19 +- .../data/handler/DoubanGameLoadHandler.ts | 2 +- .../data/handler/DoubanMovieLoadHandler.ts | 15 +- .../data/handler/DoubanMusicLoadHandler.ts | 2 +- .../data/handler/DoubanNoteLoadHandler.ts | 2 +- .../data/handler/DoubanSubjectLoadHandler.ts | 2 +- .../data/handler/DoubanTeleplayLoadHandler.ts | 2 +- .../data/handler/DoubanTheaterLoadHandler.ts | 2 +- .../wanxp/douban/data/model/DoubanSubject.ts | 3 + .../wanxp/douban/data/model/HandleContext.ts | 1 + .../wanxp/douban/setting/SettingsManager.ts | 20 ++- .../setting/model/DoubanPluginOnlineData.ts | 6 + .../model/DoubanPluginOnlineSettings.ts | 5 + .../setting/model/DoubanPluginSetting.ts | 2 + .../model/DoubanPluginSubjectProperty.ts | 5 + .../sync/handler/DoubanAbstractSyncHandler.ts | 8 +- src/org/wanxp/lang/locale/en.ts | 1 + src/org/wanxp/lang/locale/zh-cn.ts | 1 + src/org/wanxp/main.ts | 35 +++- src/org/wanxp/utils/GithubUtil.ts | 19 ++ src/org/wanxp/utils/HtmlUtil.ts | 26 +++ src/org/wanxp/utils/HttpUtil.ts | 12 +- 27 files changed, 366 insertions(+), 64 deletions(-) create mode 100644 src/org/wanxp/douban/setting/model/DoubanPluginOnlineData.ts create mode 100644 src/org/wanxp/douban/setting/model/DoubanPluginOnlineSettings.ts create mode 100644 src/org/wanxp/douban/setting/model/DoubanPluginSubjectProperty.ts create mode 100644 src/org/wanxp/utils/GithubUtil.ts create mode 100644 src/org/wanxp/utils/HtmlUtil.ts diff --git a/src/org/wanxp/constant/Constsant.ts b/src/org/wanxp/constant/Constsant.ts index 4a69b00..b1c71ad 100644 --- a/src/org/wanxp/constant/Constsant.ts +++ b/src/org/wanxp/constant/Constsant.ts @@ -1,6 +1,7 @@ import {i18nHelper} from "../lang/helper"; import DoubanSearchResultSubject from "../douban/data/model/DoubanSearchResultSubject"; import StringUtil from "../utils/StringUtil"; +import {DoubanPluginOnlineSettings} from "../douban/setting/model/DoubanPluginOnlineSettings"; /** * 常量池 @@ -39,19 +40,19 @@ export const BasicConst = { * 预估程序处理时间长度 * @private */ -export const ESTIMATE_TIME_PER:number = 2000; +export const ESTIMATE_TIME_PER: number = 2000; /** * 预估每次请求+处理时间长度(慢速模式) * @private */ -export const ESTIMATE_TIME_PER_WITH_REQUEST_SLOW:number = ESTIMATE_TIME_PER + BasicConst.CALL_DOUBAN_DELAY_SLOW + BasicConst.CALL_DOUBAN_DELAY_RANGE_SLOW / 2; +export const ESTIMATE_TIME_PER_WITH_REQUEST_SLOW: number = ESTIMATE_TIME_PER + BasicConst.CALL_DOUBAN_DELAY_SLOW + BasicConst.CALL_DOUBAN_DELAY_RANGE_SLOW / 2; /** * 预估每次请求+处理时间长度(正常模式) * @private */ -export const ESTIMATE_TIME_PER_WITH_REQUEST:number = ESTIMATE_TIME_PER + BasicConst.CALL_DOUBAN_DELAY + BasicConst.CALL_DOUBAN_DELAY_RANGE / 2; +export const ESTIMATE_TIME_PER_WITH_REQUEST: number = ESTIMATE_TIME_PER + BasicConst.CALL_DOUBAN_DELAY + BasicConst.CALL_DOUBAN_DELAY_RANGE / 2; /** * 模板类型 @@ -89,7 +90,7 @@ export enum PersonNameMode { * */ export enum TemplateKey { - movieTemplateFile= 'movieTemplateFile', + movieTemplateFile = 'movieTemplateFile', bookTemplateFile = 'bookTemplateFile', musicTemplateFile = 'musicTemplateFile', noteTemplateFile = 'noteTemplateFile', @@ -98,14 +99,81 @@ export enum TemplateKey { } export enum SupportType { - ALL = "ALL", - MOVIE = 'MOVIE', - BOOK = 'BOOK', - MUSIC = 'MUSIC', - NOTE = 'NOTE', - GAME = 'GAME', - TELEPLAY = 'TELEPLAY', - THEATER = 'THEATER', + ALL = "All", + MOVIE = 'movie', + BOOK = 'book', + MUSIC = 'music', + NOTE = 'note', + GAME = 'game', + TELEPLAY = 'teleplay', + THEATER = 'theater', +} + +export enum PropertyName { + //base + id = "id", + title = "title", + type = "type", + score = "score", + image = "image", + imageUrl = "imageUrl", + url = "url", + desc = "desc", + publisher = "publisher", + datePublished = "datePublished", + genre = "genre", + + //user + tags = "tags", + rate = "rate", + state = "state", + collectionDate = "collectionDate", + comment = "comment", + + + //book + author = "author", + translator = "translator", + isbn = "isbn", + originalTitle = "originalTitle", + subTitle = "subTitle", + totalPage = "totalPage", + series = "series", + menu = "menu", + price = "price", + binding = "binding", + producer = "producer", + + //movie + director = "director", + actor = "actor", + aggregateRating = "aggregateRating", + aliases = "aliases", + country = "country", + language = "language", + time = "time", + IMDb = "IMDb", + + //music + albumType = "albumType", + medium = "medium", + records = "records", + barcode = "barcode", + + //game + platform = "platform", + developer = "developer", + + + //teleplay + episode = "episode", + + //theater + + + //note + authorUrl = "authorUrl", + content = "content", } /** @@ -118,11 +186,11 @@ export const PersonNameModeRecords: { [key in PersonNameMode]: string } = { } export enum SyncType { - movie= 'movie', - book= 'book', - broadcast= 'broadcast', - note= 'note', - music= 'music', + movie = 'movie', + book = 'book', + broadcast = 'broadcast', + note = 'note', + music = 'music', } /** @@ -140,31 +208,31 @@ export const SyncTypeRecords: { [key in SyncType]: string } = { /** * 同步豆瓣每页的大小 */ -export const PAGE_SIZE:number = 30; +export const PAGE_SIZE: number = 30; /** * 多少条后同步速率变慢，防止403 */ - /** * 动作 */ export enum Action { - SearchAndReplace='SearchAndReplace', - SearchAndCrate='SearchAndCrate', - Sync='Sync', - SearchEditorAndReplace='SearchEditorAndReplace' + SearchAndReplace = 'SearchAndReplace', + SearchAndCrate = 'SearchAndCrate', + Sync = 'Sync', + SearchEditorAndReplace = 'SearchEditorAndReplace' } export enum SyncItemStatus { - exists= 'exists', - replace= 'replace', - create= 'create', - fail= 'fail', - unHandle='unHandle', + exists = 'exists', + replace = 'replace', + create = 'create', + fail = 'fail', + unHandle = 'unHandle', } + export enum NavigateType { previous = "previous", next = "next", @@ -172,7 +240,7 @@ export enum NavigateType { nextNeedLogin = "nextNeedLogin" } -export const DoubanSearchResultSubjectPreviousPage:DoubanSearchResultSubject = { +export const DoubanSearchResultSubjectPreviousPage: DoubanSearchResultSubject = { cast: "", datePublished: undefined, desc: "", @@ -187,7 +255,7 @@ export const DoubanSearchResultSubjectPreviousPage:DoubanSearchResultSubject = { url: NavigateType.previous } -export const DoubanSearchResultSubjectNextPage:DoubanSearchResultSubject = { +export const DoubanSearchResultSubjectNextPage: DoubanSearchResultSubject = { cast: "", datePublished: undefined, desc: "", @@ -202,7 +270,7 @@ export const DoubanSearchResultSubjectNextPage:DoubanSearchResultSubject = { url: NavigateType.next } -export const DoubanSearchResultSubjectNextPageNeedLogin:DoubanSearchResultSubject = { +export const DoubanSearchResultSubjectNextPageNeedLogin: DoubanSearchResultSubject = { cast: "", datePublished: undefined, desc: "", @@ -217,13 +285,13 @@ export const DoubanSearchResultSubjectNextPageNeedLogin:DoubanSearchResultSubjec url: NavigateType.nextNeedLogin } -export const SEARCH_ITEM_PAGE_SIZE:number = 20; +export const SEARCH_ITEM_PAGE_SIZE: number = 20; /** * 豆瓣默认请求头 * @type {string} **/ -export const DEFAULT_DOUBAN_HEADERS = StringUtil.parseHeaders(` +export const DEFAULT_DOUBAN_HEADERS = StringUtil.parseHeaders(` Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7 Accept-Language: zh-CN,zh;q=0.9 Cache-Control: max-age=0 @@ -240,6 +308,36 @@ sec-ch-ua-mobile: ?0 sec-ch-ua-platform: "Windows" `) +/** + * 属性设置 + */ +export const ONLINE_SETTING_DEFAULT: DoubanPluginOnlineSettings = { + properties: [ + { + type: SupportType.BOOK, + name: PropertyName.comment, + selectors: ['#interest_sect_level > div > span:nth-child(7)' + ] + }, + { + type: SupportType.MOVIE, + name: PropertyName.comment, + selectors: ['#interest_sect_level > div > span:nth-child(8)', + '#interest_sect_level > div > span:nth-child(7)', + '#interest_sect_level > div > span:nth-child(9)' + ] + } + ] +} + +/** + * 处理状态 + */ +export enum SubjectHandledStatus { + init = 'init', + saved = 'saved', + syncTypeDiffAbort = 'syncTypeDiffAbort', +} diff --git a/src/org/wanxp/constant/DefaultSettings.ts b/src/org/wanxp/constant/DefaultSettings.ts index 8809a2c..98f1429 100644 --- a/src/org/wanxp/constant/DefaultSettings.ts +++ b/src/org/wanxp/constant/DefaultSettings.ts @@ -2,6 +2,8 @@ import {DoubanPluginSetting} from "../douban/setting/model/DoubanPluginSetting"; import {PersonNameMode, SupportType} from "./Constsant"; export const DEFAULT_SETTINGS: DoubanPluginSetting = { + onlineSettingsFileName: "obsidian_douban_plugin_online_settings.json", + onlineSettingsGistId: "35693f9ece9bd6abba98f94e81afde19", movieTemplateFile: ``, bookTemplateFile: ``, musicTemplateFile: ``, @@ -30,5 +32,5 @@ export const DEFAULT_SETTINGS: DoubanPluginSetting = { cacheImage: true, cacheHighQuantityImage: true, attachmentPath: 'assets', - syncHandledDataArray: [], + syncHandledDataArray: [] } diff --git a/src/org/wanxp/constant/DoubanUserState.ts b/src/org/wanxp/constant/DoubanUserState.ts index fd5c724..fb56f45 100644 --- a/src/org/wanxp/constant/DoubanUserState.ts +++ b/src/org/wanxp/constant/DoubanUserState.ts @@ -114,6 +114,35 @@ export const DoubanSubjectStateRecords_MUSIC_SYNC: { [key in DoubanSubjectState] [DoubanSubjectState.collect]: i18nHelper.getMessage('500404'), } +export const DoubanSubjectStateRecords_KEY_WORD_TYPE: Map = new Map ( + [['我看过这部电视剧', SupportType.TELEPLAY], + ['我最近看过这部电视剧', SupportType.TELEPLAY], + ['我想看这部电视剧', SupportType.TELEPLAY], + ['我在看这部电视剧', SupportType.TELEPLAY], + ['我最近在看这部电视剧', SupportType.TELEPLAY], + + ['我最近看过这部电影', SupportType.MOVIE], + ['我看过这部电影', SupportType.MOVIE], + ['我想看这部电影', SupportType.MOVIE], + + ['我读过这本书', SupportType.BOOK], + ['我想读这本书', SupportType.BOOK], + ['我在读这本书', SupportType.BOOK], + ['我最近在读这本书', SupportType.BOOK], + + ['我最近听过这张唱片', SupportType.MUSIC], + ['我听过这张唱片', SupportType.MUSIC], + ['我想听这张唱片', SupportType.MUSIC], + ['我在听这张唱片', SupportType.MUSIC], + ['我最近在听这张唱片', SupportType.MUSIC], + + ['我最近玩过这个游戏', SupportType.GAME], + ['我玩过这个游戏', SupportType.GAME], + ['我想玩这个游戏', SupportType.GAME], + ['我在玩这个游戏', SupportType.GAME], + ['我最近在玩这个游戏', SupportType.GAME],] +) + diff --git a/src/org/wanxp/douban/component/DoubanSyncModal.ts b/src/org/wanxp/douban/component/DoubanSyncModal.ts index e00763e..96c4936 100644 --- a/src/org/wanxp/douban/component/DoubanSyncModal.ts +++ b/src/org/wanxp/douban/component/DoubanSyncModal.ts @@ -150,6 +150,7 @@ ${syncStatus.getHandle() == 0? '...' : i18nHelper.getMessage('110042') + ':' + T private updateContextByConfig(syncConfig: SyncConfig) { const { context} = this; context.syncConfig = syncConfig; + context.syncActive = true; } private showConfigPan(contentEl: HTMLElement, config:SyncConfig, disable:boolean) { diff --git a/src/org/wanxp/douban/data/handler/DoubanAbstractLoadHandler.ts b/src/org/wanxp/douban/data/handler/DoubanAbstractLoadHandler.ts index 892340c..e33518a 100644 --- a/src/org/wanxp/douban/data/handler/DoubanAbstractLoadHandler.ts +++ b/src/org/wanxp/douban/data/handler/DoubanAbstractLoadHandler.ts @@ -8,7 +8,7 @@ import {CheerioAPI, load} from "cheerio"; import YamlUtil from "../../../utils/YamlUtil"; import { BasicConst, - PersonNameMode, + PersonNameMode, PropertyName, SearchHandleMode, SupportType, TemplateKey, @@ -20,12 +20,17 @@ import {getDefaultTemplateContent} from "../../../constant/DefaultTemplateConten import StringUtil from "../../../utils/StringUtil"; import {DEFAULT_SETTINGS} from "../../../constant/DefaultSettings"; import {DoubanUserParameter, UserStateSubject} from "../model/UserStateSubject"; -import {DoubanSubjectState, DoubanSubjectStateRecords} from "../../../constant/DoubanUserState"; +import { + DoubanSubjectState, + DoubanSubjectStateRecords, + DoubanSubjectStateRecords_KEY_WORD_TYPE +} from "../../../constant/DoubanUserState"; import DoubanLoginModel from "../../component/DoubanLoginModel"; import DoubanHumanCheckModel from "../../component/DoubanHumanCheckModel"; import DoubanMovieSubject from "../model/DoubanMovieSubject"; import {Person} from "schema-dts"; import HttpUtil from "../../../utils/HttpUtil"; +import HtmlUtil from "../../../utils/HtmlUtil"; export default abstract class DoubanAbstractLoadHandler implements DoubanSubjectLoadHandler { @@ -139,15 +144,20 @@ export default abstract class DoubanAbstractLoadHandler abstract support(extract: DoubanSubject): boolean; - async handle(url: string, context: HandleContext): Promise { + async handle(url: string, context: HandleContext): Promise { context.plugin.settingsManager.debug(`开始请求地址:${url}`) context.plugin.settingsManager.debug(`(注意:请勿向任何人透露你的Cookie,此处若需要截图请**打码**)请求header:${context.settings.loginHeadersContent}`) - await HttpUtil.httpRequestGet(url, context.plugin.settingsManager.getHeaders(), context.plugin.settingsManager) + return await HttpUtil.httpRequestGet(url, context.plugin.settingsManager.getHeaders(), context.plugin.settingsManager) .then(load) .then(data => this.analysisUserState(data, context)) .then(({data, userState}) => { + let guessType = this.getSupportType(); + if (context.syncActive) { + guessType = this.getGuessType(data); + } let sub = this.parseSubjectFromHtml(data, context); sub.userState = userState; + sub.guessType = guessType; return sub; }) .then(content => this.toEditor(context, content)) @@ -160,11 +170,31 @@ export default abstract class DoubanAbstractLoadHandler }else { context.syncStatusHolder?context.syncStatusHolder.syncStatus.handled(1):null; } + return e; }); ; } + /** + * 通过判断 data中是否包含关键字符串 “我看过的电视剧” 判断是不是 movie + * @param data + * @private + */ + private getGuessType(data: CheerioAPI):SupportType { + if (data) { + let text = data.html(); + if (text) { + for (let [key, value] of DoubanSubjectStateRecords_KEY_WORD_TYPE) { + if (text.indexOf(key) >= 0) { + return value; + } + } + } + } + return null; + } + abstract parseSubjectFromHtml(data: CheerioAPI, context: HandleContext): T | undefined; @@ -501,4 +531,8 @@ export default abstract class DoubanAbstractLoadHandler } }) } + + protected getPropertyValue(html: CheerioAPI, name: PropertyName): string { + return HtmlUtil.getHtmlText(html, this.doubanPlugin.settingsManager.getSelector(this.getSupportType(), name)); + } } diff --git a/src/org/wanxp/douban/data/handler/DoubanBookLoadHandler.ts b/src/org/wanxp/douban/data/handler/DoubanBookLoadHandler.ts index a5dc665..ae67ff4 100644 --- a/src/org/wanxp/douban/data/handler/DoubanBookLoadHandler.ts +++ b/src/org/wanxp/douban/data/handler/DoubanBookLoadHandler.ts @@ -3,7 +3,7 @@ import DoubanAbstractLoadHandler from "./DoubanAbstractLoadHandler"; import DoubanBookSubject, {DoubanBookParameter} from "../model/DoubanBookSubject"; import DoubanPlugin from "../../../main"; import DoubanSubject from "../model/DoubanSubject"; -import {SupportType, TemplateTextMode} from "../../../constant/Constsant"; +import {PropertyName, SupportType, TemplateTextMode} from "../../../constant/Constsant"; import HandleContext from "../model/HandleContext"; import StringUtil from "../../../utils/StringUtil"; import {UserStateSubject} from "../model/UserStateSubject"; @@ -57,7 +57,7 @@ export default class DoubanBookLoadHandler extends DoubanAbstractLoadHandler div.a_stars > span.mr10').text().trim(); let collectionDateStr = html('div#interest_sect_level > div.a_stars > span.mr10').next().text().trim(); let userState1 = DoubanAbstractLoadHandler.getUserState(stateWord); - let component = html('span#rating').next().next().next().text().trim(); + let comment = this.getComment(html); const userState: UserStateSubject = { @@ -65,7 +65,7 @@ export default class DoubanBookLoadHandler extends DoubanAbstractLoadHandler = new Map( [['作者', 'author'], ['出版社:', 'publisher'], diff --git a/src/org/wanxp/douban/data/handler/DoubanGameLoadHandler.ts b/src/org/wanxp/douban/data/handler/DoubanGameLoadHandler.ts index 7d6eb4b..11022df 100644 --- a/src/org/wanxp/douban/data/handler/DoubanGameLoadHandler.ts +++ b/src/org/wanxp/douban/data/handler/DoubanGameLoadHandler.ts @@ -89,7 +89,7 @@ export default class DoubanGameLoadHandler extends DoubanAbstractLoadHandler { @@ -52,7 +51,7 @@ export default class DoubanMovieLoadHandler extends DoubanAbstractLoadHandler div.a_stars > span.mr10').text().trim(); let collectionDateStr = html('div#interest_sect_level > div.a_stars > span.mr10 > span.collection_date').text().trim(); let userState1 = DoubanAbstractLoadHandler.getUserState(stateWord); - let component = html('div#interest_sect_level > div.a_stars > span.color_gray').next().next().text().trim(); + let component = this.getComment(html, context); const userState: UserStateSubject = { @@ -66,6 +65,14 @@ export default class DoubanMovieLoadHandler extends DoubanAbstractLoadHandler div.a_stars > span.color_gray').next().next().text().trim(); + if (component) { + return component; + } + return this.getPropertyValue(html, PropertyName.comment); + } + parseSubjectFromHtml(html: CheerioAPI, context: HandleContext): DoubanMovieSubject { const movie:DoubanMovieSubject = html('script') .get() @@ -83,7 +90,7 @@ export default class DoubanMovieLoadHandler extends DoubanAbstractLoadHandler { support(extract: DoubanSubject): boolean; - handle(url: string, context: HandleContext): void; + handle(url: string, context: HandleContext): Promise; } diff --git a/src/org/wanxp/douban/data/handler/DoubanTeleplayLoadHandler.ts b/src/org/wanxp/douban/data/handler/DoubanTeleplayLoadHandler.ts index cc0ae84..7204eec 100644 --- a/src/org/wanxp/douban/data/handler/DoubanTeleplayLoadHandler.ts +++ b/src/org/wanxp/douban/data/handler/DoubanTeleplayLoadHandler.ts @@ -83,7 +83,7 @@ export class DoubanTeleplayLoadHandler extends DoubanAbstractLoadHandler = new Map([ diff --git a/src/org/wanxp/douban/data/model/HandleContext.ts b/src/org/wanxp/douban/data/model/HandleContext.ts index 0af1ee8..eea65c0 100644 --- a/src/org/wanxp/douban/data/model/HandleContext.ts +++ b/src/org/wanxp/douban/data/model/HandleContext.ts @@ -23,6 +23,7 @@ export default interface HandleContext { action:string; syncConfig?: SyncConfig; listItem?:DoubanSubject; + syncActive?:boolean; searchPage?:SearchPageInfo; diff --git a/src/org/wanxp/douban/setting/SettingsManager.ts b/src/org/wanxp/douban/setting/SettingsManager.ts index e5b064b..6a638c0 100644 --- a/src/org/wanxp/douban/setting/SettingsManager.ts +++ b/src/org/wanxp/douban/setting/SettingsManager.ts @@ -4,7 +4,11 @@ import DoubanPlugin from "../../main"; import Logger from "../../utils/Logutil"; import { DoubanPluginSetting } from "./model/DoubanPluginSetting"; import StringUtil from "../../utils/StringUtil"; -import {DEFAULT_DOUBAN_HEADERS} from "../../constant/Constsant"; +import {DEFAULT_DOUBAN_HEADERS, ONLINE_SETTING_DEFAULT, SupportType} from "../../constant/Constsant"; +import GithubUtil from "../../utils/GithubUtil"; +import {DoubanPluginOnlineData} from "./model/DoubanPluginOnlineData"; +import {DoubanPluginOnlineSettings} from "./model/DoubanPluginOnlineSettings"; +import {DoubanPluginSubjectProperty} from "./model/DoubanPluginSubjectProperty"; export default class SettingsManager { app: App; @@ -12,8 +16,8 @@ export default class SettingsManager { settings: DoubanPluginSetting; cleanupFns: Array<() => void> = []; innerLogger: Logger = new Logger(); - cookieTemp:string; + onlineSettings: DoubanPluginOnlineSettings; constructor(app: App, plugin: DoubanPlugin) { this.app = app; @@ -21,6 +25,8 @@ export default class SettingsManager { this.settings = plugin.settings; } + + getSettingWithDefault(key: keyof DoubanPluginSetting) { return [this.settings[key], DEFAULT_SETTINGS[key]]; @@ -75,4 +81,14 @@ export default class SettingsManager { return this.cookieTemp; } + getSelector(itemType: SupportType, propertyName: string):string[] { + if (this.onlineSettings && this.onlineSettings.properties) { + const doubanPluginSubjectProperty:DoubanPluginSubjectProperty = this.onlineSettings.properties.find(subjectProperty => subjectProperty.type === itemType && subjectProperty.name === propertyName); + if(doubanPluginSubjectProperty) { + return doubanPluginSubjectProperty.selectors; + } + } + return ONLINE_SETTING_DEFAULT.properties.find(subjectProperty => subjectProperty.type === itemType && subjectProperty.name === propertyName).selectors; + } + } diff --git a/src/org/wanxp/douban/setting/model/DoubanPluginOnlineData.ts b/src/org/wanxp/douban/setting/model/DoubanPluginOnlineData.ts new file mode 100644 index 0000000..588520f --- /dev/null +++ b/src/org/wanxp/douban/setting/model/DoubanPluginOnlineData.ts @@ -0,0 +1,6 @@ +import {DoubanPluginSubjectProperty} from "./DoubanPluginSubjectProperty"; +import {DoubanPluginOnlineSettings} from "./DoubanPluginOnlineSettings"; + +export interface DoubanPluginOnlineData { + settings: DoubanPluginOnlineSettings; +} diff --git a/src/org/wanxp/douban/setting/model/DoubanPluginOnlineSettings.ts b/src/org/wanxp/douban/setting/model/DoubanPluginOnlineSettings.ts new file mode 100644 index 0000000..031ff33 --- /dev/null +++ b/src/org/wanxp/douban/setting/model/DoubanPluginOnlineSettings.ts @@ -0,0 +1,5 @@ +import {DoubanPluginSubjectProperty} from "./DoubanPluginSubjectProperty"; + +export interface DoubanPluginOnlineSettings { + properties: DoubanPluginSubjectProperty[]; +} diff --git a/src/org/wanxp/douban/setting/model/DoubanPluginSetting.ts b/src/org/wanxp/douban/setting/model/DoubanPluginSetting.ts index 687ace9..51789cb 100644 --- a/src/org/wanxp/douban/setting/model/DoubanPluginSetting.ts +++ b/src/org/wanxp/douban/setting/model/DoubanPluginSetting.ts @@ -2,6 +2,8 @@ import {CustomProperty} from "./CustomProperty"; import {SyncHandledData} from "./SyncHandledData"; export interface DoubanPluginSetting { + onlineSettingsFileName: string; + onlineSettingsGistId: string; movieTemplateFile: string, bookTemplateFile: string, musicTemplateFile: string, diff --git a/src/org/wanxp/douban/setting/model/DoubanPluginSubjectProperty.ts b/src/org/wanxp/douban/setting/model/DoubanPluginSubjectProperty.ts new file mode 100644 index 0000000..e0f967b --- /dev/null +++ b/src/org/wanxp/douban/setting/model/DoubanPluginSubjectProperty.ts @@ -0,0 +1,5 @@ +export interface DoubanPluginSubjectProperty { + type: string; + name: string; + selectors: string[]; +} diff --git a/src/org/wanxp/douban/sync/handler/DoubanAbstractSyncHandler.ts b/src/org/wanxp/douban/sync/handler/DoubanAbstractSyncHandler.ts index 335c0e7..d43d37b 100644 --- a/src/org/wanxp/douban/sync/handler/DoubanAbstractSyncHandler.ts +++ b/src/org/wanxp/douban/sync/handler/DoubanAbstractSyncHandler.ts @@ -1,9 +1,7 @@ -import {CheerioAPI} from "cheerio"; -import DoubanSyncSubject from "../model/DoubanSyncSubject"; import DoubanPlugin from "../../../main"; -import {BasicConst, SyncType} from "../../../constant/Constsant"; +import {BasicConst, SubjectHandledStatus, SyncType} from "../../../constant/Constsant"; import {DoubanSyncHandler} from "./DoubanSyncHandler"; -import { SyncConfig } from "../model/SyncConfig"; +import {SyncConfig} from "../model/SyncConfig"; import HandleContext from "../../data/model/HandleContext"; import {SubjectListItem} from "../../data/model/SubjectListItem"; import {sleepRange} from "../../../utils/TimeUtil"; @@ -74,7 +72,7 @@ export abstract class DoubanAbstractSyncHandler implem return; } if(syncStatus.shouldSync(item.id)) { - await this.doubanSubjectLoadHandler.handle(item.url, context); + let subject: DoubanSubject = await this.doubanSubjectLoadHandler.handle(item.url, context); await sleepRange(BasicConst.CALL_DOUBAN_DELAY, BasicConst.CALL_DOUBAN_DELAY + BasicConst.CALL_DOUBAN_DELAY_RANGE); }else { syncStatus.unHandle(item.id, item.title); diff --git a/src/org/wanxp/lang/locale/en.ts b/src/org/wanxp/lang/locale/en.ts index 91ac4b6..0dd513b 100644 --- a/src/org/wanxp/lang/locale/en.ts +++ b/src/org/wanxp/lang/locale/en.ts @@ -234,6 +234,7 @@ PS: This file could be delete if you want to. //error '130101': `Fetch Data Error, {0}`, '140101': `Not support for current type. You can add Issues at Github:Wanxp/obsidian-douban`, + '140102': `subject type is different, will not sync this, chosen sync type is {0} but this {1} subject type is {2}`, '130105': `Can not use Douban this time, Please try again after 12 hour or 24 hour. Or you can reset your connection `, '130106': `Can not use Douban this time, Please try Login In Douban Plugin. If not working please again after 12 hour or 24 hour. Or you can reset your connection `, diff --git a/src/org/wanxp/lang/locale/zh-cn.ts b/src/org/wanxp/lang/locale/zh-cn.ts index 051d20b..1ee2051 100644 --- a/src/org/wanxp/lang/locale/zh-cn.ts +++ b/src/org/wanxp/lang/locale/zh-cn.ts @@ -235,6 +235,7 @@ export default { '130102': `Obsidian Douban插件错误提示:`, '130103': `Obsidian Douban插件异常提示:`, '140101': `当前版本暂不支持该类型导入,请升级Obsidian Douban或至github提交issuess获取帮助`, + '140102': `同步的对象和选择对象不一致，将不会同步, 现在选择同步的类型：{0}，但是获取[{1}]的类型：{2}`, '130105': `由于多次频繁请求数据，豆瓣当前暂时不可用. 请于12小时或24小时后再重试，或重置你的网络(如重新拨号或更换网络) `, '130106': `请尝试在Douban插件中登录后操作. 若还是无效果则尝试于12小时或24小时后再重试，或重置你的网络(如重新拨号或更换网络) `, diff --git a/src/org/wanxp/main.ts b/src/org/wanxp/main.ts index df1b599..3531c73 100644 --- a/src/org/wanxp/main.ts +++ b/src/org/wanxp/main.ts @@ -1,9 +1,9 @@ -import {Action, BasicConst, SearchHandleMode, SyncTypeRecords} from "./constant/Constsant"; +import {Action, BasicConst, SearchHandleMode, SubjectHandledStatus, SyncTypeRecords} from "./constant/Constsant"; import {Editor, Notice, Plugin} from "obsidian"; import {DEFAULT_SETTINGS} from "./constant/DefaultSettings"; import {DoubanFuzzySuggester} from "./douban/data/search/DoubanSearchFuzzySuggestModal"; -import { DoubanPluginSetting } from "./douban/setting/model/DoubanPluginSetting"; +import {DoubanPluginSetting} from "./douban/setting/model/DoubanPluginSetting"; import {DoubanSearchChooseItemHandler} from "./douban/data/handler/DoubanSearchChooseItemHandler"; import {DoubanSearchModal} from "./douban/data/search/DoubanSearchModal"; import DoubanSearchResultSubject from "./douban/data/model/DoubanSearchResultSubject"; @@ -24,6 +24,8 @@ import SyncHandler from "./douban/sync/handler/SyncHandler"; import UserComponent from "./douban/user/UserComponent"; import {i18nHelper} from './lang/helper'; import {log} from "src/org/wanxp/utils/Logutil"; +import GithubUtil from "./utils/GithubUtil"; +import {DoubanPluginOnlineData} from "./douban/setting/model/DoubanPluginOnlineData"; export default class DoubanPlugin extends Plugin { public settings: DoubanPluginSetting; @@ -34,6 +36,7 @@ export default class DoubanPlugin extends Plugin { public settingsManager: SettingsManager; public netFileHandler: NetFileHandler; public statusHolder: GlobalStatusHolder; + public onlineData: DoubanPluginOnlineData; async putToObsidian(context: HandleContext, extract: DoubanSubject) { @@ -44,6 +47,11 @@ export default class DoubanPlugin extends Plugin { log.warn(i18nHelper.getMessage('140101')); return; } + if (context.syncActive && extract.guessType && extract.guessType != extract.type) { + extract.handledStatus = SubjectHandledStatus.syncTypeDiffAbort; + console.log(i18nHelper.getMessage('140102', extract.type, extract.title, extract.guessType)); + return; + } if (Action.Sync == context.action) { this.showStatus(i18nHelper.getMessage('140207', syncStatus.getHasHandle(), syncStatus.getTotal(), extract.title)); }else { @@ -52,6 +60,7 @@ export default class DoubanPlugin extends Plugin { const result = await this.doubanExtractHandler.parseText(extract, context) if (result) { await this.putContentToObsidian(context, result); + extract.handledStatus = SubjectHandledStatus.saved; } if (Action.Sync == context.action) { this.showStatus(i18nHelper.getMessage('140208', syncStatus.getHasHandle(), syncStatus.getTotal(), extract.title)); @@ -205,6 +214,7 @@ export default class DoubanPlugin extends Plugin { }); this.settingsManager = new SettingsManager(app, this); + this.fetchOnlineData(this.settingsManager); this.userComponent = new UserComponent(this.settingsManager); this.netFileHandler = new NetFileHandler(this.fileHandler); if (this.userComponent.needLogin()) { @@ -276,7 +286,8 @@ export default class DoubanPlugin extends Plugin { } finally { await context.plugin.statusHolder.completeSync(); this.clearStatusBarDelay(); - } + context.syncActive = false; + } } async checkLogin(context: HandleContext):Promise { @@ -300,5 +311,23 @@ export default class DoubanPlugin extends Plugin { syncConfig.attachmentPath = syncConfig.attachmentPath ? syncConfig.attachmentPath : DEFAULT_SETTINGS.attachmentPath; syncConfig.dataFileNamePath = syncConfig.dataFileNamePath ? syncConfig.dataFileNamePath : DEFAULT_SETTINGS.dataFileNamePath; } + + private fetchOnlineData(settingsManager: SettingsManager) { + // const gistId: string = this.settings.onlineSettingsGistId??DEFAULT_SETTINGS.onlineSettingsGistId; + // const fileName: string = this.settings.onlineSettingsFileName??DEFAULT_SETTINGS.onlineSettingsFileName; + const gistId: string = DEFAULT_SETTINGS.onlineSettingsGistId; + const fileName: string = DEFAULT_SETTINGS.onlineSettingsFileName; + GithubUtil.getGistsData(gistId, fileName) + .then(data => { + this.onlineData = JSON.parse(data); + this.fillOnlineData(this.onlineData, settingsManager); + }) + } + + private fillOnlineData(onlineData: DoubanPluginOnlineData, settingsManager: SettingsManager) { + if (onlineData.settings) { + settingsManager.onlineSettings = onlineData.settings; + } + } } diff --git a/src/org/wanxp/utils/GithubUtil.ts b/src/org/wanxp/utils/GithubUtil.ts new file mode 100644 index 0000000..64287a4 --- /dev/null +++ b/src/org/wanxp/utils/GithubUtil.ts @@ -0,0 +1,19 @@ +export default class GithubUtil { + + /** + * get gists content as object + * @param gistId + * @param fileName + */ + public static async getGistsData(gistId:string, fileName:string): Promise { + const gistUrl = `https://api.github.com/gists/${gistId}` + return fetch(gistUrl) + .then(response => response.json()) + .then(data => { + const files = Object.keys(data.files); + const file = files[0]; + return data.files[file].content; + }) + .catch(error => console.error(error)); + } +} diff --git a/src/org/wanxp/utils/HtmlUtil.ts b/src/org/wanxp/utils/HtmlUtil.ts new file mode 100644 index 0000000..efbad69 --- /dev/null +++ b/src/org/wanxp/utils/HtmlUtil.ts @@ -0,0 +1,26 @@ +import {CheerioAPI} from "cheerio"; + +export default class HtmlUtil { + + /** + * 获取html text内容 + * @param html + * @param selector + */ + public static getHtmlText(html: CheerioAPI, selector: string | string[]): string { + if (typeof selector == 'string') { + return html(selector).text().trim(); + }else { + let s = '', text = ''; + for (s of selector) { + text = this.getHtmlText(html, s); + if (text) { + return text; + } + } + } + + } + + +} diff --git a/src/org/wanxp/utils/HttpUtil.ts b/src/org/wanxp/utils/HttpUtil.ts index dc8290c..0a9726f 100644 --- a/src/org/wanxp/utils/HttpUtil.ts +++ b/src/org/wanxp/utils/HttpUtil.ts @@ -23,7 +23,7 @@ export default class HttpUtil { let options = { headers: headersInner } - + settingsManager.debug(`Obsidian-Douban:从网络获取网页开始:\nurl:${url}\nheaders:${JSON.stringify(headers)}`); return new Promise((resolve, rejects) => { https.get(url, { ...options }, function (response: any) { let chunks: any = [], @@ -39,6 +39,9 @@ export default class HttpUtil { response.on("end", function () { let data = Buffer.concat(chunks, size) let html = data.toString() + if (settingsManager) { + settingsManager.debug(`Obsidian-Douban:从网络获取网页完成:\nhtml:\n${html}`); + } resolve(html) }) }) @@ -55,7 +58,9 @@ export default class HttpUtil { let options = { headers: headers } - + if (settingsManager) { + settingsManager.debug(`Obsidian-Douban:从网络获取文件开始:\n${url}`); + } return new Promise((resolve, rejects) => { https.get(url, { ...options }, function (response: any) { let chunks: any = [], @@ -70,6 +75,9 @@ export default class HttpUtil { response.on("end", function () { let data = Buffer.concat(chunks, size) + if (settingsManager) { + settingsManager.debug(`Obsidian-Douban:从网络获取文件完成:\n${url}`); + } resolve(data) }) })