change: the way of request

This commit is contained in:
wanxp 2023-09-20 13:16:24 +08:00
parent a97b6a7364
commit 01a386dd6b
15 changed files with 1917 additions and 2838 deletions

4424
package-lock.json generated

File diff suppressed because it is too large Load Diff

@ -13,7 +13,6 @@
"author": "", "author": "",
"license": "MIT", "license": "MIT",
"devDependencies": { "devDependencies": {
"@types/express": "4.11.1",
"@types/jest": "^28.1.4", "@types/jest": "^28.1.4",
"@types/node": "^16.11.6", "@types/node": "^16.11.6",
"@typescript-eslint/eslint-plugin": "^5.2.0", "@typescript-eslint/eslint-plugin": "^5.2.0",
@ -29,10 +28,8 @@
"dependencies": { "dependencies": {
"@notable/html2markdown": "^1.1.3", "@notable/html2markdown": "^1.1.3",
"@popperjs/core": "^2.11.6", "@popperjs/core": "^2.11.6",
"axios": "^1.5.0",
"cheerio": "^1.0.0-rc.11", "cheerio": "^1.0.0-rc.11",
"express": "4.15.5", "follow-redirects": "^1.15.3",
"https": "^1.0.0",
"schema-dts": "^1.1.0" "schema-dts": "^1.1.0"
} }
} }

@ -1,5 +1,6 @@
import {i18nHelper} from "../lang/helper"; import {i18nHelper} from "../lang/helper";
import DoubanSearchResultSubject from "../douban/data/model/DoubanSearchResultSubject"; import DoubanSearchResultSubject from "../douban/data/model/DoubanSearchResultSubject";
import StringUtil from "../utils/StringUtil";
/** /**
* *
@ -218,6 +219,26 @@ export const DoubanSearchResultSubjectNextPageNeedLogin:DoubanSearchResultSubjec
export const SEARCH_ITEM_PAGE_SIZE:number = 20; export const SEARCH_ITEM_PAGE_SIZE:number = 20;
/**
*
* @type {string}
**/
export const DEFAULT_DOUBAN_HEADERS = StringUtil.parseHeaders(`
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7
Accept-Language: zh-CN,zh;q=0.9
Cache-Control: max-age=0
Connection: keep-alive
Host: www.douban.com
Sec-Fetch-Dest: document
Sec-Fetch-Mode: navigate
Sec-Fetch-Site: none
Sec-Fetch-User: ?1
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36
sec-ch-ua: "Not/A)Brand";v="99", "Google Chrome";v="115", "Chromium";v="115"
sec-ch-ua-mobile: ?0
sec-ch-ua-platform: "Windows"
`)

@ -1,6 +1,5 @@
import {DoubanPluginSetting} from "../douban/setting/model/DoubanPluginSetting"; import {DoubanPluginSetting} from "../douban/setting/model/DoubanPluginSetting";
import {PersonNameMode, SupportType} from "./Constsant"; import {PersonNameMode, SupportType} from "./Constsant";
import {doubanHeaders} from "./Douban";
export const DEFAULT_SETTINGS: DoubanPluginSetting = { export const DEFAULT_SETTINGS: DoubanPluginSetting = {
movieTemplateFile: ``, movieTemplateFile: ``,
@ -10,7 +9,6 @@ export const DEFAULT_SETTINGS: DoubanPluginSetting = {
gameTemplateFile: ``, gameTemplateFile: ``,
teleplayTemplateFile: ``, teleplayTemplateFile: ``,
searchUrl: 'https://www.douban.com/search?q=', searchUrl: 'https://www.douban.com/search?q=',
searchHeaders: JSON.stringify(doubanHeaders),
dateFormat: "yyyy-MM-DD", dateFormat: "yyyy-MM-DD",
timeFormat: "HH:mm:ss", timeFormat: "HH:mm:ss",
arraySpilt: ", ", arraySpilt: ", ",
@ -28,6 +26,7 @@ export const DEFAULT_SETTINGS: DoubanPluginSetting = {
{name: 'myType', value: 'teleplay', field: SupportType.TELEPLAY}, {name: 'myType', value: 'teleplay', field: SupportType.TELEPLAY},
], ],
loginCookiesContent: '', loginCookiesContent: '',
loginHeadersContent: '',
cacheImage: true, cacheImage: true,
cacheHighQuantityImage: true, cacheHighQuantityImage: true,
attachmentPath: 'assets', attachmentPath: 'assets',

@ -59,16 +59,14 @@ export default class DoubanLoginModel {
const session = this.modal.webContents.session; const session = this.modal.webContents.session;
const filter = { const filter = {
urls: ['https://www.douban.com/','https://accounts.douban.com/','https://accounts.douban.com/passport/login'] urls: ['https://www.douban.com/']
}; };
session.webRequest.onSendHeaders(filter, async (details:any) => { session.webRequest.onSendHeaders(filter, async (details:any) => {
this.settingsManager.debug(`配置界面:登录界面请求头检测:${details.url}`) this.settingsManager.debug(`配置界面:登录界面请求头检测:${details.url}`)
const cookies = details.requestHeaders['Cookie']; const headers = details.requestHeaders;
const cookieArr = this.parseCookies(cookies); if (headers) {
// const wr_name = cookieArr.find((cookie) => cookie.name == 'wr_name').value;
if (cookieArr) {
this.settingsManager.debug(`配置界面:登录界面请求检测获取到Cookie`) this.settingsManager.debug(`配置界面:登录界面请求检测获取到Cookie`)
let user = await settingsManager.plugin.userComponent.loginCookie(cookieArr); let user = await settingsManager.plugin.userComponent.loginHeaders(headers);
if (user && user.login) { if (user && user.login) {
this.settingsManager.debug(`配置界面:登录界面豆瓣登录成功, 信息:id:${StringUtil.confuse(user.id)}:, 用户名:${StringUtil.confuse(user.name)}`) this.settingsManager.debug(`配置界面:登录界面豆瓣登录成功, 信息:id:${StringUtil.confuse(user.id)}:, 用户名:${StringUtil.confuse(user.name)}`)
session.clearStorageData(() => { session.clearStorageData(() => {
@ -77,7 +75,7 @@ export default class DoubanLoginModel {
this.onClose(); this.onClose();
return; return;
} }
this.settingsManager.debug(`配置界面:登录界面豆瓣登录失败, cookies未能成功获取用户信息`) this.settingsManager.debug(`配置界面:登录界面豆瓣登录失败, headers未能成功获取用户信息`)
} }
}); });
} }

@ -140,11 +140,9 @@ export default abstract class DoubanAbstractLoadHandler<T extends DoubanSubject>
abstract support(extract: DoubanSubject): boolean; abstract support(extract: DoubanSubject): boolean;
async handle(url: string, context: HandleContext): Promise<void> { async handle(url: string, context: HandleContext): Promise<void> {
let headers = JSON.parse(context.settings.searchHeaders);
headers.Cookie = context.settings.loginCookiesContent;
context.plugin.settingsManager.debug(`开始请求地址:${url}`) context.plugin.settingsManager.debug(`开始请求地址:${url}`)
context.plugin.settingsManager.debug(`(注意:请勿向任何人透露你的Cookie,此处若需要截图请**打码**)请求cookie:${context.settings.loginCookiesContent}`) context.plugin.settingsManager.debug(`(注意:请勿向任何人透露你的Cookie,此处若需要截图请**打码**)请求header:${context.settings.loginHeadersContent}`)
await HttpUtil.httpRequestGet(url, headers, context.plugin.settingsManager) await HttpUtil.httpRequestGet(url, context.plugin.settingsManager.getHeaders(), context.plugin.settingsManager)
.then(load) .then(load)
.then(data => this.analysisUserState(data, context)) .then(data => this.analysisUserState(data, context))
.then(({data, userState}) => { .then(({data, userState}) => {

@ -10,66 +10,26 @@ import User from "../../user/User";
import {i18nHelper} from "../../../lang/helper"; import {i18nHelper} from "../../../lang/helper";
import {load} from 'cheerio'; import {load} from 'cheerio';
import {log} from 'src/org/wanxp/utils/Logutil'; import {log} from 'src/org/wanxp/utils/Logutil';
import HttpUtil from "../../../utils/HttpUtil";
export default class Searcher { export default class Searcher {
static search(searchItem: string, doubanSettings: DoubanPluginSetting, settingsManager:SettingsManager): Promise<DoubanSearchResultSubject[]> { static search(searchItem: string, doubanSettings: DoubanPluginSetting, settingsManager:SettingsManager): Promise<DoubanSearchResultSubject[]> {
const myHeaders:Record<string, string> = JSON.parse(doubanSettings.searchHeaders); return HttpUtil.httpRequestGet(DEFAULT_SETTINGS.searchUrl + searchItem, settingsManager.getHeaders(), settingsManager)
if (doubanSettings.loginCookiesContent) {
myHeaders.Cookie = doubanSettings.loginCookiesContent
}
let requestUrlParam: RequestUrlParam = {
url: DEFAULT_SETTINGS.searchUrl + searchItem,
method: "GET",
headers: myHeaders,
throw: true
};
return requestUrl(requestUrlParam)
.then(requestUrlResponse => {
if (requestUrlResponse.status == 403) {
throw new Error(i18nHelper.getMessage('130106'));
}
return requestUrlResponse.text;
})
.then(load) .then(load)
.then(SearchParserHandler.parseSearch) .then(SearchParserHandler.parseSearch)
.catch(e => { .catch(e => {
if(e.toString().indexOf('403') > 0) { throw log.error(i18nHelper.getMessage('130101').replace('{0}', e.toString()), e);
throw new Error(i18nHelper.getMessage('130106'));
}else {
throw log.error(i18nHelper.getMessage('130101').replace('{0}', e.toString()), e);
}
}); });
;
}; };
static loadSearchItem(searchItem: string, start:number, doubanSettings: DoubanPluginSetting, settingsManager:SettingsManager): Promise<SearchPage> { static loadSearchItem(searchItem: string, start:number, doubanSettings: DoubanPluginSetting, settingsManager:SettingsManager): Promise<SearchPage> {
const myHeaders:Record<string, string> = JSON.parse(doubanSettings.searchHeaders);
if (doubanSettings.loginCookiesContent) {
myHeaders.Cookie = doubanSettings.loginCookiesContent
}
const url:string = `https://www.douban.com/j/search?q=${searchItem}&start=${start}&subtype=item`; const url:string = `https://www.douban.com/j/search?q=${searchItem}&start=${start}&subtype=item`;
log.debug(`请求更多页面:${url}`); log.debug(`请求更多页面:${url}`);
let requestUrlParam: RequestUrlParam = { return HttpUtil.httpRequestGet(url, settingsManager.getHeaders(), settingsManager)
url: url,
method: "GET",
headers: myHeaders,
throw: true
};
return requestUrl(requestUrlParam)
.then(requestUrlResponse => {
if (requestUrlResponse.status == 403) {
throw new Error(i18nHelper.getMessage('130106'));
}
return requestUrlResponse.text;
})
.then(e=>SearchParserHandler.parseSearchJson(e, start)) .then(e=>SearchParserHandler.parseSearchJson(e, start))
.catch(e => { .catch(e => {
if(e.toString().indexOf('403') > 0) { throw log.error(i18nHelper.getMessage('130101').replace('{0}', e.toString()), e);
throw new Error(i18nHelper.getMessage('130106'));
}else {
throw log.error(i18nHelper.getMessage('130101').replace('{0}', e.toString()), e);
}
}); });
; ;

@ -255,6 +255,7 @@ ${i18nHelper.getMessage('100125')}`;
.onClick(async () => { .onClick(async () => {
button.setDisabled(true); button.setDisabled(true);
manager.updateSetting('loginCookiesContent', ''); manager.updateSetting('loginCookiesContent', '');
manager.updateSetting('loginHeadersContent', '');
constructDoubanTokenSettingsUI(containerEl, manager); constructDoubanTokenSettingsUI(containerEl, manager);
}); });
}); });

@ -3,6 +3,8 @@ import { DEFAULT_SETTINGS } from "src/org/wanxp/constant/DefaultSettings";
import DoubanPlugin from "../../main"; import DoubanPlugin from "../../main";
import Logger from "../../utils/Logutil"; import Logger from "../../utils/Logutil";
import { DoubanPluginSetting } from "./model/DoubanPluginSetting"; import { DoubanPluginSetting } from "./model/DoubanPluginSetting";
import StringUtil from "../../utils/StringUtil";
import {DEFAULT_DOUBAN_HEADERS} from "../../constant/Constsant";
export default class SettingsManager { export default class SettingsManager {
app: App; app: App;
@ -29,6 +31,16 @@ export default class SettingsManager {
return this.settings[key]; return this.settings[key];
} }
getHeaders():object {
if (this.settings.loginHeadersContent) {
// return StringUtil.parseHeaders(this.settings.loginHeadersContent);
return JSON.parse(this.settings.loginHeadersContent);
}else {
return DEFAULT_DOUBAN_HEADERS;
}
}
async updateSetting(key: keyof DoubanPluginSetting, value:any) { async updateSetting(key: keyof DoubanPluginSetting, value:any) {

@ -12,7 +12,6 @@ export interface DoubanPluginSetting {
timeFormat: string, timeFormat: string,
searchUrl: string, searchUrl: string,
arraySpilt: string, arraySpilt: string,
searchHeaders?: string,
personNameMode: string, personNameMode: string,
dataFilePath: string, dataFilePath: string,
dataFileNamePath: string, dataFileNamePath: string,
@ -20,6 +19,7 @@ export interface DoubanPluginSetting {
debugMode: boolean, debugMode: boolean,
customProperties: CustomProperty[], customProperties: CustomProperty[],
loginCookiesContent: string, loginCookiesContent: string,
loginHeadersContent: string,
cacheImage: boolean, cacheImage: boolean,
cacheHighQuantityImage: boolean, cacheHighQuantityImage: boolean,

@ -46,9 +46,7 @@ export default abstract class DoubanAbstractListHandler implements DoubanListHan
abstract getSyncType():string; abstract getSyncType():string;
async getPageList(url: string, context: HandleContext):Promise<SubjectListItem[]> { async getPageList(url: string, context: HandleContext):Promise<SubjectListItem[]> {
let headers = JSON.parse(context.settings.searchHeaders); return HttpUtil.httpRequestGet(url, context.plugin.settingsManager.getHeaders(), context.plugin.settingsManager)
headers.Cookie = context.settings.loginCookiesContent;
return HttpUtil.httpRequestGet(url, headers, context.plugin.settingsManager)
.then(load) .then(load)
.then(data => this.parseSubjectFromHtml(data, context)) .then(data => this.parseSubjectFromHtml(data, context))
.catch(e => log .catch(e => log

@ -9,6 +9,7 @@ import {DEFAULT_SETTINGS} from "../../constant/DefaultSettings";
import {doubanHeaders} from "../../constant/Douban"; import {doubanHeaders} from "../../constant/Douban";
import { request } from "https"; import { request } from "https";
import HttpUtil from "../../utils/HttpUtil"; import HttpUtil from "../../utils/HttpUtil";
import {DEFAULT_DOUBAN_HEADERS} from "../../constant/Constsant";
export default class UserComponent { export default class UserComponent {
private settingsManager: SettingsManager; private settingsManager: SettingsManager;
@ -38,13 +39,15 @@ export default class UserComponent {
} }
this.user = null; this.user = null;
this.settingsManager.updateSetting('loginCookiesContent', ''); this.settingsManager.updateSetting('loginCookiesContent', '');
this.settingsManager.updateSetting('loginHeadersContent', '');
} }
needLogin() { needLogin() {
const cookie:any = this.settingsManager.getSetting('loginCookiesContent') ; const headers:any = this.settingsManager.getSetting('loginHeadersContent') ;
if(!cookie) { if(!headers) {
return false; return false;
} }
return !this.isLogin(); return !this.isLogin();
@ -63,6 +66,43 @@ export default class UserComponent {
}); });
return this.user; return this.user;
} }
async loginByHeaders() {
// @ts-ignore
let headersStr:string = this.settingsManager.getSetting('loginHeadersContent');
if(!headersStr) {
this.settingsManager.debug('主界面:loginByCookie:无豆瓣headers信息获取用户信息失败');
return new User();
}
this.settingsManager.debug('主界面:loginByCookie:豆瓣cookies信息正常尝试获取用户信息');
const headers:object = JSON.parse(headersStr);
await this.loadUserInfoByHeaders(headers).then(user => {
this.user = user;
this.settingsManager.debug(`主界面:loginByCookie:豆瓣cookies信息正常${user&&user.id?'获取用户信息成功id:'+ StringUtil.confuse(user.id) + ',用户名:'+ StringUtil.confuse(user.name) :'获取用户信息失败'}`);
});
return this.user;
}
async loginHeaders(headers: object):Promise<User> {
if(!headers) {
return new User();
}
this.settingsManager.debug('配置界面:loginCookie:豆瓣headers信息正常尝试获取用户信息,headers:' + headers);
await this.loadUserInfoByHeaders(headers).then(user => {
this.user = user;
this.settingsManager.debug(`配置界面:loginCookie:豆瓣headers信息正常${user&&user.id?'获取用户信息成功id:'+ StringUtil.confuse(user.id) + ',用户名:'+ StringUtil.confuse(user.name) :'获取用户信息失败'}`);
});
if(this.user) {
this.settingsManager.updateSetting('loginHeadersContent', JSON.stringify(headers));
}
return this.user;
}
async loadUserInfoByHeaders(headers: object): Promise<User> {
return HttpUtil.httpRequestGet('https://www.douban.com/mine/', headers, this.settingsManager)
.then(load)
.then(this.getUserInfo);
}
async loginCookie(cookie: any):Promise<User> { async loginCookie(cookie: any):Promise<User> {
if(!cookie) { if(!cookie) {
@ -82,13 +122,9 @@ export default class UserComponent {
async loadUserInfo(cookie: any): Promise<User> { async loadUserInfo(cookie: any): Promise<User> {
const headers1 = { const headers1 = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', ...DEFAULT_DOUBAN_HEADERS,
'Accept-Language': 'zh-CN,zh;q=0.9', Cookie: cookie
'Cookie': cookie,
'Referer': 'https://accounts.douban.com/',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36'
} }
// const headers1 = StringUtil.parseHeaders(cookie)
return HttpUtil.httpRequestGet('https://www.douban.com/mine/', headers1, this.settingsManager) return HttpUtil.httpRequestGet('https://www.douban.com/mine/', headers1, this.settingsManager)
.then(load) .then(load)
.then(this.getUserInfo); .then(this.getUserInfo);
@ -120,4 +156,5 @@ export default class UserComponent {
}; };
}; };
} }

@ -280,6 +280,12 @@ export default class DoubanPlugin extends Plugin {
} }
async checkLogin(context: HandleContext):Promise<boolean> { async checkLogin(context: HandleContext):Promise<boolean> {
// return await this.checkLoginByCookie(context);
return await this.checkLoginByHeaders(context);
}
private async checkLoginByCookie(context: HandleContext) {
this.settingsManager.debug('主界面:同步时的登录状态检测'); this.settingsManager.debug('主界面:同步时的登录状态检测');
if (!context.userComponent.needLogin()) { if (!context.userComponent.needLogin()) {
this.settingsManager.debug('主界面:同步时的登录状态检测完成: 无用户信息, 尝试使用cookie获取用户信息'); this.settingsManager.debug('主界面:同步时的登录状态检测完成: 无用户信息, 尝试使用cookie获取用户信息');
@ -293,6 +299,20 @@ export default class DoubanPlugin extends Plugin {
return true; return true;
} }
private async checkLoginByHeaders(context: HandleContext) {
this.settingsManager.debug('主界面:同步时的登录状态检测');
if (!context.userComponent.needLogin()) {
this.settingsManager.debug('主界面:同步时的登录状态检测完成: 无用户信息, 尝试使用headers获取用户信息');
await context.userComponent.loginByHeaders();
}
if (!context.userComponent.isLogin()) {
this.settingsManager.debug('主界面:同步时的登录状态检测完成: 尝试使用headers获取用户信息失败');
new Notice(i18nHelper.getMessage('140303'));
return false;
}
return true;
}
private initSyncDefaultSettings(syncConfig: SyncConfig) { private initSyncDefaultSettings(syncConfig: SyncConfig) {
syncConfig.dataFilePath = syncConfig.dataFilePath ? syncConfig.dataFilePath : DEFAULT_SETTINGS.dataFilePath; syncConfig.dataFilePath = syncConfig.dataFilePath ? syncConfig.dataFilePath : DEFAULT_SETTINGS.dataFilePath;
syncConfig.templateFile = syncConfig.templateFile ? syncConfig.templateFile : ''; syncConfig.templateFile = syncConfig.templateFile ? syncConfig.templateFile : '';

@ -4,6 +4,7 @@ import {i18nHelper} from "../lang/helper";
import FileHandler from "../file/FileHandler"; import FileHandler from "../file/FileHandler";
import {FileUtil} from "../utils/FileUtil"; import {FileUtil} from "../utils/FileUtil";
import HandleContext from "../douban/data/model/HandleContext"; import HandleContext from "../douban/data/model/HandleContext";
import HttpUtil from "../utils/HttpUtil";
export default class NetFileHandler { export default class NetFileHandler {
private fileHandler: FileHandler; private fileHandler: FileHandler;
@ -13,24 +14,10 @@ export default class NetFileHandler {
} }
async downloadFile(url: string, folder:string, filename: string, context:HandleContext, showError:boolean, headers?:any): Promise<{ success: boolean, error:string, filepath: string }> { async downloadFile(url: string, folder:string, filename: string, context:HandleContext, showError:boolean, headers?:any): Promise<{ success: boolean, error:string, filepath: string }> {
const headersCookie = {Cookie: context.settings.loginCookiesContent}
const headersInner = {};
if(headers) {
Object.assign(headersInner, headers, headersCookie);
}else {
Object.assign(headersInner, headersCookie);
}
const requestUrlParam: RequestUrlParam = {
url: url,
method: "GET",
throw: true,
headers: headersInner
};
const filePath:string = FileUtil.join(folder, filename); const filePath:string = FileUtil.join(folder, filename);
return requestUrl(requestUrlParam) return HttpUtil.httpRequestGetBuffer(url, headers, context.plugin.settingsManager)
.then((response) => { .then((buffer) => {
this.fileHandler.creatAttachmentWithData(filePath, response.arrayBuffer); this.fileHandler.creatAttachmentWithData(filePath, buffer);
}).then(() => { }).then(() => {
return {success: true, error: '', filepath: filePath}; return {success: true, error: '', filepath: filePath};
}) })

@ -1,4 +1,3 @@
import axios from "axios";
import { log } from "./Logutil"; import { log } from "./Logutil";
import { i18nHelper } from "../lang/helper"; import { i18nHelper } from "../lang/helper";
import SettingsManager from "../douban/setting/SettingsManager"; import SettingsManager from "../douban/setting/SettingsManager";
@ -6,28 +5,11 @@ import { request, RequestUrlParam } from "obsidian";
import DoubanHumanCheckModel from "../douban/component/DoubanHumanCheckModel"; import DoubanHumanCheckModel from "../douban/component/DoubanHumanCheckModel";
import StringUtil from './StringUtil' import StringUtil from './StringUtil'
const https = require("https"); const {https} = require("follow-redirects");
export default class HttpUtil { export default class HttpUtil {
public static headers = StringUtil.parseHeaders(`
GET / HTTP/1.1
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7
Accept-Language: zh-CN,zh;q=0.9
Cache-Control: max-age=0
Connection: keep-alive
Cookie: bid=7VX1Gyng3bk; ll="108296"; _pk_id.100001.8cb4=d613740a418b86c8.1695103115.; _pk_ses.100001.8cb4=1; __utma=30149280.540314287.1695103116.1695103116.1695103116.1; __utmc=30149280; __utmz=30149280.1695103116.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmt=1; __utmb=30149280.3.10.1695103116
Host: www.douban.com
Sec-Fetch-Dest: document
Sec-Fetch-Mode: navigate
Sec-Fetch-Site: none
Sec-Fetch-User: ?1
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36
sec-ch-ua: "Not/A)Brand";v="99", "Google Chrome";v="115", "Chromium";v="115"
sec-ch-ua-mobile: ?0
sec-ch-ua-platform: "Windows"
`)
/** /**
* get请求 * get请求
@ -37,17 +19,18 @@ sec-ch-ua-platform: "Windows"
*/ */
// Cookie: 'll="108296"; bid=xHRJLeWBrjQ; _pk_id.100001.8cb4=f8f83e81ec224a1a.1691572669.; __utmv=30149280.13103; __yadk_uid=ce95W7OsgT0iKFceWgbMSUdw1oOqxNTk; __gads=ID=62585f60f3f637d0-2234f63fc6e200a5:T=1691572672:RT=1691572672:S=ALNI_MaIqTxSWHsfpnX9nAmMHcPQEsaezg; __gpi=UID=00000c29a9f98e5b:T=1691572672:RT=1691572672:S=ALNI_MbLAq8XNoKrRPKNqGCMdgXSPZvidw; ap_v=0,6.0; __utma=30149280.135860784.1691572641.1691572641.1694509646.2; __utmc=30149280; __utmz=30149280.1694509646.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1694509648%2C%22https%3A%2F%2Fmovie.douban.com%2Ftv%2F%22%5D; _pk_ses.100001.8cb4=1; __utmt=1; dbcl2="131038721:LUssju34QFw"; ck=dCQj; push_noty_num=0; push_doumail_num=0; __utmb=30149280.3.10.1694509646' // Cookie: 'll="108296"; bid=xHRJLeWBrjQ; _pk_id.100001.8cb4=f8f83e81ec224a1a.1691572669.; __utmv=30149280.13103; __yadk_uid=ce95W7OsgT0iKFceWgbMSUdw1oOqxNTk; __gads=ID=62585f60f3f637d0-2234f63fc6e200a5:T=1691572672:RT=1691572672:S=ALNI_MaIqTxSWHsfpnX9nAmMHcPQEsaezg; __gpi=UID=00000c29a9f98e5b:T=1691572672:RT=1691572672:S=ALNI_MbLAq8XNoKrRPKNqGCMdgXSPZvidw; ap_v=0,6.0; __utma=30149280.135860784.1691572641.1691572641.1694509646.2; __utmc=30149280; __utmz=30149280.1694509646.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1694509648%2C%22https%3A%2F%2Fmovie.douban.com%2Ftv%2F%22%5D; _pk_ses.100001.8cb4=1; __utmt=1; dbcl2="131038721:LUssju34QFw"; ck=dCQj; push_noty_num=0; push_doumail_num=0; __utmb=30149280.3.10.1694509646'
public static httpRequestGet(url: string, headers: any, settingsManager?: SettingsManager): Promise<string> { public static httpRequestGet(url: string, headers: any, settingsManager?: SettingsManager): Promise<string> {
const {['Accept-Encoding']: acceptEncoding, ...headersInner} = headers;
let options = { let options = {
headers: { headers: headersInner
...this.headers,
Cookie: headers.Cookie
}
} }
return new Promise((resolve, rejects) => { return new Promise((resolve, rejects) => {
https.get(url, { ...options }, function (response: any) { https.get(url, { ...options }, function (response: any) {
let chunks: any = [], let chunks: any = [],
size = 0 size = 0;
if (response.status == 403) {
rejects(new Error(i18nHelper.getMessage('130106')));
}
response.on("data", function (chunk: any) { response.on("data", function (chunk: any) {
chunks.push(chunk) chunks.push(chunk)
size += chunk.length size += chunk.length
@ -63,7 +46,43 @@ sec-ch-ua-platform: "Windows"
}) })
} }
/** /**
* get请求
* @param url
* @param headers
* @param settingsManager
*/
// Cookie: 'll="108296"; bid=xHRJLeWBrjQ; _pk_id.100001.8cb4=f8f83e81ec224a1a.1691572669.; __utmv=30149280.13103; __yadk_uid=ce95W7OsgT0iKFceWgbMSUdw1oOqxNTk; __gads=ID=62585f60f3f637d0-2234f63fc6e200a5:T=1691572672:RT=1691572672:S=ALNI_MaIqTxSWHsfpnX9nAmMHcPQEsaezg; __gpi=UID=00000c29a9f98e5b:T=1691572672:RT=1691572672:S=ALNI_MbLAq8XNoKrRPKNqGCMdgXSPZvidw; ap_v=0,6.0; __utma=30149280.135860784.1691572641.1691572641.1694509646.2; __utmc=30149280; __utmz=30149280.1694509646.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1694509648%2C%22https%3A%2F%2Fmovie.douban.com%2Ftv%2F%22%5D; _pk_ses.100001.8cb4=1; __utmt=1; dbcl2="131038721:LUssju34QFw"; ck=dCQj; push_noty_num=0; push_doumail_num=0; __utmb=30149280.3.10.1694509646'
public static httpRequestGetBuffer(url: string, headers: any, settingsManager?: SettingsManager): Promise<ArrayBuffer> {
let options = {
headers: headers
}
return new Promise((resolve, rejects) => {
https.get(url, { ...options }, function (response: any) {
let chunks: any = [],
size = 0;
if (response.status == 403) {
rejects(new Error(i18nHelper.getMessage('130106')));
}
response.on("data", function (chunk: any) {
chunks.push(chunk)
size += chunk.length
})
response.on("end", function () {
let data = Buffer.concat(chunks, size)
resolve(data)
})
})
})
}
/**
* get请求 * get请求
* @param url * @param url
* @param headers * @param headers
@ -102,46 +121,16 @@ sec-ch-ua-platform: "Windows"
public static async humanCheck(html: any, url: string, settingsManager?: SettingsManager): Promise<any> { public static async humanCheck(html: any, url: string, settingsManager?: SettingsManager): Promise<any> {
if (settingsManager) { if (settingsManager) {
settingsManager.debug(html); settingsManager.debug(html);
} }
if (html && html.indexOf("<title>禁止访问</title>") != -1) { if (html && html.indexOf("<title>禁止访问</title>") != -1) {
const loginModel = new DoubanHumanCheckModel(url); const loginModel = new DoubanHumanCheckModel(url);
await loginModel.load(); await loginModel.load();
return ''; return '';
} else { } else {
return html; return html;
} }
}
/**
* get请求
* @param url
* @param settingsManager
*/
public static httpRequestGetUrl(url: string, settingsManager?: SettingsManager): Promise<string> {
return axios.get(url)
.then(res => res.data)
.then(data => {
if (data && data.indexOf('https://sec.douban.com/a') > 0) {
log.notice(i18nHelper.getMessage('130105'))
if (settingsManager) {
settingsManager.debug(`Obsidian-Douban:获取异常网页如下:\n${data}`);
}
}
settingsManager.debug(`Obsidian-Douban:获取网页如下:\n${data}`);
return data;
})
.catch(e => {
if (e.toString().indexOf('403') > 0) {
throw log.error(i18nHelper.getMessage('130105'), e)
} else {
throw log.error(i18nHelper.getMessage('130101').replace('{0}', e.toString()), e)
}
})
} }
} }