diff --git a/package-lock.json b/package-lock.json index 3153cd3..5723152 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@notable/html2markdown": "^1.1.3", "@popperjs/core": "^2.11.6", "cheerio": "^1.0.0-rc.11", - "follow-redirects": "^1.15.3", + "follow-redirects": "^1.15.6", "schema-dts": "^1.1.0" }, "devDependencies": { @@ -3073,9 +3073,9 @@ "peer": true }, "node_modules/follow-redirects": { - "version": "1.15.3", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", - "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "funding": [ { "type": "individual", @@ -7697,9 +7697,9 @@ "peer": true }, "follow-redirects": { - "version": "1.15.3", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", - "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==" + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==" }, "fs.realpath": { "version": "1.0.0", diff --git a/package.json b/package.json index d2309c2..9cc3452 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "@notable/html2markdown": "^1.1.3", "@popperjs/core": "^2.11.6", "cheerio": "^1.0.0-rc.11", - "follow-redirects": "^1.15.3", + "follow-redirects": "^1.15.6", "schema-dts": "^1.1.0" } } diff --git a/src/org/wanxp/utils/HttpUtil.ts b/src/org/wanxp/utils/HttpUtil.ts index 5faf26c..01842e1 100644 --- a/src/org/wanxp/utils/HttpUtil.ts +++ b/src/org/wanxp/utils/HttpUtil.ts @@ -1,57 +1,32 @@ -import { log } from "./Logutil"; -import { i18nHelper } from "../lang/helper"; import SettingsManager from "../douban/setting/SettingsManager"; -import {request, requestUrl, RequestUrlParam} from "obsidian"; -import DoubanHumanCheckModel from "../douban/component/DoubanHumanCheckModel"; +import {Platform, RequestUrlResponse} from "obsidian"; +import DesktopHttpUtil from "./desktop/DesktopHttpUtil"; +import MobileHttpUtil from "./mobile/MobileHttpUtil"; -// const {https} = require("follow-redirects"); export default class HttpUtil { + /** + * get请求 + * @param url 请求地址 + * @param headers 请求参数 + * @param settingsManager 设置管理器 + */ + // Cookie: 'll="108296"; bid=xHRJLeWBrjQ; _pk_id.100001.8cb4=f8f83e81ec224a1a.1691572669.; __utmv=30149280.13103; __yadk_uid=ce95W7OsgT0iKFceWgbMSUdw1oOqxNTk; __gads=ID=62585f60f3f637d0-2234f63fc6e200a5:T=1691572672:RT=1691572672:S=ALNI_MaIqTxSWHsfpnX9nAmMHcPQEsaezg; __gpi=UID=00000c29a9f98e5b:T=1691572672:RT=1691572672:S=ALNI_MbLAq8XNoKrRPKNqGCMdgXSPZvidw; ap_v=0,6.0; __utma=30149280.135860784.1691572641.1691572641.1694509646.2; __utmc=30149280; __utmz=30149280.1694509646.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1694509648%2C%22https%3A%2F%2Fmovie.douban.com%2Ftv%2F%22%5D; _pk_ses.100001.8cb4=1; __utmt=1; dbcl2="131038721:LUssju34QFw"; ck=dCQj; push_noty_num=0; push_doumail_num=0; __utmb=30149280.3.10.1694509646' + public static httpRequestGet(url: string, headers: any, settingsManager?: SettingsManager): Promise { + settingsManager.debug(`请求地址:${url}`); + const {['Accept-Encoding']: acceptEncoding, ...headersInner} = headers; + settingsManager.debug(`Obsidian-Douban:从网络获取网页开始:\nurl:${url}\nheaders:${JSON.stringify(headers)}`); + if (Platform.isDesktopApp) { + return DesktopHttpUtil.httpRequestGet(url, headers, settingsManager) + }else { + return MobileHttpUtil.httpRequestGet(url, headers, settingsManager).then((response:RequestUrlResponse) => { + return response.text; + }); + } - /** - * get请求 - * @param url 请求地址 - * @param headers 请求参数 - * @param settingsManager 设置管理器 - */ - // Cookie: 'll="108296"; bid=xHRJLeWBrjQ; _pk_id.100001.8cb4=f8f83e81ec224a1a.1691572669.; __utmv=30149280.13103; __yadk_uid=ce95W7OsgT0iKFceWgbMSUdw1oOqxNTk; __gads=ID=62585f60f3f637d0-2234f63fc6e200a5:T=1691572672:RT=1691572672:S=ALNI_MaIqTxSWHsfpnX9nAmMHcPQEsaezg; __gpi=UID=00000c29a9f98e5b:T=1691572672:RT=1691572672:S=ALNI_MbLAq8XNoKrRPKNqGCMdgXSPZvidw; ap_v=0,6.0; __utma=30149280.135860784.1691572641.1691572641.1694509646.2; __utmc=30149280; __utmz=30149280.1694509646.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1694509648%2C%22https%3A%2F%2Fmovie.douban.com%2Ftv%2F%22%5D; _pk_ses.100001.8cb4=1; __utmt=1; dbcl2="131038721:LUssju34QFw"; ck=dCQj; push_noty_num=0; push_doumail_num=0; __utmb=30149280.3.10.1694509646' - public static httpRequestGet(url: string, headers: any, settingsManager?: SettingsManager): Promise { - settingsManager.debug(`请求地址:${url}`); - const {['Accept-Encoding']: acceptEncoding, ...headersInner} = headers; - let options = { - headers: headersInner - } - settingsManager.debug(`Obsidian-Douban:从网络获取网页开始:\nurl:${url}\nheaders:${JSON.stringify(headers)}`); - // return new Promise((resolve, rejects) => { - // https.get(url, { ...options }, function (response: any) { - // let chunks: any = [], - // size = 0; - // if (response.status == 403) { - // rejects(new Error(i18nHelper.getMessage('130106'))); - // } - // response.on("data", function (chunk: any) { - // chunks.push(chunk) - // size += chunk.length - // }) - // - // response.on("end", function () { - // const data = Buffer.concat(chunks, size) - // const html = data.toString() - // if (settingsManager) { - // settingsManager.debug(`Obsidian-Douban:从网络获取网页完成:\nhtml:\n${html}`); - // } - // if (LoginUtil.contentNeedLogin(html)) { - // rejects(new Error(i18nHelper.getMessage('140304'))); - // } - // resolve(html) - // }) - // }) - // }) - return this.httpRequestGet1Inner(url, headers, 0, settingsManager); - - } + } /** * get请求 @@ -62,37 +37,14 @@ export default class HttpUtil { // Cookie: 'll="108296"; bid=xHRJLeWBrjQ; _pk_id.100001.8cb4=f8f83e81ec224a1a.1691572669.; __utmv=30149280.13103; __yadk_uid=ce95W7OsgT0iKFceWgbMSUdw1oOqxNTk; __gads=ID=62585f60f3f637d0-2234f63fc6e200a5:T=1691572672:RT=1691572672:S=ALNI_MaIqTxSWHsfpnX9nAmMHcPQEsaezg; __gpi=UID=00000c29a9f98e5b:T=1691572672:RT=1691572672:S=ALNI_MbLAq8XNoKrRPKNqGCMdgXSPZvidw; ap_v=0,6.0; __utma=30149280.135860784.1691572641.1691572641.1694509646.2; __utmc=30149280; __utmz=30149280.1694509646.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1694509648%2C%22https%3A%2F%2Fmovie.douban.com%2Ftv%2F%22%5D; _pk_ses.100001.8cb4=1; __utmt=1; dbcl2="131038721:LUssju34QFw"; ck=dCQj; push_noty_num=0; push_doumail_num=0; __utmb=30149280.3.10.1694509646' public static httpRequestGetJson(url: string, headers: any, settingsManager?: SettingsManager): Promise { const {['Accept-Encoding']: acceptEncoding, ...headersInner} = headers; - const options = { - headers: headersInner - } settingsManager.debug(`Obsidian-Douban:从网络获取json开始:\nurl:${url}\nheaders:${JSON.stringify(headers)}`); - // return new Promise((resolve, rejects) => { - // https.get(url, { ...options }, function (response: any) { - // const chunks: any = []; - // let size = 0; - // if (response.status == 403) { - // rejects(new Error(i18nHelper.getMessage('130106'))); - // } - // response.on("data", function (chunk: any) { - // chunks.push(chunk) - // size += chunk.length - // }) - // - // response.on("end", function () { - // const data = Buffer.concat(chunks, size) - // const html = data.toString() - // if (settingsManager) { - // settingsManager.debug(`Obsidian-Douban:从网络获取网页完成:\nhtml:\n${html}`); - // } - // if (LoginUtil.contentNeedLogin(html)) { - // rejects(new Error(i18nHelper.getMessage('140304'))); - // } - // resolve(html) - // }) - // }) - // }) - return this.httpRequestGet1Inner(url, headers, 0, settingsManager); - + if (Platform.isDesktopApp) { + return DesktopHttpUtil.httpRequestGetJson(url, headers, settingsManager) + }else { + return MobileHttpUtil.httpRequestGet(url, headers, settingsManager).then((response:RequestUrlResponse) => { + return response.json; + }); + } } /** @@ -102,107 +54,14 @@ export default class HttpUtil { * @param settingsManager 设置管理器 */ public static httpRequestGetBuffer(url: string, headers: any, settingsManager?: SettingsManager): Promise { - let options = { - headers: headers + if (Platform.isDesktopApp) { + return DesktopHttpUtil.httpRequestGetBuffer(url, headers, settingsManager) + }else { + return MobileHttpUtil.httpRequestGet(url, headers, settingsManager).then((response: RequestUrlResponse) => { + return response.arrayBuffer; + }); } - // if (settingsManager) { - // settingsManager.debug(`Obsidian-Douban:从网络获取文件开始:\n${url}`); - // } - // return new Promise((resolve, rejects) => { - // https.get(url, { ...options }, function (response: any) { - // let chunks: any = [], - // size = 0; - // if (response.status == 403) { - // rejects(new Error(i18nHelper.getMessage('130106'))); - // } - // response.on("data", function (chunk: any) { - // chunks.push(chunk) - // size += chunk.length - // }) - // - // response.on("end", function () { - // let data = Buffer.concat(chunks, size) - // if (settingsManager) { - // settingsManager.debug(`Obsidian-Douban:从网络获取文件完成:\n${url}`); - // } - // resolve(data) - // }) - // }) - // }) - // return this.httpRequestGet1Inner(url, headers, 0, settingsManager); - return this.httpRequestGet1Inner(url, headers, 0, settingsManager).then(s => { - return new TextEncoder().encode(s); - }); } - - - - /** - * get请求 - * @param url 请求地址 - * @param headers 请求参数 - * @param settingsManager 设置管理器 - */ - public static httpRequestGet1(url: string, headers: any, settingsManager?: SettingsManager): Promise { - return this.httpRequestGet1Inner(url, headers, 0, settingsManager); - } - private static httpRequestGet1Inner(url: string, headers: any, times:number, settingsManager?: SettingsManager): Promise { - let requestUrlParam: RequestUrlParam = { - url: url, - method: "GET", - headers: headers, - throw: true - }; - return requestUrl(requestUrlParam) - // .then(res => res.text) - .then(response => { - if (response && response.text.indexOf('https://sec.douban.com/a') > 0) { - log.notice(i18nHelper.getMessage('130105')) - if (settingsManager) { - settingsManager.debug(`Obsidian-Douban:获取异常网页如下:\n${response}`); - } - } - if (response.status == 301 || response.status == 302 || response.status == 303 || response.status == 307) { - if (times > 2) { - throw new Error('重定向次数过多'); - } - let location = response.headers['location']; - settingsManager.debug(`Obsidian-Douban:获取重定向地址如下:\n${location}`); - if (location.indexOf('http') != 0) { - return this.httpRequestGet1Inner(location, headers, times + 1, settingsManager); - } else { - throw new Error('重定地址错误'); - } - } - settingsManager.debug(`Obsidian-Douban:获取网页如下:\n${response}`); - return response.text; - }) - .then(s => this.humanCheck(s, url, settingsManager)) - .catch(e => { - if (e.toString().indexOf('403') > 0) { - throw log.error(i18nHelper.getMessage('130105'), e) - } else { - throw log.error(i18nHelper.getMessage('130101').replace('{0}', e.toString()), e) - } - }) - - - } - - - public static async humanCheck(html: any, url: string, settingsManager?: SettingsManager): Promise { - if (settingsManager) { - settingsManager.debug(html); - } - if (html && html.indexOf("禁止访问") != -1) { - const loginModel = new DoubanHumanCheckModel(url); - await loginModel.load(); - return ''; - } else { - return html; - } - } - } diff --git a/src/org/wanxp/utils/desktop/DesktopHttpUtil.ts b/src/org/wanxp/utils/desktop/DesktopHttpUtil.ts new file mode 100644 index 0000000..bf96441 --- /dev/null +++ b/src/org/wanxp/utils/desktop/DesktopHttpUtil.ts @@ -0,0 +1,148 @@ +import SettingsManager from "../../douban/setting/SettingsManager"; +import {i18nHelper} from "../../lang/helper"; +import {LoginUtil} from "../LoginUtil"; +import DoubanHumanCheckModel from "../../douban/component/DoubanHumanCheckModel"; + + +const {https} = require("follow-redirects"); + +export default class DesktopHttpUtil { + + + + /** + * get请求 + * @param url 请求地址 + * @param headers 请求参数 + * @param settingsManager 设置管理器 + */ + // Cookie: 'll="108296"; bid=xHRJLeWBrjQ; _pk_id.100001.8cb4=f8f83e81ec224a1a.1691572669.; __utmv=30149280.13103; __yadk_uid=ce95W7OsgT0iKFceWgbMSUdw1oOqxNTk; __gads=ID=62585f60f3f637d0-2234f63fc6e200a5:T=1691572672:RT=1691572672:S=ALNI_MaIqTxSWHsfpnX9nAmMHcPQEsaezg; __gpi=UID=00000c29a9f98e5b:T=1691572672:RT=1691572672:S=ALNI_MbLAq8XNoKrRPKNqGCMdgXSPZvidw; ap_v=0,6.0; __utma=30149280.135860784.1691572641.1691572641.1694509646.2; __utmc=30149280; __utmz=30149280.1694509646.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1694509648%2C%22https%3A%2F%2Fmovie.douban.com%2Ftv%2F%22%5D; _pk_ses.100001.8cb4=1; __utmt=1; dbcl2="131038721:LUssju34QFw"; ck=dCQj; push_noty_num=0; push_doumail_num=0; __utmb=30149280.3.10.1694509646' + public static httpRequestGet(url: string, headers: any, settingsManager?: SettingsManager): Promise { + settingsManager.debug(`请求地址:${url}`); + const {['Accept-Encoding']: acceptEncoding, ...headersInner} = headers; + let options = { + headers: headersInner + } + settingsManager.debug(`Obsidian-Douban:从网络获取网页开始:\nurl:${url}\nheaders:${JSON.stringify(headers)}`); + return new Promise((resolve, rejects) => { + https.get(url, { ...options }, function (response: any) { + let chunks: any = [], + size = 0; + if (response.status == 403) { + rejects(new Error(i18nHelper.getMessage('130106'))); + } + response.on("data", function (chunk: any) { + chunks.push(chunk) + size += chunk.length + }) + + response.on("end", function () { + const data = Buffer.concat(chunks, size) + const html = data.toString() + if (settingsManager) { + settingsManager.debug(`Obsidian-Douban:从网络获取网页完成:\nhtml:\n${html}`); + } + if (LoginUtil.contentNeedLogin(html)) { + rejects(new Error(i18nHelper.getMessage('140304'))); + } + resolve(html) + }) + }) + }) + } + + /** + * get请求 + * @param url 请求地址 + * @param headers 请求参数 + * @param settingsManager 设置管理器 + */ + // Cookie: 'll="108296"; bid=xHRJLeWBrjQ; _pk_id.100001.8cb4=f8f83e81ec224a1a.1691572669.; __utmv=30149280.13103; __yadk_uid=ce95W7OsgT0iKFceWgbMSUdw1oOqxNTk; __gads=ID=62585f60f3f637d0-2234f63fc6e200a5:T=1691572672:RT=1691572672:S=ALNI_MaIqTxSWHsfpnX9nAmMHcPQEsaezg; __gpi=UID=00000c29a9f98e5b:T=1691572672:RT=1691572672:S=ALNI_MbLAq8XNoKrRPKNqGCMdgXSPZvidw; ap_v=0,6.0; __utma=30149280.135860784.1691572641.1691572641.1694509646.2; __utmc=30149280; __utmz=30149280.1694509646.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1694509648%2C%22https%3A%2F%2Fmovie.douban.com%2Ftv%2F%22%5D; _pk_ses.100001.8cb4=1; __utmt=1; dbcl2="131038721:LUssju34QFw"; ck=dCQj; push_noty_num=0; push_doumail_num=0; __utmb=30149280.3.10.1694509646' + public static httpRequestGetJson(url: string, headers: any, settingsManager?: SettingsManager): Promise { + const {['Accept-Encoding']: acceptEncoding, ...headersInner} = headers; + const options = { + headers: headersInner + } + settingsManager.debug(`Obsidian-Douban:从网络获取json开始:\nurl:${url}\nheaders:${JSON.stringify(headers)}`); + return new Promise((resolve, rejects) => { + https.get(url, { ...options }, function (response: any) { + const chunks: any = []; + let size = 0; + if (response.status == 403) { + rejects(new Error(i18nHelper.getMessage('130106'))); + } + response.on("data", function (chunk: any) { + chunks.push(chunk) + size += chunk.length + }) + + response.on("end", function () { + const data = Buffer.concat(chunks, size) + const html = data.toString() + if (settingsManager) { + settingsManager.debug(`Obsidian-Douban:从网络获取json完成:\nhtml:\n${html}`); + } + if (LoginUtil.contentNeedLogin(html)) { + rejects(new Error(i18nHelper.getMessage('140304'))); + } + resolve(html) + }) + }) + }) + } + + /** + * get请求 + * @param url 请求地址 + * @param headers 请求参数 + * @param settingsManager 设置管理器 + */ + public static httpRequestGetBuffer(url: string, headers: any, settingsManager?: SettingsManager): Promise { + let options = { + headers: headers + } + if (settingsManager) { + settingsManager.debug(`Obsidian-Douban:从网络获取文件开始:\n${url}`); + } + return new Promise((resolve, rejects) => { + https.get(url, { ...options }, function (response: any) { + let chunks: any = [], + size = 0; + if (response.status == 403) { + rejects(new Error(i18nHelper.getMessage('130106'))); + } + response.on("data", function (chunk: any) { + chunks.push(chunk) + size += chunk.length + }) + + response.on("end", function () { + let data = Buffer.concat(chunks, size) + if (settingsManager) { + settingsManager.debug(`Obsidian-Douban:从网络获取文件完成:\n${url}`); + } + resolve(data) + }) + }) + }) + } + + + + + + + public static async humanCheck(html: any, url: string, settingsManager?: SettingsManager): Promise { + if (settingsManager) { + settingsManager.debug(html); + } + if (html && html.indexOf("禁止访问") != -1) { + const loginModel = new DoubanHumanCheckModel(url); + await loginModel.load(); + return ''; + } else { + return html; + } + } + +} diff --git a/src/org/wanxp/utils/mobile/MobileHttpUtil.ts b/src/org/wanxp/utils/mobile/MobileHttpUtil.ts new file mode 100644 index 0000000..e6c976e --- /dev/null +++ b/src/org/wanxp/utils/mobile/MobileHttpUtil.ts @@ -0,0 +1,71 @@ +import SettingsManager from "../../douban/setting/SettingsManager"; +import {requestUrl, RequestUrlParam, RequestUrlResponse} from "obsidian"; +import {log} from "../Logutil"; +import {i18nHelper} from "../../lang/helper"; +import DoubanHumanCheckModel from "../../douban/component/DoubanHumanCheckModel"; + +export default class MobileHttpUtil { + /** + * get请求 + * @param url 请求地址 + * @param headers 请求参数 + * @param settingsManager 设置管理器 + */ + public static httpRequestGet(url: string, headers: any, settingsManager?: SettingsManager): Promise { + return this.httpRequestGetInner(url, headers, 0, settingsManager); + } + private static httpRequestGetInner(url: string, headers: any, times:number, settingsManager?: SettingsManager): Promise { + let requestUrlParam: RequestUrlParam = { + url: url, + method: "GET", + headers: { ...headers}, + throw: true + }; + return requestUrl(requestUrlParam) + // .then(res => res.text) + .then(response => { + if (response && response.text.indexOf('https://sec.douban.com/a') > 0) { + log.notice(i18nHelper.getMessage('130105')) + if (settingsManager) { + settingsManager.debug(`Obsidian-Douban:获取异常网页如下:\n${response}`); + } + } + if (response.status == 301 || response.status == 302 || response.status == 303 || response.status == 307) { + if (times > 2) { + throw new Error('重定向次数过多'); + } + let location = response.headers['location']; + settingsManager.debug(`Obsidian-Douban:获取重定向地址如下:\n${location}`); + if (location.indexOf('http') != 0) { + return this.httpRequestGetInner(location, headers, times + 1, settingsManager); + } else { + throw new Error('重定地址错误'); + } + } + settingsManager.debug(`Obsidian-Douban:获取网页如下:\n${response}`); + return response; + }) + .then(s => this.humanCheck(s, url, settingsManager)) + .catch(e => { + if (e.toString().indexOf('403') > 0) { + throw log.error(i18nHelper.getMessage('130105'), e) + } else { + throw log.error(i18nHelper.getMessage('130101').replace('{0}', e.toString()), e) + } + }) + } + + + private static async humanCheck(html: any, url: string, settingsManager?: SettingsManager): Promise { + if (settingsManager) { + settingsManager.debug(html); + } + if (html && html.indexOf("禁止访问") != -1) { + const loginModel = new DoubanHumanCheckModel(url); + await loginModel.load(); + return ''; + } else { + return html; + } + } +}