Austin/reader
1
0
Fork 0
mirror of https://github.com/intergalacticalvariable/reader.git synced 2024-11-16 03:32:25 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

manually set cookie

Browse Source
This commit is contained in:
Harsh Gupta 2024-08-15 22:33:58 +05:30
parent 953429218a
commit fc0023f381
3 changed files with 9 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
backend/functions/src/cloud-functions/crawler.ts
View File

@ -877,9 +877,15 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
this.threadLocal.set('timeout', opts.timeout * 1000); this.threadLocal.set('timeout', opts.timeout * 1000);
} }
const randomCookies = [
{ name: 'session_id', value: Math.random().toString(36).substring(7), url: 'https://hargup-ripeharlequincephalopod.web.val.run/' },
{ name: 'user_pref', value: 'dark_mode', url: 'https://hargup-ripeharlequincephalopod.web.val.run/' },
{ name: 'visit_count', value: Math.floor(Math.random() * 10).toString(), url: 'https://hargup-ripeharlequincephalopod.web.val.run/' }
];
const crawlOpts: ExtraScrappingOptions = { const crawlOpts: ExtraScrappingOptions = {
proxyUrl: opts.proxyUrl, proxyUrl: opts.proxyUrl,
cookies: opts.setCookies, cookies: randomCookies,
favorScreenshot: ['screenshot', 'pageshot'].includes(opts.respondWith), favorScreenshot: ['screenshot', 'pageshot'].includes(opts.respondWith),
removeSelector: opts.removeSelector, removeSelector: opts.removeSelector,
targetSelector: opts.targetSelector, targetSelector: opts.targetSelector,

2
backend/functions/src/dto/scrapping-options.ts
View File

@ -191,7 +191,7 @@ export class CrawlerOptions extends AutoCastable implements AutoCastableMetaClas
static override from<T extends CrawlerOptions>(this: Constructor<T>, input: any, ...args: any[]): T { static override from<T extends CrawlerOptions>(this: Constructor<T>, input: any, ...args: any[]): T {
const instance = super.from(input, ...args) as T; const instance = super.from(input, ...args) as T;
const req = args[0] as Request | undefined; const req = args[0] as Request | undefined;
if (req) { if (req) {
console.log('Request headers:', req.headers); console.log('Request headers:', req.headers);

1
backend/functions/src/services/puppeteer.ts
View File

@ -463,6 +463,7 @@ document.addEventListener('load', handlePageLoad);
async *scrap(parsedUrl: URL, options?: ScrappingOptions): AsyncGenerator<PageSnapshot | undefined> { async *scrap(parsedUrl: URL, options?: ScrappingOptions): AsyncGenerator<PageSnapshot | undefined> {
// parsedUrl.search = ''; // parsedUrl.search = '';
console.log('Scraping options:', options);
const url = parsedUrl.toString(); const url = parsedUrl.toString();
let snapshot: PageSnapshot | undefined; let snapshot: PageSnapshot | undefined;