Austin/reader
1
0
Fork 0
mirror of https://github.com/intergalacticalvariable/reader.git synced 2024-11-16 03:32:25 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: Remove auth and user info

Browse Source
This commit is contained in:
Harsh Gupta (aider) 2024-08-14 13:48:29 +05:30
parent d380599986
commit fbdc266660
1 changed files with 3 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
backend/functions/src/cloud-functions/crawler.ts
View File

@ -603,21 +603,17 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
req: Request, req: Request,
res: Response, res: Response,
}, },
auth: JinaEmbeddingsAuthDTO,
crawlerOptionsHeaderOnly: CrawlerOptionsHeaderOnly, crawlerOptionsHeaderOnly: CrawlerOptionsHeaderOnly,
crawlerOptionsParamsAllowed: CrawlerOptions, crawlerOptionsParamsAllowed: CrawlerOptions,
) { ) {
const uid = await auth.solveUID();
let chargeAmount = 0;
const noSlashURL = ctx.req.url.slice(1); const noSlashURL = ctx.req.url.slice(1);
const crawlerOptions = ctx.req.method === 'GET' ? crawlerOptionsHeaderOnly : crawlerOptionsParamsAllowed; const crawlerOptions = ctx.req.method === 'GET' ? crawlerOptionsHeaderOnly : crawlerOptionsParamsAllowed;
if (!noSlashURL && !crawlerOptions.url) { if (!noSlashURL && !crawlerOptions.url) {
const latestUser = uid ? await auth.assertUser() : undefined;
if (!ctx.req.accepts('text/plain') && (ctx.req.accepts('text/json') || ctx.req.accepts('application/json'))) { if (!ctx.req.accepts('text/plain') && (ctx.req.accepts('text/json') || ctx.req.accepts('application/json'))) {
return this.getIndex(latestUser); return this.getIndex();
} }
return assignTransferProtocolMeta(`${this.getIndex(latestUser)}`, return assignTransferProtocolMeta(`${this.getIndex()}`,
{ contentType: 'text/plain', envelope: null } { contentType: 'text/plain', envelope: null }
); );
} }
@ -656,22 +652,7 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
}); });
} }
if (!uid) { // Remove auth check
if (urlToCrawl.protocol === 'http:' && (!urlToCrawl.pathname || urlToCrawl.pathname === '/') &&
crawlerOptions.respondWith !== 'default') {
throw new Error(`Your request is categorized as abuse. Please don't abuse our service. If you are sure you are not abusing, please authenticate yourself with an API key.`);
}
const blockade = (await DomainBlockade.fromFirestoreQuery(
DomainBlockade.COLLECTION
.where('domain', '==', urlToCrawl.hostname.toLowerCase())
.where('expireAt', '>=', new Date())
.limit(1)
))[0];
if (blockade) {
throw new Error(`Domain ${urlToCrawl.hostname} blocked until ${blockade.expireAt || 'Eternally'} due to previous abuse found on ${blockade.triggerUrl || 'site'}: ${blockade.triggerReason}`);
}
}
const crawlOpts = this.configure(crawlerOptions); const crawlOpts = this.configure(crawlerOptions);