Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 03:32:22 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #64 from mendableai/feat/allowed-urls

Browse Source 
[Feat] Added allowed urls
This commit is contained in:
Rafael Miller 2024-04-25 08:43:16 -03:00 committed by GitHub
commit abf6918642
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 8 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
apps/api/src/scraper/WebScraper/utils/blocklist.ts
View File

@ -14,6 +14,14 @@ const socialMediaBlocklist = [
'telegram.org', 'telegram.org',
]; ];
const allowedUrls = [
'linkedin.com/pulse'
];
export function isUrlBlocked(url: string): boolean { export function isUrlBlocked(url: string): boolean {
if (allowedUrls.some(allowedUrl => url.includes(allowedUrl))) {
return false;
}
return socialMediaBlocklist.some(domain => url.includes(domain)); return socialMediaBlocklist.some(domain => url.includes(domain));
} }