Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 03:32:22 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

added iframe links to extractLinksFromHTML

Browse Source
This commit is contained in:
rafaelsideguide 2024-10-31 10:53:47 -03:00
parent fe02101a12
commit 367af9512f
1 changed files with 10 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -333,6 +333,16 @@ export class WebCrawler {
}
});
// Extract links from iframes with inline src
$("iframe").each((_, element) => {
const src = $(element).attr("src");
if (src && src.startsWith("data:text/html")) {
const iframeHtml = decodeURIComponent(src.split(",")[1]);
const iframeLinks = this.extractLinksFromHTML(iframeHtml, url);
links = links.concat(iframeLinks);
}
});
return links;
}