Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 03:32:22 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Added tests per @nicks request

Browse Source
This commit is contained in:
Caleb Peffer 2024-07-16 21:15:59 -07:00
parent 98c788ca7a
commit 0b3c0ede49
1 changed files with 77 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
apps/api/src/scraper/WebScraper/__tests__/single_url.test.ts
View File

@ -22,6 +22,81 @@ describe('scrapSingleUrl', () => {
}, 10000);
});
import { scrapSingleUrl } from '../single_url';
import { PageOptions } from '../../../lib/entities';
// Mock the fetchHtmlContent function
jest.mock('../single_url', () => {
const originalModule = jest.requireActual('../single_url');
originalModule.fetchHtmlContent = jest.fn().mockResolvedValue(`
<html>
<head><title>Test Page</title></head>
<body>
<a href="https://example.com">Absolute Link</a>
<a href="/relative">Relative Link</a>
<a href="page">Page Link</a>
<a href="#fragment">Fragment Link</a>
<a href="mailto:test@example.com">Email Link</a>
</body>
</html>
`);
return originalModule;
});
describe('scrapSingleUrl with linksOnPage', () => {
const baseUrl = 'https://test.com';
it('should not include linksOnPage when option is false', async () => {
const pageOptions: PageOptions = {};
const result = await scrapSingleUrl(baseUrl, pageOptions);
expect(result.linksOnPage).toBeUndefined();
});
it('should include linksOnPage when option is true', async () => {
const pageOptions: PageOptions = { };
const result = await scrapSingleUrl(baseUrl, pageOptions);
expect(result.linksOnPage).toBeDefined();
expect(Array.isArray(result.linksOnPage)).toBe(true);
});
it('should correctly handle absolute URLs', async () => {
const pageOptions: PageOptions = { };
const result = await scrapSingleUrl(baseUrl, pageOptions);
expect(result.linksOnPage).toContain('https://example.com');
});
it('should correctly handle relative URLs', async () => {
const pageOptions: PageOptions = { };
const result = await scrapSingleUrl(baseUrl, pageOptions);
expect(result.linksOnPage).toContain('https://test.com/relative');
});
it('should correctly handle page URLs', async () => {
const pageOptions: PageOptions = { };
const result = await scrapSingleUrl(baseUrl, pageOptions);
expect(result.linksOnPage).toContain('https://test.com/page');
});
it('should not include fragment-only links', async () => {
const pageOptions: PageOptions = { };
const result = await scrapSingleUrl(baseUrl, pageOptions);
expect(result.linksOnPage).not.toContain('#fragment');
expect(result.linksOnPage).not.toContain('https://test.com/#fragment');
});
it('should include mailto links', async () => {
const pageOptions: PageOptions = { };
const result = await scrapSingleUrl(baseUrl, pageOptions);
expect(result.linksOnPage).toContain('mailto:test@example.com');
});
it('should return unique links', async () => {
const pageOptions: PageOptions = { };
const result = await scrapSingleUrl(baseUrl, pageOptions);
const uniqueLinks = new Set(result.linksOnPage);
expect(result.linksOnPage?.length).toBe(uniqueLinks.size);
});
});
it('should return a list of links on the mendable.ai page', async () => {
const url = 'https://mendable.ai';
@ -36,3 +111,5 @@ it('should return a list of links on the mendable.ai page', async () => {
}, 10000);