From ae38c26fa8834b3cf93bcc0211a195ad6cd24bd1 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Fri, 30 Aug 2024 10:09:39 -0300 Subject: [PATCH] feat(v1-sdks): async crawl node, python websocket + async crawl + example --- apps/js-sdk/README.md | 136 --------------- apps/js-sdk/example.js | 95 ++++++----- apps/js-sdk/example.ts | 40 +++-- apps/js-sdk/exampleV0.js | 85 ---------- apps/js-sdk/exampleV0.ts | 93 ---------- apps/js-sdk/firecrawl/README.md | 42 ++++- apps/js-sdk/test.ts | 28 --- apps/python-sdk/README.md | 59 +++++-- apps/python-sdk/example.py | 160 ++++++++++++------ apps/python-sdk/examplev0.py | 75 --------- apps/python-sdk/firecrawl/firecrawl.py | 225 ++++++++++++++----------- apps/python-sdk/requirements.txt | 5 +- 12 files changed, 394 insertions(+), 649 deletions(-) delete mode 100644 apps/js-sdk/README.md delete mode 100644 apps/js-sdk/exampleV0.js delete mode 100644 apps/js-sdk/exampleV0.ts delete mode 100644 apps/js-sdk/test.ts delete mode 100644 apps/python-sdk/examplev0.py diff --git a/apps/js-sdk/README.md b/apps/js-sdk/README.md deleted file mode 100644 index 0368677a..00000000 --- a/apps/js-sdk/README.md +++ /dev/null @@ -1,136 +0,0 @@ -# Firecrawl Node SDK - -The Firecrawl Node SDK is a library that allows you to easily scrape and crawl websites, and output the data in a format ready for use with language models (LLMs). It provides a simple and intuitive interface for interacting with the Firecrawl API. - -## Installation - -To install the Firecrawl Node SDK, you can use npm: - -```bash -npm install @mendable/firecrawl-js -``` - -## Usage - -1. Get an API key from [firecrawl.dev](https://firecrawl.dev) -2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `FirecrawlApp` class. - -Here's an example of how to use the SDK with error handling: - -```js -import FirecrawlApp, { CrawlParams, CrawlStatusResponse } from '@mendable/firecrawl-js'; - -const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"}); - -// Scrape a website -const scrapeResponse = await app.scrapeUrl('https://firecrawl.dev', { - formats: ['markdown', 'html'], -}); - -if (scrapeResponse) { - console.log(scrapeResponse) -} - -// Crawl a website -const crawlResponse = await app.crawlUrl('https://firecrawl.dev', { - limit: 100, - scrapeOptions: { - formats: ['markdown', 'html'], - } -} as CrawlParams, true, 30) as CrawlStatusResponse; - -if (crawlResponse) { - console.log(crawlResponse) -} -``` - -### Scraping a URL - -To scrape a single URL with error handling, use the `scrapeUrl` method. It takes the URL as a parameter and returns the scraped data as a dictionary. - -```js -const url = "https://example.com"; -const scrapedData = await app.scrapeUrl(url); -``` - -### Crawling a Website - -To crawl a website with error handling, use the `crawlUrl` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format. - -```js -const crawlResponse = await app.crawlUrl('https://firecrawl.dev', { - limit: 100, - scrapeOptions: { - formats: ['markdown', 'html'], - } -} as CrawlParams, true, 30) as CrawlStatusResponse; - -if (crawlResponse) { - console.log(crawlResponse) -} -``` - -### Checking Crawl Status - -To check the status of a crawl job with error handling, use the `checkCrawlStatus` method. It takes the job ID as a parameter and returns the current status of the crawl job. - -```js -const status = await app.checkCrawlStatus(id); -``` - -### Extracting structured data from a URL - -With LLM extraction, you can easily extract structured data from any URL. We support zod schema to make it easier for you too. Here is how you to use it: - -```js -import FirecrawlApp from "@mendable/firecrawl-js"; -import { z } from "zod"; - -const app = new FirecrawlApp({ - apiKey: "fc-YOUR_API_KEY", -}); - -// Define schema to extract contents into -const schema = z.object({ - top: z - .array( - z.object({ - title: z.string(), - points: z.number(), - by: z.string(), - commentsURL: z.string(), - }) - ) - .length(5) - .describe("Top 5 stories on Hacker News"), -}); - -const scrapeResult = await app.scrapeUrl("https://firecrawl.dev", { - extractorOptions: { extractionSchema: schema }, -}); - -console.log(scrapeResult.data["llm_extraction"]); -``` - -### Map a Website - -Use `map_url` to generate a list of URLs from a website. The `params` argument let you customize the mapping process, including options to exclude subdomains or to utilize the sitemap. - -```js -const mapResult = await app.mapUrl('https://example.com') as MapResponse; -console.log(mapResult) -``` - -## Error Handling - -The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message. The examples above demonstrate how to handle these errors using `try/catch` blocks. - -## License - -The Firecrawl Node SDK is licensed under the MIT License. This means you are free to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the SDK, subject to the following conditions: - -- The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -Please note that while this SDK is MIT licensed, it is part of a larger project which may be under different licensing terms. Always refer to the license information in the root directory of the main project for overall licensing details. diff --git a/apps/js-sdk/example.js b/apps/js-sdk/example.js index 5698a017..eb4bc489 100644 --- a/apps/js-sdk/example.js +++ b/apps/js-sdk/example.js @@ -1,49 +1,62 @@ -import FirecrawlApp from './firecrawl/src/index'; //'@mendable/firecrawl-js'; +import FirecrawlApp from '@mendable/firecrawl-js'; const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"}); -// Scrape a website: -const scrapeResult = await app.scrapeUrl('firecrawl.dev'); +const main = async () => { -if (scrapeResult.data) { - console.log(scrapeResult.data.markdown) -} + // Scrape a website: + const scrapeResult = await app.scrapeUrl('firecrawl.dev'); -// Crawl a website: -const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludes: ['blog/*'], limit: 5}}, false); -console.log(crawlResult) - -const jobId = await crawlResult['jobId']; -console.log(jobId); - -let job; -while (true) { - job = await app.checkCrawlStatus(jobId); - if (job.status === 'completed') { - break; + if (scrapeResult.success) { + console.log(scrapeResult.markdown) } - await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second + + // Crawl a website: + const crawlResult = await app.crawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5}); + console.log(crawlResult); + + // Asynchronously crawl a website: + const asyncCrawlResult = await app.asyncCrawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5}); + + if (asyncCrawlResult.success) { + const id = asyncCrawlResult.id; + console.log(id); + + let checkStatus; + if (asyncCrawlResult.success) { + while (true) { + checkStatus = await app.checkCrawlStatus(id); + if (checkStatus.success && checkStatus.status === 'completed') { + break; + } + await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second + } + + if (checkStatus.success && checkStatus.data) { + console.log(checkStatus.data[0].markdown); + } + } + } + + // Map a website: + const mapResult = await app.mapUrl('https://firecrawl.dev'); + console.log(mapResult) + + + // Crawl a website with WebSockets: + const watch = await app.crawlUrlAndWatch('mendable.ai', { excludePaths: ['blog/*'], limit: 5}); + + watch.addEventListener("document", doc => { + console.log("DOC", doc.detail); + }); + + watch.addEventListener("error", err => { + console.error("ERR", err.detail.error); + }); + + watch.addEventListener("done", state => { + console.log("DONE", state.detail.status); + }); } -if (job.data) { - console.log(job.data[0].markdown); -} - -// Map a website: -const mapResult = await app.map('https://firecrawl.dev'); -console.log(mapResult) - -// Crawl a website with WebSockets: -const watch = await app.crawlUrlAndWatch('mendable.ai', { excludePaths: ['blog/*'], limit: 5}); - -watch.addEventListener("document", doc => { - console.log("DOC", doc.detail); -}); - -watch.addEventListener("error", err => { - console.error("ERR", err.detail.error); -}); - -watch.addEventListener("done", state => { - console.log("DONE", state.detail.status); -}); +main() diff --git a/apps/js-sdk/example.ts b/apps/js-sdk/example.ts index 80589f5a..4142416f 100644 --- a/apps/js-sdk/example.ts +++ b/apps/js-sdk/example.ts @@ -1,4 +1,4 @@ -import FirecrawlApp, { CrawlStatusResponse, CrawlResponse } from '@mendable/firecrawl-js'; +import FirecrawlApp, { CrawlStatusResponse, ErrorResponse } from '@mendable/firecrawl-js'; const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"}); @@ -7,29 +7,35 @@ const main = async () => { // Scrape a website: const scrapeResult = await app.scrapeUrl('firecrawl.dev'); - if (scrapeResult) { + if (scrapeResult.success) { console.log(scrapeResult.markdown) } // Crawl a website: - // @ts-ignore - const crawlResult = await app.crawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5}, false) as CrawlResponse; - console.log(crawlResult) + const crawlResult = await app.crawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5}); + console.log(crawlResult); - const id = crawlResult.id; - console.log(id); + // Asynchronously crawl a website: + const asyncCrawlResult = await app.asyncCrawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5}); + + if (asyncCrawlResult.success) { + const id = asyncCrawlResult.id; + console.log(id); - let checkStatus: CrawlStatusResponse; - while (true) { - checkStatus = await app.checkCrawlStatus(id); - if (checkStatus.status === 'completed') { - break; + let checkStatus: CrawlStatusResponse | ErrorResponse; + if (asyncCrawlResult.success) { + while (true) { + checkStatus = await app.checkCrawlStatus(id); + if (checkStatus.success && checkStatus.status === 'completed') { + break; + } + await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second + } + + if (checkStatus.success && checkStatus.data) { + console.log(checkStatus.data[0].markdown); + } } - await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second - } - - if (checkStatus.data) { - console.log(checkStatus.data[0].markdown); } // Map a website: diff --git a/apps/js-sdk/exampleV0.js b/apps/js-sdk/exampleV0.js deleted file mode 100644 index 7f198598..00000000 --- a/apps/js-sdk/exampleV0.js +++ /dev/null @@ -1,85 +0,0 @@ -import { v4 as uuidv4 } from 'uuid'; -import FirecrawlApp from '@mendable/firecrawl-js'; -import { z } from "zod"; - -const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"}); - -// Scrape a website: -const scrapeResult = await app.scrapeUrl('firecrawl.dev'); -console.log(scrapeResult.data.content) - -// Crawl a website: -const idempotencyKey = uuidv4(); // optional -const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludes: ['blog/*'], limit: 5}}, false, 2, idempotencyKey); -console.log(crawlResult) - -const jobId = await crawlResult['jobId']; -console.log(jobId); - -let job; -while (true) { - job = await app.checkCrawlStatus(jobId); - if (job.status == 'completed') { - break; - } - await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second -} - -console.log(job.data[0].content); - -// Search for a query: -const query = 'what is mendable?' -const searchResult = await app.search(query) -console.log(searchResult) - -// LLM Extraction: -// Define schema to extract contents into using zod schema -const zodSchema = z.object({ - top: z - .array( - z.object({ - title: z.string(), - points: z.number(), - by: z.string(), - commentsURL: z.string(), - }) - ) - .length(5) - .describe("Top 5 stories on Hacker News"), -}); - -let llmExtractionResult = await app.scrapeUrl("https://news.ycombinator.com", { - extractorOptions: { extractionSchema: zodSchema }, -}); - -console.log(llmExtractionResult.data.llm_extraction); - -// Define schema to extract contents into using json schema -const jsonSchema = { - "type": "object", - "properties": { - "top": { - "type": "array", - "items": { - "type": "object", - "properties": { - "title": {"type": "string"}, - "points": {"type": "number"}, - "by": {"type": "string"}, - "commentsURL": {"type": "string"} - }, - "required": ["title", "points", "by", "commentsURL"] - }, - "minItems": 5, - "maxItems": 5, - "description": "Top 5 stories on Hacker News" - } - }, - "required": ["top"] -} - -llmExtractionResult = await app.scrapeUrl("https://news.ycombinator.com", { - extractorOptions: { extractionSchema: jsonSchema }, -}); - -console.log(llmExtractionResult.data.llm_extraction); \ No newline at end of file diff --git a/apps/js-sdk/exampleV0.ts b/apps/js-sdk/exampleV0.ts deleted file mode 100644 index cecaaf24..00000000 --- a/apps/js-sdk/exampleV0.ts +++ /dev/null @@ -1,93 +0,0 @@ -import FirecrawlApp, { ScrapeResponseV0, CrawlStatusResponseV0, SearchResponseV0 } from './firecrawl/src/index' //'@mendable/firecrawl-js'; -import { z } from "zod"; - -const app = new FirecrawlApp<"v0">({apiKey: "fc-YOUR_API_KEY", version: "v0"}) - -// Scrape a website: -const scrapeResult = await app.scrapeUrl('firecrawl.dev'); - -if (scrapeResult.data) { - console.log(scrapeResult.data.content) -} - -// Crawl a website: -const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludes: ['blog/*'], limit: 5}}, false); -console.log(crawlResult) - -const jobId: string = await crawlResult['jobId']; -console.log(jobId); - -let job: CrawlStatusResponseV0; -while (true) { - job = await app.checkCrawlStatus(jobId) as CrawlStatusResponseV0; - if (job.status === 'completed') { - break; - } - await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second -} - -if (job.data) { - console.log(job.data[0].content); -} - -// Search for a query: -const query = 'what is mendable?' -const searchResult = await app.search(query) as SearchResponseV0; -if (searchResult.data) { - console.log(searchResult.data[0].content) -} - -// LLM Extraction: -// Define schema to extract contents into using zod schema -const zodSchema = z.object({ - top: z - .array( - z.object({ - title: z.string(), - points: z.number(), - by: z.string(), - commentsURL: z.string(), - }) - ) - .length(5) - .describe("Top 5 stories on Hacker News"), -}); - -let llmExtractionResult = await app.scrapeUrl("https://news.ycombinator.com"); - -if (llmExtractionResult.data) { - console.log(llmExtractionResult.data[0].llm_extraction); -} - -// Define schema to extract contents into using json schema -const jsonSchema = { - "type": "object", - "properties": { - "top": { - "type": "array", - "items": { - "type": "object", - "properties": { - "title": {"type": "string"}, - "points": {"type": "number"}, - "by": {"type": "string"}, - "commentsURL": {"type": "string"} - }, - "required": ["title", "points", "by", "commentsURL"] - }, - "minItems": 5, - "maxItems": 5, - "description": "Top 5 stories on Hacker News" - } - }, - "required": ["top"] -} - -llmExtractionResult = await app.scrapeUrl("https://news.ycombinator.com", { - extractorOptions: { extractionSchema: jsonSchema }, -}); - -if (llmExtractionResult.data) { - console.log(llmExtractionResult.data[0].llm_extraction); -} - diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.md index 0368677a..0f3a6824 100644 --- a/apps/js-sdk/firecrawl/README.md +++ b/apps/js-sdk/firecrawl/README.md @@ -37,11 +37,9 @@ const crawlResponse = await app.crawlUrl('https://firecrawl.dev', { scrapeOptions: { formats: ['markdown', 'html'], } -} as CrawlParams, true, 30) as CrawlStatusResponse; +}) -if (crawlResponse) { - console.log(crawlResponse) -} +console.log(crawlResponse) ``` ### Scraping a URL @@ -63,16 +61,21 @@ const crawlResponse = await app.crawlUrl('https://firecrawl.dev', { scrapeOptions: { formats: ['markdown', 'html'], } -} as CrawlParams, true, 30) as CrawlStatusResponse; +}) +``` -if (crawlResponse) { - console.log(crawlResponse) -} + +### Asynchronous Crawl + +To initiate an asynchronous crawl of a website, utilize the AsyncCrawlURL method. This method requires the starting URL and optional parameters as inputs. The params argument enables you to define various settings for the asynchronous crawl, such as the maximum number of pages to crawl, permitted domains, and the output format. Upon successful initiation, this method returns an ID, which is essential for subsequently checking the status of the crawl. + +```js +const asyncCrawlResult = await app.asyncCrawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5}); ``` ### Checking Crawl Status -To check the status of a crawl job with error handling, use the `checkCrawlStatus` method. It takes the job ID as a parameter and returns the current status of the crawl job. +To check the status of a crawl job with error handling, use the `checkCrawlStatus` method. It takes the job ID as a parameter and returns the current status of the crawl job` ```js const status = await app.checkCrawlStatus(id); @@ -121,6 +124,27 @@ const mapResult = await app.mapUrl('https://example.com') as MapResponse; console.log(mapResult) ``` +### Crawl a website with WebSockets + +To crawl a website with WebSockets, use the `crawlUrlAndWatch` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format. + +```js +// Crawl a website with WebSockets: +const watch = await app.crawlUrlAndWatch('mendable.ai', { excludePaths: ['blog/*'], limit: 5}); + +watch.addEventListener("document", doc => { + console.log("DOC", doc.detail); +}); + +watch.addEventListener("error", err => { + console.error("ERR", err.detail.error); +}); + +watch.addEventListener("done", state => { + console.log("DONE", state.detail.status); +}); +``` + ## Error Handling The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message. The examples above demonstrate how to handle these errors using `try/catch` blocks. diff --git a/apps/js-sdk/test.ts b/apps/js-sdk/test.ts deleted file mode 100644 index 5419c2d5..00000000 --- a/apps/js-sdk/test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import FirecrawlApp from "@mendable/firecrawl-js"; -import { z } from "zod"; - -async function a() { - const app = new FirecrawlApp({ - apiKey: "fc-YOUR_API_KEY", - }); - - // Define schema to extract contents into - const schema = z.object({ - top: z - .array( - z.object({ - title: z.string(), - points: z.number(), - by: z.string(), - commentsURL: z.string(), - }) - ) - .length(5) - .describe("Top 5 stories on Hacker News"), - }); - const scrapeResult = await app.scrapeUrl("https://firecrawl.dev", { - extractorOptions: { extractionSchema: schema }, - }); - console.log(scrapeResult.data["llm_extraction"]); -} -a(); diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.md index 0cf36e9c..dcf44b25 100644 --- a/apps/python-sdk/README.md +++ b/apps/python-sdk/README.md @@ -81,22 +81,20 @@ print(data["llm_extraction"]) To crawl a website, use the `crawl_url` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format. -The `wait_until_done` parameter determines whether the method should wait for the crawl job to complete before returning the result. If set to `True`, the method will periodically check the status of the crawl job until it is completed or the specified `timeout` (in seconds) is reached. If set to `False`, the method will return immediately with the job ID, and you can manually check the status of the crawl job using the `check_crawl_status` method. - ```python -crawl_status = app.crawl_url( - 'https://firecrawl.dev', - params={ - 'limit': 100, - 'scrapeOptions': {'formats': ['markdown', 'html']} - }, - wait_until_done=True, - poll_interval=30 -) -print(crawl_status) +idempotency_key = str(uuid.uuid4()) # optional idempotency key +crawl_result = app.crawl_url('firecrawl.dev', {'excludePaths': ['blog/*']}, 2, idempotency_key) +print(crawl_result) ``` -If `wait_until_done` is set to `True`, the `crawl_url` method will return the crawl result once the job is completed. If the job fails or is stopped, an exception will be raised. +### Asynchronous Crawl a Website + +To crawl a website asynchronously, use the `async_crawl_url` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format. + +```python +crawl_result = app.async_crawl_url('firecrawl.dev', {'excludePaths': ['blog/*']}, "") +print(crawl_result) +``` ### Checking Crawl Status @@ -117,6 +115,41 @@ map_result = app.map_url('https://example.com') print(map_result) ``` +### Crawl a website with WebSockets + +To crawl a website with WebSockets, use the `crawl_url_and_watch` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format. + +```python +# inside an async function... +nest_asyncio.apply() + +# Define event handlers +def on_document(detail): + print("DOC", detail) + +def on_error(detail): + print("ERR", detail['error']) + +def on_done(detail): + print("DONE", detail['status']) + + # Function to start the crawl and watch process +async def start_crawl_and_watch(): + # Initiate the crawl job and get the watcher + watcher = app.crawl_url_and_watch('firecrawl.dev', { 'excludePaths': ['blog/*'], 'limit': 5 }) + + # Add event listeners + watcher.add_event_listener("document", on_document) + watcher.add_event_listener("error", on_error) + watcher.add_event_listener("done", on_done) + + # Start the watcher + await watcher.connect() + +# Run the event loop +await start_crawl_and_watch() +``` + ## Error Handling The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message. diff --git a/apps/python-sdk/example.py b/apps/python-sdk/example.py index d80fa795..efb13939 100644 --- a/apps/python-sdk/example.py +++ b/apps/python-sdk/example.py @@ -1,3 +1,5 @@ +import time +import nest_asyncio import uuid from firecrawl.firecrawl import FirecrawlApp @@ -9,67 +11,119 @@ print(scrape_result['markdown']) # Crawl a website: idempotency_key = str(uuid.uuid4()) # optional idempotency key -crawl_result = app.crawl_url('mendable.ai', {'crawlerOptions': {'excludes': ['blog/*']}}, True, 2, idempotency_key) +crawl_result = app.crawl_url('firecrawl.dev', {'excludePaths': ['blog/*']}, 2, idempotency_key) print(crawl_result) +# Asynchronous Crawl a website: +async_result = app.async_crawl_url('firecrawl.dev', {'excludePaths': ['blog/*']}, "") +print(async_result) + +crawl_status = app.check_crawl_status(async_result['id']) +print(crawl_status) + +attempts = 15 +while attempts > 0 and crawl_status['status'] != 'completed': + print(crawl_status) + crawl_status = app.check_crawl_status(async_result['id']) + attempts -= 1 + time.sleep(1) + +crawl_status = app.get_crawl_status(async_result['id']) +print(crawl_status) + # LLM Extraction: # Define schema to extract contents into using pydantic -from pydantic import BaseModel, Field -from typing import List +# from pydantic import BaseModel, Field +# from typing import List -class ArticleSchema(BaseModel): - title: str - points: int - by: str - commentsURL: str +# class ArticleSchema(BaseModel): +# title: str +# points: int +# by: str +# commentsURL: str -class TopArticlesSchema(BaseModel): - top: List[ArticleSchema] = Field(..., max_items=5, description="Top 5 stories") +# class TopArticlesSchema(BaseModel): +# top: List[ArticleSchema] = Field(..., max_items=5, description="Top 5 stories") -llm_extraction_result = app.scrape_url('https://news.ycombinator.com', { - 'extractorOptions': { - 'extractionSchema': TopArticlesSchema.model_json_schema(), - 'mode': 'llm-extraction' - }, - 'pageOptions':{ - 'onlyMainContent': True - } -}) +# llm_extraction_result = app.scrape_url('https://news.ycombinator.com', { +# 'extractorOptions': { +# 'extractionSchema': TopArticlesSchema.model_json_schema(), +# 'mode': 'llm-extraction' +# }, +# 'pageOptions':{ +# 'onlyMainContent': True +# } +# }) -print(llm_extraction_result['llm_extraction']) +# print(llm_extraction_result['llm_extraction']) -# Define schema to extract contents into using json schema -json_schema = { - "type": "object", - "properties": { - "top": { - "type": "array", - "items": { - "type": "object", - "properties": { - "title": {"type": "string"}, - "points": {"type": "number"}, - "by": {"type": "string"}, - "commentsURL": {"type": "string"} - }, - "required": ["title", "points", "by", "commentsURL"] - }, - "minItems": 5, - "maxItems": 5, - "description": "Top 5 stories on Hacker News" - } - }, - "required": ["top"] -} +# # Define schema to extract contents into using json schema +# json_schema = { +# "type": "object", +# "properties": { +# "top": { +# "type": "array", +# "items": { +# "type": "object", +# "properties": { +# "title": {"type": "string"}, +# "points": {"type": "number"}, +# "by": {"type": "string"}, +# "commentsURL": {"type": "string"} +# }, +# "required": ["title", "points", "by", "commentsURL"] +# }, +# "minItems": 5, +# "maxItems": 5, +# "description": "Top 5 stories on Hacker News" +# } +# }, +# "required": ["top"] +# } -llm_extraction_result = app.scrape_url('https://news.ycombinator.com', { - 'extractorOptions': { - 'extractionSchema': json_schema, - 'mode': 'llm-extraction' - }, - 'pageOptions':{ - 'onlyMainContent': True - } -}) +# llm_extraction_result = app.scrape_url('https://news.ycombinator.com', { +# 'extractorOptions': { +# 'extractionSchema': json_schema, +# 'mode': 'llm-extraction' +# }, +# 'pageOptions':{ +# 'onlyMainContent': True +# } +# }) -print(llm_extraction_result['llm_extraction']) \ No newline at end of file +# print(llm_extraction_result['llm_extraction']) + + +# Map a website: +map_result = app.map_url('https://firecrawl.dev', { 'search': 'blog' }) +print(map_result) + +# Crawl a website with WebSockets: +# inside an async function... +nest_asyncio.apply() + +# Define event handlers +def on_document(detail): + print("DOC", detail) + +def on_error(detail): + print("ERR", detail['error']) + +def on_done(detail): + print("DONE", detail['status']) + + # Function to start the crawl and watch process +async def start_crawl_and_watch(): + # Initiate the crawl job and get the watcher + watcher = app.crawl_url_and_watch('firecrawl.dev', { 'excludePaths': ['blog/*'], 'limit': 5 }) + + # Add event listeners + watcher.add_event_listener("document", on_document) + watcher.add_event_listener("error", on_error) + watcher.add_event_listener("done", on_done) + + # Start the watcher + await watcher.connect() + +# Run the event loop +await start_crawl_and_watch() \ No newline at end of file diff --git a/apps/python-sdk/examplev0.py b/apps/python-sdk/examplev0.py deleted file mode 100644 index d80fa795..00000000 --- a/apps/python-sdk/examplev0.py +++ /dev/null @@ -1,75 +0,0 @@ -import uuid -from firecrawl.firecrawl import FirecrawlApp - -app = FirecrawlApp(api_key="fc-YOUR_API_KEY") - -# Scrape a website: -scrape_result = app.scrape_url('firecrawl.dev') -print(scrape_result['markdown']) - -# Crawl a website: -idempotency_key = str(uuid.uuid4()) # optional idempotency key -crawl_result = app.crawl_url('mendable.ai', {'crawlerOptions': {'excludes': ['blog/*']}}, True, 2, idempotency_key) -print(crawl_result) - -# LLM Extraction: -# Define schema to extract contents into using pydantic -from pydantic import BaseModel, Field -from typing import List - -class ArticleSchema(BaseModel): - title: str - points: int - by: str - commentsURL: str - -class TopArticlesSchema(BaseModel): - top: List[ArticleSchema] = Field(..., max_items=5, description="Top 5 stories") - -llm_extraction_result = app.scrape_url('https://news.ycombinator.com', { - 'extractorOptions': { - 'extractionSchema': TopArticlesSchema.model_json_schema(), - 'mode': 'llm-extraction' - }, - 'pageOptions':{ - 'onlyMainContent': True - } -}) - -print(llm_extraction_result['llm_extraction']) - -# Define schema to extract contents into using json schema -json_schema = { - "type": "object", - "properties": { - "top": { - "type": "array", - "items": { - "type": "object", - "properties": { - "title": {"type": "string"}, - "points": {"type": "number"}, - "by": {"type": "string"}, - "commentsURL": {"type": "string"} - }, - "required": ["title", "points", "by", "commentsURL"] - }, - "minItems": 5, - "maxItems": 5, - "description": "Top 5 stories on Hacker News" - } - }, - "required": ["top"] -} - -llm_extraction_result = app.scrape_url('https://news.ycombinator.com', { - 'extractorOptions': { - 'extractionSchema': json_schema, - 'mode': 'llm-extraction' - }, - 'pageOptions':{ - 'onlyMainContent': True - } -}) - -print(llm_extraction_result['llm_extraction']) \ No newline at end of file diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.py index 89c51803..b7a0bff6 100644 --- a/apps/python-sdk/firecrawl/firecrawl.py +++ b/apps/python-sdk/firecrawl/firecrawl.py @@ -12,29 +12,30 @@ Classes: import logging import os import time -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, List +import asyncio +import json import requests +import websockets logger : logging.Logger = logging.getLogger("firecrawl") class FirecrawlApp: - def __init__(self, api_key: Optional[str] = None, api_url: Optional[str] = None, version: str = 'v1') -> None: + def __init__(self, api_key: Optional[str] = None, api_url: Optional[str] = None) -> None: """ - Initialize the FirecrawlApp instance with API key, API URL, and version. + Initialize the FirecrawlApp instance with API key, API URL. Args: api_key (Optional[str]): API key for authenticating with the Firecrawl API. api_url (Optional[str]): Base URL for the Firecrawl API. - version (str): API version, either 'v0' or 'v1'. """ self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY') self.api_url = api_url or os.getenv('FIRECRAWL_API_URL', 'https://api.firecrawl.dev') - self.version = version if self.api_key is None: logger.warning("No API key provided") raise ValueError('No API key provided') - logger.debug(f"Initialized FirecrawlApp with API key: {self.api_key} and version: {self.version}") + logger.debug(f"Initialized FirecrawlApp with API key: {self.api_key}") def scrape_url(self, url: str, params: Optional[Dict[str, Any]] = None) -> Any: """ @@ -74,7 +75,7 @@ class FirecrawlApp: if key != 'extractorOptions': scrape_params[key] = value - endpoint = f'/{self.version}/scrape' + endpoint = f'/v1/scrape' # Make the POST request with the prepared headers and JSON data response = requests.post( f'{self.api_url}{endpoint}', @@ -102,35 +103,14 @@ class FirecrawlApp: Any: The search results if the request is successful. Raises: + NotImplementedError: If the search request is attempted on API version v1. Exception: If the search request fails. """ - if self.version == 'v1': - raise NotImplementedError("Search is not supported in v1") - - headers = self._prepare_headers() - json_data = {'query': query} - if params: - json_data.update(params) - response = requests.post( - f'{self.api_url}/v0/search', - headers=headers, - json=json_data - ) - if response.status_code == 200: - response = response.json() - - if response['success'] and 'data' in response: - return response['data'] - else: - raise Exception(f'Failed to search. Error: {response["error"]}') - - else: - self._handle_error(response, 'search') + raise NotImplementedError("Search is not supported in v1.") def crawl_url(self, url: str, params: Optional[Dict[str, Any]] = None, - wait_until_done: bool = True, - poll_interval: int = 2, + poll_interval: Optional[int] = 2, idempotency_key: Optional[str] = None) -> Any: """ Initiate a crawl job for the specified URL using the Firecrawl API. @@ -138,8 +118,7 @@ class FirecrawlApp: Args: url (str): The URL to crawl. params (Optional[Dict[str, Any]]): Additional parameters for the crawl request. - wait_until_done (bool): Whether to wait until the crawl job is completed. - poll_interval (int): Time in seconds between status checks when waiting for job completion. + poll_interval (Optional[int]): Time in seconds between status checks when waiting for job completion. Defaults to 2 seconds. idempotency_key (Optional[str]): A unique uuid key to ensure idempotency of requests. Returns: @@ -148,28 +127,40 @@ class FirecrawlApp: Raises: Exception: If the crawl job initiation or monitoring fails. """ - endpoint = f'/{self.version}/crawl' + endpoint = f'/v1/crawl' headers = self._prepare_headers(idempotency_key) json_data = {'url': url} if params: json_data.update(params) response = self._post_request(f'{self.api_url}{endpoint}', json_data, headers) if response.status_code == 200: - if self.version == 'v0': - id = response.json().get('jobId') - else: - id = response.json().get('id') + id = response.json().get('id') + return self._monitor_job_status(id, headers, poll_interval) - if wait_until_done: - check_url = None - if self.version == 'v1': - check_url = response.json().get('url') - return self._monitor_job_status(id, headers, poll_interval, check_url) - else: - if self.version == 'v0': - return {'jobId': id} - else: - return {'id': id} + else: + self._handle_error(response, 'start crawl job') + + + def async_crawl_url(self, url: str, params: Optional[Dict[str, Any]] = None, idempotency_key: Optional[str] = None) -> Dict[str, Any]: + """ + Initiate a crawl job asynchronously. + + Args: + url (str): The URL to crawl. + params (Optional[Dict[str, Any]]): Additional parameters for the crawl request. + idempotency_key (Optional[str]): A unique uuid key to ensure idempotency of requests. + + Returns: + Dict[str, Any]: The response from the crawl initiation request. + """ + endpoint = f'/v1/crawl' + headers = self._prepare_headers(idempotency_key) + json_data = {'url': url} + if params: + json_data.update(params) + response = self._post_request(f'{self.api_url}{endpoint}', json_data, headers) + if response.status_code == 200: + return response.json() else: self._handle_error(response, 'start crawl job') @@ -186,50 +177,56 @@ class FirecrawlApp: Raises: Exception: If the status check request fails. """ - - if self.version == 'v0': - endpoint = f'/{self.version}/crawl/status/{id}' - else: - endpoint = f'/{self.version}/crawl/{id}' + endpoint = f'/v1/crawl/{id}' headers = self._prepare_headers() response = self._get_request(f'{self.api_url}{endpoint}', headers) if response.status_code == 200: data = response.json() - if self.version == 'v0': - return { - 'success': True, - 'status': data.get('status'), - 'current': data.get('current'), - 'current_url': data.get('current_url'), - 'current_step': data.get('current_step'), - 'total': data.get('total'), - 'data': data.get('data'), - 'partial_data': data.get('partial_data') if not data.get('data') else None, - } - elif self.version == 'v1': - return { - 'success': True, - 'status': data.get('status'), - 'total': data.get('total'), - 'completed': data.get('completed'), - 'creditsUsed': data.get('creditsUsed'), - 'expiresAt': data.get('expiresAt'), - 'next': data.get('next'), - 'data': data.get('data'), - 'error': data.get('error') - } + return { + 'success': True, + 'status': data.get('status'), + 'total': data.get('total'), + 'completed': data.get('completed'), + 'creditsUsed': data.get('creditsUsed'), + 'expiresAt': data.get('expiresAt'), + 'next': data.get('next'), + 'data': data.get('data'), + 'error': data.get('error') + } else: self._handle_error(response, 'check crawl status') + def crawl_url_and_watch(self, url: str, params: Optional[Dict[str, Any]] = None, idempotency_key: Optional[str] = None) -> 'CrawlWatcher': + """ + Initiate a crawl job and return a CrawlWatcher to monitor the job via WebSocket. + + Args: + url (str): The URL to crawl. + params (Optional[Dict[str, Any]]): Additional parameters for the crawl request. + idempotency_key (Optional[str]): A unique uuid key to ensure idempotency of requests. + + Returns: + CrawlWatcher: An instance of CrawlWatcher to monitor the crawl job. + """ + crawl_response = self.async_crawl_url(url, params, idempotency_key) + if crawl_response['success'] and 'id' in crawl_response: + return CrawlWatcher(crawl_response['id'], self) + else: + raise Exception("Crawl job failed to start") + def map_url(self, url: str, params: Optional[Dict[str, Any]] = None) -> Any: """ Perform a map search using the Firecrawl API. + + Args: + url (str): The URL to perform the map search on. + params (Optional[Dict[str, Any]]): Additional parameters for the map search. + + Returns: + Any: The result of the map search, typically a dictionary containing mapping data. """ - if self.version == 'v0': - raise NotImplementedError("Map is not supported in v0") - - endpoint = f'/{self.version}/map' + endpoint = f'/v1/map' headers = self._prepare_headers() # Prepare the base scrape parameters with the URL @@ -331,7 +328,7 @@ class FirecrawlApp: return response return response - def _monitor_job_status(self, id: str, headers: Dict[str, str], poll_interval: int, check_url: Optional[str] = None) -> Any: + def _monitor_job_status(self, id: str, headers: Dict[str, str], poll_interval: int) -> Any: """ Monitor the status of a crawl job until completion. @@ -339,7 +336,6 @@ class FirecrawlApp: id (str): The ID of the crawl job. headers (Dict[str, str]): The headers to include in the status check requests. poll_interval (int): Secounds between status checks. - check_url (Optional[str]): The URL to check for the crawl job. Returns: Any: The crawl results if the job is completed successfully. @@ -347,27 +343,14 @@ class FirecrawlApp: Exception: If the job fails or an error occurs during status checks. """ while True: - api_url = '' - if (self.version == 'v0'): - if check_url: - api_url = check_url - else: - api_url = f'{self.api_url}/v0/crawl/status/{id}' - else: - if check_url: - api_url = check_url - else: - api_url = f'{self.api_url}/v1/crawl/{id}' + api_url = f'{self.api_url}/v1/crawl/{id}' status_response = self._get_request(api_url, headers) if status_response.status_code == 200: status_data = status_response.json() if status_data['status'] == 'completed': if 'data' in status_data: - if self.version == 'v0': - return status_data['data'] - else: - return status_data + return status_data else: raise Exception('Crawl job completed but no data was returned') elif status_data['status'] in ['active', 'paused', 'pending', 'queued', 'waiting', 'scraping']: @@ -405,4 +388,50 @@ class FirecrawlApp: # Raise an HTTPError with the custom message and attach the response raise requests.exceptions.HTTPError(message, response=response) - \ No newline at end of file + +class CrawlWatcher: + def __init__(self, id: str, app: FirecrawlApp): + self.id = id + self.app = app + self.data: List[Dict[str, Any]] = [] + self.status = "scraping" + self.ws_url = f"{app.api_url.replace('http', 'ws')}/v1/crawl/{id}" + self.event_handlers = { + 'done': [], + 'error': [], + 'document': [] + } + + async def connect(self): + async with websockets.connect(self.ws_url, extra_headers={"Authorization": f"Bearer {self.app.api_key}"}) as websocket: + await self._listen(websocket) + + async def _listen(self, websocket): + async for message in websocket: + msg = json.loads(message) + await self._handle_message(msg) + + def add_event_listener(self, event_type: str, handler): + if event_type in self.event_handlers: + self.event_handlers[event_type].append(handler) + + def dispatch_event(self, event_type: str, detail: Dict[str, Any]): + if event_type in self.event_handlers: + for handler in self.event_handlers[event_type]: + handler(detail) + + async def _handle_message(self, msg: Dict[str, Any]): + if msg['type'] == 'done': + self.status = 'completed' + self.dispatch_event('done', {'status': self.status, 'data': self.data}) + elif msg['type'] == 'error': + self.status = 'failed' + self.dispatch_event('error', {'status': self.status, 'data': self.data, 'error': msg['error']}) + elif msg['type'] == 'catchup': + self.status = msg['data']['status'] + self.data.extend(msg['data'].get('data', [])) + for doc in self.data: + self.dispatch_event('document', doc) + elif msg['type'] == 'document': + self.data.append(msg['data']) + self.dispatch_event('document', msg['data']) \ No newline at end of file diff --git a/apps/python-sdk/requirements.txt b/apps/python-sdk/requirements.txt index 1bed5881..94971fde 100644 --- a/apps/python-sdk/requirements.txt +++ b/apps/python-sdk/requirements.txt @@ -1,3 +1,6 @@ requests pytest -python-dotenv \ No newline at end of file +python-dotenv +websockets +asyncio +nest-asyncio \ No newline at end of file