From 4edd9a34c7f79dce402826735a9b33f25fe9e159 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 28 Aug 2024 14:46:53 -0300 Subject: [PATCH] Nick: totalCount -> total, completedCount -> completed --- README.md | 2 +- apps/api/src/controllers/v1/crawl-status-ws.ts | 3 ++- apps/api/src/controllers/v1/crawl-status.ts | 3 ++- apps/api/src/controllers/v1/types.ts | 3 ++- apps/js-sdk/firecrawl/package.json | 2 +- .../src/__tests__/v1/e2e_withAuth/index.test.ts | 16 ++++++++-------- apps/js-sdk/firecrawl/src/index.ts | 6 ++++-- apps/python-sdk/firecrawl/__init__.py | 2 +- .../firecrawl/__tests__/v1/e2e_withAuth/test.py | 16 ++++++++-------- apps/python-sdk/firecrawl/firecrawl.py | 3 ++- 10 files changed, 31 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index e2197ded..21f480cc 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ curl -X GET https://api.firecrawl.dev/v1/crawl/123-456-789 \ ```json { "status": "completed", - "totalCount": 36, + "total": 36, "creditsUsed": 36, "expiresAt": "2024-00-00T00:00:00.000Z", "data": [ diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.ts index bf54cf49..551948de 100644 --- a/apps/api/src/controllers/v1/crawl-status-ws.ts +++ b/apps/api/src/controllers/v1/crawl-status-ws.ts @@ -102,7 +102,8 @@ async function crawlStatusWS(ws: WebSocket, req: RequestWithAuth legacyDocumentConverter(x)), diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts index 3cbacea3..1fe2fd9a 100644 --- a/apps/api/src/controllers/v1/crawl-status.ts +++ b/apps/api/src/controllers/v1/crawl-status.ts @@ -102,7 +102,8 @@ export async function crawlStatusController(req: RequestWithAuth { const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); const response = await app.crawlUrl('https://roastmywebsite.ai', {}, true, 30) as CrawlStatusResponse; expect(response).not.toBeNull(); - expect(response).toHaveProperty("totalCount"); - expect(response.totalCount).toBeGreaterThan(0); + expect(response).toHaveProperty("total"); + expect(response.total).toBeGreaterThan(0); expect(response).toHaveProperty("creditsUsed"); expect(response.creditsUsed).toBeGreaterThan(0); expect(response).toHaveProperty("expiresAt"); @@ -175,8 +175,8 @@ describe('FirecrawlApp E2E Tests', () => { } } as CrawlParams, true, 30) as CrawlStatusResponse; expect(response).not.toBeNull(); - expect(response).toHaveProperty("totalCount"); - expect(response.totalCount).toBeGreaterThan(0); + expect(response).toHaveProperty("total"); + expect(response.total).toBeGreaterThan(0); expect(response).toHaveProperty("creditsUsed"); expect(response.creditsUsed).toBeGreaterThan(0); expect(response).toHaveProperty("expiresAt"); @@ -231,12 +231,12 @@ describe('FirecrawlApp E2E Tests', () => { expect(statusResponse).not.toHaveProperty("partial_data"); // v0 expect(statusResponse).not.toHaveProperty("current"); // v0 expect(statusResponse).toHaveProperty("data"); - expect(statusResponse).toHaveProperty("totalCount"); + expect(statusResponse).toHaveProperty("total"); expect(statusResponse).toHaveProperty("creditsUsed"); expect(statusResponse).toHaveProperty("expiresAt"); expect(statusResponse).toHaveProperty("status"); expect(statusResponse).toHaveProperty("next"); - expect(statusResponse.totalCount).toBeGreaterThan(0); + expect(statusResponse.total).toBeGreaterThan(0); expect(statusResponse.creditsUsed).toBeGreaterThan(0); expect(statusResponse.expiresAt.getTime()).toBeGreaterThan(Date.now()); expect(statusResponse.status).toBe("scraping"); @@ -246,8 +246,8 @@ describe('FirecrawlApp E2E Tests', () => { } expect(statusResponse).not.toBeNull(); - expect(statusResponse).toHaveProperty("totalCount"); - expect(statusResponse.totalCount).toBeGreaterThan(0); + expect(statusResponse).toHaveProperty("total"); + expect(statusResponse.total).toBeGreaterThan(0); expect(statusResponse).toHaveProperty("creditsUsed"); expect(statusResponse.creditsUsed).toBeGreaterThan(0); expect(statusResponse).toHaveProperty("expiresAt"); diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 390b3701..4f3f820f 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -235,7 +235,8 @@ export interface CrawlResponseV0 { */ export interface CrawlStatusResponse { success: boolean; - totalCount: number; + total: number; + completed: number; creditsUsed: number; expiresAt: Date; status: "scraping" | "completed" | "failed"; @@ -530,7 +531,8 @@ export default class FirecrawlApp { return ({ success: true, status: response.data.status, - totalCount: response.data.totalCount, + total: response.data.total, + completed: response.data.completed, creditsUsed: response.data.creditsUsed, expiresAt: new Date(response.data.expiresAt), next: response.data.next, diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py index fbb2bdbf..1beaa043 100644 --- a/apps/python-sdk/firecrawl/__init__.py +++ b/apps/python-sdk/firecrawl/__init__.py @@ -13,7 +13,7 @@ import os from .firecrawl import FirecrawlApp -__version__ = "0.0.16" +__version__ = "1.0.0" # Define the logger for the Firecrawl project logger: logging.Logger = logging.getLogger("firecrawl") diff --git a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py b/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py index 5fb2c674..12fa10ce 100644 --- a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py +++ b/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py @@ -147,8 +147,8 @@ def test_crawl_url_wait_for_completion_e2e(): app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY) response = app.crawl_url('https://roastmywebsite.ai', {'excludePaths': ['blog/*']}, True, 30) assert response is not None - assert 'totalCount' in response - assert response['totalCount'] > 0 + assert 'total' in response + assert response['total'] > 0 assert 'creditsUsed' in response assert response['creditsUsed'] > 0 assert 'expiresAt' in response @@ -192,8 +192,8 @@ def test_crawl_url_with_options_and_wait_for_completion(): } }, True, 30) assert response is not None - assert 'totalCount' in response - assert response['totalCount'] > 0 + assert 'total' in response + assert response['total'] > 0 assert 'creditsUsed' in response assert response['creditsUsed'] > 0 assert 'expiresAt' in response @@ -247,12 +247,12 @@ def test_check_crawl_status_e2e(): assert 'partial_data' not in status_response assert 'current' not in status_response assert 'data' in status_response - assert 'totalCount' in status_response + assert 'total' in status_response assert 'creditsUsed' in status_response assert 'expiresAt' in status_response assert 'status' in status_response assert 'next' in status_response - assert status_response['totalCount'] > 0 + assert status_response['total'] > 0 assert status_response['creditsUsed'] > 0 assert datetime.strptime(status_response['expiresAt'], '%Y-%m-%dT%H:%M:%S.%fZ') > datetime.now() assert status_response['status'] == 'scraping' @@ -261,8 +261,8 @@ def test_check_crawl_status_e2e(): checks += 1 assert status_response is not None - assert 'totalCount' in status_response - assert status_response['totalCount'] > 0 + assert 'total' in status_response + assert status_response['total'] > 0 assert 'creditsUsed' in status_response assert status_response['creditsUsed'] > 0 assert 'expiresAt' in status_response diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.py index 4f71cc78..89c51803 100644 --- a/apps/python-sdk/firecrawl/firecrawl.py +++ b/apps/python-sdk/firecrawl/firecrawl.py @@ -211,7 +211,8 @@ class FirecrawlApp: return { 'success': True, 'status': data.get('status'), - 'totalCount': data.get('totalCount'), + 'total': data.get('total'), + 'completed': data.get('completed'), 'creditsUsed': data.get('creditsUsed'), 'expiresAt': data.get('expiresAt'), 'next': data.get('next'),