Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 03:32:22 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

add some types

Browse Source
This commit is contained in:
Jeff Pereira 2024-06-25 12:20:25 -07:00
parent f84fb4b331
commit 199cbe8bcb
2 changed files with 24 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
apps/api/src/main/runWebScraper.ts
View File

@ -1,5 +1,5 @@
import { Job } from "bull";
import { CrawlResult, WebScraperOptions } from "../types";
import { CrawlResult, WebScraperOptions, RunWebScraperParams, RunWebScraperResult } from "../types";
import { WebScraperDataProvider } from "../scraper/WebScraper";
import { DocumentUrl, Progress } from "../lib/entities";
import { billTeam } from "../services/billing/credit_billing";
@ -45,21 +45,7 @@ export async function runWebScraper({
onError,
team_id,
bull_job_id,
}: {
url: string;
mode: "crawl" | "single_urls" | "sitemap";
crawlerOptions: any;
pageOptions?: any;
inProgress: (progress: any) => void;
onSuccess: (result: any) => void;
onError: (error: any) => void;
team_id: string;
bull_job_id: string;
}): Promise<{
success: boolean;
message: string;
docs: Document[] | DocumentUrl[];
}> {
}: RunWebScraperParams): Promise<RunWebScraperResult> {
try {
const provider = new WebScraperDataProvider();
if (mode === "crawl") {

24
apps/api/src/types.ts
View File

@ -1,4 +1,6 @@
import { ExtractorOptions, Document } from "./lib/entities";
import { ExtractorOptions, Document, DocumentUrl } from "./lib/entities";
type Mode = "crawl" | "single_urls" | "sitemap";
export interface CrawlResult {
source: string;
@ -20,13 +22,31 @@ export interface IngestResult {
export interface WebScraperOptions {
url: string;
mode: "crawl" | "single_urls" | "sitemap";
mode: Mode;
crawlerOptions: any;
pageOptions: any;
team_id: string;
origin?: string;
}
export interface RunWebScraperParams {
url: string;
mode: Mode;
crawlerOptions: any;
pageOptions?: any;
inProgress: (progress: any) => void;
onSuccess: (result: any) => void;
onError: (error: Error) => void;
team_id: string;
bull_job_id: string;
}
export interface RunWebScraperResult {
success: boolean;
message: string;
docs: Document[] | DocumentUrl[];
}
export interface FirecrawlJob {
success: boolean;
message: string;