firecrawl/.github/workflows/fly.yml

name: Fly Deploy
on:
  push:
    branches:
      - main
  schedule:
    - cron: '0 */2 * * *'

env:
  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
  BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
  FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
  HOST: ${{ secrets.HOST }}
  LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
  LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
  POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
  POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
  NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}
  PORT: ${{ secrets.PORT }}
  REDIS_URL: ${{ secrets.REDIS_URL }}
  SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }}
  SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
  SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
  SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
  TEST_API_KEY: ${{ secrets.TEST_API_KEY }}

jobs:
  pre-deploy:
    name: Pre-deploy checks
    runs-on: ubuntu-latest
    services:
      redis:
        image: redis
        ports:
          - 6379:6379
    steps:
      - uses: actions/checkout@v3
      - name: Set up Node.js
        uses: actions/setup-node@v3
        with:
          node-version: "20"
      - name: Install pnpm
        run: npm install -g pnpm
      - name: Install dependencies
        run: pnpm install
        working-directory: ./apps/api
      - name: Start the application
        run: npm start &
        working-directory: ./apps/api
        id: start_app
      - name: Start workers
        run: npm run workers &
        working-directory: ./apps/api
        id: start_workers
      - name: Run E2E tests
        run: |
          npm run test:prod          
        working-directory: ./apps/api

  pre-deploy-test-suite:
    name: Test Suite
    needs: pre-deploy
    runs-on: ubuntu-latest
    services:
      redis:
        image: redis
        ports:
          - 6379:6379
    steps:
      - uses: actions/checkout@v3
      - name: Set up Node.js
        uses: actions/setup-node@v3
        with:
          node-version: "20"
      - name: Install pnpm
        run: npm install -g pnpm
      - name: Install dependencies
        run: pnpm install
        working-directory: ./apps/api
      - name: Start the application
        run: npm start &
        working-directory: ./apps/api
        id: start_app
      - name: Start workers
        run: npm run workers &
        working-directory: ./apps/api
        id: start_workers
      - name: Install dependencies
        run: pnpm install
        working-directory: ./apps/test-suite
      - name: Run E2E tests
        run: |
          npm run test          
        working-directory: ./apps/test-suite
      - name: Set up Python ${{ matrix.python-version }}  
        uses: actions/setup-python@v4  
        with:  
          python-version: ${{ matrix.python-version }}
      - name: Install Python dependencies  
        run: |  
          python -m pip install --upgrade pip  
          pip install -r requirements.txt
        working-directory: ./apps/python-sdk
      - name: Run E2E tests for Python SDK
        run: |  
          pytest firecrawl/__tests__/e2e_withAuth/test.py 
        working-directory: ./apps/python-sdk
      - name: Install dependencies for JavaScript SDK
        run: pnpm install
        working-directory: ./apps/js-sdk/firecrawl
      - name: Run E2E tests for JavaScript SDK
        run: npm run test
        working-directory: ./apps/js-sdk/firecrawl
        
  deploy:
    name: Deploy app
    runs-on: ubuntu-latest
    needs: pre-deploy-test-suite
    steps:
      - uses: actions/checkout@v3
      - name: Change directory
        run: cd apps/api
      - uses: superfly/flyctl-actions/setup-flyctl@master
      - run: flyctl deploy ./apps/api --remote-only -a firecrawl-scraper-js
        env:
          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}