mirror of
https://github.com/mendableai/firecrawl.git
synced 2024-11-16 03:32:22 +08:00
Merge pull request #265 from mendableai/feat/issue-264
[Feat] Added route to clean completed jobs and a github action cron that triggers every 24h
This commit is contained in:
commit
f93231499f
20
.github/workflows/clean-before-24h-complete-jobs.yml
vendored
Normal file
20
.github/workflows/clean-before-24h-complete-jobs.yml
vendored
Normal file
|
@ -0,0 +1,20 @@
|
|||
name: Clean Before 24h Completed Jobs
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 0 * * *'
|
||||
|
||||
env:
|
||||
BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
|
||||
|
||||
jobs:
|
||||
clean-jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Send GET request to clean jobs
|
||||
run: |
|
||||
response=$(curl --write-out '%{http_code}' --silent --output /dev/null https://api.firecrawl.dev/admin/${{ secrets.BULL_AUTH_KEY }}/clean-before-24h-complete-jobs)
|
||||
if [ "$response" -ne 200 ]; then
|
||||
echo "Failed to clean jobs. Response: $response"
|
||||
exit 1
|
||||
fi
|
||||
echo "Successfully cleaned jobs. Response: $response"
|
|
@ -164,6 +164,27 @@ app.get('/serverHealthCheck/notify', async (req, res) => {
|
|||
}
|
||||
});
|
||||
|
||||
app.get(`/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`, async (req, res) => {
|
||||
try {
|
||||
const webScraperQueue = getWebScraperQueue();
|
||||
const completedJobs = await webScraperQueue.getJobs(['completed']);
|
||||
const before24hJobs = completedJobs.filter(job => job.finishedOn < Date.now() - 24 * 60 * 60 * 1000);
|
||||
const jobIds = before24hJobs.map(job => job.id) as string[];
|
||||
let count = 0;
|
||||
for (const jobId of jobIds) {
|
||||
try {
|
||||
await webScraperQueue.removeJobs(jobId);
|
||||
count++;
|
||||
} catch (jobError) {
|
||||
console.error(`Failed to remove job with ID ${jobId}:`, jobError);
|
||||
}
|
||||
}
|
||||
res.status(200).send(`Removed ${count} completed jobs.`);
|
||||
} catch (error) {
|
||||
console.error('Failed to clean last 24h complete jobs:', error);
|
||||
res.status(500).send('Failed to clean jobs');
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/is-production", (req, res) => {
|
||||
res.send({ isProduction: global.isProduction });
|
||||
|
|
Loading…
Reference in New Issue
Block a user