Merge pull request #265 from mendableai/feat/issue-264

[Feat] Added route to clean completed jobs and a github action cron that triggers every 24h
This commit is contained in:
Nicolas 2024-06-11 21:33:52 -07:00 committed by GitHub
commit f93231499f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 41 additions and 0 deletions

View File

@ -0,0 +1,20 @@
name: Clean Before 24h Completed Jobs
on:
schedule:
- cron: '0 0 * * *'
env:
BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
jobs:
clean-jobs:
runs-on: ubuntu-latest
steps:
- name: Send GET request to clean jobs
run: |
response=$(curl --write-out '%{http_code}' --silent --output /dev/null https://api.firecrawl.dev/admin/${{ secrets.BULL_AUTH_KEY }}/clean-before-24h-complete-jobs)
if [ "$response" -ne 200 ]; then
echo "Failed to clean jobs. Response: $response"
exit 1
fi
echo "Successfully cleaned jobs. Response: $response"

View File

@ -164,6 +164,27 @@ app.get('/serverHealthCheck/notify', async (req, res) => {
}
});
app.get(`/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`, async (req, res) => {
try {
const webScraperQueue = getWebScraperQueue();
const completedJobs = await webScraperQueue.getJobs(['completed']);
const before24hJobs = completedJobs.filter(job => job.finishedOn < Date.now() - 24 * 60 * 60 * 1000);
const jobIds = before24hJobs.map(job => job.id) as string[];
let count = 0;
for (const jobId of jobIds) {
try {
await webScraperQueue.removeJobs(jobId);
count++;
} catch (jobError) {
console.error(`Failed to remove job with ID ${jobId}:`, jobError);
}
}
res.status(200).send(`Removed ${count} completed jobs.`);
} catch (error) {
console.error('Failed to clean last 24h complete jobs:', error);
res.status(500).send('Failed to clean jobs');
}
});
app.get("/is-production", (req, res) => {
res.send({ isProduction: global.isProduction });