Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

load testing for crawl

Browse Source
This commit is contained in:
rafaelsideguide 2024-05-22 14:26:29 -03:00
parent 04a0bef0fb
commit ff147f1f51
3 changed files with 952 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

907
apps/test-suite/load-test-results/test-run-report.json Normal file
View File

@ -0,0 +1,907 @@
{
"aggregate": {
"counters": {
"vusers.created_by_name.Crawl a URL": 10,
"vusers.created": 10,
"http.requests": 20,
"http.codes.200": 20,
"http.responses": 20,
"http.downloaded_bytes": 0,
"plugins.metrics-by-endpoint./v0/crawl.codes.200": 10,
"plugins.metrics-by-endpoint./v0/crawl/status/ea785a25-fb05-4ac6-85d0-744b95084c93.codes.200": 1,
"vusers.failed": 0,
"vusers.completed": 10,
"plugins.metrics-by-endpoint./v0/crawl/status/c07c910d-4d50-44fa-80aa-fac9869dee38.codes.200": 1,
"plugins.metrics-by-endpoint./v0/crawl/status/15f48c84-cdf7-4918-8a97-926d2887debd.codes.200": 1,
"plugins.metrics-by-endpoint./v0/crawl/status/c2d3009c-efcb-41b1-bd1e-d8112f80c1ad.codes.200": 1,
"plugins.metrics-by-endpoint./v0/crawl/status/3b22323d-8cab-47d0-ab45-8ed597a12778.codes.200": 1,
"plugins.metrics-by-endpoint./v0/crawl/status/1b7ac23b-9929-42dd-8180-a631b99f3341.codes.200": 1,
"plugins.metrics-by-endpoint./v0/crawl/status/af5c6205-08a5-4564-aaea-ad704d9e5449.codes.200": 1,
"plugins.metrics-by-endpoint./v0/crawl/status/90492ac5-4057-4a44-bc11-7613fd89a3b4.codes.200": 1,
"plugins.metrics-by-endpoint./v0/crawl/status/26f37b1d-0ef0-4107-9f4b-9ccb97826b65.codes.200": 1,
"plugins.metrics-by-endpoint./v0/crawl/status/2e703b76-c2e9-4726-972b-d08761ad6c0e.codes.200": 1
},
"rates": {
"http.request_rate": 1
},
"firstCounterAt": 1716393506465,
"firstHistogramAt": 1716393507755,
"lastCounterAt": 1716393527474,
"lastHistogramAt": 1716393527474,
"firstMetricAt": 1716393506465,
"lastMetricAt": 1716393527474,
"period": 1716393520000,
"summaries": {
"http.response_time": {
"min": 757,
"max": 1499,
"count": 20,
"mean": 966.2,
"p50": 907,
"median": 907,
"p75": 1002.4,
"p90": 1176.4,
"p95": 1274.3,
"p99": 1274.3,
"p999": 1274.3
},
"plugins.metrics-by-endpoint.response_time./v0/crawl": {
"min": 815,
"max": 1275,
"count": 10,
"mean": 996.6,
"p50": 963.1,
"median": 963.1,
"p75": 1022.7,
"p90": 1176.4,
"p95": 1176.4,
"p99": 1176.4,
"p999": 1176.4
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/ea785a25-fb05-4ac6-85d0-744b95084c93": {
"min": 943,
"max": 943,
"count": 1,
"mean": 943,
"p50": 944,
"median": 944,
"p75": 944,
"p90": 944,
"p95": 944,
"p99": 944,
"p999": 944
},
"vusers.session_length": {
"min": 11864.6,
"max": 13144.5,
"count": 10,
"mean": 12233.9,
"p50": 12213.1,
"median": 12213.1,
"p75": 12213.1,
"p90": 12459.8,
"p95": 12459.8,
"p99": 12459.8,
"p999": 12459.8
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/c07c910d-4d50-44fa-80aa-fac9869dee38": {
"min": 904,
"max": 904,
"count": 1,
"mean": 904,
"p50": 907,
"median": 907,
"p75": 907,
"p90": 907,
"p95": 907,
"p99": 907,
"p999": 907
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/15f48c84-cdf7-4918-8a97-926d2887debd": {
"min": 888,
"max": 888,
"count": 1,
"mean": 888,
"p50": 889.1,
"median": 889.1,
"p75": 889.1,
"p90": 889.1,
"p95": 889.1,
"p99": 889.1,
"p999": 889.1
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/c2d3009c-efcb-41b1-bd1e-d8112f80c1ad": {
"min": 1499,
"max": 1499,
"count": 1,
"mean": 1499,
"p50": 1495.5,
"median": 1495.5,
"p75": 1495.5,
"p90": 1495.5,
"p95": 1495.5,
"p99": 1495.5,
"p999": 1495.5
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/3b22323d-8cab-47d0-ab45-8ed597a12778": {
"min": 873,
"max": 873,
"count": 1,
"mean": 873,
"p50": 871.5,
"median": 871.5,
"p75": 871.5,
"p90": 871.5,
"p95": 871.5,
"p99": 871.5,
"p999": 871.5
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/1b7ac23b-9929-42dd-8180-a631b99f3341": {
"min": 757,
"max": 757,
"count": 1,
"mean": 757,
"p50": 757.6,
"median": 757.6,
"p75": 757.6,
"p90": 757.6,
"p95": 757.6,
"p99": 757.6,
"p999": 757.6
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/af5c6205-08a5-4564-aaea-ad704d9e5449": {
"min": 892,
"max": 892,
"count": 1,
"mean": 892,
"p50": 889.1,
"median": 889.1,
"p75": 889.1,
"p90": 889.1,
"p95": 889.1,
"p99": 889.1,
"p999": 889.1
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/90492ac5-4057-4a44-bc11-7613fd89a3b4": {
"min": 839,
"max": 839,
"count": 1,
"mean": 839,
"p50": 837.3,
"median": 837.3,
"p75": 837.3,
"p90": 837.3,
"p95": 837.3,
"p99": 837.3,
"p999": 837.3
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/26f37b1d-0ef0-4107-9f4b-9ccb97826b65": {
"min": 979,
"max": 979,
"count": 1,
"mean": 979,
"p50": 982.6,
"median": 982.6,
"p75": 982.6,
"p90": 982.6,
"p95": 982.6,
"p99": 982.6,
"p999": 982.6
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/2e703b76-c2e9-4726-972b-d08761ad6c0e": {
"min": 784,
"max": 784,
"count": 1,
"mean": 784,
"p50": 788.5,
"median": 788.5,
"p75": 788.5,
"p90": 788.5,
"p95": 788.5,
"p99": 788.5,
"p999": 788.5
}
},
"histograms": {
"http.response_time": {
"min": 757,
"max": 1499,
"count": 20,
"mean": 966.2,
"p50": 907,
"median": 907,
"p75": 1002.4,
"p90": 1176.4,
"p95": 1274.3,
"p99": 1274.3,
"p999": 1274.3
},
"plugins.metrics-by-endpoint.response_time./v0/crawl": {
"min": 815,
"max": 1275,
"count": 10,
"mean": 996.6,
"p50": 963.1,
"median": 963.1,
"p75": 1022.7,
"p90": 1176.4,
"p95": 1176.4,
"p99": 1176.4,
"p999": 1176.4
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/ea785a25-fb05-4ac6-85d0-744b95084c93": {
"min": 943,
"max": 943,
"count": 1,
"mean": 943,
"p50": 944,
"median": 944,
"p75": 944,
"p90": 944,
"p95": 944,
"p99": 944,
"p999": 944
},
"vusers.session_length": {
"min": 11864.6,
"max": 13144.5,
"count": 10,
"mean": 12233.9,
"p50": 12213.1,
"median": 12213.1,
"p75": 12213.1,
"p90": 12459.8,
"p95": 12459.8,
"p99": 12459.8,
"p999": 12459.8
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/c07c910d-4d50-44fa-80aa-fac9869dee38": {
"min": 904,
"max": 904,
"count": 1,
"mean": 904,
"p50": 907,
"median": 907,
"p75": 907,
"p90": 907,
"p95": 907,
"p99": 907,
"p999": 907
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/15f48c84-cdf7-4918-8a97-926d2887debd": {
"min": 888,
"max": 888,
"count": 1,
"mean": 888,
"p50": 889.1,
"median": 889.1,
"p75": 889.1,
"p90": 889.1,
"p95": 889.1,
"p99": 889.1,
"p999": 889.1
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/c2d3009c-efcb-41b1-bd1e-d8112f80c1ad": {
"min": 1499,
"max": 1499,
"count": 1,
"mean": 1499,
"p50": 1495.5,
"median": 1495.5,
"p75": 1495.5,
"p90": 1495.5,
"p95": 1495.5,
"p99": 1495.5,
"p999": 1495.5
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/3b22323d-8cab-47d0-ab45-8ed597a12778": {
"min": 873,
"max": 873,
"count": 1,
"mean": 873,
"p50": 871.5,
"median": 871.5,
"p75": 871.5,
"p90": 871.5,
"p95": 871.5,
"p99": 871.5,
"p999": 871.5
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/1b7ac23b-9929-42dd-8180-a631b99f3341": {
"min": 757,
"max": 757,
"count": 1,
"mean": 757,
"p50": 757.6,
"median": 757.6,
"p75": 757.6,
"p90": 757.6,
"p95": 757.6,
"p99": 757.6,
"p999": 757.6
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/af5c6205-08a5-4564-aaea-ad704d9e5449": {
"min": 892,
"max": 892,
"count": 1,
"mean": 892,
"p50": 889.1,
"median": 889.1,
"p75": 889.1,
"p90": 889.1,
"p95": 889.1,
"p99": 889.1,
"p999": 889.1
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/90492ac5-4057-4a44-bc11-7613fd89a3b4": {
"min": 839,
"max": 839,
"count": 1,
"mean": 839,
"p50": 837.3,
"median": 837.3,
"p75": 837.3,
"p90": 837.3,
"p95": 837.3,
"p99": 837.3,
"p999": 837.3
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/26f37b1d-0ef0-4107-9f4b-9ccb97826b65": {
"min": 979,
"max": 979,
"count": 1,
"mean": 979,
"p50": 982.6,
"median": 982.6,
"p75": 982.6,
"p90": 982.6,
"p95": 982.6,
"p99": 982.6,
"p999": 982.6
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/2e703b76-c2e9-4726-972b-d08761ad6c0e": {
"min": 784,
"max": 784,
"count": 1,
"mean": 784,
"p50": 788.5,
"median": 788.5,
"p75": 788.5,
"p90": 788.5,
"p95": 788.5,
"p99": 788.5,
"p999": 788.5
}
}
},
"intermediate": [
{
"counters": {
"vusers.created_by_name.Crawl a URL": 4,
"vusers.created": 4,
"http.requests": 4,
"http.codes.200": 3,
"http.responses": 3,
"plugins.metrics-by-endpoint./v0/crawl.codes.200": 3
},
"rates": {
"http.request_rate": 1
},
"http.request_rate": null,
"firstCounterAt": 1716393506465,
"firstHistogramAt": 1716393507755,
"lastCounterAt": 1716393509466,
"lastHistogramAt": 1716393509354,
"firstMetricAt": 1716393506465,
"lastMetricAt": 1716393509466,
"period": "1716393500000",
"summaries": {
"http.response_time": {
"min": 815,
"max": 1183,
"count": 3,
"mean": 1041.3,
"p50": 1130.2,
"median": 1130.2,
"p75": 1130.2,
"p90": 1130.2,
"p95": 1130.2,
"p99": 1130.2,
"p999": 1130.2
},
"plugins.metrics-by-endpoint.response_time./v0/crawl": {
"min": 815,
"max": 1183,
"count": 3,
"mean": 1041.3,
"p50": 1130.2,
"median": 1130.2,
"p75": 1130.2,
"p90": 1130.2,
"p95": 1130.2,
"p99": 1130.2,
"p999": 1130.2
}
},
"histograms": {
"http.response_time": {
"min": 815,
"max": 1183,
"count": 3,
"mean": 1041.3,
"p50": 1130.2,
"median": 1130.2,
"p75": 1130.2,
"p90": 1130.2,
"p95": 1130.2,
"p99": 1130.2,
"p999": 1130.2
},
"plugins.metrics-by-endpoint.response_time./v0/crawl": {
"min": 815,
"max": 1183,
"count": 3,
"mean": 1041.3,
"p50": 1130.2,
"median": 1130.2,
"p75": 1130.2,
"p90": 1130.2,
"p95": 1130.2,
"p99": 1130.2,
"p999": 1130.2
}
}
},
{
"counters": {
"vusers.created_by_name.Crawl a URL": 6,
"vusers.created": 6,
"http.requests": 9,
"http.codes.200": 9,
"http.responses": 9,
"plugins.metrics-by-endpoint./v0/crawl.codes.200": 7,
"plugins.metrics-by-endpoint./v0/crawl/status/ea785a25-fb05-4ac6-85d0-744b95084c93.codes.200": 1,
"vusers.failed": 0,
"vusers.completed": 2,
"plugins.metrics-by-endpoint./v0/crawl/status/c07c910d-4d50-44fa-80aa-fac9869dee38.codes.200": 1
},
"rates": {
"http.request_rate": 1
},
"http.request_rate": null,
"firstCounterAt": 1716393510470,
"firstHistogramAt": 1716393510798,
"lastCounterAt": 1716393519880,
"lastHistogramAt": 1716393519880,
"firstMetricAt": 1716393510470,
"lastMetricAt": 1716393519880,
"period": "1716393510000",
"summaries": {
"http.response_time": {
"min": 833,
"max": 1275,
"count": 9,
"mean": 965.4,
"p50": 944,
"median": 944,
"p75": 1002.4,
"p90": 1022.7,
"p95": 1022.7,
"p99": 1022.7,
"p999": 1022.7
},
"plugins.metrics-by-endpoint.response_time./v0/crawl": {
"min": 833,
"max": 1275,
"count": 7,
"mean": 977.4,
"p50": 963.1,
"median": 963.1,
"p75": 1002.4,
"p90": 1022.7,
"p95": 1022.7,
"p99": 1022.7,
"p999": 1022.7
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/ea785a25-fb05-4ac6-85d0-744b95084c93": {
"min": 943,
"max": 943,
"count": 1,
"mean": 943,
"p50": 944,
"median": 944,
"p75": 944,
"p90": 944,
"p95": 944,
"p99": 944,
"p999": 944
},
"vusers.session_length": {
"min": 12404.1,
"max": 12542,
"count": 2,
"mean": 12473.1,
"p50": 12459.8,
"median": 12459.8,
"p75": 12459.8,
"p90": 12459.8,
"p95": 12459.8,
"p99": 12459.8,
"p999": 12459.8
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/c07c910d-4d50-44fa-80aa-fac9869dee38": {
"min": 904,
"max": 904,
"count": 1,
"mean": 904,
"p50": 907,
"median": 907,
"p75": 907,
"p90": 907,
"p95": 907,
"p99": 907,
"p999": 907
}
},
"histograms": {
"http.response_time": {
"min": 833,
"max": 1275,
"count": 9,
"mean": 965.4,
"p50": 944,
"median": 944,
"p75": 1002.4,
"p90": 1022.7,
"p95": 1022.7,
"p99": 1022.7,
"p999": 1022.7
},
"plugins.metrics-by-endpoint.response_time./v0/crawl": {
"min": 833,
"max": 1275,
"count": 7,
"mean": 977.4,
"p50": 963.1,
"median": 963.1,
"p75": 1002.4,
"p90": 1022.7,
"p95": 1022.7,
"p99": 1022.7,
"p999": 1022.7
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/ea785a25-fb05-4ac6-85d0-744b95084c93": {
"min": 943,
"max": 943,
"count": 1,
"mean": 943,
"p50": 944,
"median": 944,
"p75": 944,
"p90": 944,
"p95": 944,
"p99": 944,
"p999": 944
},
"vusers.session_length": {
"min": 12404.1,
"max": 12542,
"count": 2,
"mean": 12473.1,
"p50": 12459.8,
"median": 12459.8,
"p75": 12459.8,
"p90": 12459.8,
"p95": 12459.8,
"p99": 12459.8,
"p999": 12459.8
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/c07c910d-4d50-44fa-80aa-fac9869dee38": {
"min": 904,
"max": 904,
"count": 1,
"mean": 904,
"p50": 907,
"median": 907,
"p75": 907,
"p90": 907,
"p95": 907,
"p99": 907,
"p999": 907
}
}
},
{
"counters": {
"http.codes.200": 8,
"http.responses": 8,
"plugins.metrics-by-endpoint./v0/crawl/status/15f48c84-cdf7-4918-8a97-926d2887debd.codes.200": 1,
"vusers.failed": 0,
"vusers.completed": 8,
"http.requests": 7,
"plugins.metrics-by-endpoint./v0/crawl/status/c2d3009c-efcb-41b1-bd1e-d8112f80c1ad.codes.200": 1,
"plugins.metrics-by-endpoint./v0/crawl/status/3b22323d-8cab-47d0-ab45-8ed597a12778.codes.200": 1,
"plugins.metrics-by-endpoint./v0/crawl/status/1b7ac23b-9929-42dd-8180-a631b99f3341.codes.200": 1,
"plugins.metrics-by-endpoint./v0/crawl/status/af5c6205-08a5-4564-aaea-ad704d9e5449.codes.200": 1,
"plugins.metrics-by-endpoint./v0/crawl/status/90492ac5-4057-4a44-bc11-7613fd89a3b4.codes.200": 1,
"plugins.metrics-by-endpoint./v0/crawl/status/26f37b1d-0ef0-4107-9f4b-9ccb97826b65.codes.200": 1,
"plugins.metrics-by-endpoint./v0/crawl/status/2e703b76-c2e9-4726-972b-d08761ad6c0e.codes.200": 1
},
"rates": {
"http.request_rate": 1
},
"http.request_rate": null,
"firstCounterAt": 1716393520311,
"firstHistogramAt": 1716393520311,
"lastCounterAt": 1716393527474,
"lastHistogramAt": 1716393527474,
"firstMetricAt": 1716393520311,
"lastMetricAt": 1716393527474,
"period": "1716393520000",
"summaries": {
"http.response_time": {
"min": 757,
"max": 1499,
"count": 8,
"mean": 938.9,
"p50": 871.5,
"median": 871.5,
"p75": 889.1,
"p90": 982.6,
"p95": 982.6,
"p99": 982.6,
"p999": 982.6
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/15f48c84-cdf7-4918-8a97-926d2887debd": {
"min": 888,
"max": 888,
"count": 1,
"mean": 888,
"p50": 889.1,
"median": 889.1,
"p75": 889.1,
"p90": 889.1,
"p95": 889.1,
"p99": 889.1,
"p999": 889.1
},
"vusers.session_length": {
"min": 11864.6,
"max": 13144.5,
"count": 8,
"mean": 12174.1,
"p50": 11971.2,
"median": 11971.2,
"p75": 12213.1,
"p90": 12213.1,
"p95": 12213.1,
"p99": 12213.1,
"p999": 12213.1
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/c2d3009c-efcb-41b1-bd1e-d8112f80c1ad": {
"min": 1499,
"max": 1499,
"count": 1,
"mean": 1499,
"p50": 1495.5,
"median": 1495.5,
"p75": 1495.5,
"p90": 1495.5,
"p95": 1495.5,
"p99": 1495.5,
"p999": 1495.5
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/3b22323d-8cab-47d0-ab45-8ed597a12778": {
"min": 873,
"max": 873,
"count": 1,
"mean": 873,
"p50": 871.5,
"median": 871.5,
"p75": 871.5,
"p90": 871.5,
"p95": 871.5,
"p99": 871.5,
"p999": 871.5
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/1b7ac23b-9929-42dd-8180-a631b99f3341": {
"min": 757,
"max": 757,
"count": 1,
"mean": 757,
"p50": 757.6,
"median": 757.6,
"p75": 757.6,
"p90": 757.6,
"p95": 757.6,
"p99": 757.6,
"p999": 757.6
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/af5c6205-08a5-4564-aaea-ad704d9e5449": {
"min": 892,
"max": 892,
"count": 1,
"mean": 892,
"p50": 889.1,
"median": 889.1,
"p75": 889.1,
"p90": 889.1,
"p95": 889.1,
"p99": 889.1,
"p999": 889.1
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/90492ac5-4057-4a44-bc11-7613fd89a3b4": {
"min": 839,
"max": 839,
"count": 1,
"mean": 839,
"p50": 837.3,
"median": 837.3,
"p75": 837.3,
"p90": 837.3,
"p95": 837.3,
"p99": 837.3,
"p999": 837.3
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/26f37b1d-0ef0-4107-9f4b-9ccb97826b65": {
"min": 979,
"max": 979,
"count": 1,
"mean": 979,
"p50": 982.6,
"median": 982.6,
"p75": 982.6,
"p90": 982.6,
"p95": 982.6,
"p99": 982.6,
"p999": 982.6
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/2e703b76-c2e9-4726-972b-d08761ad6c0e": {
"min": 784,
"max": 784,
"count": 1,
"mean": 784,
"p50": 788.5,
"median": 788.5,
"p75": 788.5,
"p90": 788.5,
"p95": 788.5,
"p99": 788.5,
"p999": 788.5
}
},
"histograms": {
"http.response_time": {
"min": 757,
"max": 1499,
"count": 8,
"mean": 938.9,
"p50": 871.5,
"median": 871.5,
"p75": 889.1,
"p90": 982.6,
"p95": 982.6,
"p99": 982.6,
"p999": 982.6
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/15f48c84-cdf7-4918-8a97-926d2887debd": {
"min": 888,
"max": 888,
"count": 1,
"mean": 888,
"p50": 889.1,
"median": 889.1,
"p75": 889.1,
"p90": 889.1,
"p95": 889.1,
"p99": 889.1,
"p999": 889.1
},
"vusers.session_length": {
"min": 11864.6,
"max": 13144.5,
"count": 8,
"mean": 12174.1,
"p50": 11971.2,
"median": 11971.2,
"p75": 12213.1,
"p90": 12213.1,
"p95": 12213.1,
"p99": 12213.1,
"p999": 12213.1
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/c2d3009c-efcb-41b1-bd1e-d8112f80c1ad": {
"min": 1499,
"max": 1499,
"count": 1,
"mean": 1499,
"p50": 1495.5,
"median": 1495.5,
"p75": 1495.5,
"p90": 1495.5,
"p95": 1495.5,
"p99": 1495.5,
"p999": 1495.5
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/3b22323d-8cab-47d0-ab45-8ed597a12778": {
"min": 873,
"max": 873,
"count": 1,
"mean": 873,
"p50": 871.5,
"median": 871.5,
"p75": 871.5,
"p90": 871.5,
"p95": 871.5,
"p99": 871.5,
"p999": 871.5
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/1b7ac23b-9929-42dd-8180-a631b99f3341": {
"min": 757,
"max": 757,
"count": 1,
"mean": 757,
"p50": 757.6,
"median": 757.6,
"p75": 757.6,
"p90": 757.6,
"p95": 757.6,
"p99": 757.6,
"p999": 757.6
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/af5c6205-08a5-4564-aaea-ad704d9e5449": {
"min": 892,
"max": 892,
"count": 1,
"mean": 892,
"p50": 889.1,
"median": 889.1,
"p75": 889.1,
"p90": 889.1,
"p95": 889.1,
"p99": 889.1,
"p999": 889.1
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/90492ac5-4057-4a44-bc11-7613fd89a3b4": {
"min": 839,
"max": 839,
"count": 1,
"mean": 839,
"p50": 837.3,
"median": 837.3,
"p75": 837.3,
"p90": 837.3,
"p95": 837.3,
"p99": 837.3,
"p999": 837.3
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/26f37b1d-0ef0-4107-9f4b-9ccb97826b65": {
"min": 979,
"max": 979,
"count": 1,
"mean": 979,
"p50": 982.6,
"median": 982.6,
"p75": 982.6,
"p90": 982.6,
"p95": 982.6,
"p99": 982.6,
"p999": 982.6
},
"plugins.metrics-by-endpoint.response_time./v0/crawl/status/2e703b76-c2e9-4726-972b-d08761ad6c0e": {
"min": 784,
"max": 784,
"count": 1,
"mean": 784,
"p50": 788.5,
"median": 788.5,
"p75": 788.5,
"p90": 788.5,
"p95": 788.5,
"p99": 788.5,
"p999": 788.5
}
}
}
]
}

85
apps/test-suite/load-test.yml
View File

@ -3,57 +3,60 @@ config:
http:
timeout: 30
phases:
- duration: 60
arrivalRate: 10 # Initial load
- duration: 120
arrivalRate: 20 # Increased load
- duration: 180
arrivalRate: 30 # Peak load
- duration: 60
arrivalRate: 10 # Cool down
# /scrape
# - duration: 60
# arrivalRate: 10 # Initial load
# - duration: 120
# arrivalRate: 20 # Increased load
# - duration: 180
# arrivalRate: 30 # Peak load
# - duration: 60
# arrivalRate: 10 # Cool down
# /crawl
- duration: 10
arrivalRate: 1
defaults:
headers:
Authorization: "Bearer {{ $env.TEST_API_KEY }}"
scenarios:
- name: Scrape a URL
flow:
- post:
url: "/scrape"
json:
url: "https://www.scrapethissite.com"
pageOptions:
onlyMainContent: true
capture:
- json: "$.data.markdown"
as: markdown_content
# - name: Crawl a URL
# - name: Scrape a URL
# flow:
# - post:
# url: "/crawl"
# url: "/scrape"
# json:
# url: "https://mendable.ai"
# crawlerOptions:
# includes: []
# excludes: ["blog/*"]
# limit: 100
# url: "https://www.scrapethissite.com"
# pageOptions:
# onlyMainContent: true
# capture:
# - json: "$.jobId"
# as: job_id
# - get:
# url: "/crawl/status/{{ job_id }}"
# capture:
# - json: "$.status"
# as: crawl_status
# until:
# - condition: "equals"
# value: "completed"
# variable: "crawl_status"
# retry:
# count: 20
# wait: 10
# - json: "$.data.markdown"
# as: markdown_content
- name: Crawl a URL
flow:
- post:
url: "/crawl"
json:
url: "https://spider.cloud"
crawlerOptions:
limit: 100
pageOptions:
onlyMainContent: true
capture:
- json: "$.jobId"
as: job_id
- think: 10
- get:
url: "/crawl/status/{{ job_id }}"
capture:
- json: "$.status"
as: crawl_status
until:
- condition: "equals"
value: "completed"
variable: "crawl_status"
retry:
count: 20
wait: 10
# - name: Search for a query
# flow:

2
apps/test-suite/package.json
View File

@ -4,7 +4,7 @@
"description": "",
"scripts": {
"test:suite": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false",
"test:load": "artillery run --output ./load-test-results/test-1-5/assets/test-run-report.json load-test.yml",
"test:load": "artillery run --output ./load-test-results/test-run-report.json load-test.yml",
"test:scrape": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathPattern=tests/scrape.test.ts",
"test:crawl": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathPattern=tests/crawl.test.ts"
},