{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":703131946,"defaultBranch":"main","name":"llmperf","ownerLogin":"ray-project","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2023-10-10T16:49:58.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/22125274?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1708810843.0","currentOid":""},"activityList":{"items":[{"before":"5acd7b1a75828be841af8e7d81aacc72b48f408d","after":"afd137ae21b4494f6be6b6bcd4bb526bcb7e03ed","ref":"refs/heads/main","pushedAt":"2024-04-08T02:09:35.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"rickyyx","name":"Ricky Xu ","path":"/rickyyx","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11676094?s=80&v=4"},"commit":{"message":"Fix decoding throughput computation (#45)\n\n* Fix tokens/s\r\n\r\n* remove print","shortMessageHtmlLink":"Fix decoding throughput computation (#45)"}},{"before":"ca6e5894aae6a2a48dd45e9baf032a77b73acc3d","after":"5acd7b1a75828be841af8e7d81aacc72b48f408d","ref":"refs/heads/main","pushedAt":"2024-02-24T21:53:05.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"xieus","name":"Liguang Xie","path":"/xieus","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12957223?s=80&v=4"},"commit":{"message":"Fix Installation instructions in README.md (#26)\n\nSigned-off-by: Michael Goin ","shortMessageHtmlLink":"Fix Installation instructions in README.md (#26)"}},{"before":null,"after":"6b8eaa5f40bf8ee6f4f5e216bddbbdb06fc694aa","ref":"refs/heads/xieus-patch-1","pushedAt":"2024-02-24T21:40:43.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"xieus","name":"Liguang Xie","path":"/xieus","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12957223?s=80&v=4"},"commit":{"message":"Update README.md\n\nSigned-off-by: Liguang Xie ","shortMessageHtmlLink":"Update README.md"}},{"before":"46eb92d4eb7940daa5d093a0220acd8d4f4834ba","after":null,"ref":"refs/heads/pr-typo-and-caveats","pushedAt":"2023-12-29T23:44:01.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"rickyyx","name":"Ricky Xu ","path":"/rickyyx","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11676094?s=80&v=4"}},{"before":"02b0613638d19625a16250a1843c85e50c089d9e","after":"ca6e5894aae6a2a48dd45e9baf032a77b73acc3d","ref":"refs/heads/main","pushedAt":"2023-12-29T23:44:00.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"rickyyx","name":"Ricky Xu ","path":"/rickyyx","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11676094?s=80&v=4"},"commit":{"message":"Fix typos and add caveats (#27)\n\n\r\n---------\r\n\r\nSigned-off-by: rickyyx ","shortMessageHtmlLink":"Fix typos and add caveats (#27)"}},{"before":"14348942fc1a8795a42c2613fc2857421b945ee1","after":"46eb92d4eb7940daa5d093a0220acd8d4f4834ba","ref":"refs/heads/pr-typo-and-caveats","pushedAt":"2023-12-29T23:01:24.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"rickyyx","name":"Ricky Xu ","path":"/rickyyx","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11676094?s=80&v=4"},"commit":{"message":"update\n\nSigned-off-by: rickyyx ","shortMessageHtmlLink":"update"}},{"before":"623f7854dd0bcfdfa2b8012a241d12dff3013567","after":"14348942fc1a8795a42c2613fc2857421b945ee1","ref":"refs/heads/pr-typo-and-caveats","pushedAt":"2023-12-29T19:49:56.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"rickyyx","name":"Ricky Xu ","path":"/rickyyx","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11676094?s=80&v=4"},"commit":{"message":"nits\n\nSigned-off-by: rickyyx ","shortMessageHtmlLink":"nits"}},{"before":"2c9f74b1d37bb9ead5a51ea4a079a5b714098527","after":"623f7854dd0bcfdfa2b8012a241d12dff3013567","ref":"refs/heads/pr-typo-and-caveats","pushedAt":"2023-12-29T19:45:57.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"rickyyx","name":"Ricky Xu ","path":"/rickyyx","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11676094?s=80&v=4"},"commit":{"message":"update\n\nSigned-off-by: rickyyx ","shortMessageHtmlLink":"update"}},{"before":null,"after":"2c9f74b1d37bb9ead5a51ea4a079a5b714098527","ref":"refs/heads/pr-typo-and-caveats","pushedAt":"2023-12-29T19:06:30.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"rickyyx","name":"Ricky Xu ","path":"/rickyyx","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11676094?s=80&v=4"},"commit":{"message":"update\n\nSigned-off-by: rickyyx ","shortMessageHtmlLink":"update"}},{"before":"1eac866f91773bff401f96e74c1cf20c38778329","after":"02b0613638d19625a16250a1843c85e50c089d9e","ref":"refs/heads/main","pushedAt":"2023-12-22T02:29:18.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"kylehh","name":"kYLe","path":"/kylehh","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/24217337?s=80&v=4"},"commit":{"message":"fix typo (#22)\n\nSigned-off-by: Aniket Maurya ","shortMessageHtmlLink":"fix typo (#22)"}},{"before":"fab9d1ea077592aff8547b5aae186958eb66f2ea","after":"5beb24c46a6ce3a30b03f712ede86704394477b4","ref":"refs/heads/kh/add-json-mode-prob","pushedAt":"2023-12-20T02:48:24.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"kouroshHakha","name":"kourosh hakhamaneshi","path":"/kouroshHakha","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/31483498?s=80&v=4"},"commit":{"message":"adding schema for testing\n\nSigned-off-by: Kourosh Hakhamaneshi ","shortMessageHtmlLink":"adding schema for testing"}},{"before":null,"after":"fab9d1ea077592aff8547b5aae186958eb66f2ea","ref":"refs/heads/kh/add-json-mode-prob","pushedAt":"2023-12-19T21:56:37.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"kouroshHakha","name":"kourosh hakhamaneshi","path":"/kouroshHakha","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/31483498?s=80&v=4"},"commit":{"message":"Added json mode probe to set with what probablity should queries get sent with a json mode response format. This feature allows us to benchmark clients under a mix of json mode and normal requests\n\nSigned-off-by: Kourosh Hakhamaneshi ","shortMessageHtmlLink":"Added json mode probe to set with what probablity should queries get …"}},{"before":"1e42cb3fc45134ab25d1994b4e2d87b9fcbdd5f5","after":null,"ref":"refs/heads/llmperfv2","pushedAt":"2023-12-06T01:32:59.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"avnishn","name":"Avnish Narayan","path":"/avnishn","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/38871737?s=80&v=4"}},{"before":"ae8a418477f45e1737162d9e2c59b7da569dd962","after":"1eac866f91773bff401f96e74c1cf20c38778329","ref":"refs/heads/main","pushedAt":"2023-12-06T01:32:58.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"avnishn","name":"Avnish Narayan","path":"/avnishn","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/38871737?s=80&v=4"},"commit":{"message":"LLMPerfV2 (#19)\n\n* LLMPerfV2\r\n\r\nThe latest version of LLMPerf brings a suite of significant updates designed to provide more in-depth and customizable benchmarking capabilities for LLM inference. These updates include:\r\n\r\n- Expanded metrics with quantile distribution (P25-99): Comprehensive data representation for deeper insights.\r\n- Customizable benchmarking parameters: Tailor parameters to fit specific use case scenarios.\r\n- Introduction of load test and correctness test: Assessing performance and accuracy under stress.\r\n- Broad compatibility: Supports a range of products including [Anyscale Endpoints](https://www.anyscale.com/endpoints), [OpenAI](https://openai.com/blog/openai-api), [Anthropic](https://docs.anthropic.com/claude/reference/getting-started-with-the-api), [together.ai](http://together.ai/), [Fireworks.ai](https://app.fireworks.ai/), [Perplexity](https://www.perplexity.ai/), [Huggingface](https://huggingface.co/inference-endpoints), [Lepton AI](https://www.lepton.ai/docs/overview/model_apis), and various APIs supported by the [LiteLLM project](https://litellm.ai/)).\r\n- Easy addition of new LLMs via the LLMClient API.\r\n\r\nSigned-off-by: Avnish Narayan ","shortMessageHtmlLink":"LLMPerfV2 (#19)"}},{"before":"4d1f8efb4d1c3217fc7f0ce1bbb4617d3f40173d","after":"1e42cb3fc45134ab25d1994b4e2d87b9fcbdd5f5","ref":"refs/heads/llmperfv2","pushedAt":"2023-12-05T02:43:19.000Z","pushType":"push","commitsCount":5,"pusher":{"login":"avnishn","name":"Avnish Narayan","path":"/avnishn","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/38871737?s=80&v=4"},"commit":{"message":"Merge master\n\nSigned-off-by: Avnish Narayan ","shortMessageHtmlLink":"Merge master"}},{"before":"15c97b56a01d0eef755dfd22736a3e4b1280281b","after":null,"ref":"refs/heads/kylehh-patch-1","pushedAt":"2023-12-04T23:55:16.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"rickyyx","name":"Ricky Xu ","path":"/rickyyx","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11676094?s=80&v=4"}},{"before":"ccd460557763b97670d1c67c0cb7d84161f87f8b","after":"ae8a418477f45e1737162d9e2c59b7da569dd962","ref":"refs/heads/main","pushedAt":"2023-12-04T23:55:15.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"rickyyx","name":"Ricky Xu ","path":"/rickyyx","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11676094?s=80&v=4"},"commit":{"message":"Update llmperf.py (#20)\n\n* Update llmperf.py\r\n\r\ncorrect merge with #15\r\n\r\nSigned-off-by: kYLe \r\n\r\n* Update llmperf.py\r\n\r\nSigned-off-by: kYLe \r\n\r\n---------\r\n\r\nSigned-off-by: kYLe ","shortMessageHtmlLink":"Update llmperf.py (#20)"}},{"before":"29b770fc625c70c92eca6236f6527cb020458de8","after":"15c97b56a01d0eef755dfd22736a3e4b1280281b","ref":"refs/heads/kylehh-patch-1","pushedAt":"2023-12-04T23:44:41.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"kylehh","name":"kYLe","path":"/kylehh","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/24217337?s=80&v=4"},"commit":{"message":"Update llmperf.py\n\nSigned-off-by: kYLe ","shortMessageHtmlLink":"Update llmperf.py"}},{"before":null,"after":"29b770fc625c70c92eca6236f6527cb020458de8","ref":"refs/heads/kylehh-patch-1","pushedAt":"2023-12-04T23:40:40.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"kylehh","name":"kYLe","path":"/kylehh","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/24217337?s=80&v=4"},"commit":{"message":"Update llmperf.py\n\ncorrect merge with #15\n\nSigned-off-by: kYLe ","shortMessageHtmlLink":"Update llmperf.py"}},{"before":"e7a02dde4521a22c779f08070d9508a947a95c21","after":"ccd460557763b97670d1c67c0cb7d84161f87f8b","ref":"refs/heads/main","pushedAt":"2023-12-04T23:29:55.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"kylehh","name":"kYLe","path":"/kylehh","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/24217337?s=80&v=4"},"commit":{"message":"Refactor endpoint config (#15)\n\n* refactored endpoint config and framework handling\r\n\r\n* corrected minor bug\r\n\r\n* newline at the end\r\n\r\n* merged changes from upstream\r\n\r\n---------\r\n\r\nSigned-off-by: kYLe \r\nCo-authored-by: kYLe ","shortMessageHtmlLink":"Refactor endpoint config (#15)"}},{"before":"a6159b862a812521add1fc52e29717f129550812","after":"e7a02dde4521a22c779f08070d9508a947a95c21","ref":"refs/heads/main","pushedAt":"2023-12-04T23:24:34.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"kylehh","name":"kYLe","path":"/kylehh","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/24217337?s=80&v=4"},"commit":{"message":"Cleanup args handling (#14)\n\n* removed global args dependency and added explicit params\r\n\r\n* updated openai dependency-new version has breaking changes\r\n\r\n* fix minor spelling error in prompt\r\n\r\n* removed unused import","shortMessageHtmlLink":"Cleanup args handling (#14)"}},{"before":"fec74f4c87e1609ef1ff1304342030dc0b12d99d","after":"4d1f8efb4d1c3217fc7f0ce1bbb4617d3f40173d","ref":"refs/heads/llmperfv2","pushedAt":"2023-12-04T22:22:55.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"avnishn","name":"Avnish Narayan","path":"/avnishn","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/38871737?s=80&v=4"},"commit":{"message":"Add notice back in\n\nSigned-off-by: Avnish Narayan ","shortMessageHtmlLink":"Add notice back in"}},{"before":null,"after":"fec74f4c87e1609ef1ff1304342030dc0b12d99d","ref":"refs/heads/llmperfv2","pushedAt":"2023-12-04T22:18:03.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"avnishn","name":"Avnish Narayan","path":"/avnishn","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/38871737?s=80&v=4"},"commit":{"message":"LLMPerfV2\n\nSigned-off-by: Avnish Narayan ","shortMessageHtmlLink":"LLMPerfV2"}},{"before":"a6159b862a812521add1fc52e29717f129550812","after":"1392907f9602af190889dfa99f98f9e544713832","ref":"refs/heads/pr-leader-board-readme","pushedAt":"2023-12-04T17:55:22.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"rickyyx","name":"Ricky Xu ","path":"/rickyyx","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11676094?s=80&v=4"},"commit":{"message":"update\n\nSigned-off-by: Ricky Xu ","shortMessageHtmlLink":"update"}},{"before":"1bc383ca2ec84a9bfc0dd08b8ecd370b0ef2cdd2","after":"a6159b862a812521add1fc52e29717f129550812","ref":"refs/heads/pr-leader-board-readme","pushedAt":"2023-12-04T17:52:35.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"rickyyx","name":"Ricky Xu ","path":"/rickyyx","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11676094?s=80&v=4"},"commit":{"message":"Support HF TGI API endpoints (#3)","shortMessageHtmlLink":"Support HF TGI API endpoints (#3)"}},{"before":null,"after":"1bc383ca2ec84a9bfc0dd08b8ecd370b0ef2cdd2","ref":"refs/heads/pr-leader-board-readme","pushedAt":"2023-12-04T17:51:38.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"rickyyx","name":"Ricky Xu ","path":"/rickyyx","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11676094?s=80&v=4"},"commit":{"message":"update\n\nSigned-off-by: Ricky Xu ","shortMessageHtmlLink":"update"}},{"before":"1bc383ca2ec84a9bfc0dd08b8ecd370b0ef2cdd2","after":null,"ref":"refs/heads/pr-leader-board-readme","pushedAt":"2023-12-04T17:48:59.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"rickyyx","name":"Ricky Xu ","path":"/rickyyx","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11676094?s=80&v=4"}},{"before":"ecd4c2d0050f3af6ace2f81257ecda47b76f0fee","after":"1bc383ca2ec84a9bfc0dd08b8ecd370b0ef2cdd2","ref":"refs/heads/pr-leader-board-readme","pushedAt":"2023-11-29T19:30:29.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"rickyyx","name":"Ricky Xu ","path":"/rickyyx","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11676094?s=80&v=4"},"commit":{"message":"update\n\nSigned-off-by: Ricky Xu ","shortMessageHtmlLink":"update"}},{"before":"72ee3a94e7b40f7ed226b9e4c2ae14bd2a11a0b0","after":"ecd4c2d0050f3af6ace2f81257ecda47b76f0fee","ref":"refs/heads/pr-leader-board-readme","pushedAt":"2023-11-29T19:26:13.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"rickyyx","name":"Ricky Xu ","path":"/rickyyx","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11676094?s=80&v=4"},"commit":{"message":"update\n\nSigned-off-by: Ricky Xu ","shortMessageHtmlLink":"update"}},{"before":"847b0eb4894400edd394ad94b4d6f3d6de58e23a","after":"72ee3a94e7b40f7ed226b9e4c2ae14bd2a11a0b0","ref":"refs/heads/pr-leader-board-readme","pushedAt":"2023-11-29T19:23:52.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"rickyyx","name":"Ricky Xu ","path":"/rickyyx","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11676094?s=80&v=4"},"commit":{"message":"update\n\nSigned-off-by: Ricky Xu ","shortMessageHtmlLink":"update"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAEKm-TYAA","startCursor":null,"endCursor":null}},"title":"Activity · ray-project/llmperf"}