notebooks/run_batch.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run the ReAction agent in batch\n",
    "\n",
    "Run for all:\n",
    "- Tasks\n",
    "- Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from datetime import date, timedelta\n",
    "import time\n",
    "\n",
    "import papermill as pm\n",
    "\n",
    "from react_repl_agent.utils.timedelta import format_timedelta"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup notebook to run and export directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "notebook_to_run_path = Path('./react_repl_agent.ipynb').resolve()\n",
    "assert notebook_to_run_path.exists()\n",
    "print(notebook_to_run_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "notebooks_export_dir = Path(\n",
    "    \"./export/papermill_notebooks_export_\"+ date.today().isoformat()\n",
    ").expanduser()\n",
    "notebooks_export_dir.mkdir(parents=True, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define batch parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = [\"gpt-3.5-turbo\", \"text-davinci-003\", \"gpt-4\"]\n",
    "tasks = [\n",
    "    {\n",
    "        \"name\": \"A_print_date\",\n",
    "        \"description\": \"Print the current date.\",\n",
    "    },\n",
    "    {\n",
    "        \"name\": \"B_Weather_MyLocation_Discord\",\n",
    "        \"description\": \"Send a Discord message with the current weather for my location.\",\n",
    "    },\n",
    "    {\n",
    "        \"name\": \"C_Weather_London_Slack\",\n",
    "        \"description\": \"Send a Slack message to the #experimenting-with-simple-ai-agents channel with the current weather for London, UK.\",\n",
    "    },\n",
    "    {\n",
    "        \"name\": \"D_Weather_Emoji_MyLocation_Discord\",\n",
    "        \"description\": \"Send a descriptive Discord message with the weather forecast for the next 5 days for my location. Describe the weather each day with an appropriate emoji, include the dates in the message.\",\n",
    "    },\n",
    "    {\n",
    "        \"name\": \"E_NASA_Discord\",\n",
    "        \"description\": \"Lookup the NASA picture-of-the day, resize to half it's size, and send to discord with a message starting with 'Hello from Space!' and providing additional information about the image.\",\n",
    "    },\n",
    "    {\n",
    "        \"name\": \"F_MSFT_Stock_Slack\",\n",
    "        \"description\": \"What week in the last 3 months had the lowest MSFT stock closing price? Send the result including price and date to the #experimenting-with-simple-ai-agents Slack channel.\",\n",
    "    },\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for model in models:\n",
    "    for task in tasks:\n",
    "        print(f\"\\n# Running '{model}' for '{task['name']}'\")\n",
    "        notebook_export_path = notebooks_export_dir / model / f\"{model}_{task['name']}.ipynb\"\n",
    "        notebook_export_path = notebook_export_path.resolve()\n",
    "        notebook_export_path.parent.mkdir(parents=True, exist_ok=True)\n",
    "        print(f\"Exporting to '{notebook_export_path!s}'.\")\n",
    "        try:\n",
    "            start_time = time.time()\n",
    "            pm.execute_notebook(\n",
    "                notebook_to_run_path,\n",
    "                notebook_export_path,\n",
    "                parameters=dict(\n",
    "                    llm_model=model,\n",
    "                    task_name=task[\"name\"],\n",
    "                    task_description=task[\"description\"],\n",
    "                ),\n",
    "            )\n",
    "            runtime = timedelta(seconds=time.time() - start_time)\n",
    "            print(f\"SUCCESS in {format_timedelta(runtime)}!\")\n",
    "        except pm.PapermillExecutionError as exc:\n",
    "            print(f\"ERROR!  {exc.ename}: {exc.evalue}\")\n",
    "print(\"DONE!\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "react_repl_agent",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}